def test_list_of_dictionaries_function_02(self):
dataList = [{
"owner": "daisy",
"pet": "dog",
"address": "belfast, uk"
}, {
"owner": "john",
"pet": "snake",
"address": "the moon"
}, {
"owner": "susan",
"pet": "crocodile",
"address": "larne"
}]
from fundamentals.renderer import list_of_dictionaries
dataSet = list_of_dictionaries(log=log, listOfDictionaries=dataList)
print dataSet.table()
dataSet.table(filepath=pathToOutputDir + "myData.dat")
print dataSet.csv()
dataSet.csv(filepath=pathToOutputDir + "myData.csv")
print dataSet.mysql(tableName="TNS")
dataSet.mysql(tableName="TNS",
filepath=pathToOutputDir + "myData.mysql")
print dataSet.json()
dataSet.json(filepath=pathToOutputDir + "myData.json")
print dataSet.yaml()
dataSet.yaml(filepath=pathToOutputDir + "myData.yaml")
print dataSet.markdown()
dataSet.markdown(filepath=pathToOutputDir + "myData.md")
def search(self):
"""
*Return the results of the database conesearch*
**Return**
- ``conesearch``
**Usage**
See class usage.
"""
self.log.debug('starting the ``get`` method')
sqlQuery = self._get_on_trixel_sources_from_database_query()
databaseRows = self._execute_query(sqlQuery)
matchIndies, matches = self._list_crossmatch(databaseRows)
from fundamentals.renderer import list_of_dictionaries
matches = list_of_dictionaries(log=self.log,
listOfDictionaries=matches,
reDatetime=self.reDatetime)
self.log.debug('completed the ``get`` method')
return matchIndies, matches
def search(self):
"""
*Return the results of the database conesearch*
**Return:**
- ``conesearch``
**Usage:**
See class usage.
"""
self.log.info('starting the ``get`` method')
sqlQuery = self._get_on_trixel_sources_from_database_query()
databaseRows = self._execute_query(sqlQuery)
matchIndies, matches = self._list_crossmatch(databaseRows)
from fundamentals.renderer import list_of_dictionaries
matches = list_of_dictionaries(
log=self.log,
listOfDictionaries=matches
)
self.log.info('completed the ``get`` method')
return matchIndies, matches
def test_list_of_dictionaries_function_exception(self):
from fundamentals.renderer import list_of_dictionaries
try:
this = list_of_dictionaries(log=log,
listOfDictionaries=listOfDictionaries,
fakeKey="break the code")
this.get()
assert False
except Exception, e:
assert True
print str(e)
def test_list_of_dictionaries_function_exception(self):
from fundamentals.renderer import list_of_dictionaries
try:
this = list_of_dictionaries(
log=log,
listOfDictionaries=listOfDictionaries,
fakeKey="break the code"
)
this.get()
assert False
except Exception, e:
assert True
print str(e)
def test_list_of_dictionaries_function(self):
from fundamentals.renderer import list_of_dictionaries
dataSet = list_of_dictionaries(log=log,
listOfDictionaries=listOfDictionaries)
print dataSet.table()
dataSet.table(filepath=pathToOutputDir + "myData.dat")
print dataSet.csv()
dataSet.csv(filepath=pathToOutputDir + "myData.csv")
print dataSet.mysql(tableName="TNS")
dataSet.mysql(tableName="TNS",
filepath=pathToOutputDir + "myData.mysql")
print dataSet.json()
dataSet.json(filepath=pathToOutputDir + "myData.json")
print dataSet.yaml()
dataSet.yaml(filepath=pathToOutputDir + "myData.yaml")
print dataSet.markdown()
dataSet.markdown(filepath=pathToOutputDir + "myData.md")
def test_list_of_dictionaries_function(self):
from fundamentals.renderer import list_of_dictionaries
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=listOfDictionaries
)
print dataSet.table()
dataSet.table(filepath=pathToOutputDir + "myData.dat")
print dataSet.csv()
dataSet.csv(filepath=pathToOutputDir + "myData.csv")
print dataSet.mysql(tableName="TNS")
dataSet.mysql(tableName="TNS",
filepath=pathToOutputDir + "myData.mysql")
print dataSet.json()
dataSet.json(filepath=pathToOutputDir + "myData.json")
print dataSet.yaml()
dataSet.yaml(filepath=pathToOutputDir + "myData.yaml")
print dataSet.markdown()
dataSet.markdown(filepath=pathToOutputDir + "myData.md")
def test_list_of_dictionaries_function_02(self):
dataList = [
{
"owner": "daisy",
"pet": "dog",
"address": "belfast, uk"
},
{
"owner": "john",
"pet": "snake",
"address": "the moon"
},
{
"owner": "susan",
"pet": "crocodile",
"address": "larne"
}
]
from fundamentals.renderer import list_of_dictionaries
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=dataList
)
print dataSet.table()
dataSet.table(filepath=pathToOutputDir + "myData.dat")
print dataSet.csv()
dataSet.csv(filepath=pathToOutputDir + "myData.csv")
print dataSet.mysql(tableName="TNS")
dataSet.mysql(tableName="TNS",
filepath=pathToOutputDir + "myData.mysql")
print dataSet.json()
dataSet.json(filepath=pathToOutputDir + "myData.json")
print dataSet.yaml()
dataSet.yaml(filepath=pathToOutputDir + "myData.yaml")
print dataSet.markdown()
dataSet.markdown(filepath=pathToOutputDir + "myData.md")
def get(self):
"""
*get the cone_search object*
**Return**
- ``results`` -- the results of the conesearch
"""
self.log.debug('starting the ``get`` method')
# sort results by angular separation
from operator import itemgetter
results = list(self.squareResults)
results = sorted(results,
key=itemgetter('separation_arcsec'),
reverse=True)
# order of results
headers = [
"sdss_name", "type", "ra", "dec", "specz", "specz_err", "photoz",
"photoz_err", "separation_arcsec", "separation_north_arcsec",
"separation_east_arcsec"
]
import collections
orderDict = collections.OrderedDict(sorted({}.items()))
# filter out results greater than the search radius
filteredResults = []
for row in results:
if float(row["separation_arcsec"]) < self.searchRadius:
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if h in list(row.keys()):
orderDict[h] = row[h]
filteredResults.append(orderDict)
else:
pass
if self.nearest and len(filteredResults):
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if h in list(filteredResults[0].keys()):
orderDict[h] = row[h]
filteredResults = [orderDict]
# filteredResults = [filteredResults[0]]
if not len(filteredResults):
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if self.galaxyType == "all" or self.galaxyType == False or (
self.galaxyType == "specz"
and h not in ["photoz_err", "photoz"]) or (
self.galaxyType == "photoz"
and h not in ["specz", "specz_err"]):
orderDict[h] = ""
filteredResults = [orderDict]
# pretty format print
dataSet = list_of_dictionaries(log=self.log,
listOfDictionaries=list(
reversed(filteredResults)))
if self.outputFormat == "csv":
results = dataSet.csv()
else:
results = dataSet.table()
# sdss only allows 60 hits per minute
sleep(1)
self.log.debug('completed the ``get`` method')
return results
class workspace():
"""
*tools for sorting, archiving and indexing tasks and maintaining the contents of all taskpaper files within a given workspace*
**Key Arguments:**
- ``log`` -- logger
- ``fileOrWorkspacePath`` -- the root path of the workspace you wish to sort the taskpaper docs within, or the path to a single taskpaper file
- ``settings`` -- the settings dictionary
**Usage:**
To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).
To initiate a taskpaper workspace object, use the following:
.. code-block:: python
from tastic.workspace import workspace
ws = workspace(
log=log,
settings=settings,
fileOrWorkspacePath="/path/to/root/of/workspace"
)
or to target a single taskpaper document use instead the path to the file:
.. code-block:: python
from tastic.workspace import workspace
ws = workspace(
log=log,
settings=settings,
fileOrWorkspacePath="/path/to/doc.taskpaper"
)
"""
# Initialisation
def __init__(self, log, fileOrWorkspacePath, settings=False):
self.log = log
log.debug("instansiating a new 'sort' object")
self.settings = settings
self.taskpaperPath = False
self.workspaceRoot = False
# xt-self-arg-tmpx
# INITIAL ACTIONS
# ARE WE DEALING WITH A WORKSPACE DIRECTORY OR SINGLE FILE
if os.path.isfile(fileOrWorkspacePath):
self.taskpaperPath = fileOrWorkspacePath
else:
self.workspaceRoot = fileOrWorkspacePath
self.taskpaperFiles = self._get_all_taskpaper_files()
return None
def sort(self):
"""
*sort the workspace or individual taskpaper document via the workflow tags found in the settings file*
**Usage:**
To sort all of the taskpaper documents in the workspace via the workflow tag set with the settings file, for example:
.. code-block:: yaml
workflowTags: "@due, @flag, @hold, @next, @someday, @wait"
use the ``sort()`` method:
.. code-block:: python
ws.sort()
"""
self.log.info('starting the ``sort`` method')
for f in self.taskpaperFiles:
self._sort_tp_file(f)
self.log.info('completed the ``sort`` method')
return None
def archive_done(self):
"""*move done tasks from the document's 'Archive' project into an adjacent markdown tasklog file*
**Usage:**
To move the archived tasks within a workspace's taskpaper docs into ``-tasklog.md`` files use the ``archive_done()`` method:
.. code-block:: python
ws.archive_done()
"""
self.log.info('starting the ``archive_done`` method')
for f in self.taskpaperFiles:
self._archive_tp_file_done_tasks(f)
self.log.info('completed the ``archive_done`` method')
return None
def _get_all_taskpaper_files(self):
"""*get a list of all the taskpaper filepaths in the workspace*
**Return:**
- ``taskpaperFiles`` -- a list of paths to all the taskpaper files within the workspace
"""
self.log.info('starting the ``_get_all_taskpaper_files`` method')
if self.workspaceRoot:
from fundamentals.files import recursive_directory_listing
theseFiles = recursive_directory_listing(
log=self.log,
baseFolderPath=self.workspaceRoot,
whatToList="files" # all | files | dirs
)
taskpaperFiles = []
taskpaperFiles[:] = [
f for f in theseFiles if os.path.splitext(f)[1] == ".taskpaper"
]
else:
taskpaperFiles = [self.taskpaperPath]
self.log.info('completed the ``_get_all_taskpaper_files`` method')
return taskpaperFiles
def _sort_tp_file(self, taskpaperPath):
"""*sort individual taskpaper documents*
**Key Arguments:**
- ``taskpaperPath`` -- path to a taskpaper file
**Return:**
- None
"""
self.log.info('starting the ``_sort_tp_file`` method')
# OPEN TASKPAPER FILE
self.log.info("sorting taskpaper file %(taskpaperPath)s" % locals())
doc = document(taskpaperPath)
doc.tidy()
doc.sort_tasks(self.settings["workflowTags"])
doc.sort_projects(self.settings["workflowTags"])
doc.save()
self.log.info('completed the ``_sort_tp_file`` method')
return None
def _archive_tp_file_done_tasks(self, taskpaperPath):
"""* archive tp file done tasks*
**Key Arguments:**
- ``taskpaperPath`` -- path to a taskpaper file
**Return:**
- None
"""
self.log.info('starting the ``_archive_tp_file_done_tasks`` method')
self.log.info("archiving taskpaper file %(taskpaperPath)s" % locals())
taskLog = {}
mdArchiveFile = taskpaperPath.replace(".taskpaper", "-tasklog.md")
exists = os.path.exists(mdArchiveFile)
if exists:
pathToReadFile = mdArchiveFile
try:
self.log.debug("attempting to open the file %s" %
(pathToReadFile, ))
readFile = codecs.open(pathToReadFile,
encoding='utf-8',
mode='r')
thisData = readFile.read()
readFile.close()
except IOError, e:
message = 'could not open the file %s' % (pathToReadFile, )
self.log.critical(message)
raise IOError(message)
readFile.close()
table = False
for l in thisData.split("\n"):
l = l.encode("utf-8")
if ":---" in l:
table = True
continue
if table == True and len(l) and l[0] == "|":
dictt = collections.OrderedDict(sorted({}.items()))
columns = l.split("|")
dictt["task"] = columns[1].strip().decode("utf-8")
dictt["completed"] = columns[2].strip().decode("utf-8")
dictt["project"] = columns[3].strip().decode("utf-8")
taskLog[dictt["task"] + dictt["completed"] +
dictt["project"]] = dictt
doc = document(taskpaperPath)
aProject = doc.get_project("Archive")
if not aProject:
return
doneTasks = aProject.tagged_tasks("@done")
for task in doneTasks:
dateCompleted = ""
project = ""
for t in task.tags:
if "done" in t:
dateCompleted = t.replace("done",
"").replace("(",
"").replace(")", "")
if "project(" in t:
project = t.replace("project",
"").replace("(", "").replace(")", "")
dictt = collections.OrderedDict(sorted({}.items()))
notes = ""
if task.notes:
for n in task.notes:
if len(notes) and notes[-2:] != ". ":
if notes[-1] == ".":
notes += " "
else:
notes += ". "
notes += n.title
if len(notes):
notes = "<br><br>**NOTES:**<br>" + \
"<br>".join(textwrap.wrap(
notes, 120, break_long_words=True))
dictt["task"] = "<br>".join(
textwrap.wrap(task.title[2:], 120,
break_long_words=True)) + notes
dictt["task"] = dictt["task"].encode("utf-8")
dictt["completed"] = dateCompleted
dictt["project"] = project
# SET ENCODE ERROR RETURN VALUE
# RECODE INTO ASCII
dictt["task"] = dictt["task"].decode("utf-8")
dictt["completed"] = dictt["completed"].decode("utf-8")
dictt["project"] = dictt["project"].decode("utf-8")
taskLog[dictt["task"] + dictt["completed"] +
dictt["project"]] = dictt
taskLog = taskLog.values()
taskLog = sorted(taskLog, key=itemgetter('task'), reverse=True)
taskLog = sorted(taskLog, key=itemgetter('project'), reverse=True)
taskLog = sorted(taskLog, key=itemgetter('completed'), reverse=True)
dataSet = list_of_dictionaries(log=self.log,
listOfDictionaries=taskLog)
markdownData = dataSet.markdown(filepath=None)
try:
self.log.debug("attempting to open the file %s" %
(mdArchiveFile, ))
writeFile = codecs.open(mdArchiveFile, encoding='utf-8', mode='w')
except IOError, e:
message = 'could not open the file %s' % (mdArchiveFile, )
self.log.critical(message)
raise IOError(message)
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="rockfinder",
defaultSettingsFile=True
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# CALL FUNCTIONS/OBJECTS
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/rockfinder/rockfinder.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
if where and orbfitFlag:
from rockfinder import orbfit_ephemeris
eph = orbfit_ephemeris(
log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
settings=settings,
verbose=extraFlag
)
else:
from rockfinder import jpl_horizons_ephemeris
eph = jpl_horizons_ephemeris(
log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
verbose=extraFlag
)
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=eph
)
# xfundamentals-render-list-of-dictionaries
output = dataSet.table(filepath=None)
if csv:
output = dataSet.csv(filepath=None)
elif json:
output = dataSet.json(filepath=None)
elif yaml:
output = dataSet.yaml(filepath=None)
elif md:
output = dataSet.markdown(filepath=None)
elif rst:
output = dataSet.reST(filepath=None)
print output
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def parse_panstarrs_nightlogs(
self,
updateAll=False):
"""*download and parse the ps1 night logs from the range of time a wave survey campaign is active*
The night-log data is added to the ps1_nightlogs table
**Key Arguments:**
- ``updateAll`` -- update all of the PS1 nightlogs. This will take sometime, the default is to lift the logs from the last 7 days. Default *False*.
**Return:**
- None
**Usage:**
.. todo::
- add usage info
- create a sublime snippet for usage
- update package tutorial if needed
.. code-block:: python
usage code
"""
self.log.debug('starting the ``parse_panstarrs_nightlogs`` method')
# CONVERTER TO CONVERT MJD TO DATE
converter = conversions(
log=self.log
)
createStatement = """
CREATE TABLE `ps1_nightlogs` (
`primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter',
`airm` double DEFAULT NULL,
`comments` varchar(200) DEFAULT NULL,
`decDeg` double DEFAULT NULL,
`etime` double DEFAULT NULL,
`f` varchar(10) DEFAULT NULL,
`filesetID` varchar(100) DEFAULT NULL,
`raDeg` double DEFAULT NULL,
`telescope_pointing` varchar(200) DEFAULT NULL,
`time_registered` datetime DEFAULT NULL,
`type` varchar(100) DEFAULT NULL,
`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,
`dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,
`updated` varchar(45) DEFAULT '0',
PRIMARY KEY (`primaryId`),
UNIQUE KEY `filesetid` (`filesetID`)
) ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=latin1;
"""
from astrocalc.times import now
mjdNow = now(
log=self.log
).get_mjd()
# WAVE METADATA FOUND IN SETTINGS FILE
for wave in self.settings["gravitational waves"]:
# GIVE A 3 DAY WINDOW EITHER SIDE OF WAVE TIME-RANGE
mjdLower = int(self.settings["gravitational waves"][
wave]["mjd"] - 21. - 3.)
mjdUpper = int(self.settings["gravitational waves"][
wave]["mjd"] + 31. + 3.)
if updateAll == False:
if mjdUpper < mjdNow - 7.:
continue
if mjdUpper > mjdNow:
mjdUpper = int(mjdNow)
if mjdLower < mjdNow - 7.:
mjdLower = int(mjdNow - 7.)
# METRIC NIGHT LOGS FOR EACH NIGHT FOUND AT A URL SIMILAR TO :
# "http://ipp0022.ifa.hawaii.edu/ps1sc/metrics/2016-12-14/index.html"
urls = []
for i in range(mjdUpper - mjdLower + 3):
mjd = i + mjdLower
utDate = converter.mjd_to_ut_datetime(
mjd=mjd,
sqlDate=False,
datetimeObject=True
)
utDate = utDate.strftime("%Y-%m-%d")
urls.append("http://ipp0022.ifa.hawaii.edu/ps1sc/metrics/%(utDate)s/index.html" % locals(
))
localUrls = multiobject_download(
urlList=urls,
downloadDirectory="/tmp",
log=self.log,
timeStamp=True,
timeout=180,
concurrentDownloads=2,
resetFilename=False,
credentials=False, # { 'username' : "...", "password", "..." }
longTime=True,
indexFilenames=False
)
for url in localUrls:
if not url:
continue
pathToReadFile = url
try:
self.log.debug("attempting to open the file %s" %
(pathToReadFile,))
readFile = codecs.open(
pathToReadFile, encoding='utf-8', mode='r')
thisData = readFile.read()
readFile.close()
except IOError, e:
message = 'could not open the file %s' % (pathToReadFile,)
self.log.critical(message)
raise IOError(message)
readFile.close()
regex = re.compile(r'<pre>\s*# (filesetID.*?)</pre>', re.S)
matchObject = re.finditer(
regex,
thisData
)
for match in matchObject:
csvReader = csv.DictReader(
io.StringIO(match.group(1)), delimiter='|')
nightLog = []
for row in csvReader:
cleanDict = {}
for k, v in row.iteritems():
cleanDict[k.strip().replace(" ", "_")] = v.strip()
if "telescope_pointing" in cleanDict:
cleanDict["raDeg"] = cleanDict["telescope_pointing"].split()[
0]
cleanDict["decDeg"] = cleanDict["telescope_pointing"].split()[
1]
if "time_registered" in cleanDict:
cleanDict["time_registered"] = cleanDict[
"time_registered"].replace("Z", "")
nightLog.append(cleanDict)
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=nightLog
)
# Recursively create missing directories
if not os.path.exists("/tmp/ps1_nightlogs"):
os.makedirs("/tmp/ps1_nightlogs")
mysqlData = dataSet.mysql(
tableName="ps1_nightlogs", filepath="/tmp/ps1_nightlogs/ps1_nightlog_%(utDate)s.sql" % locals(), createStatement=createStatement)
directory_script_runner(
log=self.log,
pathToScriptDirectory="/tmp/ps1_nightlogs",
databaseName=self.settings["database settings"][
"ligo_virgo_waves"]["db"],
loginPath=self.settings["database settings"][
"ligo_virgo_waves"]["loginPath"],
successRule="delete",
failureRule="failed"
)
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="rockfinder",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# CALL FUNCTIONS/OBJECTS
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/rockfinder/rockfinder.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
if where and orbfitFlag:
from rockfinder import orbfit_ephemeris
eph = orbfit_ephemeris(log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
settings=settings,
verbose=extraFlag)
else:
from rockfinder import jpl_horizons_ephemeris
eph = jpl_horizons_ephemeris(log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
verbose=extraFlag)
dataSet = list_of_dictionaries(log=log, listOfDictionaries=eph)
# xfundamentals-render-list-of-dictionaries
output = dataSet.table(filepath=None)
if csv:
output = dataSet.csv(filepath=None)
elif json:
output = dataSet.json(filepath=None)
elif yaml:
output = dataSet.yaml(filepath=None)
elif md:
output = dataSet.markdown(filepath=None)
elif rst:
output = dataSet.reST(filepath=None)
print(output)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def get(self):
"""
*get the cone_search object*
**Return:**
- ``results`` -- the results of the conesearch
"""
self.log.info('starting the ``get`` method')
# sort results by angular separation
from operator import itemgetter
results = list(self.squareResults)
results = sorted(
results, key=itemgetter('separation_arcsec'), reverse=True)
# order of results
headers = ["sdss_name", "type", "ra", "dec", "specz", "specz_err", "photoz",
"photoz_err", "separation_arcsec", "separation_north_arcsec", "separation_east_arcsec"]
import collections
orderDict = collections.OrderedDict(sorted({}.items()))
# filter out results greater than the search radius
filteredResults = []
for row in results:
if float(row["separation_arcsec"]) < self.searchRadius:
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if h in row.keys():
orderDict[h] = row[h]
filteredResults.append(orderDict)
else:
pass
if self.nearest and len(filteredResults):
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if h in filteredResults[0].keys():
orderDict[h] = row[h]
filteredResults = [orderDict]
# filteredResults = [filteredResults[0]]
if not len(filteredResults):
orderDict = collections.OrderedDict(sorted({}.items()))
for h in headers:
if self.galaxyType == "all" or self.galaxyType == False or (self.galaxyType == "specz" and h not in ["photoz_err", "photoz"]) or (self.galaxyType == "photoz" and h not in ["specz", "specz_err"]):
orderDict[h] = ""
filteredResults = [orderDict]
# pretty format print
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=list(reversed(filteredResults))
)
if self.outputFormat == "csv":
results = dataSet.csv()
else:
results = dataSet.table()
# sdss only allows 60 hits per minute
sleep(1)
self.log.info('completed the ``get`` method')
return results
def main(arguments=None):
"""
The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command
.. todo ::
- update key arguments values and definitions with defaults
- update return values and definitions
- update usage examples and text
- update docstring text
- check sublime snippet exists
- clip any useful text to docs mindmap
- regenerate the docs and check redendering of this docstring
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="sherlock",
distributionName="qub-sherlock")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.debug('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# call the worker function
# x-if-settings-or-database-credientials
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/sherlock/sherlock.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if match or dbmatch:
if verboseFlag:
verbose = 2
else:
verbose = 1
if skipNedUpdateFlag:
updateNed = False
else:
updateNed = True
if skipMagUpdateFlag:
updatePeakMags = False
else:
updatePeakMags = True
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verbose,
update=updateFlag,
updateNed=updateNed,
updatePeakMags=updatePeakMags)
classifier.classify()
if clean:
cleaner = database_cleaner(log=log, settings=settings)
cleaner.clean()
if wiki:
updateWiki = update_wiki_pages(log=log, settings=settings)
updateWiki.update()
if iimport and ned:
ned = nedStreamImporter(log=log,
settings=settings,
coordinateList=["%(ra)s %(dec)s" % locals()],
radiusArcsec=radiusArcsec)
ned.ingest()
if iimport and cat:
if cat_name == "veron":
catalogue = veronImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if "ned_d" in cat_name:
catalogue = nedImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if iimport and stream:
if "marshall" in stream_name:
stream = marshallImporter(
log=log,
settings=settings,
)
stream.ingest()
if "ifs" in stream_name:
stream = ifsImporter(log=log, settings=settings)
stream.ingest()
if not init and not match and not clean and not wiki and not iimport and ra:
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verboseFlag)
classifier.classify()
if info:
print "sherlock-catalogues"
wiki = update_wiki_pages(log=log, settings=settings)
table = list(wiki._get_table_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
print
print "Crossmatch Streams"
table = list(wiki._get_stream_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
print
print "Views on Catalogues and Streams"
table = list(wiki._get_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.debug(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def get(self):
"""
*get the survey footprint stats and print to screen/file*
**Return:**
- ``None``
"""
self.log.debug('starting the ``get`` method')
# GRAB METADATA FROM THE DATABASES
this = plot_wave_observational_timelines(
log=self.log, settings=self.settings)
plotParameters, ps1Transients, ps1Pointings, altasPointings, atlasTransients = this.get_gw_parameters_from_settings(
gwid=self.gwid,
stackOnly=False)
if self.telescope == "atlas":
pointings = altasPointings
pointingSide = 5.46
if self.telescope == "ps1":
pointings = ps1Pointings
pointingSide = 0.4
telescope = self.telescope.upper()
# SORT ALL POINTINGS VIA MJD
pointings = sorted(list(pointings),
key=itemgetter('mjd'), reverse=False)
nside, hpixArea, aMap, healpixIds, wr, wd = self._create_healpixid_coordinate_grid()
print "EXPID, RA, DEC, MJD, EXPTIME, FILTER, LIM-MAG, EXP-AREA, EXP-LIKELIHOOD, CUM-AREA, CUM-LIKELIHOOD" % locals()
allHealpixIds = np.array([])
dictList = []
iindex = 0
count = len(pointings)
cumArea = 0
cumProb = 0
for pti, pt in enumerate(pointings):
pti = pti + 1
if pti > 1:
# Cursor up one line and clear line
sys.stdout.write("\x1b[1A\x1b[2K")
percent = (float(pti) / float(count)) * 100.
print '%(pti)s/%(count)s (%(percent)1.1f%% done): summing total area and likelihood covered by %(telescope)s' % locals()
thisDict = collections.OrderedDict(sorted({}.items()))
pra = pt["raDeg"]
pdec = pt["decDeg"]
pmjd = pt["mjd"]
pexpid = pt["exp_id"]
pexptime = pt["exp_time"]
pfilter = pt["filter"]
plim = pt["limiting_magnitude"]
# DETERMINE THE CORNERS FOR EACH ATLAS EXPOSURE AS MAPPED TO THE
# SKY
decCorners = (pdec - pointingSide / 2,
pdec + pointingSide / 2)
corners = []
for d in decCorners:
if d > 90.:
d = 180. - d
elif d < -90.:
d = -180 - d
raCorners = (pra - (pointingSide / 2) / np.cos(d * self.DEG_TO_RAD_FACTOR),
pra + (pointingSide / 2) / np.cos(d * self.DEG_TO_RAD_FACTOR))
for r in raCorners:
if r > 360.:
r = 720. - r
elif r < 0.:
r = 360. + r
corners.append(hp.ang2vec(r, d, lonlat=True))
# FLIP CORNERS 3 & 4 SO HEALPY UNDERSTANDS POLYGON SHAPE
corners = [corners[0], corners[1],
corners[3], corners[2]]
# RETURN HEALPIXELS IN EXPOSURE AREA
expPixels = hp.query_polygon(nside, np.array(
corners))
expProb = []
expProb[:] = [aMap[i] for i in expPixels]
expProb = sum(expProb)
expArea = len(expPixels) * hpixArea
if expProb / expArea < 2e-6:
continue
pindex = "%(iindex)05d" % locals()
iindex += 1
allHealpixIds = np.append(allHealpixIds, expPixels)
allHealpixIds = np.unique(allHealpixIds)
cumProb = []
cumProb[:] = [aMap[int(i)] for i in allHealpixIds]
cumProb = sum(cumProb)
cumArea = len(allHealpixIds) * hpixArea
thisDict["INDEX"] = pindex
thisDict["EXPID"] = pexpid
thisDict["RA"] = "%(pra)5.5f" % locals()
thisDict["DEC"] = "%(pdec)5.5f" % locals()
thisDict["MJD"] = "%(pmjd)6.6f" % locals()
thisDict["EXPTIME"] = "%(pexptime)02.1f" % locals()
thisDict["FILTER"] = pfilter
try:
thisDict["LIM-MAG"] = "%(plim)5.2f" % locals()
except:
thisDict["LIM-MAG"] = "NaN"
# thisDict["EXP-AREA"] = expArea
# thisDict["EXP-LIKELIHOOD"] = expProb
thisDict["CUM-AREA"] = "%(cumArea)05.2f" % locals()
thisDict["CUM-LIKELIHOOD"] = "%(cumProb)05.2f" % locals()
dictList.append(thisDict)
if not len(dictList):
thisDict = {}
thisDict["INDEX"] = "NULL"
thisDict["EXPID"] = "NULL"
thisDict["RA"] = "NULL"
thisDict["DEC"] = "NULL"
thisDict["MJD"] = "NULL"
thisDict["EXPTIME"] = "NULL"
thisDict["FILTER"] = "NULL"
thisDict["LIM-MAG"] = "NULL"
dictList.append(thisDict)
print "AREA: %(cumArea)0.2f. PROB: %(cumProb)0.5f" % locals()
printFile = self.settings["output directory"] + "/" + \
self.gwid + "/" + self.gwid + "-" + self.telescope + "-coverage-stats.csv"
# RECURSIVELY CREATE MISSING DIRECTORIES
if not os.path.exists(self.settings["output directory"] + "/" + self.gwid):
os.makedirs(self.settings["output directory"] + "/" + self.gwid)
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=dictList,
)
csvData = dataSet.csv(filepath=printFile)
print "The coverage stats file was written to `%(printFile)s`" % locals()
self.log.debug('completed the ``get`` method')
return None
def convert_sqlite_to_mysql(self):
"""*copy the contents of the sqlite database into the mysql database*
See class docstring for usage
"""
from fundamentals.renderer import list_of_dictionaries
from fundamentals.mysql import directory_script_runner
self.log.debug('starting the ``convert_sqlite_to_mysql`` method')
con = lite.connect(self.pathToSqlite)
con.row_factory = lite.Row
cur = con.cursor()
# GET ALL TABLE NAMES
cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
tables = cur.fetchall()
createStatements = []
inserts = []
for table in tables:
table = table['name']
if table == "sqlite_sequence":
continue
# CREATE TABLE collection_books (folder_id, fingerprint, primary key(folder_id, fingerprint));
# GENEREATE THE MYSQL CREATE STATEMENTS FOR EACH TABLE
cur.execute(
"SELECT sql FROM sqlite_master WHERE name = '%(table)s';" %
locals())
createStatement = cur.fetchone()
createStatement = createStatement[0].replace('"', '`') + ";"
if "DEFAULT" not in createStatement:
if "primary key(" in createStatement:
tmp = createStatement.split("primary key(")
tmp[0] = tmp[0].replace(",", " varchar(150) DEFAULT NULL,")
createStatement = ("primary key(").join(tmp)
if "primary key," in createStatement:
tmp = createStatement.split("primary key,")
tmp[1] = tmp[1].replace(",", " varchar(150) DEFAULT NULL,")
tmp[1] = tmp[1].replace(");",
" varchar(150) DEFAULT NULL);")
createStatement = ("primary key,").join(tmp)
createStatement = createStatement.replace(
"INTEGER PRIMARY KEY", "INTEGER AUTO_INCREMENT PRIMARY KEY")
createStatement = createStatement.replace("AUTOINCREMENT",
"AUTO_INCREMENT")
createStatement = createStatement.replace("DEFAULT 't'",
"DEFAULT '1'")
createStatement = createStatement.replace("DEFAULT 'f'",
"DEFAULT '0'")
createStatement = createStatement.replace(",'t'", ",'1'")
createStatement = createStatement.replace(",'f'", ",'0'")
if "CREATE TABLE `" in createStatement:
createStatement = createStatement.replace(
"CREATE TABLE `",
"CREATE TABLE IF NOT EXISTS `" + self.tablePrefix)
else:
createStatement = createStatement.replace(
"CREATE TABLE ",
"CREATE TABLE IF NOT EXISTS " + self.tablePrefix)
if ", primary key(" in createStatement:
createStatement = createStatement.replace(
", primary key(", """,
`dateLastModified` datetime DEFAULT NULL,
`updated` tinyint(4) DEFAULT '0',
primary key(""")
else:
createStatement = createStatement.replace(
");", """,
`dateLastModified` datetime DEFAULT NULL,
`updated` tinyint(4) DEFAULT '0');
""")
createStatement = createStatement.replace(
" text primary key", " varchar(100) primary key")
createStatement = createStatement.replace(
"`EntryText` TEXT NOT NULL,", "`EntryText` TEXT,")
createStatement = createStatement.replace(
"`SelectionText` TEXT NOT NULL", "`SelectionText` TEXT")
createStatement = createStatement.replace(
"`Filename` INTEGER NOT NULL,", "`Filename` TEXT NOT NULL,")
createStatement = createStatement.replace(
"`SessionPartUUID` TEXT NOT NULL UNIQUE,",
"`SessionPartUUID` VARCHAR(100) NOT NULL UNIQUE,")
createStatement = createStatement.replace(
"`Name` TEXT PRIMARY KEY NOT NULL",
"`Name` VARCHAR(100) PRIMARY KEY NOT NULL")
# GRAB THE DATA TO ADD TO THE MYSQL DATABASE TABLES
cur.execute("SELECT * from '%(table)s';" % locals())
rows = cur.fetchall()
allRows = []
for row in rows:
allRows.append(dict(row))
# RECURSIVELY CREATE MISSING DIRECTORIES
if not os.path.exists("/tmp/headjack/"):
os.makedirs("/tmp/headjack/")
# DUMP THE DATA INTO A MYSQL DATABASE
dataSet = list_of_dictionaries(log=self.log,
listOfDictionaries=allRows)
originalList = dataSet.list
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S%f.sql")
mysqlData = dataSet.mysql(tableName=self.tablePrefix + table,
filepath="/tmp/headjack/" + now,
createStatement=createStatement)
directory_script_runner(
log=self.log,
pathToScriptDirectory="/tmp/headjack/",
databaseName=self.settings["database settings"]["db"],
loginPath=self.settings["database settings"]["loginPath"],
successRule="delete",
failureRule="failed")
con.close()
self.log.debug('completed the ``convert_sqlite_to_mysql`` method')
return None
def update_gravity_event_annotations(
self):
"""*update gravity event annotations*
**Key Arguments:**
# -
**Return:**
- None
**Usage:**
.. todo::
- add usage info
- create a sublime snippet for usage
- write a command-line tool for this method
- update package tutorial with command-line tool info if needed
.. code-block:: python
usage code
"""
self.log.debug(
'completed the ````update_gravity_event_annotations`` method')
from breaker.transients import annotator
# CREATE THE ANNOTATION HELPER TABLES IF THEY DON"T EXIST
moduleDirectory = os.path.dirname(__file__)
mysql_scripts = moduleDirectory + "/resources/mysql"
for db in ["ps1gw", "ps13pi", "atlas"]:
directory_script_runner(
log=self.log,
pathToScriptDirectory=mysql_scripts,
databaseName=self.settings["database settings"][db]["db"],
loginPath=self.settings["database settings"][db]["loginPath"],
waitForResult=True,
successRule=False,
failureRule=False
)
for db in ["ligo_virgo_waves"]:
directory_script_runner(
log=self.log,
pathToScriptDirectory=mysql_scripts + "/ps1_skycell_help_tables",
databaseName=self.settings["database settings"][db]["db"],
loginPath=self.settings["database settings"][db]["loginPath"],
waitForResult=True,
successRule=False,
failureRule=False
)
# UPDATE THE TABLE WITH THE METADATA OF EACH GRAVITY EVENT
sqlQuery = ""
for g in self.settings["gravitational waves"]:
h = self.settings["gravitational waves"][g]["human-name"]
m = self.settings["gravitational waves"][g]["mjd"]
cmd = """insert ignore into tcs_gravity_events (`gracedb_id`, `gravity_event_id`, `mjd`) VALUES ("%(g)s", "%(h)s", %(m)s) on duplicate key update mjd=%(m)s;\n""" % locals(
)
sqlQuery += cmd
for db in [self.atlasDbConn, self.ps1gwDbConn, self.ps13piDbConn]:
writequery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=db
)
sqlQuery = sqlQuery.replace("tcs_gravity_events", "gravity_events")
writequery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=self.ligo_virgo_wavesDbConn,
)
for db in ["ps1gw", "ps13pi", "atlas"]:
directory_script_runner(
log=self.log,
pathToScriptDirectory=mysql_scripts,
databaseName=self.settings["database settings"][db]["db"],
loginPath=self.settings["database settings"][db]["loginPath"],
waitForResult=True,
successRule=False,
failureRule=False
)
for db in ["ligo_virgo_waves"]:
directory_script_runner(
log=self.log,
pathToScriptDirectory=mysql_scripts + "/ps1_skycell_help_tables",
databaseName=self.settings["database settings"][db]["db"],
loginPath=self.settings["database settings"][db]["loginPath"],
waitForResult=True,
successRule=False,
failureRule=False
)
dbDict = {
"ps1gw": self.ps1gwDbConn,
"atlas": self.atlasDbConn,
"ps13pi": self.ps13piDbConn,
"ligo_virgo_waves": self.ligo_virgo_wavesDbConn
}
for db in dbDict.keys():
for g in self.settings["gravitational waves"]:
h = self.settings["gravitational waves"][g]["human-name"]
print "Annotating new transients associated with gravity event %(h)s" % locals()
m = self.settings["gravitational waves"][g]["mjd"]
mapPath = self.settings["gravitational waves"][g]["mapPath"]
mapName = os.path.basename(mapPath)
thisDbConn = dbDict[db]
if thisDbConn in [self.ps1gwDbConn, self.ps13piDbConn]:
sqlQuery = u"""
SELECT
a.transient_object_id, a.gracedb_id, t.ra_psf, t.dec_psf
FROM
tcs_transient_objects t,
tcs_gravity_event_annotations a
WHERE
a.transient_object_id = t.id
AND t.detection_list_id != 0
AND (a.map_name !=
"%(mapName)s" or a.map_name is null)
AND a.gracedb_id="%(g)s";
""" % locals()
rows = readquery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=thisDbConn,
quiet=False
)
transients = {}
for r in rows:
transients[r["transient_object_id"]] = (
r["ra_psf"], r["dec_psf"])
an = annotator(
log=self.log,
settings=self.settings,
gwid=g
)
transientNames, probs = an.annotate(transients)
if thisDbConn in [self.atlasDbConn]:
sqlQuery = u"""
SELECT
a.transient_object_id, a.gracedb_id, t.ra, t.dec
FROM
atlas_diff_objects t,
tcs_gravity_event_annotations a
WHERE
a.transient_object_id = t.id
AND t.detection_list_id != 0
AND (a.map_name !=
"%(mapName)s" or a.map_name is null)
AND a.gracedb_id="%(g)s";
""" % locals()
rows = readquery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=thisDbConn,
quiet=False
)
transients = {}
for r in rows:
transients[r["transient_object_id"]] = (
r["ra"], r["dec"])
an = annotator(
log=self.log,
settings=self.settings,
gwid=g
)
transientNames, probs = an.annotate(transients)
if thisDbConn in [self.ligo_virgo_wavesDbConn]:
# PANSTARRS SKYCELLS
sqlQuery = u"""
SELECT
a.skycell_id, a.gracedb_id, t.raDeg, t.decDeg
FROM
ps1_skycell_map t,
ps1_skycell_gravity_event_annotations a
WHERE
a.skycell_id = t.skycell_id
AND (a.map_name != "%(mapName)s" or a.map_name is null)
AND a.gracedb_id="%(g)s";
""" % locals()
rows = readquery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=thisDbConn,
quiet=False
)
exposures = {}
for r in rows:
exposures[r["skycell_id"]] = (
r["raDeg"], r["decDeg"])
stats = survey_footprint(
log=self.log,
settings=self.settings,
gwid=g
)
exposureIDs, probs = stats.annotate_exposures(
exposures=exposures,
pointingSide=0.4
)
dataList = []
for p, t in zip(probs, exposureIDs):
dataList.append({
"skycell_id": t,
"prob_coverage": p,
"gracedb_id": g,
"map_name": mapName
})
tableName = "ps1_skycell_gravity_event_annotations"
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=dataList,
reDatetime=re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')
)
# RECURSIVELY CREATE MISSING DIRECTORIES
if not os.path.exists("/tmp/mysqlinsert/%(db)s" % locals()):
os.makedirs("/tmp/mysqlinsert/%(db)s" % locals())
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S%f")
mysqlData = dataSet.mysql(
tableName=tableName, filepath="/tmp/mysqlinsert/%(db)s/%(now)s.sql" % locals(), createStatement=False)
# ATLAS EXPOSURES
sqlQuery = u"""
SELECT
atlas_object_id, gracedb_id, raDeg, decDeg
FROM
atlas_exposure_gravity_event_annotations
WHERE
(map_name != "%(mapName)s" or map_name is null)
AND gracedb_id="%(g)s";
""" % locals()
rows = readquery(
log=self.log,
sqlQuery=sqlQuery,
dbConn=thisDbConn,
quiet=False
)
exposures = {}
for r in rows:
exposures[r["atlas_object_id"]] = (
r["raDeg"], r["decDeg"])
stats = survey_footprint(
log=self.log,
settings=self.settings,
gwid=g
)
exposureIDs, probs = stats.annotate_exposures(
exposures=exposures,
pointingSide=5.46
)
dataList = []
for p, t in zip(probs, exposureIDs):
dataList.append({
"atlas_object_id": t,
"prob_coverage": p,
"gracedb_id": g,
"map_name": mapName
})
tableName = "atlas_exposure_gravity_event_annotations"
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=dataList,
reDatetime=re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')
)
# RECURSIVELY CREATE MISSING DIRECTORIES
if not os.path.exists("/tmp/mysqlinsert/%(db)s" % locals()):
os.makedirs("/tmp/mysqlinsert/%(db)s" % locals())
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S%f")
mysqlData = dataSet.mysql(
tableName=tableName, filepath="/tmp/mysqlinsert/%(db)s/%(now)s.sql" % locals(), createStatement=False)
if thisDbConn not in [self.ligo_virgo_wavesDbConn]:
dataList = []
for p, t in zip(probs, transientNames):
dataList.append({
"transient_object_id": t,
"enclosing_contour": p,
"gracedb_id": g,
"map_name": mapName
})
tableName = "tcs_gravity_event_annotations"
dataSet = list_of_dictionaries(
log=self.log,
listOfDictionaries=dataList,
reDatetime=re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')
)
# RECURSIVELY CREATE MISSING DIRECTORIES
if not os.path.exists("/tmp/mysqlinsert/%(db)s" % locals()):
os.makedirs("/tmp/mysqlinsert/%(db)s" % locals())
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S%f")
mysqlData = dataSet.mysql(
tableName=tableName, filepath="/tmp/mysqlinsert/%(db)s/%(now)s.sql" % locals(), createStatement=False)
for db in dbDict.keys():
directory_script_runner(
log=self.log,
pathToScriptDirectory="/tmp/mysqlinsert/%(db)s" % locals(),
databaseName=self.settings["database settings"][db]["db"],
loginPath=self.settings["database settings"][db]["loginPath"],
waitForResult=True,
successRule=False,
failureRule=False
)
self.log.debug(
'completed the ``update_gravity_event_annotations`` method')
return None
def main(arguments=None):
"""
*The main function used when ``find_atlas_exposure_containing_ssobject.py`` is run as a single script from the cl*
"""
# SETUP VARIABLES
# MAKE SURE HEALPIX SMALL ENOUGH TO MATCH FOOTPRINTS CORRECTLY
nside = 1024
pi = (4 * math.atan(1.0))
DEG_TO_RAD_FACTOR = pi / 180.0
RAD_TO_DEG_FACTOR = 180.0 / pi
tileSide = 5.46
i = 0
outputList = []
rsyncContent = []
obscodes = {"02": "T05", "01": "T08"}
# SETUP THE COMMAND-LINE UTIL SETTINGS
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False)
arguments, settings, log, dbConn = su.setup()
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
dbSettings = {
'host': '127.0.0.1',
'user': '******',
'tunnel': {
'remote ip': 'starbase.mp.qub.ac.uk',
'remote datbase host': 'dormammu',
'remote user': '******',
'port': 5003
},
'password': '******',
'db': 'atlas_moving_objects'
}
# SETUP DATABASE CONNECTIONS
dbConn = database(log=log, dbSettings=dbSettings).connect()
# GRAB THE EXPOSURE LISTING
for expPrefix, obscode in obscodes.iteritems():
exposureList = []
mjds = []
sqlQuery = "select * from atlas_exposures where expname like '%(expPrefix)s%%'" % locals(
)
connected = 0
while connected == 0:
try:
rows = readquery(log=log,
sqlQuery=sqlQuery,
dbConn=dbConn,
quiet=False)
connected = 1
except:
# SETUP DATABASE CONNECTIONS
dbConn = database(log=log, dbSettings=dbSettings).connect()
print "Can't connect to DB - try again"
time.sleep(2)
t = len(rows)
print "There are %(t)s '%(expPrefix)s' exposures to check - hang tight" % locals(
)
for row in rows:
row["mjd"] = row["mjd"] + row["exp_time"] / (2. * 60 * 60 * 24)
exposureList.append(row)
mjds.append(row["mjd"])
results = []
batchSize = 500
total = len(mjds[1:])
batches = int(total / batchSize)
start = 0
end = 0
theseBatches = []
for i in range(batches + 1):
end = end + batchSize
start = i * batchSize
thisBatch = mjds[start:end]
theseBatches.append(thisBatch)
i = 0
totalLen = len(theseBatches)
index = 0
for batch in theseBatches:
i += 1
if index > 1:
# Cursor up one line and clear line
sys.stdout.write("\x1b[1A\x1b[2K")
print "Requesting batch %(i)04d/%(totalLen)s from JPL" % locals()
index += 1
eph = jpl_horizons_ephemeris(log=log,
objectId=[ssobject],
mjd=batch,
obscode=obscode,
verbose=False)
for b in batch:
match = 0
# print b
for row in eph:
if math.floor(row["mjd"] * 10000 +
0.01) == math.floor(b * 10000 + 0.01):
match = 1
results.append(row)
if match == 0:
for row in eph:
if math.floor(row["mjd"] * 10000) == math.floor(b *
10000):
match = 1
results.append(row)
if match == 0:
results.append(None)
this = math.floor(b * 10000 + 0.01)
print "MJD %(b)s (%(this)s) is missing" % locals()
for row in eph:
print math.floor(row["mjd"] * 10000 + 0.00001)
print ""
print "Finding the exopsures containing the SS object"
for e, r in zip(exposureList, results):
# CALCULATE SEPARATION IN ARCSEC
if not r:
continue
calculator = separations(
log=log,
ra1=r["ra_deg"],
dec1=r["dec_deg"],
ra2=e["raDeg"],
dec2=e["decDeg"],
)
angularSeparation, north, east = calculator.get()
sep = float(angularSeparation) / 3600.
if sep < 5.:
# THE SKY-LOCATION AS A HEALPIXEL ID
pinpoint = hp.ang2pix(nside,
theta=r["ra_deg"],
phi=r["dec_deg"],
lonlat=True)
decCorners = (e["decDeg"] - tileSide / 2,
e["decDeg"] + tileSide / 2)
corners = []
for d in decCorners:
if d > 90.:
d = 180. - d
elif d < -90.:
d = -180 - d
raCorners = (
e["raDeg"] -
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR),
e["raDeg"] +
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR))
for rc in raCorners:
if rc > 360.:
rc = 720. - rc
elif rc < 0.:
rc = 360. + rc
corners.append(hp.ang2vec(rc, d, lonlat=True))
# NEAR THE POLES RETURN SQUARE INTO TRIANGE - ALMOST DEGENERATE
pole = False
for d in decCorners:
if d > 87.0 or d < -87.0:
pole = True
if pole == True:
corners = corners[1:]
else:
# FLIP CORNERS 3 & 4 SO HEALPY UNDERSTANDS POLYGON SHAPE
corners = [corners[0], corners[1], corners[3], corners[2]]
# RETURN HEALPIXELS IN EXPOSURE AREA
expPixels = hp.query_polygon(nside, np.array(corners))
if pinpoint in expPixels:
outputList.append({
"obs": e["expname"],
"mjd": e["mjd"],
"raDeg": r["ra_deg"],
"decDeg": r["dec_deg"],
"mag": r["apparent_mag"],
"sep": sep
})
thisMjd = int(math.floor(e["mjd"]))
expname = e["expname"]
ssobject_ = ssobject.replace(" ", "_")
raStr = r["ra_deg"]
decStr = r["dec_deg"]
rsyncContent.append(
"rsync -av [email protected]:/atlas/red/%(expPrefix)sa/%(thisMjd)s/%(expname)s.fits.fz %(ssobject_)s_atlas_exposures/"
% locals())
rsyncContent.append(
"touch %(ssobject_)s_atlas_exposures/%(expname)s.location"
% locals())
rsyncContent.append(
'echo "_RAJ2000,_DEJ2000,OBJECT\n%(raStr)s,%(decStr)s,%(ssobject)s" > %(ssobject_)s_atlas_exposures/%(expname)s.location'
% locals())
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=outputList,
# use re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T') for mysql
reDatetime=False)
ssobject = ssobject.replace(" ", "_")
csvData = dataSet.csv(
filepath="./%(ssobject)s_atlas_exposure_matches.csv" % locals())
rsyncContent = ("\n").join(rsyncContent)
pathToWriteFile = "./%(ssobject)s_atlas_exposure_rsync.sh" % locals()
try:
log.debug("attempting to open the file %s" % (pathToWriteFile, ))
writeFile = codecs.open(pathToWriteFile, encoding='utf-8', mode='w')
except IOError, e:
message = 'could not open the file %s' % (pathToWriteFile, )
log.critical(message)
raise IOError(message)
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
distributionName="qub-sherlock",
projectName="sherlock",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/sherlock/sherlock.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
init = a["init"]
match = a["match"]
dbmatch = a["dbmatch"]
clean = a["clean"]
wiki = a["wiki"]
iimport = a["import"]
ned = a["ned"]
cat = a["cat"]
stream = a["stream"]
info = a["info"]
ra = a["ra"]
dec = a["dec"]
radiusArcsec = a["radiusArcsec"]
cat_name = a["cat_name"]
stream_name = a["stream_name"]
skipNedUpdateFlag = a["skipNedUpdateFlag"]
skipMagUpdateFlag = a["skipMagUpdateFlag"]
settingsFlag = a["settingsFlag"]
verboseFlag = a["verboseFlag"]
updateFlag = a["updateFlag"]
# CALL FUNCTIONS/OBJECTS
if match or dbmatch:
if verboseFlag:
verbose = 2
else:
verbose = 1
if skipNedUpdateFlag:
updateNed = False
else:
updateNed = True
if skipMagUpdateFlag:
updatePeakMags = False
else:
updatePeakMags = True
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verbose,
update=updateFlag,
updateNed=updateNed,
updatePeakMags=updatePeakMags)
classifier.classify()
if clean:
cleaner = database_cleaner(log=log, settings=settings)
cleaner.clean()
if wiki:
updateWiki = update_wiki_pages(log=log, settings=settings)
updateWiki.update()
if iimport and ned:
ned = nedStreamImporter(log=log,
settings=settings,
coordinateList=["%(ra)s %(dec)s" % locals()],
radiusArcsec=radiusArcsec)
ned.ingest()
if iimport and cat:
if cat_name == "veron":
catalogue = veronImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if "ned_d" in cat_name:
catalogue = nedImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if iimport and stream:
if "ifs" in stream_name:
stream = ifsImporter(log=log, settings=settings)
stream.ingest()
if not init and not match and not clean and not wiki and not iimport and ra:
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verboseFlag)
classifier.classify()
if info:
print("sherlock-catalogues")
wiki = update_wiki_pages(log=log, settings=settings)
table = list(wiki._get_table_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
print()
print("Crossmatch Streams")
table = list(wiki._get_stream_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
print()
print("Views on Catalogues and Streams")
table = list(wiki._get_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
