def refresh_database(self):
"""
*Refresh the unit test database*
"""
from fundamentals.mysql import directory_script_runner
from fundamentals import tools
packageDirectory = self.get_project_root()
su = tools(
arguments={"settingsFile": packageDirectory +
"/test_settings.yaml"},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName=None,
defaultSettingsFile=False
)
arguments, settings, log, dbConn = su.setup()
directory_script_runner(
log=log,
pathToScriptDirectory=packageDirectory + "/tests/input",
dbConn=dbConn,
successRule=None,
failureRule=None
)
# DATABASE IMPORT WAS STALLING UNITTESTS
import time
time.sleep(20)
def main(arguments=None):
"""
*The main function used when ``mjd_to_date.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE mjd_to_date.py AT %s' %
(startTime,))
# call the worker function
# x-if-settings-or-database-credientials
thisDate = mjd_to_date(
log=log,
mjd=float(mjd),
fraction=fractionFlag,
sqlDate=sqlDateFlag
)
print thisDate.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE mjd_to_date.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``convert_mysql_database_to_innodb.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
from fundamentals import tools, times
su = tools(
arguments=arguments,
docString=__doc__
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE convert_mysql_database_to_innodb.py AT %s' %
(startTime,))
if "tableSchema" not in locals():
tableSchema = False
# call the worker function
# x-if-settings-or-database-credientials
convert_mysql_database_to_innodb(
log=log,
dbConn=dbConn,
tableSchema=tableSchema,
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE convert_mysql_database_to_innodb.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
The main function used when ``yaml_to_database.py`` when installed as a cl tool
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
if os.path.isfile(pathToYaml):
from fundamentals.mysql import yaml_to_database
# PARSE YAML FILE CONTENTS AND ADD TO DATABASE
yaml2db = yaml_to_database(
log=log,
settings=settings,
dbConn=dbConn
)
yaml2db.add_yaml_file_content_to_database(
filepath=pathToYaml,
deleteFile=deleteFlag
)
basename = os.path.basename(pathToYaml)
print "Content of %(basename)s added to database" % locals()
else:
from fundamentals.mysql import yaml_to_database
yaml2db = yaml_to_database(
log=log,
settings=settings,
dbConn=dbConn,
pathToInputDir=pathToYaml,
deleteFiles=deleteFlag
)
yaml2db.ingest()
print "Content of %(pathToYaml)s directory added to database" % locals()
return
def main(arguments=None):
"""
*The main function used when ``get_angular_separation.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=True
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE get_angular_separation.py AT %s' %
(startTime,))
# call the worker function
# x-if-settings-or-database-credientials
angularSeparation, north, east = get_angular_separation(
log=log,
ra1=ra1,
dec1=dec1,
ra2=ra2,
dec2=dec2
)
print """%(angularSeparation)6.4f\" (%(north)6.4f\" N, %(east)6.4f\" E)""" % locals()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE get_angular_separation.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
The main function used when ``yaml_to_database.py`` when installed as a cl tool
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
if os.path.isfile(pathToYaml):
from fundamentals.mysql import yaml_to_database
# PARSE YAML FILE CONTENTS AND ADD TO DATABASE
yaml2db = yaml_to_database(log=log, settings=settings, dbConn=dbConn)
yaml2db.add_yaml_file_content_to_database(filepath=pathToYaml,
deleteFile=deleteFlag)
basename = os.path.basename(pathToYaml)
print("Content of %(basename)s added to database" % locals())
else:
from fundamentals.mysql import yaml_to_database
yaml2db = yaml_to_database(log=log,
settings=settings,
dbConn=dbConn,
pathToInputDir=pathToYaml,
deleteFiles=deleteFlag)
yaml2db.ingest()
print("Content of %(pathToYaml)s directory added to database" %
locals())
return
def main(arguments=None):
"""
*The main function used when ``convert_excel_workbook_to_binary_fits_table.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE convert_excel_workbook_to_binary_fits_table.py AT %s' %
(startTime,))
# call the worker function
fitsFile = convert_excel_workbook_to_binary_fits_table(
log=log,
pathToWorkbook=pathToExcelFile,
pathToOutputFits=pathToOutputFits
)
fitsFile.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE convert_excel_workbook_to_binary_fits_table.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
The main function used when ``directory_script_runner.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="fundmentals"
)
arguments, settings, log, dbConn = su.setup()
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
if successFlag and successFlag.lower() == "none":
successFlag = None
if failureFlag and failureFlag.lower() == "none":
failureFlag = None
directory_script_runner(
log=log,
pathToScriptDirectory=pathToDirectory,
databaseName=databaseName,
loginPath=loginPath,
successRule=successFlag,
failureRule=failureFlag
)
return
def main(arguments=None):
"""
The main function used when ``directory_script_runner.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="fundmentals"
)
arguments, settings, log, dbConn = su.setup()
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
if successFlag and successFlag.lower() == "none":
successFlag = None
if failureFlag and failureFlag.lower() == "none":
failureFlag = None
directory_script_runner(
log=log,
pathToScriptDirectory=pathToDirectory,
databaseName=databaseName,
loginPath=loginPath,
successRule=successFlag,
failureRule=failureFlag
)
return
def main(arguments=None):
"""
The main function used when ``yaml_to_database.py`` when installed as a cl tool
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
from fundamentals.mysql import sqlite2mysql
converter = sqlite2mysql(
log=log,
settings=settings,
pathToSqlite=pathToSqliteDB,
tablePrefix=tablePrefix,
dbConn=dbConn
)
converter.convert_sqlite_to_mysql()
return
def main(arguments=None):
"""
The main function used when ``yaml_to_database.py`` when installed as a cl tool
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
from fundamentals.mysql import sqlite2mysql
converter = sqlite2mysql(log=log,
settings=settings,
pathToSqlite=pathToSqliteDB,
tablePrefix=tablePrefix,
dbConn=dbConn)
converter.convert_sqlite_to_mysql()
return
def main(arguments=None):
"""
*The main function used when ``import-pdfs-into-papers.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="muppet"
)
arguments, settings, log, dbConn = su.setup()
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
# CREATE APPLESCRIPT RETURN STRING WITH THE DETAILS I WANT
asProperties = {
"abstract": "abstract",
"title": "title",
"tags": "keyword names",
"papersId": "id",
"papersUrl": "item url",
"author": "author names",
"subtype": "subtitle",
"rating": "my rating",
"url": "publication url",
"path": "full path of primary file item",
"reference": "formatted reference",
"originalPath": "original path of primary file item"
}
returnString = "return "
for i, v in enumerate(asProperties.values()):
if i + 1 == len(asProperties):
returnString += """ %(v)s of thisItem""" % locals()
else:
returnString += """ %(v)s of thisItem & "||" & """ % locals()
elif action == "archive":
if len(inputMetadata["tags"]) == 0:
print "%(filePath)s needs tags added" % locals()
continue
outputMetadata = import_media_and_return_metadata(
log=log,
mediaPath=pdfPath,
returnString=returnString,
asProperties=asProperties,
)
# CHECK FILE WAS INDEED IMPORTED
if outputMetadata["originalPath"] != pdfPath:
originalPath = outputMetadata["originalPath"]
log.error(
'this is not the original file: "%(originalPath)s" ' % locals())
sys.exit(0)
else:
updates = ""
sqlUpdates = ""
sqlTags = ""
thisId = outputMetadata["papersId"]
for k, v in inputMetadata.iteritems():
if k == "tags":
tags = v.split(",")
for tag in tags:
tag = tag.strip().lower().replace(" ", "-")
updates += """
set this to (make new keyword item with properties {name:"%(tag)s"})
add keywords {this} to thisItem
""" % locals()
sqlTags += """insert ignore into `media-tags` (mediaTable,mediaId,tag) values("reading-list", %(databaseid)s , "%(tag)s");\n""" % locals(
)
else:
if k in asProperties:
prop = asProperties[k]
updates += """set %(prop)s of thisItem to "%(v)s"\n""" % locals()
if len(sqlUpdates):
sqlUpdates += """, %(k)s = "%(v)s" """ % locals(
)
else:
sqlUpdates += """update `reading-list` set %(k)s = "%(v)s" """ % locals(
)
if len(thisData):
import pyperclip
pyperclip.copy(thisData)
applescript = u"""
tell application "Papers"
repeat with i from (count of publication items) to 1 by -1
if id of item i of publication items is equal to "%(thisId)s" then
set thisItem to item i of publication items
exit repeat
end if
end repeat
%(updates)s
set m to selected mode of front library window
set selected mode of front library window to Library Mode
tell front library window to set selected publications to (thisItem)
set i to selected inspector tab of front library window
set selected inspector tab of front library window to Notes Tab
set papersId to id of thisItem
end tell
tell application "System Events"
activate
set the_results to (display dialog "Paste notes to 'General Note' section on the right" buttons {"cancel", "skip", "notes added"} default button "skip")
set BUTTON_Returned to button returned of the_results
if ((BUTTON_Returned) is equal to "notes added") then
set BUTTON_Returned to papersId
end if
return BUTTON_Returned
end tell
""" % locals()
cmd = "\n".join(
["osascript << EOT", applescript, "EOT"])
p = Popen(cmd, stdout=PIPE,
stdin=PIPE, shell=True)
output = p.communicate()[0].strip()
if output.lower() == "cancel":
print "user cancelled the import script"
sys.exit(0)
elif output.lower() == "skip":
continue
elif "error" in output.lower():
print output
sys.exit(0)
elif len(output) == 0:
print "Error in applescript:\n"
print applescript
sys.exit(0)
papersId = output
if databaseid and databaseid.lower() != "false":
sqlUpdates += """, papersId = "%(papersId)s", workflowStage = "archived", notes = "%(thisData)s" where primaryId = %(databaseid)s """ % locals()
dms.execute_mysql_write_query(
sqlQuery=sqlUpdates,
dbConn=dbConn,
log=log
)
dms.execute_mysql_write_query(
sqlQuery=sqlTags,
dbConn=dbConn,
log=log
)
# Recursively create missing directories
if not os.path.exists(pdfFolderPath + "/trash"):
os.makedirs(pdfFolderPath + "/trash")
try:
source = filePath
destination = ("/").join(filePath.split("/")
[0:-1]) + "/trash/" + filePath.split("/")[-1]
log.debug("attempting to rename file %s to %s" %
(source, destination))
os.rename(source, destination)
except Exception, e:
log.error("could not rename file %s to %s - failed with this error: %s " %
(source, destination, str(e),))
sys.exit(0)
try:
source = pdfPath
destination = ("/").join(pdfPath.split("/")
[0:-1]) + "/trash/" + pdfPath.split("/")[-1]
log.debug("attempting to rename file %s to %s" %
(source, destination))
os.rename(source, destination)
except Exception, e:
log.error("could not rename file %s to %s - failed with this error: %s " %
(source, destination, str(e),))
sys.exit(0)
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False
)
arguments, settings, log, dbConn = su.setup()
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
privateFlag = True
# set options interactively if user requests
if interactiveFlag:
if interactiveFlag == "create":
create = True
location = ""
while location != "g" and location != "b":
location = raw_input(
"github or bitbucket [g/b]? \n > ")
if location == "g":
location = "github"
else:
location = "bitbucket"
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/%(location)s.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
if "pathToHostDirectory" in previousSettings:
default = previousSettings["pathToHostDirectory"]
pathToHostDirectory = raw_input(
"path to the host directory? (%(default)s)\n > " % locals())
if not len(pathToHostDirectory):
pathToHostDirectory = default
else:
pathToHostDirectory = raw_input(
"path to the host directory?\n > " % locals())
if "yamlSettingsFlag" in previousSettings:
default = previousSettings["yamlSettingsFlag"]
yamlSettingsFlag = raw_input(
"path to the credentials file? (%(default)s)\n > " % locals())
if not len(yamlSettingsFlag):
yamlSettingsFlag = default
else:
yamlSettingsFlag = raw_input(
"path to the credentials file?\n > " % locals())
if "projectName" in previousSettings:
default = previousSettings["projectName"]
projectName = raw_input(
"name of new git repo? (%(default)s)\n > " % locals())
if not len(projectName):
projectName = default
else:
projectName = raw_input(
"name of new git repo?\n > " % locals())
while branchesFlag != "y" and branchesFlag != "n":
branchesFlag = raw_input(
"create dev and bug branches [y/n]? \n > ")
if branchesFlag == "y":
branchesFlag = True
else:
branchesFlag = False
while privateFlag != "y" and privateFlag != "n":
privateFlag = raw_input(
"private repo [y/n]? \n > ")
if privateFlag == "y":
privateFlag = True
else:
privateFlag = False
if "strapline" in previousSettings:
default = previousSettings["strapline"]
strapline = raw_input(
"give a short description of the project (%(default)s)\n > " % locals())
if not len(strapline):
strapline = default
else:
strapline = raw_input(
"give a short description of the project\n > " % locals())
while wikiFlag != "y" and wikiFlag != "n":
wikiFlag = raw_input(
"add a wiki [y/n]? \n > ")
if wikiFlag == "y":
wiki = ""
while wiki != "y" and wiki != "n":
wiki = raw_input(
"do you want to make a seperate repo for wiki/issues [y/n]? \n > ")
if wiki == "y":
wiki = "seperate"
else:
wiki = "same"
else:
wiki = False
# save the most recently used requests
pickleMeObjects = [
"pathToHostDirectory", "projectName", "strapline", "yamlSettingsFlag"]
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
else:
if wikiFlag:
wiki = seperateOrSame
else:
wiki = False
# Create command ...
if create and pathToHostDirectory and projectName:
create_project_folder(
log=log,
pathToHostDirectory=pathToHostDirectory,
wiki=wiki,
projectName=projectName
)
pathToProjectRoot = """%(pathToHostDirectory)s/%(projectName)s""" % locals(
)
create_local_git_repo(
log=log,
pathToProjectRoot=pathToProjectRoot,
branches=branchesFlag
)
wikiUrl = False
if location == "bb" or location == "bitbucket":
repoUrl = add_git_repo_to_bitbucket(
log=log,
pathToProject=pathToProjectRoot,
strapline=strapline,
pathToCredentials=yamlSettingsFlag,
private=privateFlag,
wiki=wiki
)
if wiki:
thisWiki = clone_bitbucket_repo_wiki(
log=log,
projectName=projectName,
pathToHostDirectory=pathToHostDirectory,
branches=branchesFlag,
strapline=strapline,
wiki=wiki,
pathToCredentials=yamlSettingsFlag
)
wikiUrl, pathToWikiRoot = thisWiki.get()
add_git_repo_to_tower(
log=log,
pathToProjectRoot=pathToWikiRoot
)
if wikiUrl:
webbrowser.open_new_tab(wikiUrl)
elif location == "gh" or location == "github":
repoUrl = add_git_repo_to_github(
log=log,
pathToProject=pathToProjectRoot,
strapline=strapline,
private=privateFlag,
pathToCredentials=yamlSettingsFlag,
wiki=wiki
)
if wiki:
thisWiki = clone_github_repo_wiki(
log=log,
projectName=projectName,
pathToHostDirectory=pathToHostDirectory,
branches=branchesFlag,
strapline=strapline,
wiki=wiki,
pathToCredentials=yamlSettingsFlag
)
wikiUrl, pathToWikiRoot = thisWiki.get()
add_git_repo_to_tower(
log=log,
pathToProjectRoot=pathToWikiRoot
)
if wikiUrl:
webbrowser.open_new_tab(wikiUrl)
add_git_repo_to_tower(
log=log,
pathToProjectRoot=pathToProjectRoot
)
open_repo_in_sublime(
log=log,
pathToProjectRoot=pathToProjectRoot
)
## open in webbrowser
webbrowser.open_new_tab(repoUrl)
# hook commands ...
if hook:
if location == "bb" or location == "bitbucket":
add_hook_to_bitbucket_repo(
log,
repoName=projectName,
hookDomain=domainName,
pathToCredentials=yamlSettingsFlag
)
elif location == "gh" or location == "github":
add_hook_to_github_repo(
log,
repoName=projectName,
hookDomain=domainName,
pathToCredentials=yamlSettingsFlag
)
open_webhook_list_in_browser(
log=log,
location=location,
projectName=projectName
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
dev_flag = False
su = tools(arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="rockAtlas")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/rockAtlas/rockAtlas.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
# CALL FUNCTIONS/OBJECTS
if bookkeeping:
from rockAtlas.bookkeeping import bookkeeper
bk = bookkeeper(log=log, settings=settings, fullUpdate=fullFlag)
bk.clean_all()
if astorb:
from rockAtlas.orbital_elements import astorb
oe = astorb(log=log, settings=settings)
oe.refresh()
if pyephem:
from rockAtlas.positions import pyephemPositions
pyeph = pyephemPositions(log=log, settings=settings, dev_flag=dev_flag)
pyeph.get(singleSnapshot=oneFlag)
if orbfit:
from rockAtlas.positions import orbfitPositions
oe = orbfitPositions(log=log, settings=settings, dev_flag=dev_flag)
oe.get(singleExposure=oneFlag)
if cache:
from rockAtlas.phot import download
data = download(log=log, settings=settings, dev_flag=dev_flag)
data.get(days=days)
if dophot:
from rockAtlas.phot import dophotMatch
dp = dophotMatch(log=log, settings=settings)
dp.get()
if cycle:
from rockAtlas.phot import download
from rockAtlas.positions import pyephemPositions
from rockAtlas.positions import orbfitPositions
from rockAtlas.phot import dophotMatch
from fundamentals.mysql import readquery
# INITIAL ACTIONS
# SETUP ALL DATABASE CONNECTIONS
from rockAtlas import database
db = database(log=log, settings=settings)
dbConns, dbVersions = db.connect()
atlas3DbConn = dbConns["atlas3"]
atlas4DbConn = dbConns["atlas4"]
atlasMoversDBConn = dbConns["atlasMovers"]
while True:
if dev_flag:
o = " and dev_flag = 1"
else:
o = " "
sqlQuery = u"""
select distinct floor(mjd) from (
select mjd from atlas_exposures where dophot_match = 0 %(o)s
union all
select mjd from day_tracker where processed = 0 %(o)s) as a;
""" % locals()
rows = readquery(log=log,
sqlQuery=sqlQuery,
dbConn=atlasMoversDBConn,
quiet=False)
if len(rows) == 0:
if dev_flag:
print "Processing of the ATLAS development dataset is now complete and up to date"
else:
print "Processing of ATLAS data is now complete and up to date"
break
start_time = time.time()
data = download(log=log, settings=settings, dev_flag=dev_flag)
data.get(days=days)
print "%d seconds to download ATLAS cache of %s days\n" % (
time.time() - start_time, days)
start_time = time.time()
pyeph = pyephemPositions(log=log,
settings=settings,
dev_flag=dev_flag)
pyeph.get()
print "%d seconds to generate pyephem snapshots\n" % (time.time() -
start_time, )
start_time = time.time()
oe = orbfitPositions(log=log, settings=settings, dev_flag=dev_flag)
oe.get()
print "%d seconds to generate orbfit positions\n" % (time.time() -
start_time, )
start_time = time.time()
dp = dophotMatch(log=log, settings=settings)
dp.get()
print "%d seconds to extract dophot measurements\n" % (
time.time() - start_time, )
start_time = time.time()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``execute_mysql_script.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##
# PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED
# setup the command-line util settings
from fundamentals import tools, times
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG"
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
# Check for the force option
if "force" not in locals() or force is not True:
force = False
# SETUP A DATABASE CONNECTION BASED ON WHAT ARGUMENTS HAVE BEEN PASSED
dbConn = False
if 'settings' in locals() and "database settings" in settings:
host = settings["database settings"]["host"]
user = settings["database settings"]["user"]
passwd = settings["database settings"]["password"]
dbName = settings["database settings"]["db"]
dbConn = True
elif "host" in locals() and "dbName" in locals():
# SETUP DB CONNECTION
dbConn = True
host = arguments["--host"]
user = arguments["--user"]
passwd = arguments["--passwd"]
dbName = arguments["--dbName"]
if dbConn:
import pymysql as ms
dbConn = ms.connect(
host=host,
user=user,
passwd=passwd,
db=dbName,
use_unicode=True,
charset='utf8'
)
dbConn.autocommit(True)
log.debug('dbConn: %s' % (dbConn,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE execute_mysql_script.py AT %s' %
(startTime,))
# call the worker function
execute_mysql_script(
log=log,
user=user,
passwd=passwd,
db=dbName,
host=host,
pathToMysqlScript=pathToMysqlScript,
force=force
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE execute_mysql_script.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``find_atlas_exposure_containing_ssobject.py`` is run as a single script from the cl*
"""
# SETUP VARIABLES
# MAKE SURE HEALPIX SMALL ENOUGH TO MATCH FOOTPRINTS CORRECTLY
nside = 1024
pi = (4 * math.atan(1.0))
DEG_TO_RAD_FACTOR = pi / 180.0
RAD_TO_DEG_FACTOR = 180.0 / pi
tileSide = 5.46
i = 0
outputList = []
rsyncContent = []
obscodes = {"02": "T05", "01": "T08"}
# SETUP THE COMMAND-LINE UTIL SETTINGS
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False)
arguments, settings, log, dbConn = su.setup()
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
dbSettings = {
'host': '127.0.0.1',
'user': '******',
'tunnel': {
'remote ip': 'starbase.mp.qub.ac.uk',
'remote datbase host': 'dormammu',
'remote user': '******',
'port': 5003
},
'password': '******',
'db': 'atlas_moving_objects'
}
# SETUP DATABASE CONNECTIONS
dbConn = database(log=log, dbSettings=dbSettings).connect()
# GRAB THE EXPOSURE LISTING
for expPrefix, obscode in obscodes.iteritems():
exposureList = []
mjds = []
sqlQuery = "select * from atlas_exposures where expname like '%(expPrefix)s%%'" % locals(
)
connected = 0
while connected == 0:
try:
rows = readquery(log=log,
sqlQuery=sqlQuery,
dbConn=dbConn,
quiet=False)
connected = 1
except:
# SETUP DATABASE CONNECTIONS
dbConn = database(log=log, dbSettings=dbSettings).connect()
print "Can't connect to DB - try again"
time.sleep(2)
t = len(rows)
print "There are %(t)s '%(expPrefix)s' exposures to check - hang tight" % locals(
)
for row in rows:
row["mjd"] = row["mjd"] + row["exp_time"] / (2. * 60 * 60 * 24)
exposureList.append(row)
mjds.append(row["mjd"])
results = []
batchSize = 500
total = len(mjds[1:])
batches = int(total / batchSize)
start = 0
end = 0
theseBatches = []
for i in range(batches + 1):
end = end + batchSize
start = i * batchSize
thisBatch = mjds[start:end]
theseBatches.append(thisBatch)
i = 0
totalLen = len(theseBatches)
index = 0
for batch in theseBatches:
i += 1
if index > 1:
# Cursor up one line and clear line
sys.stdout.write("\x1b[1A\x1b[2K")
print "Requesting batch %(i)04d/%(totalLen)s from JPL" % locals()
index += 1
eph = jpl_horizons_ephemeris(log=log,
objectId=[ssobject],
mjd=batch,
obscode=obscode,
verbose=False)
for b in batch:
match = 0
# print b
for row in eph:
if math.floor(row["mjd"] * 10000 +
0.01) == math.floor(b * 10000 + 0.01):
match = 1
results.append(row)
if match == 0:
for row in eph:
if math.floor(row["mjd"] * 10000) == math.floor(b *
10000):
match = 1
results.append(row)
if match == 0:
results.append(None)
this = math.floor(b * 10000 + 0.01)
print "MJD %(b)s (%(this)s) is missing" % locals()
for row in eph:
print math.floor(row["mjd"] * 10000 + 0.00001)
print ""
print "Finding the exopsures containing the SS object"
for e, r in zip(exposureList, results):
# CALCULATE SEPARATION IN ARCSEC
if not r:
continue
calculator = separations(
log=log,
ra1=r["ra_deg"],
dec1=r["dec_deg"],
ra2=e["raDeg"],
dec2=e["decDeg"],
)
angularSeparation, north, east = calculator.get()
sep = float(angularSeparation) / 3600.
if sep < 5.:
# THE SKY-LOCATION AS A HEALPIXEL ID
pinpoint = hp.ang2pix(nside,
theta=r["ra_deg"],
phi=r["dec_deg"],
lonlat=True)
decCorners = (e["decDeg"] - tileSide / 2,
e["decDeg"] + tileSide / 2)
corners = []
for d in decCorners:
if d > 90.:
d = 180. - d
elif d < -90.:
d = -180 - d
raCorners = (
e["raDeg"] -
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR),
e["raDeg"] +
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR))
for rc in raCorners:
if rc > 360.:
rc = 720. - rc
elif rc < 0.:
rc = 360. + rc
corners.append(hp.ang2vec(rc, d, lonlat=True))
# NEAR THE POLES RETURN SQUARE INTO TRIANGE - ALMOST DEGENERATE
pole = False
for d in decCorners:
if d > 87.0 or d < -87.0:
pole = True
if pole == True:
corners = corners[1:]
else:
# FLIP CORNERS 3 & 4 SO HEALPY UNDERSTANDS POLYGON SHAPE
corners = [corners[0], corners[1], corners[3], corners[2]]
# RETURN HEALPIXELS IN EXPOSURE AREA
expPixels = hp.query_polygon(nside, np.array(corners))
if pinpoint in expPixels:
outputList.append({
"obs": e["expname"],
"mjd": e["mjd"],
"raDeg": r["ra_deg"],
"decDeg": r["dec_deg"],
"mag": r["apparent_mag"],
"sep": sep
})
thisMjd = int(math.floor(e["mjd"]))
expname = e["expname"]
ssobject_ = ssobject.replace(" ", "_")
raStr = r["ra_deg"]
decStr = r["dec_deg"]
rsyncContent.append(
"rsync -av [email protected]:/atlas/red/%(expPrefix)sa/%(thisMjd)s/%(expname)s.fits.fz %(ssobject_)s_atlas_exposures/"
% locals())
rsyncContent.append(
"touch %(ssobject_)s_atlas_exposures/%(expname)s.location"
% locals())
rsyncContent.append(
'echo "_RAJ2000,_DEJ2000,OBJECT\n%(raStr)s,%(decStr)s,%(ssobject)s" > %(ssobject_)s_atlas_exposures/%(expname)s.location'
% locals())
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=outputList,
# use re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T') for mysql
reDatetime=False)
ssobject = ssobject.replace(" ", "_")
csvData = dataSet.csv(
filepath="./%(ssobject)s_atlas_exposure_matches.csv" % locals())
rsyncContent = ("\n").join(rsyncContent)
pathToWriteFile = "./%(ssobject)s_atlas_exposure_rsync.sh" % locals()
try:
log.debug("attempting to open the file %s" % (pathToWriteFile, ))
writeFile = codecs.open(pathToWriteFile, encoding='utf-8', mode='w')
except IOError, e:
message = 'could not open the file %s' % (pathToWriteFile, )
log.critical(message)
raise IOError(message)
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="marshallEngine",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
init = a["init"]
clean = a["clean"]
iimport = a["import"]
lightcurve = a["lightcurve"]
transientBucketId = a["transientBucketId"]
survey = a["survey"]
withInLastDay = a["withInLastDay"]
settingsFlag = a["settingsFlag"]
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/marshallEngine/marshallEngine.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
# CALL FUNCTIONS/OBJECTS
# DEFAULT VALUES
if not withInLastDay:
withInLastDay = 30
# CALL FUNCTIONS/OBJECTS
if clean:
# RESCUE ORPHANED TRANSIENTS - NO MASTER ID FLAG
print("rescuing orphaned transients")
from fundamentals.mysql import writequery
procedureNames = [
"update_transients_with_no_masteridflag",
"insert_new_transients_into_transientbucketsummaries",
"resurrect_objects", "update_sherlock_xmatch_counts",
"update_inbox_auto_archiver", "update_transient_akas"
]
# CALL EACH PROCEDURE
for p in procedureNames:
sqlQuery = "CALL `%(p)s`();" % locals()
writequery(
log=log,
sqlQuery=sqlQuery,
dbConn=dbConn,
)
# UPDATE THE TRANSIENT BUCKET SUMMARY TABLE IN THE MARSHALL DATABASE
from marshallEngine.housekeeping import update_transient_summaries
updater = update_transient_summaries(log=log,
settings=settings,
dbConn=dbConn).update()
if iimport:
if survey.lower() == "panstarrs":
from marshallEngine.feeders.panstarrs.data import data
from marshallEngine.feeders.panstarrs import images
if survey.lower() == "atlas":
from marshallEngine.feeders.atlas.data import data
from marshallEngine.feeders.atlas import images
if survey.lower() == "useradded":
from marshallEngine.feeders.useradded.data import data
from marshallEngine.feeders.useradded import images
if survey.lower() == "tns":
from marshallEngine.feeders.tns.data import data
from marshallEngine.feeders.tns import images
if survey.lower() == "ztf":
from marshallEngine.feeders.ztf.data import data
from marshallEngine.feeders.ztf import images
ingester = data(log=log, settings=settings,
dbConn=dbConn).ingest(withinLastDays=withInLastDay)
cacher = images(log=log, settings=settings,
dbConn=dbConn).cache(limit=3000)
from marshallEngine.services import panstarrs_location_stamps
ps_stamp = panstarrs_location_stamps(log=log,
settings=settings,
dbConn=dbConn).get()
if lightcurve:
from marshallEngine.lightcurves import marshall_lightcurves
lc = marshall_lightcurves(log=log,
dbConn=dbConn,
settings=settings,
transientBucketIds=transientBucketId)
filepath = lc.plot()
print(
"The lightcurve plot for transient %(transientBucketId)s can be found here: %(filepath)s"
% locals())
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
import os
import nose2
import nose2
import unittest
import shutil
import yaml
from fundamentals import logs
from fundamentals.utKit import utKit
from fundamentals import tools, times
su = tools(
arguments={},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="fundamentals"
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/fundamentals/fundamentals.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
*The main function used when ``convert_spectrum_fits_to_ascii.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE convert_spectrum_fits_to_ascii.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
convert_spectrum_fits_to_ascii(
log=log,
fitsFilePath=pathToFits
).get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE convert_spectrum_fits_to_ascii.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
stream = file(
pathToInputDir + "/example_settings.yaml", 'r')
settings = yaml.load(stream)
stream.close()
su = tools(
arguments={"settingsFile":
pathToInputDir + "/example_settings.yaml"},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="HMpTy"
)
arguments, settings, log, dbConn = su.setup()
# import shutil
# try:
# shutil.rmtree(pathToOutputDir)
# except:
# pass
# Recursively create missing directories
if not os.path.exists(pathToOutputDir):
os.makedirs(pathToOutputDir)
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="polyglot"
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# for k, v in locals().iteritems():
# print k, v
if not destinationFolder:
destinationFolder = os.getcwd()
if not filenameFlag:
filenameFlag = False
if not cleanFlag:
readability = False
else:
readability = True
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/polyglot/polyglot.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if pdf and url:
filepath = printpdf.printpdf(
log=log,
settings=settings,
url=url,
folderpath=destinationFolder,
title=filenameFlag,
append=False,
readability=readability
).get()
if html and url:
cleaner = htmlCleaner.htmlCleaner(
log=log,
settings=settings,
url=url,
outputDirectory=destinationFolder,
title=filenameFlag, # SET TO FALSE TO USE WEBPAGE TITLE,
style=cleanFlag, # add polyglot's styling to the HTML document
metadata=True, # include metadata in generated HTML (e.g. title),
h1=True # include title as H1 at the top of the doc
)
filepath = cleaner.clean()
if epub:
if url:
iinput = url
else:
iinput = docx
from polyglot import ebook
epub = ebook(
log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag,
bookFormat="epub",
outputDirectory=destinationFolder
)
filepath = epub.get()
if mobi:
if url:
iinput = url
else:
iinput = docx
from polyglot import ebook
mobi = ebook(
log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag,
bookFormat="mobi",
outputDirectory=destinationFolder,
)
filepath = mobi.get()
if kindle:
if url:
iinput = url
else:
iinput = docx
from polyglot import kindle
sender = kindle(
log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag
)
success = sender.send()
if kindleNB2MD:
basename = os.path.basename(notebook)
extension = os.path.splitext(basename)[1]
filenameNoExtension = os.path.splitext(basename)[0]
if destinationFolder:
filepath = destinationFolder + "/" + filenameNoExtension + ".md"
else:
filepath = notebook.replace("." + extension, ".md")
from polyglot.markdown import kindle_notebook
nb = kindle_notebook(
log=log,
kindleExportPath=notebook,
outputPath=filepath
)
nb.convert()
if openFlag:
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``sky-tile-pinpointer.py`` is run as a single script from the cl*
"""
# MAKE SURE HEALPIX SMALL ENOUGH TO MATCH FOOTPRINTS CORRECTLY
nside = 1024
pi = (4 * math.atan(1.0))
DEG_TO_RAD_FACTOR = pi / 180.0
RAD_TO_DEG_FACTOR = 180.0 / pi
# SETUP THE COMMAND-LINE UTIL SETTINGS
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName=False)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
tileSide = float(tileSide)
# CONVERT RA AND DEC
# ASTROCALC UNIT CONVERTER OBJECT
converter = unit_conversion(log=log)
ra = converter.ra_sexegesimal_to_decimal(ra=ra)
dec = converter.dec_sexegesimal_to_decimal(dec=dec)
# THE SKY-LOCATION AS A HEALPIXEL ID
pinpoint = hp.ang2pix(nside, theta=ra, phi=dec, lonlat=True)
matchedTileIds = []
with open(pathToTileList, 'rb') as csvFile:
csvReader = csv.DictReader(csvFile,
dialect='excel',
delimiter=',',
quotechar='"')
for row in csvReader:
row["DEC"] = float(row["DEC"])
row["RA"] = float(row["RA"])
decCorners = (row["DEC"] - tileSide / 2, row["DEC"] + tileSide / 2)
corners = []
for d in decCorners:
if d > 90.:
d = 180. - d
elif d < -90.:
d = -180 - d
raCorners = (row["RA"] -
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR),
row["RA"] +
(tileSide / 2) / np.cos(d * DEG_TO_RAD_FACTOR))
for r in raCorners:
if r > 360.:
r = 720. - r
elif r < 0.:
r = 360. + r
corners.append(hp.ang2vec(r, d, lonlat=True))
# FLIP CORNERS 3 & 4 SO HEALPY UNDERSTANDS POLYGON SHAPE
corners = [corners[0], corners[1], corners[3], corners[2]]
# RETURN HEALPIXELS IN EXPOSURE AREA
expPixels = hp.query_polygon(nside, np.array(corners))
if pinpoint in expPixels:
# CALCULATE SEPARATION IN ARCSEC
calculator = separations(
log=log,
ra1=ra,
dec1=dec,
ra2=row["RA"],
dec2=row["DEC"],
)
angularSeparation, north, east = calculator.get()
angularSeparation = float(angularSeparation) / 3600
north = float(north) / 3600
east = float(east) / 3600
matchedTileIds.append(
row["EXPID"] +
": %(angularSeparation)1.4f deg from center (%(north)1.4f N, %(east)1.4f E) "
% locals())
csvFile.close()
for i in matchedTileIds:
print i
return
def main(arguments=None):
"""
The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command
.. todo ::
- update key arguments values and definitions with defaults
- update return values and definitions
- update usage examples and text
- update docstring text
- check sublime snippet exists
- clip any useful text to docs mindmap
- regenerate the docs and check redendering of this docstring
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="sherlock",
distributionName="qub-sherlock")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.debug('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# call the worker function
# x-if-settings-or-database-credientials
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/sherlock/sherlock.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if match or dbmatch:
if verboseFlag:
verbose = 2
else:
verbose = 1
if skipNedUpdateFlag:
updateNed = False
else:
updateNed = True
if skipMagUpdateFlag:
updatePeakMags = False
else:
updatePeakMags = True
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verbose,
update=updateFlag,
updateNed=updateNed,
updatePeakMags=updatePeakMags)
classifier.classify()
if clean:
cleaner = database_cleaner(log=log, settings=settings)
cleaner.clean()
if wiki:
updateWiki = update_wiki_pages(log=log, settings=settings)
updateWiki.update()
if iimport and ned:
ned = nedStreamImporter(log=log,
settings=settings,
coordinateList=["%(ra)s %(dec)s" % locals()],
radiusArcsec=radiusArcsec)
ned.ingest()
if iimport and cat:
if cat_name == "veron":
catalogue = veronImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if "ned_d" in cat_name:
catalogue = nedImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if iimport and stream:
if "marshall" in stream_name:
stream = marshallImporter(
log=log,
settings=settings,
)
stream.ingest()
if "ifs" in stream_name:
stream = ifsImporter(log=log, settings=settings)
stream.ingest()
if not init and not match and not clean and not wiki and not iimport and ra:
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verboseFlag)
classifier.classify()
if info:
print "sherlock-catalogues"
wiki = update_wiki_pages(log=log, settings=settings)
table = list(wiki._get_table_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
print
print "Crossmatch Streams"
table = list(wiki._get_stream_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
print
print "Views on Catalogues and Streams"
table = list(wiki._get_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print tableData
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.debug(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False)
arguments, settings, log, dbConn = su.setup()
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
privateFlag = True
# set options interactively if user requests
if interactiveFlag:
if interactiveFlag == "create":
create = True
location = ""
while location != "g" and location != "b":
location = raw_input("github or bitbucket [g/b]? \n > ")
if location == "g":
location = "github"
else:
location = "bitbucket"
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/%(location)s.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
if "pathToHostDirectory" in previousSettings:
default = previousSettings["pathToHostDirectory"]
pathToHostDirectory = raw_input(
"path to the host directory? (%(default)s)\n > " % locals())
if not len(pathToHostDirectory):
pathToHostDirectory = default
else:
pathToHostDirectory = raw_input(
"path to the host directory?\n > " % locals())
if "yamlSettingsFlag" in previousSettings:
default = previousSettings["yamlSettingsFlag"]
yamlSettingsFlag = raw_input(
"path to the credentials file? (%(default)s)\n > " %
locals())
if not len(yamlSettingsFlag):
yamlSettingsFlag = default
else:
yamlSettingsFlag = raw_input(
"path to the credentials file?\n > " % locals())
if "projectName" in previousSettings:
default = previousSettings["projectName"]
projectName = raw_input(
"name of new git repo? (%(default)s)\n > " % locals())
if not len(projectName):
projectName = default
else:
projectName = raw_input("name of new git repo?\n > " % locals())
while branchesFlag != "y" and branchesFlag != "n":
branchesFlag = raw_input(
"create dev and bug branches [y/n]? \n > ")
if branchesFlag == "y":
branchesFlag = True
else:
branchesFlag = False
while privateFlag != "y" and privateFlag != "n":
privateFlag = raw_input("private repo [y/n]? \n > ")
if privateFlag == "y":
privateFlag = True
else:
privateFlag = False
if "strapline" in previousSettings:
default = previousSettings["strapline"]
strapline = raw_input(
"give a short description of the project (%(default)s)\n > "
% locals())
if not len(strapline):
strapline = default
else:
strapline = raw_input(
"give a short description of the project\n > " % locals())
while wikiFlag != "y" and wikiFlag != "n":
wikiFlag = raw_input("add a wiki [y/n]? \n > ")
if wikiFlag == "y":
wiki = ""
while wiki != "y" and wiki != "n":
wiki = raw_input(
"do you want to make a seperate repo for wiki/issues [y/n]? \n > "
)
if wiki == "y":
wiki = "seperate"
else:
wiki = "same"
else:
wiki = False
# save the most recently used requests
pickleMeObjects = [
"pathToHostDirectory", "projectName", "strapline",
"yamlSettingsFlag"
]
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
else:
if wikiFlag:
wiki = seperateOrSame
else:
wiki = False
# Create command ...
if create and pathToHostDirectory and projectName:
create_project_folder(log=log,
pathToHostDirectory=pathToHostDirectory,
wiki=wiki,
projectName=projectName)
pathToProjectRoot = """%(pathToHostDirectory)s/%(projectName)s""" % locals(
)
create_local_git_repo(log=log,
pathToProjectRoot=pathToProjectRoot,
branches=branchesFlag)
wikiUrl = False
if location == "bb" or location == "bitbucket":
repoUrl = add_git_repo_to_bitbucket(
log=log,
pathToProject=pathToProjectRoot,
strapline=strapline,
pathToCredentials=yamlSettingsFlag,
private=privateFlag,
wiki=wiki)
if wiki:
thisWiki = clone_bitbucket_repo_wiki(
log=log,
projectName=projectName,
pathToHostDirectory=pathToHostDirectory,
branches=branchesFlag,
strapline=strapline,
wiki=wiki,
pathToCredentials=yamlSettingsFlag)
wikiUrl, pathToWikiRoot = thisWiki.get()
add_git_repo_to_tower(log=log,
pathToProjectRoot=pathToWikiRoot)
if wikiUrl:
webbrowser.open_new_tab(wikiUrl)
elif location == "gh" or location == "github":
repoUrl = add_git_repo_to_github(
log=log,
pathToProject=pathToProjectRoot,
strapline=strapline,
private=privateFlag,
pathToCredentials=yamlSettingsFlag,
wiki=wiki)
if wiki:
thisWiki = clone_github_repo_wiki(
log=log,
projectName=projectName,
pathToHostDirectory=pathToHostDirectory,
branches=branchesFlag,
strapline=strapline,
wiki=wiki,
pathToCredentials=yamlSettingsFlag)
wikiUrl, pathToWikiRoot = thisWiki.get()
add_git_repo_to_tower(log=log,
pathToProjectRoot=pathToWikiRoot)
if wikiUrl:
webbrowser.open_new_tab(wikiUrl)
add_git_repo_to_tower(log=log, pathToProjectRoot=pathToProjectRoot)
open_repo_in_sublime(log=log, pathToProjectRoot=pathToProjectRoot)
## open in webbrowser
webbrowser.open_new_tab(repoUrl)
# hook commands ...
if hook:
if location == "bb" or location == "bitbucket":
add_hook_to_bitbucket_repo(log,
repoName=projectName,
hookDomain=domainName,
pathToCredentials=yamlSettingsFlag)
elif location == "gh" or location == "github":
add_hook_to_github_repo(log,
repoName=projectName,
hookDomain=domainName,
pathToCredentials=yamlSettingsFlag)
open_webhook_list_in_browser(log=log,
location=location,
projectName=projectName)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
import os
import nose
import shutil
import yaml
from projector import devonthink_indexer, cl_utils
from projector.utKit import utKit
from fundamentals import tools, times
su = tools(
arguments={},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="devonthink_indexer"
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/projector/projector.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
import os
import shutil
import yaml
import unittest
from panstamps import downloader, cl_utils
from panstamps.image import image
from panstamps.utKit import utKit
from fundamentals import tools
su = tools(
arguments={"settingsFile": None},
docString=__doc__,
logLevel="DEBUG",
options_first=True,
projectName="panstamps"
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/panstamps/panstamps.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="tastic")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/tastic/tastic.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
# CALL FUNCTIONS/OBJECTS
if sort or archive:
ws = workspace(log=log,
settings=settings,
fileOrWorkspacePath=pathToFileOrWorkspace)
if sort:
ws.sort()
if archive:
ws.archive_done()
if sync:
tp = syncc(log=log,
settings=settings,
workspaceRoot=pathToWorkspace,
workspaceName=workspaceName,
syncFolder=pathToSyncFolder,
editorialRootPath=editorialRootPath,
includeFileTags=fileTagsFlag)
tp.sync()
if reminders:
r = reminderss(log=log, settings=settings)
r.import_list(listName=listName, pathToTaskpaperDoc=pathToTaskpaperDoc)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=True,
projectName="inoreader")
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
if sub:
rss = add_subscription(log,
settings=settings,
settingsFilePath=settingsFile,
rssUrl=feedUrl,
folder=folder)
rss.get()
if articles:
if favouriteFlag:
unreadOrStarred = "starred"
elif unreadFlag:
unreadOrStarred = "unread"
articles = get_articles(log,
settings=settings,
settingsFilePath=settingsFile,
stream=stream,
maxArticles=maxArticles,
unreadOrStarred=unreadOrStarred)
articles = articles.get()
if len(articles) == 0:
print "No articles found"
else:
for article in articles:
title = article["title"]
thisId = article["id"]
print "%(title)s (%(thisId)s)" % locals()
if tag:
tag = edit_tags(log,
settings=settings,
settingsFilePath=settingsFile,
removeTag=tagToRemove,
addTag=tagToAdd,
articleIdList=articleId)
tag.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="rockfinder",
defaultSettingsFile=True
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# CALL FUNCTIONS/OBJECTS
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/rockfinder/rockfinder.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
if where and orbfitFlag:
from rockfinder import orbfit_ephemeris
eph = orbfit_ephemeris(
log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
settings=settings,
verbose=extraFlag
)
else:
from rockfinder import jpl_horizons_ephemeris
eph = jpl_horizons_ephemeris(
log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
verbose=extraFlag
)
dataSet = list_of_dictionaries(
log=log,
listOfDictionaries=eph
)
# xfundamentals-render-list-of-dictionaries
output = dataSet.table(filepath=None)
if csv:
output = dataSet.csv(filepath=None)
elif json:
output = dataSet.json(filepath=None)
elif yaml:
output = dataSet.yaml(filepath=None)
elif md:
output = dataSet.markdown(filepath=None)
elif rst:
output = dataSet.reST(filepath=None)
print output
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="rockfinder",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# CALL FUNCTIONS/OBJECTS
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/rockfinder/rockfinder.yaml"
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
if where and orbfitFlag:
from rockfinder import orbfit_ephemeris
eph = orbfit_ephemeris(log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
settings=settings,
verbose=extraFlag)
else:
from rockfinder import jpl_horizons_ephemeris
eph = jpl_horizons_ephemeris(log=log,
objectId=objectId,
mjd=mjd,
obscode=obscode,
verbose=extraFlag)
dataSet = list_of_dictionaries(log=log, listOfDictionaries=eph)
# xfundamentals-render-list-of-dictionaries
output = dataSet.table(filepath=None)
if csv:
output = dataSet.csv(filepath=None)
elif json:
output = dataSet.json(filepath=None)
elif yaml:
output = dataSet.yaml(filepath=None)
elif md:
output = dataSet.markdown(filepath=None)
elif rst:
output = dataSet.reST(filepath=None)
print(output)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
import os
import nose2
import shutil
import unittest
import yaml
from rockAtlas.orbital_elements import astorb
from rockAtlas.utKit import utKit
from fundamentals import tools
su = tools(arguments={"settingsFile": None},
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="rockAtlas",
defaultSettingsFile=False)
arguments, settings, log, dbConn = su.setup()
# # load settings
stream = file("/Users/Dave/.config/rockAtlas/rockAtlas.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
# stream = file(
def main(arguments=None):
"""
*The main function used when ``shift_coordinates.py`` is run as a single script from the cl, or when installed as a cl command*
"""
from dryxPython import astrotools as dat
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE shift_coordinates.py AT %s' %
(startTime,))
# set options interactively if user requests
# if interactiveFlag:
# x-raw-input
# x-boolean-raw-input
# pass
# call the worker function
# x-if-settings-or-database-credientials
newCoords = shift_coordinates(
log=log,
ra=ra,
dec=dec,
north=float(north),
east=float(east)
)
raNew, decNew = newCoords.get()
raSex = dat.ra_to_sex(
ra=raNew,
delimiter=':'
)
decSex = dat.dec_to_sex(
dec=decNew,
delimiter=':'
)
print """%(raNew)6.4f, %(decNew)6.4f (%(raSex)s, %(decSex)s)""" % locals()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE shift_coordinates.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
import os
import nose
import shutil
import yaml
from sloancone.utKit import utKit
from fundamentals import tools
su = tools(
arguments={"settingsFile": None},
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="sloancone",
tunnel=False
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/sloancone/sloancone.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments, docString=__doc__, logLevel="DEBUG", options_first=False, projectName="projector")
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(" \t\n;")
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec (varname + " = '%s'" % (val,))
else:
exec (varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug("%s = %s" % (varname, val))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info("--- STARTING TO RUN THE cl_utils.py AT %s" % (startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if home:
userId = "*****@*****.**"
stack = "home projects"
dropboxPath = "/notes/thespacedoctor-wiki/projects/"
elif work:
userId = "*****@*****.**"
stack = "work projects"
dropboxPath = "/notes/astronotes-wiki/projects/"
# CALL FUNCTIONS/OBJECTS
if create:
gmail_utils(log=log, settings=settings, userId=userId).create_label("projects/" + projectName)
evernote_utils(log=log, settings=settings).create_notebook(nbName=projectName, stack=stack)
mkdir(log=log, settings=settings, dropboxPath=dropboxPath + projectName).get()
this = touch_file(log=log, settings=settings, destination=dropboxPath + projectName + ".remove").touch()
# try:
# devonthink_indexer(
# log=log,
# settings=settings,
# ).get()
# time.sleep(3)
# devonthink_indexer(
# log=log,
# settings=settings,
# ).get()
# except:
# pass
if remote:
setup_projects_from_file_input(log=log, settings=settings)
if auth:
evernote_utils(log=log, settings=settings).authenticate()
# if dt:
# devonthink_indexer(
# log=log,
# settings=settings,
# ).get()
if ls:
gmail_utils(log=log, settings=settings, userId=userId).list_live_projects(archive=archiveFlag)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info("-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --" % (endTime, runningTime))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="headjack"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# CALL FUNCTIONS/OBJECTS
if read and sendToKindle:
from headjack.read import sendToKindle
sender = sendToKindle(
log=log,
settings=settings,
dbConn=dbConn
)
sender.send()
# CALL FUNCTIONS/OBJECTS
if read and convert and kindleAnnotations:
from headjack.read import convertKindleNB
converter = convertKindleNB(
log=log,
settings=settings,
dbConn=dbConn
)
converter.convert()
if media and stage:
from headjack.archiver import docs
stager = docs(
log=log,
settings=settings,
dbConn=dbConn
)
stager.stash()
if media and archive:
from headjack.archiver import docs
stager = docs(
log=log,
settings=settings,
dbConn=dbConn
)
stager.archive()
if web2epub:
from headjack.read import generate_web_article_epubs
epubs = generate_web_article_epubs(
log=log,
settings=settings,
dbConn=dbConn
)
epubs.create()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
import os
import nose2
import shutil
import unittest
import yaml
from fundamentals.utKit import utKit
from fundamentals import tools
su = tools(
arguments={"settingsFile": None},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="fundamentals",
defaultSettingsFile=False
)
arguments, settings, log, dbConn = su.setup()
# # load settings
# stream = file(
# "/Users/Dave/.config/fundamentals/fundamentals.yaml", 'r')
# settings = yaml.load(stream)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="polygot")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# for k, v in locals().iteritems():
# print k, v
if not destinationFolder:
destinationFolder = os.getcwd()
if not filenameFlag:
filenameFlag = False
if not cleanFlag:
readability = False
else:
readability = True
if pdf:
filepath = printpdf.printpdf(log=log,
settings=settings,
url=url,
folderpath=destinationFolder,
title=filenameFlag,
append=False,
readability=readability).get()
if html:
cleaner = htmlCleaner.htmlCleaner(
log=log,
settings=settings,
url=url,
outputDirectory=destinationFolder,
title=filenameFlag, # SET TO FALSE TO USE WEBPAGE TITLE,
style=cleanFlag, # add polygot's styling to the HTML document
metadata=True, # include metadata in generated HTML (e.g. title),
h1=True # include title as H1 at the top of the doc
)
filepath = cleaner.clean()
if openFlag:
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=True,
projectName="panstamps")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
if ra:
try:
ra = float(ra)
except:
if ":" not in ra:
log.error(
"ERROR: ra must be in decimal degree or sexagesimal format"
)
return
if dec:
try:
dec = float(dec)
except:
if ":" not in dec:
log.error(
"ERROR: dec must be in decimal degree or sexagesimal format"
)
return
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# BUILD KEYWORD DICT
kwargs = {}
kwargs["log"] = log
kwargs["settings"] = settings
kwargs["ra"] = ra
kwargs["dec"] = dec
# FITS OPTIONS
kwargs["fits"] = True # DEFAULT
if fitsFlag == False and nofitsFlag == True:
kwargs["fits"] = False
# JPEG OPTIONS
kwargs["jpeg"] = False # DEFAULT
if jpegFlag == True and nojpegFlag == False:
kwargs["jpeg"] = True
# COLOR JPEG OPTIONS
kwargs["color"] = False # DEFAULT
if colorFlag == True and nocolorFlag == False:
kwargs["color"] = True
# WIDTH OPTION
kwargs["arcsecSize"] = 60
if widthFlag:
kwargs["arcsecSize"] = float(widthFlag) * 60.
# CHOOSE A FILTERSET
kwargs["filterSet"] = 'gri'
if filtersFlag:
kwargs["filterSet"] = filtersFlag
for i in kwargs["filterSet"]:
if i not in "grizy":
log.error(
"ERROR: the requested filter must be in the grizy filter set")
return
# WHICH IMAGE TYPE TO DOWNLOAD
if stack:
kwargs["imageType"] = "stack"
if warp:
kwargs["imageType"] = "warp"
if closestFlag:
kwargs["imageType"] = "warp"
# MJD WINDOW
kwargs["mjdStart"] = mjdStart
kwargs["mjdEnd"] = mjdEnd
kwargs["window"] = False
try:
kwargs["window"] = int(closestFlag)
except:
pass
if not kwargs["window"]:
if mjd and closestFlag == "before":
kwargs["mjdEnd"] = mjd
elif mjd and closestFlag == "after":
kwargs["mjdStart"] = mjd
else:
if mjd and kwargs["window"] < 0:
kwargs["mjdEnd"] = mjd
elif mjd and kwargs["window"] > 0:
kwargs["mjdStart"] = mjd
# DOWNLOAD LOCATION
if downloadFolderFlag:
home = expanduser("~")
downloadFolderFlag = downloadFolderFlag.replace("~", home)
kwargs["downloadDirectory"] = downloadFolderFlag
# xt-kwarg_key_and_value
# DOWNLOAD THE IMAGES
images = downloader(**kwargs)
fitsPaths, jpegPaths, colorPath = images.get()
jpegPaths += colorPath
# POST-DOWNLOAD PROCESS IMAGES
kwargs = {}
kwargs["log"] = log
kwargs["settings"] = settings
# WIDTH OPTION
kwargs["arcsecSize"] = 60
if widthFlag:
kwargs["arcsecSize"] = float(widthFlag) * 60.
# ANNOTATE JPEG OPTIONS
kwargs["crosshairs"] = True # DEFAULT
kwargs["scale"] = True
if annotateFlag == False and noannotateFlag == True:
kwargs["crosshairs"] = False # DEFAULT
kwargs["scale"] = False
# INVERT OPTIONS
kwargs["invert"] = False # DEFAULT
if invertFlag == True and noinvertFlag == False:
kwargs["invert"] = True
# GREYSCALE OPTIONS
kwargs["greyscale"] = False # DEFAULT
if greyscaleFlag == True and nogreyscaleFlag == False:
kwargs["greyscale"] = True
# TRANSIENT DOT OPTIONS
kwargs["transient"] = False # DEFAULT
if transientFlag == True and notransientFlag == False:
kwargs["transient"] = True
for j in jpegPaths:
kwargs["imagePath"] = j
# kwargs["transient"] = False
# kwargs["invert"] = False
# kwargs["greyscale"] = False
oneImage = image(**kwargs)
oneImage.get()
# CALL FUNCTIONS/OBJECTS
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
from builtins import str
from builtins import range
import os
import nose2
import shutil
import unittest
import yaml
from fundamentals.utKit import utKit
import time
from fundamentals import tools
su = tools(arguments={"settingsFile": None},
docString=__doc__,
logLevel="INFO",
options_first=False,
projectName="fundamentals",
defaultSettingsFile=False)
arguments, settings, log, dbConn = su.setup()
# # load settings
# stream = open(
# "/Users/Dave/.config/fundamentals/fundamentals.yaml", 'r')
# settings = yaml.load(stream)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="atelParser",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/atelParser/atelParser.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
count = a["count"]
download = a["download"]
parse = a["parse"]
reparseFlag = a["reparseFlag"]
# CALL FUNCTIONS/OBJECTS
if download:
from atelParser import download
atels = download(log=log, settings=settings)
atelsToDownload = atels.get_list_of_atels_still_to_download()
atels.download_list_of_atels(atelsToDownload)
if count:
from atelParser import download
atels = download(log=log, settings=settings)
latestNumber = atels.get_latest_atel_number()
from datetime import datetime, date, time
now = datetime.now()
now = now.strftime("%Y/%m/%d %H:%M:%Ss")
print("%(latestNumber)s ATels have been reported as of %(now)s" %
locals())
if parse:
from atelParser import mysql
parser = mysql(log=log, settings=settings, reParse=reparseFlag)
parser.atels_to_database()
parser.parse_atels()
parser.populate_htm_columns()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``update_request_watcher.py`` is run as a single script from the cl, or when installed as a cl command*
"""
from dryxPython import commonutils as dcu
from dryxPython.projectsetup import setup_main_clutil
from dryxPython import logs as dl
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING"
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE update_request_watcher.py AT %s' %
(startTime,))
# find all update request files in "/_updates_required_" folder
basePath = pathToGitRepos + "/_updates_required_"
for d in os.listdir(basePath):
if os.path.isfile(os.path.join(basePath, d)):
if "gitupdates" in d:
thisRepo = d.replace(".gitupdates", "")
# check local git-repo requiring an update actually exists --
# trigger pull if it does
pathToRepo = pathToGitRepos + "/" + thisRepo
if not os.path.exists(pathToRepo):
message = "the path to the Folder folder %s does not exist on this machine" % (
pathToRepo,)
log.warning(message)
else:
dcu.update_git_repos.update_git_repos(
log=log,
gitProjectRoot=pathToRepo,
branchToUpdate="master",
installClUtils=True
)
# finally delete the update request file
os.remove(os.path.join(basePath, d))
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE update_request_watcher.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
import os
import unittest
import shutil
import unittest
import yaml
from astrocalc.utKit import utKit
from fundamentals import tools
from os.path import expanduser
home = expanduser("~")
packageDirectory = utKit("").get_project_root()
settingsFile = packageDirectory + "/test_settings.yaml"
su = tools(arguments={"settingsFile": settingsFile},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName=None,
defaultSettingsFile=False)
arguments, settings, log, dbConn = su.setup()
# SETUP PATHS TO COMMON DIRECTORIES FOR TEST DATA
moduleDirectory = os.path.dirname(__file__)
pathToInputDir = moduleDirectory + "/input/"
pathToOutputDir = moduleDirectory + "/output/"
try:
shutil.rmtree(pathToOutputDir)
except:
pass
# COPY INPUT TO OUTPUT DIR
shutil.copytree(pathToInputDir, pathToOutputDir)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
stream = file(pathToInputDir + "/example_settings.yaml", 'r')
settings = yaml.load(stream)
stream.close()
su = tools(
arguments={"settingsFile": pathToInputDir + "/example_settings.yaml"},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="sherlock")
arguments, settings, log, dbConn = su.setup()
import shutil
try:
shutil.rmtree(pathToOutputDir)
except:
pass
# COPY INPUT TO OUTPUT DIR
shutil.copytree(pathToInputDir, pathToOutputDir)
# Recursively create missing directories
if not os.path.exists(pathToOutputDir):
os.makedirs(pathToOutputDir)
import os
import nose
import shutil
import yaml
from polygot import htmlCleaner, cl_utils
from polygot.utKit import utKit
from fundamentals import tools
su = tools(arguments={"settingsFile": None},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="polygot",
tunnel=False)
arguments, settings, log, dbConn = su.setup()
# # load settings
# stream = file(
# "/Users/Dave/.config/polygot/polygot.yaml", 'r')
# settings = yaml.load(stream)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
stream = file(pathToInputDir + "/example_settings.yaml", 'r')
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="qubits")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
if init:
from . import workspace
ws = workspace(log=log, pathToWorkspace=pathToWorkspace)
ws.setup()
return
# IMPORT THE SIMULATION SETTINGS
(allSettings, programSettings, limitingMags, sampleNumber,
peakMagnitudeDistributions, explosionDaysFromSettings,
extendLightCurveTail, relativeSNRates, lowerRedshiftLimit,
upperRedshiftLimit, redshiftResolution, restFrameFilter,
kCorrectionTemporalResolution, kCorPolyOrder, kCorMinimumDataPoints,
extinctionType, extinctionConstant, hostExtinctionDistributions,
galacticExtinctionDistribution, surveyCadenceSettings, snLightCurves,
surveyArea, CCSNRateFraction, transientToCCSNRateFraction,
extraSurveyConstraints, lightCurvePolyOrder,
logLevel) = cu.read_in_survey_parameters(
log, pathToSettingsFile=pathToSettingsFile)
logFilePath = pathToOutputDirectory + "/qubits.log"
del log
log = _set_up_command_line_tool(level=str(logLevel),
logFilePath=logFilePath)
# dbConn, log = cu.settings(
# pathToSettingsFile=pathToSettingsFile,
# dbConn=False,
# log=True
# )
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE qubits AT %s' % (startTime, ))
resultsDict = {}
pathToOutputPlotDirectory = pathToOutputDirectory + "/plots/"
dcu.dryx_mkdir(log, directoryPath=pathToOutputPlotDirectory)
pathToResultsFolder = pathToOutputDirectory + "/results/"
dcu.dryx_mkdir(log, directoryPath=pathToResultsFolder)
if not programSettings[
'Extract Lightcurves from Spectra'] and not programSettings[
'Generate KCorrection Database'] and not programSettings[
'Run the Simulation'] and not programSettings[
'Compile and Plot Results']:
print "All stages of the simulatation have been switched off. Please switch on at least one stage of the simulation under the 'Programming Settings' in the settings file `%(pathToSettingsFile)s`" % locals(
)
# GENERATE THE DATA FOR SIMULATIONS
if programSettings['Extract Lightcurves from Spectra']:
log.info('generating the Lightcurves')
dg.generate_model_lightcurves(
log=log,
pathToSpectralDatabase=pathToSpectralDatabase,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
explosionDaysFromSettings=explosionDaysFromSettings,
extendLightCurveTail=extendLightCurveTail,
polyOrder=lightCurvePolyOrder)
print "The lightcurve file can be found here: %(pathToOutputDirectory)stransient_light_curves.yaml" % locals(
)
print "The lightcurve plots can be found in %(pathToOutputPlotDirectory)s" % locals(
)
if programSettings['Generate KCorrection Database']:
log.info('generating the kcorrection data')
dg.generate_kcorrection_listing_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
pathToSpectralDatabase=pathToSpectralDatabase,
restFrameFilter=restFrameFilter,
temporalResolution=kCorrectionTemporalResolution,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution)
log.info('generating the kcorrection polynomials')
dg.generate_kcorrection_polynomial_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
restFrameFilter=restFrameFilter,
kCorPolyOrder=kCorPolyOrder, # ORDER OF THE POLYNOMIAL TO FIT
kCorMinimumDataPoints=kCorMinimumDataPoints,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution,
plot=programSettings['Generate KCorrection Plots'])
print "The k-correction database has been generated here: %(pathToOutputDirectory)sk_corrections" % locals(
)
if programSettings['Generate KCorrection Plots']:
print "The k-correction polynomial plots can also be found in %(pathToOutputDirectory)sk_corrections" % locals(
)
if programSettings['Run the Simulation']:
# CREATE THE OBSERVABLE UNIVERSE!
log.info('generating the redshift array')
redshiftArray = u.random_redshift_array(
log,
sampleNumber,
lowerRedshiftLimit,
upperRedshiftLimit,
redshiftResolution=redshiftResolution,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['Redshifts'] = redshiftArray.tolist()
log.info('generating the SN type array')
snTypesArray = u.random_sn_types_array(
log,
sampleNumber,
relativeSNRates,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['SN Types'] = snTypesArray.tolist()
log.info('generating peak magnitudes for the SNe')
peakMagnitudesArray = u.random_peak_magnitudes(
log,
peakMagnitudeDistributions,
snTypesArray,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN host extictions array')
hostExtinctionArray = u.random_host_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
hostExtinctionDistributions,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN galactic extictions array')
galacticExtinctionArray = u.random_galactic_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
galacticExtinctionDistribution,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the raw lightcurves for the SNe')
rawLightCurveDict = u.generate_numpy_polynomial_lightcurves(
log,
snLightCurves=snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the k-correction array for the SNe')
kCorrectionArray = u.build_kcorrection_array(
log,
redshiftArray,
snTypesArray,
snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the observed lightcurves for the SNe')
observedFrameLightCurveInfo, peakAppMagList = u.convert_lightcurves_to_observered_frame(
log,
snLightCurves=snLightCurves,
rawLightCurveDict=rawLightCurveDict,
redshiftArray=redshiftArray,
snTypesArray=snTypesArray,
peakMagnitudesArray=peakMagnitudesArray,
kCorrectionArray=kCorrectionArray,
hostExtinctionArray=hostExtinctionArray,
galacticExtinctionArray=galacticExtinctionArray,
restFrameFilter=restFrameFilter,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
polyOrder=lightCurvePolyOrder,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the survey observation cadence')
cadenceDictionary = ss.survey_cadence_arrays(
log,
surveyCadenceSettings,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if the SNe are discoverable by the survey')
discoverableList = ss.determine_if_sne_are_discoverable(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info(
'determining the day (if and) when each SN is first discoverable by the survey'
)
ripeDayList = ss.determine_when_sne_are_ripe_for_discovery(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
discoverableList=discoverableList,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
plot=programSettings['Plot Simulation Helper Plots'])
# log.info('determining the day when each SN is disappears fainter than the survey limiting mags')
# disappearDayList = determine_when_discovered_sne_disappear(
# log,
# redshiftArray=redshiftArray,
# limitingMags=limitingMags,
# ripeDayList=ripeDayList,
# observedFrameLightCurveInfo=observedFrameLightCurveInfo,
# plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if and when each SN is discovered by the survey')
lightCurveDiscoveryDayList, surveyDiscoveryDayList, snCampaignLengthList = ss.determine_if_sne_are_discovered(
log,
limitingMags=limitingMags,
ripeDayList=ripeDayList,
cadenceDictionary=cadenceDictionary,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
extraSurveyConstraints=extraSurveyConstraints,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict[
'Discoveries Relative to Peak Magnitudes'] = lightCurveDiscoveryDayList
resultsDict[
'Discoveries Relative to Survey Year'] = surveyDiscoveryDayList
resultsDict['Campaign Length'] = snCampaignLengthList
resultsDict['Cadence Dictionary'] = cadenceDictionary
resultsDict['Peak Apparent Magnitudes'] = peakAppMagList
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
fileName = pathToOutputDirectory + \
"/simulation_results_%s.yaml" % (now,)
stream = file(fileName, 'w')
yamlContent = dict(allSettings.items() + resultsDict.items())
yaml.dump(yamlContent, stream, default_flow_style=False)
stream.close()
print "The simulation output file can be found here: %(fileName)s. Remember to update your settings file 'Simulation Results File Used for Plots' parameter with this filename before compiling the results." % locals(
)
if programSettings['Plot Simulation Helper Plots']:
print "The simulation helper-plots found in %(pathToOutputPlotDirectory)s" % locals(
)
# COMPILE AND PLOT THE RESULTS
if programSettings['Compile and Plot Results']:
pathToYamlFile = pathToOutputDirectory + "/" + \
programSettings['Simulation Results File Used for Plots']
result_log = r.log_the_survey_settings(log, pathToYamlFile)
snSurveyDiscoveryTimes, lightCurveDiscoveryTimes, snTypes, redshifts, cadenceDictionary, peakAppMagList, snCampaignLengthList = r.import_results(
log, pathToYamlFile)
snRatePlotLink, totalRate, tooFaintRate, shortCampaignRate = r.determine_sn_rate(
log,
lightCurveDiscoveryTimes,
snSurveyDiscoveryTimes,
redshifts,
surveyCadenceSettings=surveyCadenceSettings,
lowerRedshiftLimit=lowerRedshiftLimit,
upperRedshiftLimit=upperRedshiftLimit,
redshiftResolution=redshiftResolution,
surveyArea=surveyArea,
CCSNRateFraction=CCSNRateFraction,
transientToCCSNRateFraction=transientToCCSNRateFraction,
peakAppMagList=peakAppMagList,
snCampaignLengthList=snCampaignLengthList,
extraSurveyConstraints=extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """
## Results ##
This simulated survey discovered a total of **%s** transients per year. An extra **%s** transients were detected but deemed too faint to constrain a positive transient identification and a further **%s** transients where detected but an observational campaign of more than **%s** days could not be completed to ensure identification. See below for the various output plots.
""" % (
totalRate, tooFaintRate, shortCampaignRate,
extraSurveyConstraints["Observable for at least ? number of days"])
cadenceWheelLink = r.plot_cadence_wheel(
log,
cadenceDictionary,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (cadenceWheelLink, )
discoveryMapLink = r.plot_sn_discovery_map(
log,
snSurveyDiscoveryTimes,
peakAppMagList,
snCampaignLengthList,
redshifts,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (discoveryMapLink, )
ratioMapLink = r.plot_sn_discovery_ratio_map(
log,
snSurveyDiscoveryTimes,
redshifts,
peakAppMagList,
snCampaignLengthList,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (ratioMapLink, )
result_log += """%s""" % (snRatePlotLink, )
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
mdLogPath = pathToResultsFolder + \
"simulation_result_log_%s.md" % (now,)
mdLog = open(mdLogPath, 'w')
mdLog.write(result_log)
mdLog.close()
dmd.convert_to_html(log=log, pathToMMDFile=mdLogPath, css="amblin")
print "Results can be found here: %(pathToResultsFolder)s" % locals()
html = mdLogPath.replace(".md", ".html")
print "Open this file in your browser: %(html)s" % locals()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="sloancone",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
search = a["search"]
covered = a["covered"]
nearest = a["nearest"]
fformat = a["format"]
galaxyType = a["galaxyType"]
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/sloancone/sloancone.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
# CALL FUNCTIONS/OBJECTS
# CALL THE WORKER FUNCTION
if search:
cs = cone_search(log=log,
ra=ra,
dec=dec,
searchRadius=float(arcsecRadius),
nearest=nearestFlag,
outputFormat=outputFormat,
galaxyType=galaxyType)
results = cs.get()
print(results)
# COVERED = TRUE | FALSE | 999 (I.E. NOT SURE)
if covered:
check = check_coverage(log=log, ra=ra, dec=dec).get()
print(check)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="HMpTy")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.items():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, str):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# CALL FUNCTIONS/OBJECTS
if index:
add_htm_ids_to_mysql_database_table(raColName=raCol,
declColName=decCol,
tableName=tableName,
dbConn=dbConn,
log=log,
primaryIdColumnName=primaryIdCol,
reindex=forceFlag)
if search:
cs = conesearch(log=log,
dbConn=dbConn,
tableName=tableName,
columns=False,
ra=ra,
dec=dec,
radiusArcsec=float(radius),
separations=True,
distinct=False,
sqlWhere=False)
matchIndies, matches = cs.search()
if not renderFlag:
print(matches.table())
elif renderFlag == "json":
print(matches.json())
elif renderFlag == "csv":
print(matches.csv())
elif renderFlag == "yaml":
print(matches.yaml())
elif renderFlag == "md":
print(matches.markdown())
elif renderFlag == "table":
print(matches.markdown())
elif renderFlag == "mysql":
print(matches.mysql(tableName=resultsTable))
if level:
from HMpTy import HTM
mesh = HTM(depth=int(level), log=log)
htmids = mesh.lookup_id(ra, dec)
print(htmids[0])
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="polyglot")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# for k, v in locals().iteritems():
# print k, v
if not destinationFolder:
destinationFolder = os.getcwd()
if not filenameFlag:
filenameFlag = False
if not cleanFlag:
readability = False
else:
readability = True
if init:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/polyglot/polyglot.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if pdf and url:
filepath = printpdf.printpdf(log=log,
settings=settings,
url=url,
folderpath=destinationFolder,
title=filenameFlag,
append=False,
readability=readability).get()
if html and url:
cleaner = htmlCleaner.htmlCleaner(
log=log,
settings=settings,
url=url,
outputDirectory=destinationFolder,
title=filenameFlag, # SET TO FALSE TO USE WEBPAGE TITLE,
style=cleanFlag, # add polyglot's styling to the HTML document
metadata=True, # include metadata in generated HTML (e.g. title),
h1=True # include title as H1 at the top of the doc
)
filepath = cleaner.clean()
if epub:
if url:
iinput = url
else:
iinput = docx
from polyglot import ebook
epub = ebook(log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag,
bookFormat="epub",
outputDirectory=destinationFolder)
filepath = epub.get()
if mobi:
if url:
iinput = url
else:
iinput = docx
from polyglot import ebook
mobi = ebook(
log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag,
bookFormat="mobi",
outputDirectory=destinationFolder,
)
filepath = mobi.get()
if kindle:
if url:
iinput = url
else:
iinput = docx
from polyglot import kindle
sender = kindle(log=log,
settings=settings,
urlOrPath=iinput,
title=filenameFlag)
success = sender.send()
if kindleNB2MD:
basename = os.path.basename(notebook)
extension = os.path.splitext(basename)[1]
filenameNoExtension = os.path.splitext(basename)[0]
if destinationFolder:
filepath = destinationFolder + "/" + filenameNoExtension + ".md"
else:
filepath = notebook.replace("." + extension, ".md")
from polyglot.markdown import kindle_notebook
nb = kindle_notebook(log=log,
kindleExportPath=notebook,
outputPath=filepath)
nb.convert()
if openFlag:
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="HMpTy",
defaultSettingsFile=True
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (varname, val,))
hostFlag = a["hostFlag"]
userFlag = a["userFlag"]
passwdFlag = a["passwdFlag"]
dbNameFlag = a["dbNameFlag"]
tableName = a["tableName"]
index = a["index"]
htmid = a["htmid"]
primaryIdCol = a["primaryIdCol"]
raCol = a["raCol"]
decCol = a["decCol"]
ra = a["ra"]
radius = a["radius"]
level = a["level"]
forceFlag = a["forceFlag"]
renderFlag = a["renderFlag"]
search = a["search"]
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/HMpTy/HMpTy.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
# CALL FUNCTIONS/OBJECTS
if index:
add_htm_ids_to_mysql_database_table(
raColName=raCol,
declColName=decCol,
tableName=tableName,
dbConn=dbConn,
log=log,
primaryIdColumnName=primaryIdCol,
reindex=forceFlag,
dbSettings=dbSettings
)
if search:
cs = conesearch(
log=log,
dbConn=dbConn,
tableName=tableName,
columns=False,
ra=ra,
dec=dec,
radiusArcsec=float(radius),
separations=True,
distinct=False,
sqlWhere=False
)
matchIndies, matches = cs.search()
if not renderFlag:
print(matches.table())
elif renderFlag == "json":
print(matches.json())
elif renderFlag == "csv":
print(matches.csv())
elif renderFlag == "yaml":
print(matches.yaml())
elif renderFlag == "md":
print(matches.markdown())
elif renderFlag == "table":
print(matches.markdown())
elif renderFlag == "mysql":
print(matches.mysql(tableName=resultsTable))
if level:
from HMpTy import HTM
mesh = HTM(
depth=int(level),
log=log
)
htmids = mesh.lookup_id(ra, dec)
print(htmids[0])
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=True,
projectName="sloancone",
tunnel=False
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# CALL FUNCTIONS/OBJECTS
# call the worker function
if search:
cs = cone_search(
log=log,
ra=ra,
dec=dec,
searchRadius=float(arcsecRadius),
nearest=nearestFlag,
outputFormat=outputFormat,
galaxyType=galaxyType
)
results = cs.get()
print results
# covered = True | False | 999 (i.e. not sure)
if covered:
check = check_coverage(
log=log,
ra=ra,
dec=dec
).get()
print check
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="dropterm"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
val = val.replace("'", "\\'")
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
if authenticate == True:
this = authenticate_dropterm.authenticate_dropterm(
log=log,
settings=settings,
pathToSettings=pathToSettingsFile
)
if move == True and contents == False:
this = move_files_or_folders.move_files_or_folders(
log=log,
settings=settings,
source=sourcePath,
destination=destinationPath
)
this.move()
if move == True and contents == True:
this = move_files_or_folders.move_files_or_folders(
log=log,
settings=settings,
source=sourcePath,
destination=destinationPath,
moveFolderContents=True
)
this.move()
if copy == True and contents == False:
this = copy_files_or_folders.copy_files_or_folders(
log=log,
settings=settings,
source=sourcePath,
destination=destinationPath
)
this.copy()
if copy == True and contents == True:
this = copy_files_or_folders.copy_files_or_folders(
log=log,
settings=settings,
source=sourcePath,
destination=destinationPath,
copyFolderContents=True
)
this.copy()
if ls == True:
this = list_directory_contents(
log=log,
settings=settings,
directoryToList=directoryPath
)
filePaths, fileNames, shareUrls, meta = this.get()
for f in fileNames:
print f
if share == True:
if lFlag == False:
lFlag = True
else:
lFlag = False
this = share_file.share_file(
log=log,
settings=settings,
filePath=filePath,
short=lFlag
)
url = this.get()
print url
if mkdir == True:
this = mkdirectory.mkdir(
log=log,
settings=settings,
dropboxPath=dirPath
).get()
if touch == True:
this = touch_file.touch_file(
log=log,
settings=settings,
destination=destinationPath
).touch()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="transientNamer"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
if search or new or cone:
if ra:
tns = transientNamer.search(
log=log,
ra=ra,
dec=dec,
radiusArcsec=arcsecRadius,
comments=withCommentsFlag
)
if name:
tns = transientNamer.search(
log=log,
name=name,
comments=withCommentsFlag
)
if discInLastDays:
tns = transientNamer.search(
log=log,
discInLastDays=discInLastDays,
comments=withCommentsFlag
)
# Recursively create missing directories
if outputFlag and not os.path.exists(outputFlag):
os.makedirs(outputFlag)
if tableNamePrefix:
sources, phot, spec, files = tns.mysql(
tableNamePrefix=tableNamePrefix, dirPath=outputFlag)
numSources = len(sources.split("\n")) - 1
elif not render or render == "table":
sources, phot, spec, files = tns.table(dirPath=outputFlag)
numSources = len(sources.split("\n")) - 4
elif render == "csv":
sources, phot, spec, files = tns.csv(dirPath=outputFlag)
numSources = len(sources.split("\n")) - 1
elif render == "json":
sources, phot, spec, files = tns.json(dirPath=outputFlag)
numSources = len(sources.split("{")) - 1
elif render == "yaml":
sources, phot, spec, files = tns.yaml(dirPath=outputFlag)
numSources = len(sources.split("\n-"))
elif render == "markdown":
sources, phot, spec, files = tns.markdown(dirPath=outputFlag)
numSources = len(sources.split("\n")) - 2
if numSources == 1:
print "%(numSources)s transient found" % locals()
elif numSources > 1:
print "%(numSources)s transients found" % locals()
if not outputFlag:
print "\n# Matched Transients"
print sources
print "\n# Transient Photometry"
print phot
print "\n# Transient Spectra"
print spec
print "\n# Transient Supplementary Files"
print files
print "\n# Original TNS Search URL"
print tns.url
# CALL FUNCTIONS/OBJECTS
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
import os
import nose
import shutil
import yaml
from panstamps import downloader, cl_utils
from panstamps.utKit import utKit
import unittest
from fundamentals import tools
su = tools(arguments={"settingsFile": None},
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="panstamps")
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file("/Users/Dave/.config/panstamps/panstamps.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
class test_downloader(unittest.TestCase):
def test_downloader_function(self):
kwargs = {}
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="HMpTy"
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# CALL FUNCTIONS/OBJECTS
if index:
add_htm_ids_to_mysql_database_table(
raColName=raCol,
declColName=decCol,
tableName=tableName,
dbConn=dbConn,
log=log,
primaryIdColumnName=primaryIdCol,
reindex=forceFlag
)
if search:
cs = conesearch(
log=log,
dbConn=dbConn,
tableName=tableName,
columns=False,
ra=ra,
dec=dec,
radiusArcsec=float(radius),
separations=True,
distinct=False,
sqlWhere=False
)
matchIndies, matches = cs.search()
if not renderFlag:
print matches.table()
elif renderFlag == "json":
print matches.json()
elif renderFlag == "csv":
print matches.csv()
elif renderFlag == "yaml":
print matches.yaml()
elif renderFlag == "md":
print matches.markdown()
elif renderFlag == "table":
print matches.markdown()
elif renderFlag == "mysql":
print matches.mysql(tableName=resultsTable)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
from __future__ import print_function
from builtins import str
import os
import nose
import shutil
import unittest
import yaml
from fundamentals.utKit import utKit
from fundamentals.files import tag
from fundamentals import tools
su = tools(arguments={"settingsFile": None},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="fundamentals")
arguments, settings, log, dbConn = su.setup()
# # load settings
# stream = open(
# "/Users/Dave/.config/fundamentals/fundamentals.yaml", 'r')
# settings = yaml.load(stream)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
import os
import nose
import shutil
import yaml
import unittest
from breaker import cl_utils
from breaker.fakers import generate_faker_catalogue
from breaker.utKit import utKit
from fundamentals import tools, times
su = tools(
arguments={},
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="breaker"
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/breaker/breaker.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
import os
import nose
import shutil
import yaml
from astrocalc.utKit import utKit
from fundamentals import tools
su = tools(
arguments={"settingsFile": None},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="astrocalc"
)
arguments, settings, log, dbConn = su.setup()
# # load settings
# stream = file(
# "/Users/Dave/.config/astrocalc/astrocalc.yaml", 'r')
# settings = yaml.load(stream)
# stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
stream = file(
import os
import nose
import shutil
import yaml
from transientNamer import search
from transientNamer.utKit import utKit
from fundamentals import tools
su = tools(
arguments={"settingsFile": None},
docString=__doc__,
logLevel="ERROR",
options_first=False,
projectName="transientNamer"
)
arguments, settings, log, dbConn = su.setup()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
# load settings
stream = file(
pathToInputDir + "/transientNamer.yaml", 'r')
settings = yaml.load(stream)
stream.close()
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
from astrocalc.coords import unit_conversion
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="CRITICAL",
options_first=True,
projectName="astrocalc",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
coordflip = a["coordflip"]
sep = a["sep"]
timeflip = a["timeflip"]
trans = a["trans"]
now = a["now"]
dist = a["dist"]
ra = a["ra"]
ra1 = a["ra1"]
ra2 = a["ra2"]
dec = a["dec"]
dec1 = a["dec1"]
dec2 = a["dec2"]
datetime = a["datetime"]
north = a["north"]
east = a["east"]
distVal = a["distVal"]
hVal = a["hcFlag"]
OmegaMatter = a["wmFlag"]
OmegaVacuum = a["wvFlag"]
mpcFlag = a["mpcFlag"]
redshiftFlag = a["redshiftFlag"]
cartesianFlag = a["cartesianFlag"]
# CALL FUNCTIONS/OBJECTS
if coordflip:
if cartesianFlag:
converter = unit_conversion(log=log)
x, y, z = converter.ra_dec_to_cartesian(ra="23 45 21.23232",
dec="+01:58:5.45341")
print(x, y, z)
return
try:
ra = float(ra)
dec = float(dec)
degree = True
except Exception as e:
degree = False
if degree is True:
converter = unit_conversion(log=log)
try:
ra = converter.ra_decimal_to_sexegesimal(ra=ra, delimiter=":")
dec = converter.dec_decimal_to_sexegesimal(dec=dec,
delimiter=":")
except Exception as e:
print(e)
sys.exit(0)
print(ra, dec)
else:
converter = unit_conversion(log=log)
try:
ra = converter.ra_sexegesimal_to_decimal(ra=ra)
dec = converter.dec_sexegesimal_to_decimal(dec=dec)
except Exception as e:
print(e)
sys.exit(0)
print(ra, dec)
if sep:
from astrocalc.coords import separations
calculator = separations(
log=log,
ra1=ra1,
dec1=dec1,
ra2=ra2,
dec2=dec2,
)
angularSeparation, north, east = calculator.get()
print("""%(angularSeparation)s arcsec (%(north)s N, %(east)s E)""" %
locals())
if timeflip:
try:
inputMjd = float(datetime)
if datetime[0] not in ["0", "1", "2"]:
inputMjd = True
else:
inputMjd = False
except:
inputMjd = False
from astrocalc.times import conversions
converter = conversions(log=log)
if inputMjd == False:
try:
mjd = converter.ut_datetime_to_mjd(utDatetime=datetime)
print(mjd)
except Exception as e:
print(e)
else:
try:
utDate = converter.mjd_to_ut_datetime(mjd=datetime)
print(utDate)
except Exception as e:
print(e)
if trans:
# TRANSLATE COORDINATES ACROSS SKY
from astrocalc.coords import translate
newRa, newDec = translate(log=log,
ra=ra,
dec=dec,
northArcsec=float(north),
eastArcsec=float(east)).get()
from astrocalc.coords import unit_conversion
converter = unit_conversion(log=log)
ra = converter.ra_decimal_to_sexegesimal(ra=newRa, delimiter=":")
dec = converter.dec_decimal_to_sexegesimal(dec=newDec, delimiter=":")
print("%(newRa)s, %(newDec)s (%(ra)s, %(dec)s)" % locals())
if now:
from astrocalc.times import now
mjd = now(log=log).get_mjd()
print(mjd)
if dist and redshiftFlag:
from astrocalc.distances import converter
c = converter(log=log)
if not hcFlag:
hcFlag = 70.
if not wmFlag:
wmFlag = 0.3
if not wvFlag:
wvFlag = 0.7
dists = c.redshift_to_distance(z=float(distVal),
WM=float(wmFlag),
WV=float(wvFlag),
H0=float(hcFlag))
print("Distance Modulus: " + str(dists["dmod"]) + " mag")
print("Luminousity Distance: " + str(dists["dl_mpc"]) + " Mpc")
print("Angular Size Scale: " + str(dists["da_scale"]) + " kpc/arcsec")
print("Angular Size Distance: " + str(dists["da_mpc"]) + " Mpc")
print("Comoving Radial Distance: " + str(dists["dcmr_mpc"]) + " Mpc")
if dist and mpcFlag:
from astrocalc.distances import converter
c = converter(log=log)
z = c.distance_to_redshift(mpc=float(distVal))
print("z = %(z)s" % locals())
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
*The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
distributionName="qub-sherlock",
projectName="sherlock",
defaultSettingsFile=True)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
# AUTOMATICALLY
a = {}
for arg, val in list(arguments.items()):
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
a[varname] = val
if arg == "--dbConn":
dbConn = val
a["dbConn"] = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
# set options interactively if user requests
if "interactiveFlag" in a and a["interactiveFlag"]:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
if a["init"]:
from os.path import expanduser
home = expanduser("~")
filepath = home + "/.config/sherlock/sherlock.yaml"
try:
cmd = """open %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
try:
cmd = """start %(filepath)s""" % locals()
p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
except:
pass
return
init = a["init"]
match = a["match"]
dbmatch = a["dbmatch"]
clean = a["clean"]
wiki = a["wiki"]
iimport = a["import"]
ned = a["ned"]
cat = a["cat"]
stream = a["stream"]
info = a["info"]
ra = a["ra"]
dec = a["dec"]
radiusArcsec = a["radiusArcsec"]
cat_name = a["cat_name"]
stream_name = a["stream_name"]
skipNedUpdateFlag = a["skipNedUpdateFlag"]
skipMagUpdateFlag = a["skipMagUpdateFlag"]
settingsFlag = a["settingsFlag"]
verboseFlag = a["verboseFlag"]
updateFlag = a["updateFlag"]
# CALL FUNCTIONS/OBJECTS
if match or dbmatch:
if verboseFlag:
verbose = 2
else:
verbose = 1
if skipNedUpdateFlag:
updateNed = False
else:
updateNed = True
if skipMagUpdateFlag:
updatePeakMags = False
else:
updatePeakMags = True
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verbose,
update=updateFlag,
updateNed=updateNed,
updatePeakMags=updatePeakMags)
classifier.classify()
if clean:
cleaner = database_cleaner(log=log, settings=settings)
cleaner.clean()
if wiki:
updateWiki = update_wiki_pages(log=log, settings=settings)
updateWiki.update()
if iimport and ned:
ned = nedStreamImporter(log=log,
settings=settings,
coordinateList=["%(ra)s %(dec)s" % locals()],
radiusArcsec=radiusArcsec)
ned.ingest()
if iimport and cat:
if cat_name == "veron":
catalogue = veronImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if "ned_d" in cat_name:
catalogue = nedImporter(log=log,
settings=settings,
pathToDataFile=pathToDataFile,
version=cat_version,
catalogueName=cat_name)
catalogue.ingest()
if iimport and stream:
if "ifs" in stream_name:
stream = ifsImporter(log=log, settings=settings)
stream.ingest()
if not init and not match and not clean and not wiki and not iimport and ra:
classifier = transient_classifier.transient_classifier(
log=log,
settings=settings,
ra=ra,
dec=dec,
name=False,
verbose=verboseFlag)
classifier.classify()
if info:
print("sherlock-catalogues")
wiki = update_wiki_pages(log=log, settings=settings)
table = list(wiki._get_table_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
print()
print("Crossmatch Streams")
table = list(wiki._get_stream_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
print()
print("Views on Catalogues and Streams")
table = list(wiki._get_view_infos(trimmed=True))
dataSet = list_of_dictionaries(log=log, listOfDictionaries=table)
tableData = dataSet.reST(filepath=None)
print(tableData)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return