def main():
"""
*The main function used when ``plotting`` run as a single script from the cl*
"""
################ > IMPORTS ################
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
import dryxPython.commonutils as cu
dbConn, log = settings(
dbConn=False,
log=True)
## START LOGGING ##
startTime = cu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE plotting AT %s' % (startTime,))
# WRITE CODE HERE
if dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = cu.get_now_sql_datetime()
runningTime = cu.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE plotting AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``add_mavericks_tags_to_voodoopad.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##
# PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED
if arguments == None:
arguments = docopt(__doc__)
# SETUP LOGGER -- DEFAULT TO CONSOLE LOGGER IF NONE PROVIDED IN SETTINGS
if "--logger" not in arguments or arguments["--logger"] is None:
log = dl.console_logger(
level="DEBUG"
)
log.debug('logger setup')
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
varname = arg.replace("--", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE add_mavericks_tags_to_voodoopad.py AT %s' %
(startTime,))
# call the worker function
add_mavericks_tags_to_voodoopad(
log=log,
pathToVpspotlight=pathToVpspotlight,
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE add_mavericks_tags_to_voodoopad.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main():
"""
*Used for debugging
Key Arguments:
-
- dbConn -- mysql database connection
- log -- logger
Return:
- None*
"""
################ > IMPORTS ################
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
import pesstoMarshallPythonPath as pp
pp.set_python_path()
import pmCommonUtils as p
import dryxPython.commonutils as cu
################ > SETUP ##################
# SETUP DB CONNECTION AND A LOGGER
dbConn, log = p.settings()
## START LOGGING ##
startTime = cu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE fitstools AT %s' % (startTime,))
################ > VARIABLE SETTINGS ######
################ >ACTION(S) ###############
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = cu.get_now_sql_datetime()
runningTime = cu.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE fitstools AT %s (RUNTIME: %s) --' %
(endTime, runningTime,))
return
def main():
"""
*The main function used when ``csvtools.py`` run as a single script from the cl*
"""
########## PRE-IMPORT SETUP ##########
relativePathToProjectRoot = "../../../"
import dryxPython.projectsetup as dps
projectSetup = dps.projectSetup(
dbConn=False,
relativePathToProjectRoot=relativePathToProjectRoot
)
global settings, contentPaths
dbConn, log, settings, contentPaths = projectSetup.get_project_atrributes()
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
import dryxPython.commonutils as cu
## START LOGGING ##
startTime = cu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE csvtools.py AT %s' % (startTime,))
# SET GLOBAL VARIABLES
# WRITE CODE HERE
if dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = cu.get_now_sql_datetime()
runningTime = cu.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE csvtools.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = setup_main_clutil(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="sherlock"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.debug(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
if match:
sherlock = classifier(
log=log,
settings=settings,
update=updateFlag,
transientIdList=[]
)
sherlock.get()
if clean:
cleaner = cleanup_database_tables(
log=log,
settings=settings
)
cleaner.get()
if wiki:
updateWiki = update_wiki_pages(
log=log,
settings=settings
)
updateWiki.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.debug('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="qubits")
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val, ))
else:
exec(varname + " = %s" % (val, ))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (
varname,
val,
))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime, ))
if init:
from . import workspace
ws = workspace(log=log, pathToWorkspace=pathToWorkspace)
ws.setup()
return
# IMPORT THE SIMULATION SETTINGS
(allSettings, programSettings, limitingMags, sampleNumber,
peakMagnitudeDistributions, explosionDaysFromSettings,
extendLightCurveTail, relativeSNRates, lowerRedshiftLimit,
upperRedshiftLimit, redshiftResolution, restFrameFilter,
kCorrectionTemporalResolution, kCorPolyOrder, kCorMinimumDataPoints,
extinctionType, extinctionConstant, hostExtinctionDistributions,
galacticExtinctionDistribution, surveyCadenceSettings, snLightCurves,
surveyArea, CCSNRateFraction, transientToCCSNRateFraction,
extraSurveyConstraints, lightCurvePolyOrder,
logLevel) = cu.read_in_survey_parameters(
log, pathToSettingsFile=pathToSettingsFile)
logFilePath = pathToOutputDirectory + "/qubits.log"
del log
log = _set_up_command_line_tool(level=str(logLevel),
logFilePath=logFilePath)
# dbConn, log = cu.settings(
# pathToSettingsFile=pathToSettingsFile,
# dbConn=False,
# log=True
# )
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE qubits AT %s' % (startTime, ))
resultsDict = {}
pathToOutputPlotDirectory = pathToOutputDirectory + "/plots/"
dcu.dryx_mkdir(log, directoryPath=pathToOutputPlotDirectory)
pathToResultsFolder = pathToOutputDirectory + "/results/"
dcu.dryx_mkdir(log, directoryPath=pathToResultsFolder)
if not programSettings[
'Extract Lightcurves from Spectra'] and not programSettings[
'Generate KCorrection Database'] and not programSettings[
'Run the Simulation'] and not programSettings[
'Compile and Plot Results']:
print "All stages of the simulatation have been switched off. Please switch on at least one stage of the simulation under the 'Programming Settings' in the settings file `%(pathToSettingsFile)s`" % locals(
)
# GENERATE THE DATA FOR SIMULATIONS
if programSettings['Extract Lightcurves from Spectra']:
log.info('generating the Lightcurves')
dg.generate_model_lightcurves(
log=log,
pathToSpectralDatabase=pathToSpectralDatabase,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
explosionDaysFromSettings=explosionDaysFromSettings,
extendLightCurveTail=extendLightCurveTail,
polyOrder=lightCurvePolyOrder)
print "The lightcurve file can be found here: %(pathToOutputDirectory)stransient_light_curves.yaml" % locals(
)
print "The lightcurve plots can be found in %(pathToOutputPlotDirectory)s" % locals(
)
if programSettings['Generate KCorrection Database']:
log.info('generating the kcorrection data')
dg.generate_kcorrection_listing_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
pathToSpectralDatabase=pathToSpectralDatabase,
restFrameFilter=restFrameFilter,
temporalResolution=kCorrectionTemporalResolution,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution)
log.info('generating the kcorrection polynomials')
dg.generate_kcorrection_polynomial_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
restFrameFilter=restFrameFilter,
kCorPolyOrder=kCorPolyOrder, # ORDER OF THE POLYNOMIAL TO FIT
kCorMinimumDataPoints=kCorMinimumDataPoints,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution,
plot=programSettings['Generate KCorrection Plots'])
print "The k-correction database has been generated here: %(pathToOutputDirectory)sk_corrections" % locals(
)
if programSettings['Generate KCorrection Plots']:
print "The k-correction polynomial plots can also be found in %(pathToOutputDirectory)sk_corrections" % locals(
)
if programSettings['Run the Simulation']:
# CREATE THE OBSERVABLE UNIVERSE!
log.info('generating the redshift array')
redshiftArray = u.random_redshift_array(
log,
sampleNumber,
lowerRedshiftLimit,
upperRedshiftLimit,
redshiftResolution=redshiftResolution,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['Redshifts'] = redshiftArray.tolist()
log.info('generating the SN type array')
snTypesArray = u.random_sn_types_array(
log,
sampleNumber,
relativeSNRates,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['SN Types'] = snTypesArray.tolist()
log.info('generating peak magnitudes for the SNe')
peakMagnitudesArray = u.random_peak_magnitudes(
log,
peakMagnitudeDistributions,
snTypesArray,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN host extictions array')
hostExtinctionArray = u.random_host_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
hostExtinctionDistributions,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN galactic extictions array')
galacticExtinctionArray = u.random_galactic_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
galacticExtinctionDistribution,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the raw lightcurves for the SNe')
rawLightCurveDict = u.generate_numpy_polynomial_lightcurves(
log,
snLightCurves=snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the k-correction array for the SNe')
kCorrectionArray = u.build_kcorrection_array(
log,
redshiftArray,
snTypesArray,
snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the observed lightcurves for the SNe')
observedFrameLightCurveInfo, peakAppMagList = u.convert_lightcurves_to_observered_frame(
log,
snLightCurves=snLightCurves,
rawLightCurveDict=rawLightCurveDict,
redshiftArray=redshiftArray,
snTypesArray=snTypesArray,
peakMagnitudesArray=peakMagnitudesArray,
kCorrectionArray=kCorrectionArray,
hostExtinctionArray=hostExtinctionArray,
galacticExtinctionArray=galacticExtinctionArray,
restFrameFilter=restFrameFilter,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
polyOrder=lightCurvePolyOrder,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the survey observation cadence')
cadenceDictionary = ss.survey_cadence_arrays(
log,
surveyCadenceSettings,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if the SNe are discoverable by the survey')
discoverableList = ss.determine_if_sne_are_discoverable(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info(
'determining the day (if and) when each SN is first discoverable by the survey'
)
ripeDayList = ss.determine_when_sne_are_ripe_for_discovery(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
discoverableList=discoverableList,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
plot=programSettings['Plot Simulation Helper Plots'])
# log.info('determining the day when each SN is disappears fainter than the survey limiting mags')
# disappearDayList = determine_when_discovered_sne_disappear(
# log,
# redshiftArray=redshiftArray,
# limitingMags=limitingMags,
# ripeDayList=ripeDayList,
# observedFrameLightCurveInfo=observedFrameLightCurveInfo,
# plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if and when each SN is discovered by the survey')
lightCurveDiscoveryDayList, surveyDiscoveryDayList, snCampaignLengthList = ss.determine_if_sne_are_discovered(
log,
limitingMags=limitingMags,
ripeDayList=ripeDayList,
cadenceDictionary=cadenceDictionary,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
extraSurveyConstraints=extraSurveyConstraints,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict[
'Discoveries Relative to Peak Magnitudes'] = lightCurveDiscoveryDayList
resultsDict[
'Discoveries Relative to Survey Year'] = surveyDiscoveryDayList
resultsDict['Campaign Length'] = snCampaignLengthList
resultsDict['Cadence Dictionary'] = cadenceDictionary
resultsDict['Peak Apparent Magnitudes'] = peakAppMagList
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
fileName = pathToOutputDirectory + \
"/simulation_results_%s.yaml" % (now,)
stream = file(fileName, 'w')
yamlContent = dict(allSettings.items() + resultsDict.items())
yaml.dump(yamlContent, stream, default_flow_style=False)
stream.close()
print "The simulation output file can be found here: %(fileName)s. Remember to update your settings file 'Simulation Results File Used for Plots' parameter with this filename before compiling the results." % locals(
)
if programSettings['Plot Simulation Helper Plots']:
print "The simulation helper-plots found in %(pathToOutputPlotDirectory)s" % locals(
)
# COMPILE AND PLOT THE RESULTS
if programSettings['Compile and Plot Results']:
pathToYamlFile = pathToOutputDirectory + "/" + \
programSettings['Simulation Results File Used for Plots']
result_log = r.log_the_survey_settings(log, pathToYamlFile)
snSurveyDiscoveryTimes, lightCurveDiscoveryTimes, snTypes, redshifts, cadenceDictionary, peakAppMagList, snCampaignLengthList = r.import_results(
log, pathToYamlFile)
snRatePlotLink, totalRate, tooFaintRate, shortCampaignRate = r.determine_sn_rate(
log,
lightCurveDiscoveryTimes,
snSurveyDiscoveryTimes,
redshifts,
surveyCadenceSettings=surveyCadenceSettings,
lowerRedshiftLimit=lowerRedshiftLimit,
upperRedshiftLimit=upperRedshiftLimit,
redshiftResolution=redshiftResolution,
surveyArea=surveyArea,
CCSNRateFraction=CCSNRateFraction,
transientToCCSNRateFraction=transientToCCSNRateFraction,
peakAppMagList=peakAppMagList,
snCampaignLengthList=snCampaignLengthList,
extraSurveyConstraints=extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """
## Results ##
This simulated survey discovered a total of **%s** transients per year. An extra **%s** transients were detected but deemed too faint to constrain a positive transient identification and a further **%s** transients where detected but an observational campaign of more than **%s** days could not be completed to ensure identification. See below for the various output plots.
""" % (
totalRate, tooFaintRate, shortCampaignRate,
extraSurveyConstraints["Observable for at least ? number of days"])
cadenceWheelLink = r.plot_cadence_wheel(
log,
cadenceDictionary,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (cadenceWheelLink, )
discoveryMapLink = r.plot_sn_discovery_map(
log,
snSurveyDiscoveryTimes,
peakAppMagList,
snCampaignLengthList,
redshifts,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (discoveryMapLink, )
ratioMapLink = r.plot_sn_discovery_ratio_map(
log,
snSurveyDiscoveryTimes,
redshifts,
peakAppMagList,
snCampaignLengthList,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (ratioMapLink, )
result_log += """%s""" % (snRatePlotLink, )
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
mdLogPath = pathToResultsFolder + \
"simulation_result_log_%s.md" % (now,)
mdLog = open(mdLogPath, 'w')
mdLog.write(result_log)
mdLog.close()
dmd.convert_to_html(log=log, pathToMMDFile=mdLogPath, css="amblin")
print "Results can be found here: %(pathToResultsFolder)s" % locals()
html = mdLogPath.replace(".md", ".html")
print "Open this file in your browser: %(html)s" % locals()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
return
def main(arguments=None):
"""
The main function used when ``importers_clutils.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = setup_main_clutil(
arguments=arguments, docString=__doc__, logLevel="WARNING", options_first=False, projectName="sherlock"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(" \t\n;")
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug("%s = %s" % (varname, val))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info("--- STARTING TO RUN THE importers_clutils.py AT %s" % (startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
if cat:
if cat_name == "milliquas":
testObject = milliquasImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if cat_name == "veron":
testObject = veronImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "sdss" in cat_name:
testObject = sdssImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "ned_d" in cat_name:
testObject = nedImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
elif stream:
if "pessto" in stream_name:
testObject = pesstoImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "ifs" in stream_name:
testObject = ifsImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info("-- FINISHED ATTEMPT TO RUN THE importers_clutils.py AT %s (RUNTIME: %s) --" % (endTime, runningTime))
return
def main(arguments=None):
"""
*The main function used when ``createpythonpackage.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##
# PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED
if arguments == None:
arguments = docopt(__doc__)
# x-unpackge-settings-in-main-function
# SETUP LOGGER -- DEFAULT TO CONSOLE LOGGER IF NONE PROVIDED IN SETTINGS
if 'settings' in locals() and "logging settings" in settings:
log = dl.setup_dryx_logging(
yaml_file=arguments["--settingsFile"]
)
elif "--logger" not in arguments or arguments["--logger"] is None:
log = dl.console_logger(
level="WARNING"
)
log.debug('logger setup')
# x-setup-database-connection-in-main-function
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
varname = arg.replace("--", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE createpythonpackage.py AT %s' %
(startTime,))
log.debug('locals(): %s' % (locals(),))
# call the worker function
if "packageName" in locals() and packageName and "location" in locals():
createpythonpackage(
log=log,
packageName=packageName,
location=location,
)
elif "subPackageName" in locals() and subPackageName:
createpythonsubpackage(
log=log,
subPackageName=subPackageName,
pathToHostDirectory=pathToHostDirectory,
)
elif "moduleName" in locals() and moduleName:
createpythonmodule(
log=log,
moduleName=moduleName,
pathToHostDirectory=pathToHostDirectory,
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE createpythonpackage.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def qubits(clArgs=None):
"""
*qubits
======================
:Summary:
The main MCS project file.
A Monte Carlo Simulator of a PS1 supernova survey
Many parameters can be set and customised in the yaml settings (see settings file)
:Author:
David Young
:Date Created:
April 18, 2013
:dryx syntax:
- ``xxx`` = come back here and do some more work
- ``_someObject`` = a 'private' object that should only be changed for debugging
:Notes:
- If you have any questions requiring this script please email me: [email protected]
Usage:
qubits -s <pathToSettingsFile> -o <pathToOutputDirectory> -d <pathToSpectralDatabase>
-h, --help show this help message
-v, --version print version
-s, --settings provide a path to the settings file
-d, --database provide the path to the root directory containing your nested-folders and files spectral database
-o, --output provide a path to an output directory for the results of the simulations*
"""
################ > IMPORTS ################
## STANDARD LIB ##
import sys
import os
from datetime import datetime, date, time
## THIRD PARTY ##
from docopt import docopt
import yaml
## LOCAL APPLICATION ##
from . import commonutils as cu
from . import surveysim as ss
from . import datagenerator as dg
from . import results as r
import dryxPython.commonutils as dcu
from . import universe as u
import dryxPython.mmd.mmd as dmd
# SETUP AN EMPTY LOGGER (IF REQUIRED)
log = _set_up_command_line_tool()
if clArgs == None:
clArgs = docopt(qubits.__doc__)
pathToOutputDirectory = clArgs["<pathToOutputDirectory>"]
pathToSettingsFile = clArgs["<pathToSettingsFile>"]
pathToSpectralDatabase = clArgs["<pathToSpectralDatabase>"]
pathToOutputDirectory = os.path.abspath(pathToOutputDirectory) + "/"
pathToSettingsFile = os.path.abspath(pathToSettingsFile)
pathToSpectralDatabase = os.path.abspath(pathToSpectralDatabase) + "/"
# IMPORT THE SIMULATION SETTINGS
(allSettings, programSettings, limitingMags, sampleNumber,
peakMagnitudeDistributions, explosionDaysFromSettings,
extendLightCurveTail, relativeSNRates, lowerRedshiftLimit,
upperRedshiftLimit, redshiftResolution, restFrameFilter,
kCorrectionTemporalResolution, kCorPolyOrder, kCorMinimumDataPoints,
extinctionType, extinctionConstant, hostExtinctionDistributions,
galacticExtinctionDistribution, surveyCadenceSettings, snLightCurves,
surveyArea, CCSNRateFraction, transientToCCSNRateFraction,
extraSurveyConstraints, lightCurvePolyOrder,
logLevel) = cu.read_in_survey_parameters(
log, pathToSettingsFile=pathToSettingsFile)
logFilePath = pathToOutputDirectory + "/qubits.log"
del log
log = _set_up_command_line_tool(level=str(logLevel),
logFilePath=logFilePath)
# dbConn, log = cu.settings(
# pathToSettingsFile=pathToSettingsFile,
# dbConn=False,
# log=True
# )
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE qubits AT %s' % (startTime, ))
resultsDict = {}
pathToOutputPlotDirectory = pathToOutputDirectory + "/plots/"
dcu.dryx_mkdir(log, directoryPath=pathToOutputPlotDirectory)
pathToResultsFolder = pathToOutputDirectory + "/results/"
dcu.dryx_mkdir(log, directoryPath=pathToResultsFolder)
# GENERATE THE DATA FOR SIMULATIONS
if programSettings['Extract Lightcurves from Spectra']:
log.info('generating the Lightcurves')
dg.generate_model_lightcurves(
log=log,
pathToSpectralDatabase=pathToSpectralDatabase,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
explosionDaysFromSettings=explosionDaysFromSettings,
extendLightCurveTail=extendLightCurveTail,
polyOrder=lightCurvePolyOrder)
if programSettings['Generate KCorrection Database']:
log.info('generating the kcorrection data')
dg.generate_kcorrection_listing_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
pathToSpectralDatabase=pathToSpectralDatabase,
restFrameFilter=restFrameFilter,
temporalResolution=kCorrectionTemporalResolution,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution)
log.info('generating the kcorrection polynomials')
dg.generate_kcorrection_polynomial_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
restFrameFilter=restFrameFilter,
kCorPolyOrder=kCorPolyOrder, # ORDER OF THE POLYNOMIAL TO FIT
kCorMinimumDataPoints=kCorMinimumDataPoints,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution,
plot=programSettings['Generate KCorrection Plots'])
if programSettings['Run the Simulation']:
# CREATE THE OBSERVABLE UNIVERSE!
log.info('generating the redshift array')
redshiftArray = u.random_redshift_array(
log,
sampleNumber,
lowerRedshiftLimit,
upperRedshiftLimit,
redshiftResolution=redshiftResolution,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['Redshifts'] = redshiftArray.tolist()
log.info('generating the SN type array')
snTypesArray = u.random_sn_types_array(
log,
sampleNumber,
relativeSNRates,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['SN Types'] = snTypesArray.tolist()
log.info('generating peak magnitudes for the SNe')
peakMagnitudesArray = u.random_peak_magnitudes(
log,
peakMagnitudeDistributions,
snTypesArray,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN host extictions array')
hostExtinctionArray = u.random_host_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
hostExtinctionDistributions,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN galactic extictions array')
galacticExtinctionArray = u.random_galactic_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
galacticExtinctionDistribution,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the raw lightcurves for the SNe')
rawLightCurveDict = u.generate_numpy_polynomial_lightcurves(
log,
snLightCurves=snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the k-correction array for the SNe')
kCorrectionArray = u.build_kcorrection_array(
log,
redshiftArray,
snTypesArray,
snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the observed lightcurves for the SNe')
observedFrameLightCurveInfo, peakAppMagList = u.convert_lightcurves_to_observered_frame(
log,
snLightCurves=snLightCurves,
rawLightCurveDict=rawLightCurveDict,
redshiftArray=redshiftArray,
snTypesArray=snTypesArray,
peakMagnitudesArray=peakMagnitudesArray,
kCorrectionArray=kCorrectionArray,
hostExtinctionArray=hostExtinctionArray,
galacticExtinctionArray=galacticExtinctionArray,
restFrameFilter=restFrameFilter,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
polyOrder=lightCurvePolyOrder,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the survey observation cadence')
cadenceDictionary = ss.survey_cadence_arrays(
log,
surveyCadenceSettings,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if the SNe are discoverable by the survey')
discoverableList = ss.determine_if_sne_are_discoverable(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info(
'determining the day (if and) when each SN is first discoverable by the survey'
)
ripeDayList = ss.determine_when_sne_are_ripe_for_discovery(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
discoverableList=discoverableList,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
plot=programSettings['Plot Simulation Helper Plots'])
# log.info('determining the day when each SN is disappears fainter than the survey limiting mags')
# disappearDayList = determine_when_discovered_sne_disappear(
# log,
# redshiftArray=redshiftArray,
# limitingMags=limitingMags,
# ripeDayList=ripeDayList,
# observedFrameLightCurveInfo=observedFrameLightCurveInfo,
# plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if and when each SN is discovered by the survey')
lightCurveDiscoveryDayList, surveyDiscoveryDayList, snCampaignLengthList = ss.determine_if_sne_are_discovered(
log,
limitingMags=limitingMags,
ripeDayList=ripeDayList,
cadenceDictionary=cadenceDictionary,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
extraSurveyConstraints=extraSurveyConstraints,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict[
'Discoveries Relative to Peak Magnitudes'] = lightCurveDiscoveryDayList
resultsDict[
'Discoveries Relative to Survey Year'] = surveyDiscoveryDayList
resultsDict['Campaign Length'] = snCampaignLengthList
resultsDict['Cadence Dictionary'] = cadenceDictionary
resultsDict['Peak Apparent Magnitudes'] = peakAppMagList
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
fileName = pathToOutputDirectory + \
"simulation_results_%s.yaml" % (now,)
stream = file(fileName, 'w')
yamlContent = dict(allSettings.items() + resultsDict.items())
yaml.dump(yamlContent, stream, default_flow_style=False)
stream.close()
# COMPILE AND PLOT THE RESULTS
if programSettings['Compile and Plot Results']:
pathToYamlFile = pathToOutputDirectory + \
programSettings['Simulation Results File Used for Plots']
result_log = r.log_the_survey_settings(log, pathToYamlFile)
snSurveyDiscoveryTimes, lightCurveDiscoveryTimes, snTypes, redshifts, cadenceDictionary, peakAppMagList, snCampaignLengthList = r.import_results(
log, pathToYamlFile)
snRatePlotLink, totalRate, tooFaintRate, shortCampaignRate = r.determine_sn_rate(
log,
lightCurveDiscoveryTimes,
snSurveyDiscoveryTimes,
redshifts,
surveyCadenceSettings=surveyCadenceSettings,
lowerRedshiftLimit=lowerRedshiftLimit,
upperRedshiftLimit=upperRedshiftLimit,
redshiftResolution=redshiftResolution,
surveyArea=surveyArea,
CCSNRateFraction=CCSNRateFraction,
transientToCCSNRateFraction=transientToCCSNRateFraction,
peakAppMagList=peakAppMagList,
snCampaignLengthList=snCampaignLengthList,
extraSurveyConstraints=extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """
## Results ##
This simulated survey discovered a total of **%s** transients per year. An extra **%s** transients were detected but deemed too faint to constrain a positive transient identification and a further **%s** transients where detected but an observational campaign of more than **%s** days could not be completed to ensure identification. See below for the various output plots.
""" % (
totalRate, tooFaintRate, shortCampaignRate,
extraSurveyConstraints["Observable for at least ? number of days"])
cadenceWheelLink = r.plot_cadence_wheel(
log,
cadenceDictionary,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (cadenceWheelLink, )
discoveryMapLink = r.plot_sn_discovery_map(
log,
snSurveyDiscoveryTimes,
peakAppMagList,
snCampaignLengthList,
redshifts,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (discoveryMapLink, )
ratioMapLink = r.plot_sn_discovery_ratio_map(
log,
snSurveyDiscoveryTimes,
redshifts,
peakAppMagList,
snCampaignLengthList,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (ratioMapLink, )
result_log += """%s""" % (snRatePlotLink, )
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
mdLogPath = pathToResultsFolder + \
"simulation_result_log_%s.md" % (now,)
mdLog = open(mdLogPath, 'w')
mdLog.write(result_log)
mdLog.close()
dmd.convert_to_html(log=log, pathToMMDFile=mdLogPath, css="amblin")
# if dbConn:
# dbConn.commit()
# dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE qubits AT %s (RUNTIME: %s) --' % (
endTime,
runningTime,
))
# TEST THE ARGUMENTS
## VARIABLES ##
return None
def convert_dictionary_to_mysql_table(
dbConn,
log,
dictionary,
dbTableName,
uniqueKeyList=[],
createHelperTables=False,
dateModified=False,
returnInsertOnly=False,
replace=False):
""" *Convert a python dictionary into a mysql table*
NOTE: ADDED TO FUNDAMENTALS
**Key Arguments:**
- ``log`` -- logger
- ``dictionary`` -- python dictionary
- ``dbConn`` -- the db connection
- ``dbTableName`` -- name of the table you wish to add the data to (or create if it does not exist)
- ``uniqueKeyList`` - a lists column names that need combined to create the primary key
- ``createHelperTables`` -- create some helper tables with the main table, detailing original keywords etc
- ``returnInsertOnly`` -- returns only the insert command (does not execute it)
- ``replace`` -- use replace instead of insert statement
**Return:**
- ``None`` """
# # >IMPORTS ##
import pymysql as mdb
import re
import yaml
import time
import datetime
from dryxPython import commonutils as dcu
# import ordereddict as c # REMOVE WHEN PYTHON 2.7 INSTALLED ON PSDB
import collections as c
import dryxPython.mysql as dms
log.debug('starting convert_dictionary_to_mysql_table')
if replace:
insertVerb = "INSERT"
else:
insertVerb = "INSERT IGNORE"
if returnInsertOnly == False:
# TEST THE ARGUMENTS
if str(type(dbConn).__name__) != "Connection":
message = 'Please use a valid MySQL DB connection.'
log.critical(message)
raise TypeError(message)
if not isinstance(dictionary, dict):
message = 'Please make sure "dictionary" argument is a dict type.'
log.critical(message)
raise TypeError(message)
if not isinstance(uniqueKeyList, list):
message = 'Please make sure "uniqueKeyList" is a list'
log.critical(message)
raise TypeError(message)
for i in uniqueKeyList:
if i not in dictionary.keys():
message = 'Please make sure values in "uniqueKeyList" are present in the "dictionary" you are tring to convert'
log.critical(message)
raise ValueError(message)
for k, v in dictionary.iteritems():
# log.debug('k: %s, v: %s' % (k, v,))
if isinstance(v, list) and len(v) != 2:
message = 'Please make sure the list values in "dictionary" 2 items in length'
log.critical("%s: in %s we have a %s (%s)" %
(message, k, v, type(v)))
raise ValueError(message)
if isinstance(v, list):
if not (isinstance(v[0], str) or isinstance(v[0], int) or isinstance(v[0], bool) or isinstance(v[0], float) or isinstance(v[0], long) or isinstance(v[0], datetime.date) or v[0] == None):
message = 'Please make sure values in "dictionary" are of an appropriate value to add to the database, must be str, float, int or bool'
log.critical("%s: in %s we have a %s (%s)" %
(message, k, v, type(v)))
raise ValueError(message)
else:
if not (isinstance(v, str) or isinstance(v, int) or isinstance(v, bool) or isinstance(v, float) or isinstance(v, long) or isinstance(v, unicode) or isinstance(v, datetime.date) or v == None):
this = type(v)
message = 'Please make sure values in "dictionary" are of an appropriate value to add to the database, must be str, float, int or bool : %(k)s is a %(this)s' % locals(
)
log.critical("%s: in %s we have a %s (%s)" %
(message, k, v, type(v)))
raise ValueError(message)
if not isinstance(createHelperTables, bool):
message = 'Please make sure "createHelperTables" is a True or False'
log.critical(message)
raise TypeError(message)
# TEST IF TABLE EXISTS
tableExists = does_mysql_table_exist(dbConn, log, dbTableName)
# CREATE THE TABLE IF IT DOES NOT EXIST
if tableExists is False:
sqlQuery = """
CREATE TABLE `%(dbTableName)s`
(`primaryId` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'An internal counter',
PRIMARY KEY (`primaryId`))
ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=latin1;
""" % locals()
dms.execute_mysql_write_query(
sqlQuery=sqlQuery,
dbConn=dbConn,
log=log
)
reFeedParserClass = re.compile('FeedParserDict')
reDatetime = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T')
reTypeTime = re.compile('struct_time')
qCreateColumn = ''
formattedKey = ''
formattedKeyList = []
myValues = []
# ADD EXTRA COLUMNS TO THE DICTIONARY
dictionary['dateCreated'] = [
str(dcu.get_now_sql_datetime()), "date row was created"]
if dateModified:
dictionary['dateModified'] = [
str(dcu.get_now_sql_datetime()), "date row was modified"]
# ITERATE THROUGH THE DICTIONARY AND GENERATE THE A TABLE COLUMN WITH THE
# NAME OF THE KEY, IF IT DOES NOT EXIST
count = len(dictionary)
i = 1
for (key, value) in dictionary.items():
if (isinstance(value, list) and value[0] is None):
del dictionary[key]
# SORT THE DICTIONARY BY KEY
odictionary = c.OrderedDict(sorted(dictionary.items()))
for (key, value) in odictionary.iteritems():
formattedKey = key.replace(" ", "_").replace("-", "_")
# DEC A KEYWORD IN MYSQL - NEED TO CHANGE BEFORE INGEST
if formattedKey == "dec":
formattedKey = "decl"
if formattedKey == "DEC":
formattedKey = "DECL"
formattedKeyList.extend([formattedKey])
if len(key) > 0:
# CONVERT LIST AND FEEDPARSER VALUES TO YAML (SO I CAN PASS IT AS A
# STRING TO MYSQL)
if isinstance(value, list) and (isinstance(value[0], list) or reFeedParserClass.search(str(type(value[0])))):
value[0] = yaml.dump(value[0])
value[0] = str(value[0])
# REMOVE CHARACTERS THAT COLLIDE WITH MYSQL
# if type(value[0]) == str or type(value[0]) == unicode:
# value[0] = value[0].replace('"', """'""")
# JOIN THE VALUES TOGETHER IN A LIST - EASIER TO GENERATE THE MYSQL
# COMMAND LATER
if isinstance(value, str):
value = value.replace('\\', '\\\\')
value = value.replace('"', '\\"')
try:
udata = value.decode("utf-8", "ignore")
value = udata.encode("ascii", "ignore")
except:
log.error('cound not decode value %(value)s' % locals())
# log.debug('udata: %(udata)s' % locals())
if isinstance(value, unicode):
value = value.replace('"', '\\"')
value = value.encode("ascii", "ignore")
if isinstance(value, list) and isinstance(value[0], unicode):
myValues.extend(['%s' % value[0].strip()])
elif isinstance(value, list):
myValues.extend(['%s' % (value[0], )])
else:
myValues.extend(['%s' % (value, )])
if returnInsertOnly == False:
# CHECK IF COLUMN EXISTS YET
colExists = \
"SELECT *\
FROM information_schema.COLUMNS\
WHERE TABLE_SCHEMA=DATABASE()\
AND COLUMN_NAME='" \
+ formattedKey + "'\
AND TABLE_NAME='" + dbTableName + """'"""
try:
# log.debug('checking if the column '+formattedKey+' exists
# in the '+dbTableName+' table')
rows = execute_mysql_read_query(
colExists,
dbConn,
log,
)
except Exception as e:
log.error('something went wrong' + str(e) + '\n')
# IF COLUMN DOESN'T EXIT - GENERATE IT
if len(rows) == 0:
qCreateColumn = """ALTER TABLE %s ADD %s""" % (
dbTableName, formattedKey)
if not isinstance(value, list):
value = [value]
if reDatetime.search(str(value[0])):
# log.debug('Ok - a datetime string was found')
qCreateColumn += ' datetime DEFAULT NULL'
elif formattedKey == 'updated_parsed' or formattedKey == 'published_parsed' or formattedKey \
== 'feedName' or formattedKey == 'title':
qCreateColumn += ' varchar(100) DEFAULT NULL'
elif (isinstance(value[0], str) or isinstance(value[0], unicode)) and len(value[0]) < 30:
qCreateColumn += ' varchar(100) DEFAULT NULL'
elif (isinstance(value[0], str) or isinstance(value[0], unicode)) and len(value[0]) >= 30 and len(value[0]) < 80:
qCreateColumn += ' varchar(100) DEFAULT NULL'
elif isinstance(value[0], str) or isinstance(value[0], unicode):
columnLength = 450 + len(value[0]) * 2
qCreateColumn += ' varchar(' + str(
columnLength) + ') DEFAULT NULL'
elif isinstance(value[0], int) and abs(value[0]) <= 9:
qCreateColumn += ' tinyint DEFAULT NULL'
elif isinstance(value[0], int):
qCreateColumn += ' int DEFAULT NULL'
elif isinstance(value[0], float) or isinstance(value[0], long):
qCreateColumn += ' double DEFAULT NULL'
elif isinstance(value[0], bool):
qCreateColumn += ' tinyint DEFAULT NULL'
elif isinstance(value[0], list):
qCreateColumn += ' varchar(1024) DEFAULT NULL'
else:
# log.debug('Do not know what format to add this key in
# MySQL - removing from dictionary: %s, %s'
# % (key, type(value[0])))
formattedKeyList.pop()
myValues.pop()
qCreateColumn = None
if qCreateColumn:
# ADD COMMENT TO GIVE THE ORGINAL KEYWORD IF formatted FOR
# MYSQL
if key is not formattedKey:
qCreateColumn += " COMMENT 'original keyword: " + \
key + """'"""
# CREATE THE COLUMN IF IT DOES NOT EXIST
try:
log.info('creating the ' +
formattedKey + ' column in the ' + dbTableName + ' table')
message = execute_mysql_write_query(
qCreateColumn,
dbConn,
log,
)
except Exception as e:
# log.debug('qCreateColumn: %s' % (qCreateColumn,
# ))
log.error('could not create the ' + formattedKey + ' column in the ' + dbTableName
+ ' table -- ' + str(e) + '\n')
if returnInsertOnly == False:
# GENERATE THE INDEX NAME - THEN CREATE INDEX IF IT DOES NOT YET EXIST
if len(uniqueKeyList):
for i in range(len(uniqueKeyList)):
uniqueKeyList[i] = uniqueKeyList[
i].replace(" ", "_").replace("-", "_")
if uniqueKeyList[i] == "dec":
uniqueKeyList[i] = "decl"
if uniqueKeyList[i] == "DEC":
uniqueKeyList[i] = "DECL"
indexName = uniqueKeyList[0].replace(" ", "_").replace("-", "_")
for i in range(len(uniqueKeyList) - 1):
indexName += '_' + uniqueKeyList[i + 1]
indexName = dcu.make_lowercase_nospace(indexName)
rows = execute_mysql_read_query(
"""SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND
TABLE_NAME = '"""
+ dbTableName + """' AND INDEX_NAME = '""" +
indexName + """'""",
dbConn,
log,
)
exists = rows[0]['COUNT(*)']
# log.debug('uniqueKeyList: %s' % (uniqueKeyList,))
if exists == 0:
if isinstance(uniqueKeyList, list):
uniqueKeyList = ','.join(uniqueKeyList)
addUniqueKey = 'ALTER TABLE ' + dbTableName + \
' ADD unique ' + indexName + """ (""" + uniqueKeyList + ')'
# log.debug('HERE IS THE COMMAND:'+addUniqueKey)
message = execute_mysql_write_query(
addUniqueKey,
dbConn,
log,
)
if returnInsertOnly == True:
myKeys = ','.join(formattedKeyList)
valueString = ("%s, " * len(myValues))[:-2]
insertCommand = insertVerb + """ INTO `""" + dbTableName + \
"""` (""" + myKeys + """) VALUES (""" + valueString + """)"""
mv = []
mv[:] = [None if m == "None" else m for m in myValues]
valueTuple = tuple(mv)
dup = ""
if replace:
dup = " ON DUPLICATE KEY UPDATE "
for k, v in zip(formattedKeyList, mv):
dup = """%(dup)s %(k)s=values(%(k)s),""" % locals()
dup = dup[:-1]
insertCommand = insertCommand + dup
return insertCommand, valueTuple
# GENERATE THE INSERT COMMAND - IGNORE DUPLICATE ENTRIES
myKeys = ','.join(formattedKeyList)
myValues = '" ,"'.join(myValues)
# log.debug(myValues+" ------ PRESTRIP")
# REMOVE SOME CONVERSION NOISE
myValues = myValues.replace('time.struct_time', '')
myValues = myValues.replace(
'- !!python/object/new:feedparser.FeedParserDict', '')
myValues = myValues.replace(
'!!python/object/new:feedparser.FeedParserDict', '')
myValues = myValues.replace('dictitems:', '')
myValues = myValues.replace('dictitems', '')
myValues = myValues.replace('!!python/unicode:', '')
myValues = myValues.replace('!!python/unicode', '')
myValues = myValues.replace('"None"', 'null')
myValues = myValues.replace('"None', 'null')
if myValues[-4:] != 'null':
myValues += '"'
dup = ""
if replace:
dup = " ON DUPLICATE KEY UPDATE "
dupValues = ('"' + myValues).split(" ,")
dupKeys = myKeys.split(",")
for k, v in zip(dupKeys, dupValues):
dup = """%(dup)s %(k)s=%(v)s,""" % locals()
dup = dup[:-1]
# log.debug(myValues+" ------ POSTSTRIP")
addValue = insertVerb + """ INTO `""" + dbTableName + \
"""` (""" + myKeys + """) VALUES (\"""" + \
myValues + """) %(dup)s """ % locals()
# log.debug(addValue)
message = ""
try:
# log.debug('adding new data to the %s table; query: %s' %
# (dbTableName, addValue))
message = execute_mysql_write_query(
addValue,
dbConn,
log
)
except Exception as e:
log.error("could not add new data added to the table '" +
dbTableName + "' : " + str(e) + '\n')
log.debug('finished convert_dictionary_to_mysql_table')
return message
feedURL, self._downloadDirectory, 0)
except Exception, e:
log.error("could not download %s xml file : %s" %
(rssFeedName, str(e)))
return -1
# INSTANTIATE THE XML FILE OBJECT
xf = xml_file()
xf.feedUrl = localUrl
xf.rssFeedName = rssFeedName
# CHANNEL ELEMENTS = TOP LEVEL XML FEED METADATA
# GRAB THE DICTIONARY OF ELEMENTS AND CREATE/APPEND TO MYSQL TABLE
xfce = xf.get_channel_elements()
# ADD EXTRA COLUMNS TO THE DICTIONARY
now = str(cu.get_now_sql_datetime())
xfce['dateCreated'] = now
xfce['dateLastModified'] = now
xfce['dateLastRead'] = now
xfce['rssFeedName'] = rssFeedName
xfce['feedURL'] = feedURL
xfce['rssFeedSource'] = rssFeedSource
if(type(uniqueColumns) is list): # I.E. DICTIONARY
xfce['feedTableUniqueKeyName'] = "_".join(uniqueColumns)
xfce['uniqueKeyCols'] = ",".join(uniqueColumns)
else:
xfce['feedTableUniqueKeyName'] = uniqueColumns
xfce['uniqueKeyCols'] = uniqueColumns
# APPEND TO MYSQL SUBSCRIPTION TABLE
try:
def main(arguments=None):
"""
*The main function used when ``xy_scatter.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##
# PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED
if arguments == None:
arguments = docopt(__doc__)
# SETUP LOGGER -- DEFAULT TO CONSOLE LOGGER IF NONE PROVIDED IN SETTINGS
if 'settings' in locals() and "logging settings" in settings:
log = dl.setup_dryx_logging(
yaml_file=arguments["--settingsFile"]
)
elif "--logger" not in arguments or arguments["--logger"] is None:
log = dl.console_logger(
level="DEBUG"
)
log.debug('logger setup')
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
varname = arg.replace("--", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE xy_scatter.py AT %s' %
(startTime,))
# if "axisLabels" not in globals():
# axisLabels = False
# if "title" not in globals():
# title = False
# if "dataLabels" not in globals():
# dataLabels = False
# call the worker function
xy_scatter(
log=log,
x=x,
y=y,
axisLabels=axisLabels,
title=title,
dataLabels=dataLabels,
colors=colors
)
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE xy_scatter.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``update_git_repos.py`` is run as a single script from the cl, or when installed as a cl command*
"""
########## IMPORTS ##########
## STANDARD LIB ##
## THIRD PARTY ##
## LOCAL APPLICATION ##
import dryxPython.commonutils as dcu
## ACTIONS BASED ON WHICH ARGUMENTS ARE RECIEVED ##
# PRINT COMMAND-LINE USAGE IF NO ARGUMENTS PASSED
if arguments == None:
arguments = docopt(__doc__)
# UNPACK SETTINGS
if "--settingsFile" in arguments and arguments["--settingsFile"]:
import yaml
stream = file(arguments["--settingsFile"], 'r')
settings = yaml.load(stream)
stream.close()
# SETUP LOGGER -- DEFAULT TO CONSOLE LOGGER IF NONE PROVIDED IN SETTINGS
if 'settings' in locals() and "logging settings" in settings:
log = dl.setup_dryx_logging(
yaml_file=arguments["--settingsFile"]
)
elif "--logger" not in arguments or arguments["--logger"] is None:
log = dl.console_logger(
level="DEBUG"
)
log.debug('logger setup')
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
varname = arg.replace("--", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE git_update_script.py AT %s' %
(startTime,))
# call the worker function
# x-if-settings-or-database-credientials
if "git repos" in settings:
for repo in settings["git repos"]:
log.debug('repo["path"]: %s' % (repo["path"],))
log.debug('repo["branchToUpdate"]: %s' % (repo["branchToUpdate"],))
update_git_repos(
log=log,
gitProjectRoot=repo["path"],
branchToUpdate=repo["branchToUpdate"]
)
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info(
'-- FINISHED ATTEMPT TO RUN THE git_update_script.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
def main(arguments=None):
"""
*The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command*
"""
# setup the command-line util settings
su = tools(
arguments=arguments,
docString=__doc__,
logLevel="WARNING",
options_first=False,
projectName="qubits"
)
arguments, settings, log, dbConn = su.setup()
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if varname == "import":
varname = "iimport"
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = times.get_now_sql_datetime()
log.info(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
if init:
from . import workspace
ws = workspace(
log=log,
pathToWorkspace=pathToWorkspace
)
ws.setup()
return
# IMPORT THE SIMULATION SETTINGS
(allSettings,
programSettings,
limitingMags,
sampleNumber,
peakMagnitudeDistributions,
explosionDaysFromSettings,
extendLightCurveTail,
relativeSNRates,
lowerRedshiftLimit,
upperRedshiftLimit,
redshiftResolution,
restFrameFilter,
kCorrectionTemporalResolution,
kCorPolyOrder,
kCorMinimumDataPoints,
extinctionType,
extinctionConstant,
hostExtinctionDistributions,
galacticExtinctionDistribution,
surveyCadenceSettings,
snLightCurves,
surveyArea,
CCSNRateFraction,
transientToCCSNRateFraction,
extraSurveyConstraints,
lightCurvePolyOrder,
logLevel) = cu.read_in_survey_parameters(
log,
pathToSettingsFile=pathToSettingsFile
)
logFilePath = pathToOutputDirectory + "/qubits.log"
del log
log = _set_up_command_line_tool(
level=str(logLevel),
logFilePath=logFilePath
)
# dbConn, log = cu.settings(
# pathToSettingsFile=pathToSettingsFile,
# dbConn=False,
# log=True
# )
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info('--- STARTING TO RUN THE qubits AT %s' % (startTime,))
resultsDict = {}
pathToOutputPlotDirectory = pathToOutputDirectory + "/plots/"
dcu.dryx_mkdir(
log,
directoryPath=pathToOutputPlotDirectory
)
pathToResultsFolder = pathToOutputDirectory + "/results/"
dcu.dryx_mkdir(
log,
directoryPath=pathToResultsFolder
)
if not programSettings['Extract Lightcurves from Spectra'] and not programSettings['Generate KCorrection Database'] and not programSettings['Run the Simulation'] and not programSettings['Compile and Plot Results']:
print "All stages of the simulatation have been switched off. Please switch on at least one stage of the simulation under the 'Programming Settings' in the settings file `%(pathToSettingsFile)s`" % locals()
# GENERATE THE DATA FOR SIMULATIONS
if programSettings['Extract Lightcurves from Spectra']:
log.info('generating the Lightcurves')
dg.generate_model_lightcurves(
log=log,
pathToSpectralDatabase=pathToSpectralDatabase,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
explosionDaysFromSettings=explosionDaysFromSettings,
extendLightCurveTail=extendLightCurveTail,
polyOrder=lightCurvePolyOrder
)
print "The lightcurve file can be found here: %(pathToOutputDirectory)stransient_light_curves.yaml" % locals()
print "The lightcurve plots can be found in %(pathToOutputPlotDirectory)s" % locals()
if programSettings['Generate KCorrection Database']:
log.info('generating the kcorrection data')
dg.generate_kcorrection_listing_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
pathToSpectralDatabase=pathToSpectralDatabase,
restFrameFilter=restFrameFilter,
temporalResolution=kCorrectionTemporalResolution,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution)
log.info('generating the kcorrection polynomials')
dg.generate_kcorrection_polynomial_database(
log,
pathToOutputDirectory=pathToOutputDirectory,
restFrameFilter=restFrameFilter,
kCorPolyOrder=kCorPolyOrder, # ORDER OF THE POLYNOMIAL TO FIT
kCorMinimumDataPoints=kCorMinimumDataPoints,
redshiftResolution=redshiftResolution,
redshiftLower=lowerRedshiftLimit,
redshiftUpper=upperRedshiftLimit + redshiftResolution,
plot=programSettings['Generate KCorrection Plots'])
print "The k-correction database has been generated here: %(pathToOutputDirectory)sk_corrections" % locals()
if programSettings['Generate KCorrection Plots']:
print "The k-correction polynomial plots can also be found in %(pathToOutputDirectory)sk_corrections" % locals()
if programSettings['Run the Simulation']:
# CREATE THE OBSERVABLE UNIVERSE!
log.info('generating the redshift array')
redshiftArray = u.random_redshift_array(
log,
sampleNumber,
lowerRedshiftLimit,
upperRedshiftLimit,
redshiftResolution=redshiftResolution,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['Redshifts'] = redshiftArray.tolist()
log.info('generating the SN type array')
snTypesArray = u.random_sn_types_array(
log,
sampleNumber,
relativeSNRates,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict['SN Types'] = snTypesArray.tolist()
log.info('generating peak magnitudes for the SNe')
peakMagnitudesArray = u.random_peak_magnitudes(
log,
peakMagnitudeDistributions,
snTypesArray,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN host extictions array')
hostExtinctionArray = u.random_host_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
hostExtinctionDistributions,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the SN galactic extictions array')
galacticExtinctionArray = u.random_galactic_extinction(
log,
sampleNumber,
extinctionType,
extinctionConstant,
galacticExtinctionDistribution,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the raw lightcurves for the SNe')
rawLightCurveDict = u.generate_numpy_polynomial_lightcurves(
log,
snLightCurves=snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the k-correction array for the SNe')
kCorrectionArray = u.build_kcorrection_array(
log,
redshiftArray,
snTypesArray,
snLightCurves,
pathToOutputDirectory=pathToOutputDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the observed lightcurves for the SNe')
observedFrameLightCurveInfo, peakAppMagList = u.convert_lightcurves_to_observered_frame(
log,
snLightCurves=snLightCurves,
rawLightCurveDict=rawLightCurveDict,
redshiftArray=redshiftArray,
snTypesArray=snTypesArray,
peakMagnitudesArray=peakMagnitudesArray,
kCorrectionArray=kCorrectionArray,
hostExtinctionArray=hostExtinctionArray,
galacticExtinctionArray=galacticExtinctionArray,
restFrameFilter=restFrameFilter,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
polyOrder=lightCurvePolyOrder,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('generating the survey observation cadence')
cadenceDictionary = ss.survey_cadence_arrays(
log,
surveyCadenceSettings,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if the SNe are discoverable by the survey')
discoverableList = ss.determine_if_sne_are_discoverable(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
pathToOutputDirectory=pathToOutputDirectory,
pathToOutputPlotDirectory=pathToOutputPlotDirectory,
plot=programSettings['Plot Simulation Helper Plots'])
log.info(
'determining the day (if and) when each SN is first discoverable by the survey')
ripeDayList = ss.determine_when_sne_are_ripe_for_discovery(
log,
redshiftArray=redshiftArray,
limitingMags=limitingMags,
discoverableList=discoverableList,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
plot=programSettings['Plot Simulation Helper Plots'])
# log.info('determining the day when each SN is disappears fainter than the survey limiting mags')
# disappearDayList = determine_when_discovered_sne_disappear(
# log,
# redshiftArray=redshiftArray,
# limitingMags=limitingMags,
# ripeDayList=ripeDayList,
# observedFrameLightCurveInfo=observedFrameLightCurveInfo,
# plot=programSettings['Plot Simulation Helper Plots'])
log.info('determining if and when each SN is discovered by the survey')
lightCurveDiscoveryDayList, surveyDiscoveryDayList, snCampaignLengthList = ss.determine_if_sne_are_discovered(
log,
limitingMags=limitingMags,
ripeDayList=ripeDayList,
cadenceDictionary=cadenceDictionary,
observedFrameLightCurveInfo=observedFrameLightCurveInfo,
extraSurveyConstraints=extraSurveyConstraints,
plot=programSettings['Plot Simulation Helper Plots'])
resultsDict[
'Discoveries Relative to Peak Magnitudes'] = lightCurveDiscoveryDayList
resultsDict[
'Discoveries Relative to Survey Year'] = surveyDiscoveryDayList
resultsDict['Campaign Length'] = snCampaignLengthList
resultsDict['Cadence Dictionary'] = cadenceDictionary
resultsDict['Peak Apparent Magnitudes'] = peakAppMagList
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
fileName = pathToOutputDirectory + \
"/simulation_results_%s.yaml" % (now,)
stream = file(fileName, 'w')
yamlContent = dict(allSettings.items() + resultsDict.items())
yaml.dump(yamlContent, stream, default_flow_style=False)
stream.close()
print "The simulation output file can be found here: %(fileName)s. Remember to update your settings file 'Simulation Results File Used for Plots' parameter with this filename before compiling the results." % locals()
if programSettings['Plot Simulation Helper Plots']:
print "The simulation helper-plots found in %(pathToOutputPlotDirectory)s" % locals()
# COMPILE AND PLOT THE RESULTS
if programSettings['Compile and Plot Results']:
pathToYamlFile = pathToOutputDirectory + "/" + \
programSettings['Simulation Results File Used for Plots']
result_log = r.log_the_survey_settings(log, pathToYamlFile)
snSurveyDiscoveryTimes, lightCurveDiscoveryTimes, snTypes, redshifts, cadenceDictionary, peakAppMagList, snCampaignLengthList = r.import_results(
log, pathToYamlFile)
snRatePlotLink, totalRate, tooFaintRate, shortCampaignRate = r.determine_sn_rate(
log,
lightCurveDiscoveryTimes,
snSurveyDiscoveryTimes,
redshifts,
surveyCadenceSettings=surveyCadenceSettings,
lowerRedshiftLimit=lowerRedshiftLimit,
upperRedshiftLimit=upperRedshiftLimit,
redshiftResolution=redshiftResolution,
surveyArea=surveyArea,
CCSNRateFraction=CCSNRateFraction,
transientToCCSNRateFraction=transientToCCSNRateFraction,
peakAppMagList=peakAppMagList,
snCampaignLengthList=snCampaignLengthList,
extraSurveyConstraints=extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """
## Results ##
This simulated survey discovered a total of **%s** transients per year. An extra **%s** transients were detected but deemed too faint to constrain a positive transient identification and a further **%s** transients where detected but an observational campaign of more than **%s** days could not be completed to ensure identification. See below for the various output plots.
""" % (totalRate, tooFaintRate, shortCampaignRate, extraSurveyConstraints["Observable for at least ? number of days"])
cadenceWheelLink = r.plot_cadence_wheel(
log,
cadenceDictionary,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (cadenceWheelLink,)
discoveryMapLink = r.plot_sn_discovery_map(
log,
snSurveyDiscoveryTimes,
peakAppMagList,
snCampaignLengthList,
redshifts,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (discoveryMapLink,)
ratioMapLink = r.plot_sn_discovery_ratio_map(
log,
snSurveyDiscoveryTimes,
redshifts,
peakAppMagList,
snCampaignLengthList,
extraSurveyConstraints,
pathToOutputPlotFolder=pathToOutputPlotDirectory)
result_log += """%s""" % (ratioMapLink,)
result_log += """%s""" % (snRatePlotLink,)
now = datetime.now()
now = now.strftime("%Y%m%dt%H%M%S")
mdLogPath = pathToResultsFolder + \
"simulation_result_log_%s.md" % (now,)
mdLog = open(mdLogPath, 'w')
mdLog.write(result_log)
mdLog.close()
dmd.convert_to_html(
log=log,
pathToMMDFile=mdLogPath,
css="amblin"
)
print "Results can be found here: %(pathToResultsFolder)s" % locals()
html = mdLogPath.replace(".md", ".html")
print "Open this file in your browser: %(html)s" % locals()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = times.get_now_sql_datetime()
runningTime = times.calculate_time_difference(startTime, endTime)
log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return