def main(arguments=None):
"""
The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = setup_main_clutil(
arguments=arguments,
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="sherlock"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(' \t\n;')
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug('%s = %s' % (varname, val,))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.debug(
'--- STARTING TO RUN THE cl_utils.py AT %s' %
(startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
if match:
sherlock = classifier(
log=log,
settings=settings,
update=updateFlag,
transientIdList=[]
)
sherlock.get()
if clean:
cleaner = cleanup_database_tables(
log=log,
settings=settings
)
cleaner.get()
if wiki:
updateWiki = update_wiki_pages(
log=log,
settings=settings
)
updateWiki.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.debug('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
(endTime, runningTime, ))
return
import os
import nose
import shutil
import yaml
from sherlock import cl_utils
from sherlock.imports import ifs_galaxies
from sherlock.utKit import utKit
from dryxPython.projectsetup import setup_main_clutil
su = setup_main_clutil(
arguments={},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="ifs_galaxies"
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/sherlock/sherlock.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
import os
import nose
import shutil
import yaml
from xxxpythonPackageNamexxx import xxxmodule_namexxx, cl_utils
from xxxpythonPackageNamexxx.utKit import utKit
from dryxPython.projectsetup import setup_main_clutil
su = setup_main_clutil(
arguments={},
docString=__doc__,
logLevel="DEBUG",
options_first=False,
projectName="xxxpythonPackageNamexxx",
tunnel=False
)
arguments, settings, log, dbConn = su.setup()
# load settings
stream = file(
"/Users/Dave/.config/xxxpythonPackageNamexxx/xxxpythonPackageNamexxx.yaml", 'r')
settings = yaml.load(stream)
stream.close()
# SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE
moduleDirectory = os.path.dirname(__file__)
utKit = utKit(moduleDirectory)
log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule()
utKit.tearDownModule()
def main(arguments=None):
"""
The main function used when ``importers_clutils.py`` is run as a single script from the cl, or when installed as a cl command
"""
# setup the command-line util settings
su = setup_main_clutil(
arguments=arguments, docString=__doc__, logLevel="WARNING", options_first=False, projectName="sherlock"
)
arguments, settings, log, dbConn = su.setup()
# tab completion for raw_input
readline.set_completer_delims(" \t\n;")
readline.parse_and_bind("tab: complete")
readline.set_completer(tab_complete)
# unpack remaining cl arguments using `exec` to setup the variable names
# automatically
for arg, val in arguments.iteritems():
if arg[0] == "-":
varname = arg.replace("-", "") + "Flag"
else:
varname = arg.replace("<", "").replace(">", "")
if isinstance(val, str) or isinstance(val, unicode):
exec(varname + " = '%s'" % (val,))
else:
exec(varname + " = %s" % (val,))
if arg == "--dbConn":
dbConn = val
log.debug("%s = %s" % (varname, val))
## START LOGGING ##
startTime = dcu.get_now_sql_datetime()
log.info("--- STARTING TO RUN THE importers_clutils.py AT %s" % (startTime,))
# set options interactively if user requests
if "interactiveFlag" in locals() and interactiveFlag:
# load previous settings
moduleDirectory = os.path.dirname(__file__) + "/resources"
pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
try:
with open(pathToPickleFile):
pass
previousSettingsExist = True
except:
previousSettingsExist = False
previousSettings = {}
if previousSettingsExist:
previousSettings = pickle.load(open(pathToPickleFile, "rb"))
# x-raw-input
# x-boolean-raw-input
# x-raw-input-with-default-value-from-previous-settings
# save the most recently used requests
pickleMeObjects = []
pickleMe = {}
theseLocals = locals()
for k in pickleMeObjects:
pickleMe[k] = theseLocals[k]
pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
# call the worker function
# x-if-settings-or-database-credientials
if cat:
if cat_name == "milliquas":
testObject = milliquasImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if cat_name == "veron":
testObject = veronImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "sdss" in cat_name:
testObject = sdssImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "ned_d" in cat_name:
testObject = nedImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
elif stream:
if "pessto" in stream_name:
testObject = pesstoImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "ifs" in stream_name:
testObject = ifsImporter(
log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name
)
testObject.get()
if "dbConn" in locals() and dbConn:
dbConn.commit()
dbConn.close()
## FINISH LOGGING ##
endTime = dcu.get_now_sql_datetime()
runningTime = dcu.calculate_time_difference(startTime, endTime)
log.info("-- FINISHED ATTEMPT TO RUN THE importers_clutils.py AT %s (RUNTIME: %s) --" % (endTime, runningTime))
return
