def main(arguments=None): """ The main function used when ``cl_utils.py`` is run as a single script from the cl, or when installed as a cl command """ # setup the command-line util settings su = setup_main_clutil( arguments=arguments, docString=__doc__, logLevel="DEBUG", options_first=False, projectName="sherlock" ) arguments, settings, log, dbConn = su.setup() # tab completion for raw_input readline.set_completer_delims(' \t\n;') readline.parse_and_bind("tab: complete") readline.set_completer(tab_complete) # unpack remaining cl arguments using `exec` to setup the variable names # automatically for arg, val in arguments.iteritems(): if arg[0] == "-": varname = arg.replace("-", "") + "Flag" else: varname = arg.replace("<", "").replace(">", "") if isinstance(val, str) or isinstance(val, unicode): exec(varname + " = '%s'" % (val,)) else: exec(varname + " = %s" % (val,)) if arg == "--dbConn": dbConn = val log.debug('%s = %s' % (varname, val,)) ## START LOGGING ## startTime = dcu.get_now_sql_datetime() log.debug( '--- STARTING TO RUN THE cl_utils.py AT %s' % (startTime,)) # set options interactively if user requests if "interactiveFlag" in locals() and interactiveFlag: # load previous settings moduleDirectory = os.path.dirname(__file__) + "/resources" pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals() try: with open(pathToPickleFile): pass previousSettingsExist = True except: previousSettingsExist = False previousSettings = {} if previousSettingsExist: previousSettings = pickle.load(open(pathToPickleFile, "rb")) # x-raw-input # x-boolean-raw-input # x-raw-input-with-default-value-from-previous-settings # save the most recently used requests pickleMeObjects = [] pickleMe = {} theseLocals = locals() for k in pickleMeObjects: pickleMe[k] = theseLocals[k] pickle.dump(pickleMe, open(pathToPickleFile, "wb")) # call the worker function # x-if-settings-or-database-credientials if match: sherlock = classifier( log=log, settings=settings, update=updateFlag, transientIdList=[] ) sherlock.get() if clean: cleaner = cleanup_database_tables( log=log, settings=settings ) cleaner.get() if wiki: updateWiki = update_wiki_pages( log=log, settings=settings ) updateWiki.get() if "dbConn" in locals() and dbConn: dbConn.commit() dbConn.close() ## FINISH LOGGING ## endTime = dcu.get_now_sql_datetime() runningTime = dcu.calculate_time_difference(startTime, endTime) log.debug('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' % (endTime, runningTime, )) return
import os import nose import shutil import yaml from sherlock import cl_utils from sherlock.imports import ifs_galaxies from sherlock.utKit import utKit from dryxPython.projectsetup import setup_main_clutil su = setup_main_clutil( arguments={}, docString=__doc__, logLevel="DEBUG", options_first=False, projectName="ifs_galaxies" ) arguments, settings, log, dbConn = su.setup() # load settings stream = file( "/Users/Dave/.config/sherlock/sherlock.yaml", 'r') settings = yaml.load(stream) stream.close() # SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE moduleDirectory = os.path.dirname(__file__) utKit = utKit(moduleDirectory) log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule() utKit.tearDownModule()
import os import nose import shutil import yaml from xxxpythonPackageNamexxx import xxxmodule_namexxx, cl_utils from xxxpythonPackageNamexxx.utKit import utKit from dryxPython.projectsetup import setup_main_clutil su = setup_main_clutil( arguments={}, docString=__doc__, logLevel="DEBUG", options_first=False, projectName="xxxpythonPackageNamexxx", tunnel=False ) arguments, settings, log, dbConn = su.setup() # load settings stream = file( "/Users/Dave/.config/xxxpythonPackageNamexxx/xxxpythonPackageNamexxx.yaml", 'r') settings = yaml.load(stream) stream.close() # SETUP AND TEARDOWN FIXTURE FUNCTIONS FOR THE ENTIRE MODULE moduleDirectory = os.path.dirname(__file__) utKit = utKit(moduleDirectory) log, dbConn, pathToInputDir, pathToOutputDir = utKit.setupModule() utKit.tearDownModule()
def main(arguments=None): """ The main function used when ``importers_clutils.py`` is run as a single script from the cl, or when installed as a cl command """ # setup the command-line util settings su = setup_main_clutil( arguments=arguments, docString=__doc__, logLevel="WARNING", options_first=False, projectName="sherlock" ) arguments, settings, log, dbConn = su.setup() # tab completion for raw_input readline.set_completer_delims(" \t\n;") readline.parse_and_bind("tab: complete") readline.set_completer(tab_complete) # unpack remaining cl arguments using `exec` to setup the variable names # automatically for arg, val in arguments.iteritems(): if arg[0] == "-": varname = arg.replace("-", "") + "Flag" else: varname = arg.replace("<", "").replace(">", "") if isinstance(val, str) or isinstance(val, unicode): exec(varname + " = '%s'" % (val,)) else: exec(varname + " = %s" % (val,)) if arg == "--dbConn": dbConn = val log.debug("%s = %s" % (varname, val)) ## START LOGGING ## startTime = dcu.get_now_sql_datetime() log.info("--- STARTING TO RUN THE importers_clutils.py AT %s" % (startTime,)) # set options interactively if user requests if "interactiveFlag" in locals() and interactiveFlag: # load previous settings moduleDirectory = os.path.dirname(__file__) + "/resources" pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals() try: with open(pathToPickleFile): pass previousSettingsExist = True except: previousSettingsExist = False previousSettings = {} if previousSettingsExist: previousSettings = pickle.load(open(pathToPickleFile, "rb")) # x-raw-input # x-boolean-raw-input # x-raw-input-with-default-value-from-previous-settings # save the most recently used requests pickleMeObjects = [] pickleMe = {} theseLocals = locals() for k in pickleMeObjects: pickleMe[k] = theseLocals[k] pickle.dump(pickleMe, open(pathToPickleFile, "wb")) # call the worker function # x-if-settings-or-database-credientials if cat: if cat_name == "milliquas": testObject = milliquasImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() if cat_name == "veron": testObject = veronImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() if "sdss" in cat_name: testObject = sdssImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() if "ned_d" in cat_name: testObject = nedImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() elif stream: if "pessto" in stream_name: testObject = pesstoImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() if "ifs" in stream_name: testObject = ifsImporter( log=log, settings=settings, pathToDataFile=pathToDataFile, version=cat_version, catalogueName=cat_name ) testObject.get() if "dbConn" in locals() and dbConn: dbConn.commit() dbConn.close() ## FINISH LOGGING ## endTime = dcu.get_now_sql_datetime() runningTime = dcu.calculate_time_difference(startTime, endTime) log.info("-- FINISHED ATTEMPT TO RUN THE importers_clutils.py AT %s (RUNTIME: %s) --" % (endTime, runningTime)) return