def workerStampCutter(num, db, listFragment, dateAndTime, firstPass, miscParameters): """thread worker function""" # Redefine the output to be a log file. options = miscParameters[8] sys.stdout = open( '%s%s_%s_%d.log' % (options.loglocation, options.logprefix, dateAndTime, num), "w") conn = dbConnect(db[3], db[0], db[1], db[2]) PSSImageRootLocation = '/' + db[3] + '/images/' + db[2] limit = miscParameters[0] mostRecent = miscParameters[1] nondetections = miscParameters[2] discoverylimit = miscParameters[3] lastdetectionlimit = miscParameters[4] requestType = miscParameters[5] ddc = miscParameters[6] wpwarp = miscParameters[7] # Call the postage stamp downloader objectsForUpdate = makeATLASObjectPostageStamps3( conn, listFragment, PSSImageRootLocation, limit=limit, mostRecent=mostRecent, nonDets=nondetections, discoveryLimit=discoverylimit, lastDetectionLimit=lastdetectionlimit, requestType=requestType, ddc=ddc, wpwarp=wpwarp, options=options) #q.put(objectsForUpdate) print("Process complete.") conn.close() print("DB Connection Closed - exiting") return 0
def workerInsert(num, db, objectListFragment, dateAndTime, firstPass, miscParameters): """thread worker function""" # Redefine the output to be a log file. options = miscParameters[0] sys.stdout = open( '%s%s_%s_%d.log' % (options.loglocationInsert, options.logprefixInsert, dateAndTime, num), "w") conn = dbConnect(db['hostname'], db['username'], db['password'], db['database'], quitOnError=True) # This is in the worker function objectsForUpdate = executeLoad(conn, options.table, objectListFragment, int(options.bundlesize)) print("Process complete.") conn.close() print("DB Connection Closed - exiting") return 0
def main(): """main. """ opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile import yaml with open(configFile) as yaml_file: config = yaml.load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] detectionList = 1 customList = None conn = dbConnect(hostname, username, password, database) update = options.update limit = int(options.limit) mostRecent = not (options.earliest) nondetections = options.nondetections discoverylimit = int(options.discoverylimit) lastdetectionlimit = int(options.lastdetectionlimit) try: requestType = REQUESTTYPES[options.requesttype] except KeyError as e: requestType = REQUESTTYPES['incremental'] objectList = [] if options.candidate is not None and len(options.candidate) > 0: for cand in options.candidate: objectList.append({'id': int(cand)}) else: if options.customlist is not None: if int(options.customlist) > 0 and int(options.customlist) < 100: customList = int(options.customlist) objectList = getObjectsByCustomList(conn, customList) else: print( "The list must be between 1 and 100 inclusive. Exiting.") sys.exit(1) else: if options.detectionlist is not None: if int(options.detectionlist) >= 0 and int( options.detectionlist) < 9: detectionList = int(options.detectionlist) objectList = getObjectsByList(conn, listId=detectionList) else: print( "The list must be between 0 and 6 inclusive. Exiting." ) sys.exit(1) print("LENGTH OF OBJECTLIST = ", len(objectList)) PSSImageRootLocation = '/' + hostname + '/images/' + database #exposureSet = getUniqueExposures(conn, objectList, limit = limit, mostRecent = mostRecent) # Only download exposures if requested. Otherwise assume we already HAVE the data. if not options.skipdownload: exposureSet = getUniqueExposures(conn, objectList, limit=limit, mostRecent=mostRecent, nonDets=nondetections, discoveryLimit=discoverylimit, lastDetectionLimit=lastdetectionlimit, ddc=options.ddc) exposureSet.sort() for row in exposureSet: print(row) downloadExposures(exposureSet) makeATLASObjectPostageStamps3(conn, objectList, PSSImageRootLocation, limit=limit, mostRecent=mostRecent, nonDets=nondetections, discoveryLimit=discoverylimit, lastDetectionLimit=lastdetectionlimit, requestType=requestType, ddc=options.ddc, wpwarp=options.wpwarp, options=options) conn.close()
def main(argv=None): """main. """ opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile import yaml with open(configFile) as yaml_file: config = yaml.load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] detectionList = 1 customList = None conn = dbConnect(hostname, username, password, database) update = options.update limit = int(options.limit) limitafter = int(options.limitafter) objectList = [] if options.candidate is not None and len(options.candidate) > 0: for cand in options.candidate: obj = getATLASObject(conn, objectId=int(cand)) if obj: objectList.append(obj) else: if options.customlist is not None: if int(options.customlist) > 0 and int(options.customlist) < 100: customList = int(options.customlist) objectList = getObjectsByCustomList(conn, customList, processingFlags=0) else: print( "The list must be between 1 and 100 inclusive. Exiting.") sys.exit(1) else: if options.detectionlist is not None: if int(options.detectionlist) >= 0 and int( options.detectionlist) < 9: detectionList = int(options.detectionlist) objectList = getObjectsByList(conn, listId=detectionList, processingFlags=0) else: print( "The list must be between 0 and 9 inclusive. Exiting." ) sys.exit(1) print("LENGTH OF OBJECTLIST = ", len(objectList)) perObjectExps, allExps = getForcedPhotometryUniqueExposures( conn, objectList, discoveryLimit=limit, cutoffLimit=limitafter, incremental=True, ddc=options.ddc, useFlagDate=options.useflagdate) if options.test: for obj in objectList: print(obj['id']) for exp in perObjectExps[obj['id']]['exps']: print(exp) return 0 if not options.skipdownload: doRsync(allExps, 'diff') doRsync(allExps, 'red', getMetadata=True, metadataExtension='.tph') fphot = doForcedPhotometry(options, objectList, perObjectExps) if options.update: insertForcedPhotometry(conn, fphot)
def main(argv=None): """main. Args: argv: """ opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile import yaml with open(configFile) as yaml_file: config = yaml.load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] MAX_NUMBER_OF_OBJECTS = int( config['postage_stamp_parameters']['max_number_of_objects']) db = [] db.append(username) db.append(password) db.append(database) db.append(hostname) detectionList = 1 customList = None conn = dbConnect(hostname, username, password, database) update = options.update limit = int(options.limit) limitafter = int(options.limitafter) mlscore = None if options.mlscore is not None: mlscore = float(options.mlscore) objectList = [] flagDate = '2015-12-20' if options.flagdate is not None: try: flagDate = '%s-%s-%s' % (options.flagdate[0:4], options.flagdate[4:6], options.flagdate[6:8]) except: flagDate = '2015-12-20' if options.candidate is not None and len(options.candidate) > 0: for cand in options.candidate: obj = getATLASObject(conn, objectId=int(cand)) if obj: objectList.append(obj) else: if options.customlist is not None: if int(options.customlist) > 0 and int(options.customlist) < 100: customList = int(options.customlist) objectList = getObjectsByCustomList(conn, customList, processingFlags=0) else: print( "The list must be between 1 and 100 inclusive. Exiting.") sys.exit(1) else: if options.detectionlist is not None: if int(options.detectionlist) >= 0 and int( options.detectionlist) < 9: detectionList = int(options.detectionlist) objectList = getObjectsByList(conn, listId=detectionList, dateThreshold=flagDate, processingFlags=0) else: print( "The list must be between 0 and 9 inclusive. Exiting." ) sys.exit(1) print("LENGTH OF OBJECTLIST = ", len(objectList)) if mlscore is not None and not ( options.candidate ): # Only do this filter if the IDs are not provided explicitly. updatedList = [] for row in objectList: if row['zooniverse_score'] is not None and row[ 'zooniverse_score'] >= mlscore: updatedList.append(row) if len(updatedList) > 0: objectList = updatedList print("LENGTH OF CLIPPED OBJECTLIST = ", len(objectList)) currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S") (year, month, day, hour, min, sec) = currentDate.split(':') dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec) # Single threaded #perObjectExps, exposureSet = getForcedPhotometryUniqueExposures(conn, objectList, discoveryLimit = limit, ddc = options.ddc, useFlagDate = options.useflagdate) perObjectExps, exposureSet = getForcedPhotometryUniqueExposures( conn, objectList, discoveryLimit=limit, cutoffLimit=limitafter, ddc=options.ddc, useFlagDate=options.useflagdate) if options.test: for obj in objectList: print(obj['id']) for exp in perObjectExps[obj['id']]['exps']: print(exp) return 0 # We'll hand the entire perObjectExps dictionary to each thread. # Download threads with multiprocessing - try 10 threads by default print("TOTAL OBJECTS = %d" % len(exposureSet)) print("Downloading exposures...") if not options.skipdownload: if len(exposureSet) > 0: nProcessors, listChunks = splitList(exposureSet, bins=int( options.downloadthreads)) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, workerExposureDownloader, miscParameters=[options], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) # Belt and braces - try again with one less thread, just in case the previous one failed. nProcessors, listChunks = splitList( exposureSet, bins=int(options.downloadthreads) - 1) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, workerExposureDownloader, miscParameters=[options], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) # Produce stamps with multiprocessing - try n(CPUs) threads by default print("Doing Forced Photometry...") if len(objectList) > 0: nProcessors, listChunks = splitList(objectList) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) objectsForUpdate = parallelProcess( db, dateAndTime, nProcessors, listChunks, workerForcedPhotometry, miscParameters=[options, perObjectExps]) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) if len(objectsForUpdate) > 0 and update: insertForcedPhotometry(conn, objectsForUpdate) conn.close() return 0
def main(): opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile regex = options.regex import yaml with open(configFile) as yaml_file: config = yaml.safe_load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] db = [] db.append(username) db.append(password) db.append(database) db.append(hostname) conn = dbConnect(hostname, username, password, database) warnings.filterwarnings("ignore") # Parse command line currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S") (year, month, day, hour, min, sec) = currentDate.split(':') dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec) pid = int(options.pid) maxjobs = int(options.maxjobs) days = int(options.days) camera = options.camera try: mjdToIngest = options.mjd except TypeError as e: mjdToIngest = None print("camera =", camera) print("regex =", regex) todayMJD = getCurrentMJD() # Use + 1 to include today! mjdthreshold = int(todayMJD) - days + 1 # Specified MJD trumps mjd Threshold, so just go as far back # as the specified date if mjdToIngest: mjdthreshold = int(mjdToIngest[0:5]) - 1 ingester = options.ingester fileList = getFiles(regex, camera, mjdToIngest=mjdToIngest, mjdthreshold=mjdthreshold, days=days, atlasroot=options.atlasroot, options=options) ingestedFiles = getFilesIngestedddc2(conn, mjdthreshold=mjdthreshold, camera=camera) fileListDict = OrderedDict() print("List of files...") for row in fileList: fileListDict[os.path.basename(row)] = row print(row) print("List of ingested files...") for row in ingestedFiles: print(row) filesToIngest = [ fileListDict[x] for x in list(set(fileListDict.keys()) - set(ingestedFiles)) ] filesToIngest.sort() print("List of files to ingest...") for row in filesToIngest: print(row) print("TOTAL OBJECTS TO CHECK = %d" % len(filesToIngest)) if len(fileList) > 0: # 2018-02-06 KWS Use half the default number of processes. This may ironically speed up ingest. nProcessors, listChunks = splitList(filesToIngest, bins=28) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, worker, miscParameters=[options], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) conn.close() return 0
def main(): opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile regex = options.regex import yaml with open(configFile) as yaml_file: config = yaml.load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] conn = dbConnect(hostname, username, password, database) warnings.filterwarnings("ignore") # Parse command line currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S") (year, month, day, hour, min, sec) = currentDate.split(':') dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec) pid = int(options.pid) maxjobs = int(options.maxjobs) days = int(options.days) camera = options.camera try: mjdToIngest = options.mjd except TypeError as e: mjdToIngest = None print("camera =", camera) print("regex =", regex) todayMJD = getCurrentMJD() mjdthreshold = int(todayMJD) - days + 1 # Specified MJD trumps mjd Threshold, so just go as far back # as the specified date if mjdToIngest: mjdthreshold = int(mjdToIngest[0:5]) - 1 ingester = options.ingester fileList = getFiles(regex, camera, mjdToIngest=mjdToIngest, mjdthreshold=mjdthreshold, days=days, options=options) ingestedFiles = getFilesIngestedddc2(conn, mjdthreshold=mjdthreshold, mjdToIngest=mjdToIngest, camera=camera) fileListDict = OrderedDict() print("List of files...") for row in fileList: fileListDict[os.path.basename(row)] = row print(row) print("List of ingested files...") for row in ingestedFiles: print(row) filesToIngest = [ fileListDict[x] for x in list(set(fileListDict.keys()) - set(ingestedFiles)) ] filesToIngest.sort() print("List of files to ingest...") for row in filesToIngest: print(row) print("TOTAL OBJECTS TO CHECK = %d" % len(filesToIngest)) ingesterWrapper(ingester, configFile, filesToIngest) conn.close() return 0
def main(): """main. """ opts = docopt(__doc__, version='0.1') opts = cleanOptions(opts) options = Struct(**opts) configFile = options.configfile import yaml with open(configFile) as yaml_file: config = yaml.load(yaml_file) username = config['databases']['local']['username'] password = config['databases']['local']['password'] database = config['databases']['local']['database'] hostname = config['databases']['local']['hostname'] MAX_NUMBER_OF_OBJECTS = int( config['postage_stamp_parameters']['max_number_of_objects']) db = [] db.append(username) db.append(password) db.append(database) db.append(hostname) detectionList = 1 customList = None conn = dbConnect(hostname, username, password, database) update = options.update limit = int(options.limit) mostRecent = not (options.earliest) nondetections = options.nondetections discoverylimit = int(options.discoverylimit) lastdetectionlimit = int(options.lastdetectionlimit) objectList = [] try: requestType = REQUESTTYPES[options.requesttype] except KeyError as e: requestType = REQUESTTYPES['incremental'] print("REQUEST TYPE = ", requestType) flagDate = '2015-12-20' if options.flagdate is not None: try: flagDate = '%s-%s-%s' % (options.flagdate[0:4], options.flagdate[4:6], options.flagdate[6:8]) except: flagDate = '2015-12-20' if options.candidate is not None and len(options.candidate) > 0: for cand in options.candidate: objectList.append({'id': int(cand)}) else: if options.customlist is not None: if int(options.customlist) > 0 and int(options.customlist) < 100: customList = int(options.customlist) objectList = getObjectsByCustomList(conn, customList) else: print( "The list must be between 1 and 100 inclusive. Exiting.") sys.exit(1) else: if options.detectionlist is not None: if int(options.detectionlist) >= 0 and int( options.detectionlist) < 9: detectionList = int(options.detectionlist) objectList = getObjectsByList(conn, listId=detectionList, dateThreshold=flagDate) else: print( "The list must be between 0 and 6 inclusive. Exiting." ) sys.exit(1) currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S") (year, month, day, hour, min, sec) = currentDate.split(':') dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec) if len(objectList) > MAX_NUMBER_OF_OBJECTS: sys.stderr.write( "The number of objects (%d) exceeds the maximum allowed (%d). Cannot continue.\n" % (len(objectList), MAX_NUMBER_OF_OBJECTS)) sys.exit(1) # Only download exposures if requested. Otherwise assume we already HAVE the data. if not options.skipdownload: exposureSet = getUniqueExposures(conn, objectList, limit=limit, mostRecent=mostRecent, nonDets=nondetections, discoveryLimit=discoverylimit, lastDetectionLimit=lastdetectionlimit, requestType=requestType, ddc=options.ddc) # Download threads with multiprocessing - try 10 threads by default print("TOTAL OBJECTS = %d" % len(exposureSet)) print("Downloading exposures...") if len(exposureSet) > 0: nProcessors, listChunks = splitList(exposureSet, bins=int( options.downloadthreads)) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, workerImageDownloader, miscParameters=[options], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) # Belt and braces. Do again, with one less thread. nProcessors, listChunks = splitList( exposureSet, bins=int(options.downloadthreads) - 1) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, workerImageDownloader, miscParameters=[options], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) # Produce stamps with multiprocessing - try n(CPUs) threads by default print("Producing stamps...") if len(objectList) > 0: nProcessors, listChunks = splitList(objectList, bins=48) print("%s Parallel Processing..." % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) parallelProcess(db, dateAndTime, nProcessors, listChunks, workerStampCutter, miscParameters=[ limit, mostRecent, nondetections, discoverylimit, lastdetectionlimit, requestType, options.ddc, options.wpwarp, options ], drainQueues=False) print("%s Done Parallel Processing" % (datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S"))) conn.close() return 0