def workerStampCutter(num, db, listFragment, dateAndTime, firstPass,
miscParameters):
"""thread worker function"""
# Redefine the output to be a log file.
options = miscParameters[8]
sys.stdout = open(
'%s%s_%s_%d.log' %
(options.loglocation, options.logprefix, dateAndTime, num), "w")
conn = dbConnect(db[3], db[0], db[1], db[2])
PSSImageRootLocation = '/' + db[3] + '/images/' + db[2]
limit = miscParameters[0]
mostRecent = miscParameters[1]
nondetections = miscParameters[2]
discoverylimit = miscParameters[3]
lastdetectionlimit = miscParameters[4]
requestType = miscParameters[5]
ddc = miscParameters[6]
wpwarp = miscParameters[7]
# Call the postage stamp downloader
objectsForUpdate = makeATLASObjectPostageStamps3(
conn,
listFragment,
PSSImageRootLocation,
limit=limit,
mostRecent=mostRecent,
nonDets=nondetections,
discoveryLimit=discoverylimit,
lastDetectionLimit=lastdetectionlimit,
requestType=requestType,
ddc=ddc,
wpwarp=wpwarp,
options=options)
#q.put(objectsForUpdate)
print("Process complete.")
conn.close()
print("DB Connection Closed - exiting")
return 0
def workerInsert(num, db, objectListFragment, dateAndTime, firstPass,
miscParameters):
"""thread worker function"""
# Redefine the output to be a log file.
options = miscParameters[0]
sys.stdout = open(
'%s%s_%s_%d.log' %
(options.loglocationInsert, options.logprefixInsert, dateAndTime, num),
"w")
conn = dbConnect(db['hostname'],
db['username'],
db['password'],
db['database'],
quitOnError=True)
# This is in the worker function
objectsForUpdate = executeLoad(conn, options.table, objectListFragment,
int(options.bundlesize))
print("Process complete.")
conn.close()
print("DB Connection Closed - exiting")
return 0
def main():
"""main.
"""
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
import yaml
with open(configFile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
detectionList = 1
customList = None
conn = dbConnect(hostname, username, password, database)
update = options.update
limit = int(options.limit)
mostRecent = not (options.earliest)
nondetections = options.nondetections
discoverylimit = int(options.discoverylimit)
lastdetectionlimit = int(options.lastdetectionlimit)
try:
requestType = REQUESTTYPES[options.requesttype]
except KeyError as e:
requestType = REQUESTTYPES['incremental']
objectList = []
if options.candidate is not None and len(options.candidate) > 0:
for cand in options.candidate:
objectList.append({'id': int(cand)})
else:
if options.customlist is not None:
if int(options.customlist) > 0 and int(options.customlist) < 100:
customList = int(options.customlist)
objectList = getObjectsByCustomList(conn, customList)
else:
print(
"The list must be between 1 and 100 inclusive. Exiting.")
sys.exit(1)
else:
if options.detectionlist is not None:
if int(options.detectionlist) >= 0 and int(
options.detectionlist) < 9:
detectionList = int(options.detectionlist)
objectList = getObjectsByList(conn, listId=detectionList)
else:
print(
"The list must be between 0 and 6 inclusive. Exiting."
)
sys.exit(1)
print("LENGTH OF OBJECTLIST = ", len(objectList))
PSSImageRootLocation = '/' + hostname + '/images/' + database
#exposureSet = getUniqueExposures(conn, objectList, limit = limit, mostRecent = mostRecent)
# Only download exposures if requested. Otherwise assume we already HAVE the data.
if not options.skipdownload:
exposureSet = getUniqueExposures(conn,
objectList,
limit=limit,
mostRecent=mostRecent,
nonDets=nondetections,
discoveryLimit=discoverylimit,
lastDetectionLimit=lastdetectionlimit,
ddc=options.ddc)
exposureSet.sort()
for row in exposureSet:
print(row)
downloadExposures(exposureSet)
makeATLASObjectPostageStamps3(conn,
objectList,
PSSImageRootLocation,
limit=limit,
mostRecent=mostRecent,
nonDets=nondetections,
discoveryLimit=discoverylimit,
lastDetectionLimit=lastdetectionlimit,
requestType=requestType,
ddc=options.ddc,
wpwarp=options.wpwarp,
options=options)
conn.close()
def main(argv=None):
"""main.
"""
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
import yaml
with open(configFile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
detectionList = 1
customList = None
conn = dbConnect(hostname, username, password, database)
update = options.update
limit = int(options.limit)
limitafter = int(options.limitafter)
objectList = []
if options.candidate is not None and len(options.candidate) > 0:
for cand in options.candidate:
obj = getATLASObject(conn, objectId=int(cand))
if obj:
objectList.append(obj)
else:
if options.customlist is not None:
if int(options.customlist) > 0 and int(options.customlist) < 100:
customList = int(options.customlist)
objectList = getObjectsByCustomList(conn,
customList,
processingFlags=0)
else:
print(
"The list must be between 1 and 100 inclusive. Exiting.")
sys.exit(1)
else:
if options.detectionlist is not None:
if int(options.detectionlist) >= 0 and int(
options.detectionlist) < 9:
detectionList = int(options.detectionlist)
objectList = getObjectsByList(conn,
listId=detectionList,
processingFlags=0)
else:
print(
"The list must be between 0 and 9 inclusive. Exiting."
)
sys.exit(1)
print("LENGTH OF OBJECTLIST = ", len(objectList))
perObjectExps, allExps = getForcedPhotometryUniqueExposures(
conn,
objectList,
discoveryLimit=limit,
cutoffLimit=limitafter,
incremental=True,
ddc=options.ddc,
useFlagDate=options.useflagdate)
if options.test:
for obj in objectList:
print(obj['id'])
for exp in perObjectExps[obj['id']]['exps']:
print(exp)
return 0
if not options.skipdownload:
doRsync(allExps, 'diff')
doRsync(allExps, 'red', getMetadata=True, metadataExtension='.tph')
fphot = doForcedPhotometry(options, objectList, perObjectExps)
if options.update:
insertForcedPhotometry(conn, fphot)
def main(argv=None):
"""main.
Args:
argv:
"""
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
import yaml
with open(configFile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
MAX_NUMBER_OF_OBJECTS = int(
config['postage_stamp_parameters']['max_number_of_objects'])
db = []
db.append(username)
db.append(password)
db.append(database)
db.append(hostname)
detectionList = 1
customList = None
conn = dbConnect(hostname, username, password, database)
update = options.update
limit = int(options.limit)
limitafter = int(options.limitafter)
mlscore = None
if options.mlscore is not None:
mlscore = float(options.mlscore)
objectList = []
flagDate = '2015-12-20'
if options.flagdate is not None:
try:
flagDate = '%s-%s-%s' % (options.flagdate[0:4],
options.flagdate[4:6],
options.flagdate[6:8])
except:
flagDate = '2015-12-20'
if options.candidate is not None and len(options.candidate) > 0:
for cand in options.candidate:
obj = getATLASObject(conn, objectId=int(cand))
if obj:
objectList.append(obj)
else:
if options.customlist is not None:
if int(options.customlist) > 0 and int(options.customlist) < 100:
customList = int(options.customlist)
objectList = getObjectsByCustomList(conn,
customList,
processingFlags=0)
else:
print(
"The list must be between 1 and 100 inclusive. Exiting.")
sys.exit(1)
else:
if options.detectionlist is not None:
if int(options.detectionlist) >= 0 and int(
options.detectionlist) < 9:
detectionList = int(options.detectionlist)
objectList = getObjectsByList(conn,
listId=detectionList,
dateThreshold=flagDate,
processingFlags=0)
else:
print(
"The list must be between 0 and 9 inclusive. Exiting."
)
sys.exit(1)
print("LENGTH OF OBJECTLIST = ", len(objectList))
if mlscore is not None and not (
options.candidate
): # Only do this filter if the IDs are not provided explicitly.
updatedList = []
for row in objectList:
if row['zooniverse_score'] is not None and row[
'zooniverse_score'] >= mlscore:
updatedList.append(row)
if len(updatedList) > 0:
objectList = updatedList
print("LENGTH OF CLIPPED OBJECTLIST = ", len(objectList))
currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")
(year, month, day, hour, min, sec) = currentDate.split(':')
dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec)
# Single threaded
#perObjectExps, exposureSet = getForcedPhotometryUniqueExposures(conn, objectList, discoveryLimit = limit, ddc = options.ddc, useFlagDate = options.useflagdate)
perObjectExps, exposureSet = getForcedPhotometryUniqueExposures(
conn,
objectList,
discoveryLimit=limit,
cutoffLimit=limitafter,
ddc=options.ddc,
useFlagDate=options.useflagdate)
if options.test:
for obj in objectList:
print(obj['id'])
for exp in perObjectExps[obj['id']]['exps']:
print(exp)
return 0
# We'll hand the entire perObjectExps dictionary to each thread.
# Download threads with multiprocessing - try 10 threads by default
print("TOTAL OBJECTS = %d" % len(exposureSet))
print("Downloading exposures...")
if not options.skipdownload:
if len(exposureSet) > 0:
nProcessors, listChunks = splitList(exposureSet,
bins=int(
options.downloadthreads))
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
workerExposureDownloader,
miscParameters=[options],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
# Belt and braces - try again with one less thread, just in case the previous one failed.
nProcessors, listChunks = splitList(
exposureSet, bins=int(options.downloadthreads) - 1)
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
workerExposureDownloader,
miscParameters=[options],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
# Produce stamps with multiprocessing - try n(CPUs) threads by default
print("Doing Forced Photometry...")
if len(objectList) > 0:
nProcessors, listChunks = splitList(objectList)
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
objectsForUpdate = parallelProcess(
db,
dateAndTime,
nProcessors,
listChunks,
workerForcedPhotometry,
miscParameters=[options, perObjectExps])
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
if len(objectsForUpdate) > 0 and update:
insertForcedPhotometry(conn, objectsForUpdate)
conn.close()
return 0
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
regex = options.regex
import yaml
with open(configFile) as yaml_file:
config = yaml.safe_load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
db = []
db.append(username)
db.append(password)
db.append(database)
db.append(hostname)
conn = dbConnect(hostname, username, password, database)
warnings.filterwarnings("ignore")
# Parse command line
currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")
(year, month, day, hour, min, sec) = currentDate.split(':')
dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec)
pid = int(options.pid)
maxjobs = int(options.maxjobs)
days = int(options.days)
camera = options.camera
try:
mjdToIngest = options.mjd
except TypeError as e:
mjdToIngest = None
print("camera =", camera)
print("regex =", regex)
todayMJD = getCurrentMJD()
# Use + 1 to include today!
mjdthreshold = int(todayMJD) - days + 1
# Specified MJD trumps mjd Threshold, so just go as far back
# as the specified date
if mjdToIngest:
mjdthreshold = int(mjdToIngest[0:5]) - 1
ingester = options.ingester
fileList = getFiles(regex,
camera,
mjdToIngest=mjdToIngest,
mjdthreshold=mjdthreshold,
days=days,
atlasroot=options.atlasroot,
options=options)
ingestedFiles = getFilesIngestedddc2(conn,
mjdthreshold=mjdthreshold,
camera=camera)
fileListDict = OrderedDict()
print("List of files...")
for row in fileList:
fileListDict[os.path.basename(row)] = row
print(row)
print("List of ingested files...")
for row in ingestedFiles:
print(row)
filesToIngest = [
fileListDict[x]
for x in list(set(fileListDict.keys()) - set(ingestedFiles))
]
filesToIngest.sort()
print("List of files to ingest...")
for row in filesToIngest:
print(row)
print("TOTAL OBJECTS TO CHECK = %d" % len(filesToIngest))
if len(fileList) > 0:
# 2018-02-06 KWS Use half the default number of processes. This may ironically speed up ingest.
nProcessors, listChunks = splitList(filesToIngest, bins=28)
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
worker,
miscParameters=[options],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
conn.close()
return 0
def main():
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
regex = options.regex
import yaml
with open(configFile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
conn = dbConnect(hostname, username, password, database)
warnings.filterwarnings("ignore")
# Parse command line
currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")
(year, month, day, hour, min, sec) = currentDate.split(':')
dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec)
pid = int(options.pid)
maxjobs = int(options.maxjobs)
days = int(options.days)
camera = options.camera
try:
mjdToIngest = options.mjd
except TypeError as e:
mjdToIngest = None
print("camera =", camera)
print("regex =", regex)
todayMJD = getCurrentMJD()
mjdthreshold = int(todayMJD) - days + 1
# Specified MJD trumps mjd Threshold, so just go as far back
# as the specified date
if mjdToIngest:
mjdthreshold = int(mjdToIngest[0:5]) - 1
ingester = options.ingester
fileList = getFiles(regex,
camera,
mjdToIngest=mjdToIngest,
mjdthreshold=mjdthreshold,
days=days,
options=options)
ingestedFiles = getFilesIngestedddc2(conn,
mjdthreshold=mjdthreshold,
mjdToIngest=mjdToIngest,
camera=camera)
fileListDict = OrderedDict()
print("List of files...")
for row in fileList:
fileListDict[os.path.basename(row)] = row
print(row)
print("List of ingested files...")
for row in ingestedFiles:
print(row)
filesToIngest = [
fileListDict[x]
for x in list(set(fileListDict.keys()) - set(ingestedFiles))
]
filesToIngest.sort()
print("List of files to ingest...")
for row in filesToIngest:
print(row)
print("TOTAL OBJECTS TO CHECK = %d" % len(filesToIngest))
ingesterWrapper(ingester, configFile, filesToIngest)
conn.close()
return 0
def main():
"""main.
"""
opts = docopt(__doc__, version='0.1')
opts = cleanOptions(opts)
options = Struct(**opts)
configFile = options.configfile
import yaml
with open(configFile) as yaml_file:
config = yaml.load(yaml_file)
username = config['databases']['local']['username']
password = config['databases']['local']['password']
database = config['databases']['local']['database']
hostname = config['databases']['local']['hostname']
MAX_NUMBER_OF_OBJECTS = int(
config['postage_stamp_parameters']['max_number_of_objects'])
db = []
db.append(username)
db.append(password)
db.append(database)
db.append(hostname)
detectionList = 1
customList = None
conn = dbConnect(hostname, username, password, database)
update = options.update
limit = int(options.limit)
mostRecent = not (options.earliest)
nondetections = options.nondetections
discoverylimit = int(options.discoverylimit)
lastdetectionlimit = int(options.lastdetectionlimit)
objectList = []
try:
requestType = REQUESTTYPES[options.requesttype]
except KeyError as e:
requestType = REQUESTTYPES['incremental']
print("REQUEST TYPE = ", requestType)
flagDate = '2015-12-20'
if options.flagdate is not None:
try:
flagDate = '%s-%s-%s' % (options.flagdate[0:4],
options.flagdate[4:6],
options.flagdate[6:8])
except:
flagDate = '2015-12-20'
if options.candidate is not None and len(options.candidate) > 0:
for cand in options.candidate:
objectList.append({'id': int(cand)})
else:
if options.customlist is not None:
if int(options.customlist) > 0 and int(options.customlist) < 100:
customList = int(options.customlist)
objectList = getObjectsByCustomList(conn, customList)
else:
print(
"The list must be between 1 and 100 inclusive. Exiting.")
sys.exit(1)
else:
if options.detectionlist is not None:
if int(options.detectionlist) >= 0 and int(
options.detectionlist) < 9:
detectionList = int(options.detectionlist)
objectList = getObjectsByList(conn,
listId=detectionList,
dateThreshold=flagDate)
else:
print(
"The list must be between 0 and 6 inclusive. Exiting."
)
sys.exit(1)
currentDate = datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")
(year, month, day, hour, min, sec) = currentDate.split(':')
dateAndTime = "%s%s%s_%s%s%s" % (year, month, day, hour, min, sec)
if len(objectList) > MAX_NUMBER_OF_OBJECTS:
sys.stderr.write(
"The number of objects (%d) exceeds the maximum allowed (%d). Cannot continue.\n"
% (len(objectList), MAX_NUMBER_OF_OBJECTS))
sys.exit(1)
# Only download exposures if requested. Otherwise assume we already HAVE the data.
if not options.skipdownload:
exposureSet = getUniqueExposures(conn,
objectList,
limit=limit,
mostRecent=mostRecent,
nonDets=nondetections,
discoveryLimit=discoverylimit,
lastDetectionLimit=lastdetectionlimit,
requestType=requestType,
ddc=options.ddc)
# Download threads with multiprocessing - try 10 threads by default
print("TOTAL OBJECTS = %d" % len(exposureSet))
print("Downloading exposures...")
if len(exposureSet) > 0:
nProcessors, listChunks = splitList(exposureSet,
bins=int(
options.downloadthreads))
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
workerImageDownloader,
miscParameters=[options],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
# Belt and braces. Do again, with one less thread.
nProcessors, listChunks = splitList(
exposureSet, bins=int(options.downloadthreads) - 1)
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
workerImageDownloader,
miscParameters=[options],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
# Produce stamps with multiprocessing - try n(CPUs) threads by default
print("Producing stamps...")
if len(objectList) > 0:
nProcessors, listChunks = splitList(objectList, bins=48)
print("%s Parallel Processing..." %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
parallelProcess(db,
dateAndTime,
nProcessors,
listChunks,
workerStampCutter,
miscParameters=[
limit, mostRecent, nondetections, discoverylimit,
lastdetectionlimit, requestType, options.ddc,
options.wpwarp, options
],
drainQueues=False)
print("%s Done Parallel Processing" %
(datetime.datetime.now().strftime("%Y:%m:%d:%H:%M:%S")))
conn.close()
return 0
