def main(user_args):
upc_session_maker, upc_engine = db_connect(upc_db)
persist = user_args.persist
log_level = user_args.log_level
namespace = user_args.namespace
try:
slurm_job_id = os.environ['SLURM_ARRAY_JOB_ID']
slurm_array_id = os.environ['SLURM_ARRAY_TASK_ID']
except:
slurm_job_id = ''
slurm_array_id = ''
inputfile = ''
context = {'job_id': slurm_job_id, 'array_id':slurm_array_id, 'inputfile': inputfile}
logger = logging.getLogger('UPC_Process')
level = logging.getLevelName(log_level)
logger.setLevel(level)
log_file_handle = logging.FileHandler(pds_log + 'Process.log')
formatter = logging.Formatter(
'%(asctime)s - %(job_id)s - %(array_id)s - %(inputfile)s - %(name)s - %(levelname)s, %(message)s')
log_file_handle.setFormatter(formatter)
logger.addHandler(log_file_handle)
logger = logging.LoggerAdapter(logger, context)
# Redis Queue Objects
RQ_main = RedisQueue('UPC_UpdateQueue', namespace)
RQ_work = RedisQueue('UPC_UpdateWorkQueue', namespace)
logger.info("UPC Update Queue: %s", RQ_main.id_name)
RQ_error = RedisQueue(upc_error_queue)
RQ_lock = RedisLock(lock_obj)
# If the queue isn't registered, add it and set it to "running"
RQ_lock.add({RQ_main.id_name: '1'})
# while there are items in the redis queue
while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name):
# get a file from the queue
item = RQ_main.Qfile2Qwork(RQ_main.getQueueName(), RQ_work.getQueueName())
item_list = literal_eval(item)
inputfile = item_list[0]
archive = item_list[1]
failing_command = item_list[2]
update_type = item_list[3]
upc_id = None
if not os.path.isfile(inputfile):
RQ_error.QueueAdd(f'Unable to locate or access {inputfile} during UPC update')
logger.debug("%s is not a file\n", inputfile)
exit()
# Build URL for edr_source
edr_source = inputfile.replace(workarea, web_base)
# Update the logger context to include inputfile
context['inputfile'] = inputfile
try:
session = upc_session_maker()
session.close()
except TypeError as e:
logger.error("Unable to create a database session/connection to the upc database: %s", e)
raise e
try:
if update_type.lower() == 'upc':
recipe_file = recipe_base + "/" + archive + '.json'
no_extension_inputfile = os.path.splitext(inputfile)[0]
cam_info_file = no_extension_inputfile + '_caminfo.pvl'
footprint_file = no_extension_inputfile + '_footprint.json'
catlab_output = no_extension_inputfile + '_catlab.pvl'
with open(recipe_file) as fp:
upc_json = json.load(fp)['upc']
# Attempt to get the optional search_term_mapping for the upc
# process
try:
search_term_mapping = upc_json['search_term_mapping']
except KeyError:
search_term_mapping = {}
# Some datasets with attached PDS labels cause PVL to hang,
# so recipe includes call to dump label using `catlab`
# If present, use the catlab output as pds_label instead of inputfile
if os.path.exists(catlab_output):
pds_label = pvl.load(catlab_output)
else:
pds_label = pvl.load(inputfile)
instrument_name = get_instrument_name(pds_label)
spacecraft_name = get_spacecraft_name(pds_label)
target_name = get_target_name(pds_label)
with session_scope(upc_session_maker) as session:
target_qobj = Targets.create(session, targetname=target_name,
displayname=target_name.title(),
system=target_name)
target_id = target_qobj.targetid
with session_scope(upc_session_maker) as session:
instrument_qobj = Instruments.create(session, instrument=instrument_name,
spacecraft=spacecraft_name)
instrument_id = instrument_qobj.instrumentid
######## Generate DataFiles Record ########
datafile_attributes = create_datafiles_atts(pds_label, edr_source, no_extension_inputfile + '.cub')
datafile_attributes['instrumentid'] = instrument_id
datafile_attributes['targetid'] = target_id
with session_scope(upc_session_maker) as session:
datafile_qobj = DataFiles.create(session, **datafile_attributes)
upc_id = datafile_qobj.upcid
######## Generate SearchTerms Record ########
search_term_attributes = create_search_terms_atts(cam_info_file, upc_id,
no_extension_inputfile + '.cub',
footprint_file, search_term_mapping)
search_term_attributes['targetid'] = target_id
search_term_attributes['instrumentid'] = instrument_id
with session_scope(upc_session_maker) as session:
SearchTerms.create(session, **search_term_attributes)
######## Generate JsonKeywords Record ########
json_keywords_attributes = create_json_keywords_atts(cam_info_file, upc_id, inputfile,
failing_command, logger)
with session_scope(upc_session_maker) as session:
JsonKeywords.create(session, **json_keywords_attributes)
# Derived Processing:
# If we don't have a upcid, get the matching ID from the database
if not upc_id:
with session_scope(upc_session_maker) as session:
src = inputfile.replace(workarea, web_base)
datafile = session.query(DataFiles).filter(or_(DataFiles.source == src,
DataFiles.detached_label == src)).first()
if not datafile:
RQ_error.QueueAdd(f'No matching upcid was found for {inputfile}, '
'derived product paths could not be added')
logger.warning(f'No matching upcid was found for %s, '
'derived product paths could not be added', inputfile)
upc_id = datafile.upcid
final_path = makedir(inputfile)
src = os.path.splitext(inputfile)[0]
derived_product = os.path.join(final_path, os.path.splitext(os.path.basename(inputfile))[0])
# If derived products exist, copy them to the derived area and add the path to the db
try:
shutil.move(src + '.browse.jpg', derived_product + '.browse.jpg')
shutil.move(src + '.thumbnail.jpg', derived_product + '.thumbnail.jpg')
add_url(derived_product, upc_id, upc_session_maker)
except FileNotFoundError:
RQ_error.QueueAdd(f'Unable to locate or access derived products for {inputfile}')
logger.warning(f'Unable to locate or access derived products for %s', inputfile)
if not persist:
# Remove all files file from the workarea except for the copied
# source file
file_prefix = os.path.splitext(inputfile)[0]
workarea_files = glob(file_prefix + '*')
# os.remove(os.path.join(workarea, 'print.prt'))
for file in workarea_files:
os.remove(file)
# Handle SQL specific database errors
except SQLAlchemyError as e:
logger.error("Database operation failed: %s \nRequeueing (%s, %s)", e, inputfile, archive)
RQ_main.QueueAdd((inputfile, archive, failing_command, update_type))
raise e
RQ_work.QueueRemove(item)
# Disconnect from the engines
upc_engine.dispose()
def main(user_args):
key = user_args.key
namespace = user_args.namespace
if namespace is None:
namespace = default_namespace
workarea = scratch + key + '/'
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_work = RedisQueue(key + '_WorkQueue', namespace)
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_final = RedisQueue('FinalQueue', namespace)
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RHash = RedisHash(key + '_info')
RHerror = RedisHash(key + '_error')
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'MAP': '1'})
if int(RQ_file.QueueSize()) == 0:
print("No Files Found in Redis Queue")
elif RQ_lock.available('MAP'):
jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(),
RQ_work.getQueueName())
# Setup system logging
basename = os.path.splitext(os.path.basename(jobFile))[0]
logger = logging.getLogger(key + '.' + basename)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + '/Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting MAP Processing')
loggyOBJ = Loggy(basename)
# File Naming
infile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub'
outfile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub'
# Recipe Stuff
status = 'success'
for element in RQ_recipe.RecipeGet():
if status == 'error':
break
elif status == 'success':
processOBJ = Process()
process = processOBJ.JSON2Process(element)
if 'gdal_translate' not in processOBJ.getProcessName():
if 'cubeatt-band' in processOBJ.getProcessName():
if '+' in jobFile:
processOBJ.updateParameter('from_', jobFile)
processOBJ.updateParameter('to', outfile)
processOBJ.ChangeProcess('cubeatt')
else:
continue
elif 'map2map' in processOBJ.getProcessName():
if '+' in jobFile:
processOBJ.updateParameter('from_', infile)
else:
processOBJ.updateParameter('from_', jobFile)
processOBJ.updateParameter('to', outfile)
elif 'cubeatt-bit' in processOBJ.getProcessName():
if RHash.OutBit() == 'unsignedbyte':
temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254'
elif RHash.OutBit() == 'signedword':
temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', temp_outfile)
processOBJ.ChangeProcess('cubeatt')
elif 'isis2pds' in processOBJ.getProcessName():
# finalfile = infile.replace('.input.cub', '_final.img')
finalfile = workarea + RHash.getMAPname() + '.img'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', finalfile)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
print(processOBJ.getProcess())
for k, v in processOBJ.getProcess().items():
func = getattr(isis, k)
subloggyOBJ = SubLoggy(k)
try:
func(**v)
logger.info('Process %s :: Success', k)
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if os.path.isfile(outfile):
os.rename(outfile, infile)
status = 'success'
except ProcessError as e:
logger.error('Process %s :: Error', k)
logger.error(e)
status = 'error'
eSTR = 'Error Executing ' + k + \
' Standard Error: ' + str(e)
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
RQ_work.QueueRemove(jobFile)
break
else:
GDALcmd = ""
for process, v, in processOBJ.getProcess().items():
subloggyOBJ = SubLoggy(process)
GDALcmd += process
for dict_key, value in v.items():
GDALcmd += ' ' + dict_key + ' ' + value
img_format = RHash.Format()
if img_format == 'GeoTiff-BigTiff':
fileext = 'tif'
elif img_format == 'GeoJPEG-2000':
fileext = 'jp2'
elif img_format == 'JPEG':
fileext = 'jpg'
elif img_format == 'PNG':
fileext = 'png'
elif img_format == 'GIF':
fileext = 'gif'
logGDALcmd = GDALcmd + ' ' + basename + '.input.cub ' + RHash.getMAPname(
) + '.' + fileext
finalfile = workarea + RHash.getMAPname() + '.' + fileext
GDALcmd += ' ' + infile + ' ' + finalfile
print(GDALcmd)
try:
subprocess.call(GDALcmd, shell=True)
logger.info('Process GDAL translate :: Success')
status = 'success'
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'www.gdal.org/gdal_translate.html')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
os.remove(infile)
except OSError as e:
logger.error('Process GDAL translate :: Error')
logger.error(e)
status = 'error'
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
'Process GDAL translate :: Error')
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
subloggyOBJ.errorOut(e)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if status == 'success':
if RHash.Format() == 'ISIS3':
finalfile = workarea + RHash.getMAPname() + '.cub'
shutil.move(infile, finalfile)
if RHash.getStatus() != 'ERROR':
RHash.Status('SUCCESS')
try:
RQ_zip.QueueAdd(finalfile)
logger.info('File Added to ZIP Queue')
except:
logger.error('File NOT Added to ZIP Queue')
try:
RQ_loggy.QueueAdd(loggyOBJ.Loggy2json())
logger.info('JSON Added to Loggy Queue')
except:
logger.error('JSON NOT Added to Loggy Queue')
RQ_work.QueueRemove(jobFile)
elif status == 'error':
RHash.Status('ERROR')
if os.path.isfile(infile):
os.remove(infile)
if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0:
try:
RQ_final.QueueAdd(key)
logger.info('Key %s Added to Final Queue: Success', key)
logger.info('Job Complete')
except:
logger.error('Key NOT Added to Final Queue')
else:
logger.warning('Queues Not Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args):
proc = user_args.proc
derived = user_args.derived
log_level = user_args.log_level
namespace = user_args.namespace
try:
slurm_job_id = os.environ['SLURM_ARRAY_JOB_ID']
slurm_array_id = os.environ['SLURM_ARRAY_TASK_ID']
except:
slurm_job_id = ''
slurm_array_id = ''
inputfile = ''
context = {'job_id': slurm_job_id, 'array_id':slurm_array_id, 'inputfile': inputfile}
logger = logging.getLogger('UPC_Process')
level = logging.getLevelName(log_level)
logger.setLevel(level)
log_file_handle = logging.FileHandler(pds_log + 'Process.log')
formatter = logging.Formatter(
'%(asctime)s - %(job_id)s - %(array_id)s - %(inputfile)s - %(name)s - %(levelname)s, %(message)s')
log_file_handle.setFormatter(formatter)
logger.addHandler(log_file_handle)
logger = logging.LoggerAdapter(logger, context)
# Redis Queue Objects
RQ_main = RedisQueue('UPC_ReadyQueue', namespace)
RQ_error = RedisQueue(upc_error_queue, namespace)
RQ_work = RedisQueue('UPC_WorkQueue', namespace)
RQ_update = RedisQueue('UPC_UpdateQueue', namespace)
logger.info("UPC Processing Queue: %s", RQ_main.id_name)
RQ_lock = RedisLock(lock_obj)
# If the queue isn't registered, add it and set it to "running"
RQ_lock.add({RQ_main.id_name: '1'})
# while there are items in the redis queue
while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name):
# get a file from the queue
item = RQ_main.Qfile2Qwork(RQ_main.getQueueName(), RQ_work.getQueueName())
inputfile = literal_eval(item)[0]
archive = literal_eval(item)[1]
if not os.path.isfile(inputfile):
RQ_error.QueueAdd(f'Unable to locate or access {inputfile} during UPC processing')
logger.debug("%s is not a file\n", inputfile)
exit()
# Update the logger context to include inputfile
context['inputfile'] = inputfile
recipe_file = recipe_base + "/" + archive + '.json'
no_extension_inputfile = os.path.splitext(inputfile)[0]
cam_info_file = no_extension_inputfile + '_caminfo.pvl'
footprint_file = no_extension_inputfile + '_footprint.json'
catlab_output = no_extension_inputfile + '_catlab.pvl'
if proc:
with open(recipe_file) as fp:
upc_json = json.load(fp)['upc']
recipe_string = json.dumps(upc_json['recipe'])
processes = generate_processes(inputfile,
recipe_string, logger,
no_extension_inputfile=no_extension_inputfile,
catlab_output=catlab_output,
cam_info_file=cam_info_file,
footprint_file=footprint_file)
failing_command, _ = process(processes, workarea, logger)
if derived:
if os.path.isfile(inputfile):
recipe_file = recipe_base + "/" + archive + '.json'
with open(recipe_file) as fp:
recipe = json.load(fp, object_pairs_hook=parse_pairs)['reduced']
recipe_string = json.dumps(recipe['recipe'])
logger.info('Starting Process: %s', inputfile)
work_dir = os.path.dirname(inputfile)
derived_product = os.path.join(work_dir, os.path.splitext(os.path.basename(inputfile))[0])
no_extension_inputfile = os.path.splitext(inputfile)[0]
processes = generate_processes(inputfile,
recipe_string,
logger,
no_extension_inputfile=no_extension_inputfile,
derived_product=derived_product)
failing_command, _ = process(processes, workarea, logger)
if failing_command:
logger.warn(logger.error('%s Processing Error: %s', inputfile, failing_command))
if proc:
RQ_update.QueueAdd((inputfile, archive, failing_command, 'upc'))
elif derived:
RQ_update.QueueAdd((inputfile, archive, failing_command, 'derived'))
RQ_work.QueueRemove(item)
logger.info("UPC processing exited")
def main(user_args):
log_level = user_args.log_level
override = user_args.override
namespace = user_args.namespace
logger = logging.getLogger('Ingest_Process')
level = logging.getLevelName(log_level)
logger.setLevel(level)
logFileHandle = logging.FileHandler(pds_log + 'Ingest.log')
print("Log File: {}Ingest.log".format(pds_log))
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info("Starting Ingest Process")
PDSinfoDICT = json.load(open(pds_info, 'r'))
RQ_main = RedisQueue('Ingest_ReadyQueue', namespace)
RQ_work = RedisQueue('Ingest_WorkQueue', namespace)
RQ_error = RedisQueue(upc_error_queue, namespace)
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({RQ_main.id_name: '1'})
try:
Session, engine = db_connect(pds_db)
session = Session()
logger.info('DataBase Connecton: Success')
except:
logger.error('DataBase Connection: Error')
return 1
index = 1
while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name):
item = RQ_main.Qfile2Qwork(RQ_main.getQueueName(), RQ_work.getQueueName())
inputfile = literal_eval(item)[0]
archive = literal_eval(item)[1]
if not os.path.isfile(inputfile):
RQ_error.QueueAdd(f'Unable to locate or access {inputfile} during ingest processing')
logger.warning("%s is not a file\n", inputfile)
continue
subfile = inputfile.replace(PDSinfoDICT[archive]['path'], '')
# Calculate checksum in chunks of 4096
f_hash = hashlib.md5()
with open(inputfile, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
f_hash.update(chunk)
filechecksum = f_hash.hexdigest()
QOBJ = session.query(Files).filter_by(filename=subfile).first()
runflag = False
if QOBJ is None or filechecksum != QOBJ.checksum:
runflag = True
if runflag or override:
date = datetime.datetime.now(
pytz.utc).strftime("%Y-%m-%d %H:%M:%S")
fileURL = inputfile.replace(archive_base, web_base)
# If all upc requirements are in 'inputfile,' flag for upc
try:
upcflag = all(req in inputfile for req in PDSinfoDICT[archive]['upc_reqs'])
except KeyError:
logger.debug("No upc_reqs found for %s\nSetting upc eligibility False for all related files.", str(archive))
upcflag = False
filesize = os.path.getsize(inputfile)
try:
ingest_entry = Files()
if QOBJ is not None and override:
ingest_entry.fileid = QOBJ.fileid
ingest_entry.archiveid = PDSinfoDICT[archive]['archiveid']
ingest_entry.filename = subfile
ingest_entry.entry_date = date
ingest_entry.checksum = filechecksum
ingest_entry.upc_required = upcflag
ingest_entry.validation_required = True
ingest_entry.header_only = False
ingest_entry.release_date = date
ingest_entry.file_url = fileURL
ingest_entry.file_size = filesize
ingest_entry.di_pass = True
ingest_entry.di_date = date
session.merge(ingest_entry)
session.flush()
RQ_work.QueueRemove(item)
index = index + 1
except Exception as e:
logger.error("Error During File Insert %s : %s", str(subfile), str(e))
elif not runflag and not override:
RQ_work.QueueRemove(item)
logger.warning("Not running ingest: file %s already present"
" in database and no override flag supplied", inputfile)
if index >= 250:
try:
session.commit()
logger.info("Commit 250 files to Database: Success")
index = 1
except Exception as e:
session.rollback()
logger.warning("Unable to commit to database: %s", str(e))
else:
logger.info("No Files Found in Ingest Queue")
try:
session.commit()
logger.info("Commit to Database: Success")
except Exception as e:
logger.error("Unable to commit to database: %s", str(e))
session.rollback()
# Close connection to database
session.close()
engine.dispose()
if RQ_main.QueueSize() == 0 and RQ_work.QueueSize() == 0:
logger.info("Process Complete All Queues Empty")
elif RQ_main.QueueSize() == 0 and RQ_work.QueueSize() != 0:
logger.warning("Process Done Work Queue NOT Empty Contains %s Files", str(
RQ_work.QueueSize()))
logger.info("Ingest Complete")
def main(key, namespace=None):
key = user_args.key
namespace = user_args.namespace
print(namespace)
if namespace is None:
namespace = default_namespace
workarea = scratch + key + '/'
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_work = RedisQueue(key + '_WorkQueue', namespace)
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_final = RedisQueue('FinalQueue', namespace)
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RHash = RedisHash(key + '_info')
RHerror = RedisHash(key + '_error')
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'POW': '1'})
if int(RQ_file.QueueSize()) == 0:
print("No Files Found in Redis Queue")
elif RQ_lock.available('POW'):
print(RQ_file.getQueueName())
jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(),
RQ_work.getQueueName())
# Setup system logging
basename = os.path.splitext(os.path.basename(jobFile))[0]
logger = logging.getLogger(key + '.' + basename)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + '/Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting POW Processing')
# set up loggy
loggyOBJ = Loggy(basename)
# File Naming
if '+' in jobFile:
bandSplit = jobFile.split('+')
inputFile = bandSplit[0]
else:
inputFile = jobFile
infile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub'
outfile = workarea + \
os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub'
status = 'success'
for element in RQ_recipe.RecipeGet():
if status == 'error':
break
elif status == 'success':
processOBJ = Process()
process = processOBJ.JSON2Process(element)
if 'gdal_translate' not in processOBJ.getProcessName():
print(processOBJ.getProcessName())
if '2isis' in processOBJ.getProcessName():
processOBJ.updateParameter('from_', inputFile)
processOBJ.updateParameter('to', outfile)
elif 'cubeatt-band' in processOBJ.getProcessName():
if '+' in jobFile:
infileB = infile + '+' + bandSplit[1]
processOBJ.updateParameter('from_', infileB)
processOBJ.updateParameter('to', outfile)
processOBJ.ChangeProcess('cubeatt')
else:
continue
elif 'cubeatt-bit' in processOBJ.getProcessName():
if RHash.OutBit() == 'unsignedbyte':
temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254'
elif RHash.OutBit() == 'signedword':
temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765'
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', temp_outfile)
processOBJ.ChangeProcess('cubeatt')
elif 'spice' in processOBJ.getProcessName():
processOBJ.updateParameter('from_', infile)
elif 'ctxevenodd' in processOBJ.getProcessName():
label = pvl.load(infile)
SS = label['IsisCube']['Instrument']['SpatialSumming']
print(SS)
if SS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'mocevenodd' in processOBJ.getProcessName():
label = pvl.load(infile)
CTS = label['IsisCube']['Instrument'][
'CrosstrackSumming']
print(CTS)
if CTS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'mocnoise50' in processOBJ.getProcessName():
label = pvl.load(infile)
CTS = label['IsisCube']['Instrument'][
'CrosstrackSumming']
if CTS != 1:
continue
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
elif 'cam2map' in processOBJ.getProcessName():
processOBJ.updateParameter('from', infile)
processOBJ.updateParameter('to', outfile)
if RHash.getGRtype() == 'smart' or RHash.getGRtype(
) == 'fill':
subloggyOBJ = SubLoggy('cam2map')
camrangeOUT = workarea + basename + '_camrange.txt'
isis.camrange(from_=infile, to=camrangeOUT)
cam = pvl.load(camrangeOUT)
if cam['UniversalGroundRange']['MaximumLatitude'] < float(RHash.getMinLat()) or \
cam['UniversalGroundRange']['MinimumLatitude'] > float(RHash.getMaxLat()) or \
cam['UniversalGroundRange']['MaximumLongitude'] < float(RHash.getMinLon()) or \
cam['UniversalGroundRange']['MinimumLongitude'] > float(RHash.getMaxLon()):
status = 'error'
eSTR = "Error Ground Range Outside Extent Range"
RHerror.addError(
os.path.splitext(
os.path.basename(jobFile))[0], eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
break
elif RHash.getGRtype() == 'smart':
if cam['UniversalGroundRange'][
'MinimumLatitude'] > float(
RHash.getMinLat()):
minlat = cam['UniversalGroundRange'][
'MinimumLatitude']
else:
minlat = RHash.getMinLat()
if cam['UniversalGroundRange'][
'MaximumLatitude'] < float(
RHash.getMaxLat()):
maxlat = cam['UniversalGroundRange'][
'MaximumLatitude']
else:
maxlat = RHash.getMaxLat()
if cam['UniversalGroundRange'][
'MinimumLongitude'] > float(
RHash.getMinLon()):
minlon = cam['UniversalGroundRange'][
'MinimumLongitude']
else:
minlon = RHash.getMinLon()
if cam['UniversalGroundRange'][
'MaximumLongitude'] < float(
RHash.getMaxLon()):
maxlon = cam['UniversalGroundRange'][
'MaximumLongitude']
else:
maxlon = RHash.getMaxLon()
elif RHash.getGRtype() == 'fill':
minlat = RHash.getMinLat()
maxlat = RHash.getMaxLat()
minlon = RHash.getMinLon()
maxlon = RHash.getMaxLon()
processOBJ.AddParameter('minlat', minlat)
processOBJ.AddParameter('maxlat', maxlat)
processOBJ.AddParameter('minlon', minlon)
processOBJ.AddParameter('maxlon', maxlon)
#os.remove(camrangeOUT)
elif 'isis2pds' in processOBJ.getProcessName():
finalfile = infile.replace('.input.cub', '_final.img')
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', finalfile)
else:
processOBJ.updateParameter('from_', infile)
processOBJ.updateParameter('to', outfile)
print(processOBJ.getProcess())
for k, v in processOBJ.getProcess().items():
func = getattr(isis, k)
subloggyOBJ = SubLoggy(k)
try:
func(**v)
logger.info('Process %s :: Success', k)
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if os.path.isfile(outfile):
os.rename(outfile, infile)
status = 'success'
except ProcessError as e:
logger.error('Process %s :: Error', k)
logger.error(e)
status = 'error'
eSTR = 'Error Executing ' + k + \
' Standard Error: ' + str(e)
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
eSTR)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(processOBJ.LogCommandline())
subloggyOBJ.setHelpLink(processOBJ.LogHelpLink())
subloggyOBJ.errorOut(eSTR)
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
else:
GDALcmd = ""
for process, v, in processOBJ.getProcess().items():
subloggyOBJ = SubLoggy(process)
GDALcmd += process
for dict_key, value in v.items():
GDALcmd += ' ' + dict_key + ' ' + value
frmt = RHash.Format()
if frmt == 'GeoTiff-BigTiff':
fileext = 'tif'
elif frmt == 'GeoJPEG-2000':
fileext = 'jp2'
elif frmt == 'JPEG':
fileext = 'jpg'
elif frmt == 'PNG':
fileext = 'png'
elif frmt == 'GIF':
fileext = 'gif'
logGDALcmd = GDALcmd + ' ' + basename + \
'.input.cub ' + basename + '_final.' + fileext
finalfile = infile.replace('.input.cub',
'_final.' + fileext)
GDALcmd += ' ' + infile + ' ' + finalfile
print(GDALcmd)
result = subprocess.call(GDALcmd, shell=True)
if result == 0:
logger.info('Process GDAL translate :: Success')
status = 'success'
subloggyOBJ.setStatus('SUCCESS')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
#os.remove(infile)
else:
errmsg = 'Error Executing GDAL translate: Error'
logger.error(errmsg)
status = 'error'
RHerror.addError(
os.path.splitext(os.path.basename(jobFile))[0],
errmsg)
subloggyOBJ.setStatus('ERROR')
subloggyOBJ.setCommand(logGDALcmd)
subloggyOBJ.setHelpLink(
'http://www.gdal.org/gdal_translate.html')
subloggyOBJ.errorOut('Process GDAL translate :: Error')
loggyOBJ.AddProcess(subloggyOBJ.getSLprocess())
if status == 'success':
if RHash.Format() == 'ISIS3':
finalfile = infile.replace('.input.cub', '_final.cub')
shutil.move(infile, finalfile)
if RHash.getStatus() != 'ERROR':
RHash.Status('SUCCESS')
try:
RQ_zip.QueueAdd(finalfile)
logger.info('File Added to ZIP Queue')
except:
logger.error('File NOT Added to ZIP Queue')
elif status == 'error':
RHash.Status('ERROR')
if os.path.isfile(infile):
#os.remove(infile)
pass
try:
RQ_loggy.QueueAdd(loggyOBJ.Loggy2json())
RQ_work.QueueRemove(jobFile)
logger.info('JSON Added to Loggy Queue')
except:
logger.error('JSON NOT Added to Loggy Queue')
if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0:
try:
RQ_final.QueueAdd(key)
logger.info('Key %s Added to Final Queue: Success', key)
logger.info(
'Both Queues Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
logger.info('JOB Complete')
except:
logger.error('Key NOT Added to Final Queue')
elif RQ_file.QueueSize() == 0 and RQ_work.QueueSize() != 0:
logger.warning(
'Work Queue Not Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args):
key = user_args.key
namespace = user_args.namespace
if namespace is None:
namespace = default_namespace
work_dir = os.path.join(workarea, key)
RQ_file = RedisQueue(key + '_FileQueue', namespace)
RQ_work = RedisQueue(key + '_WorkQueue', namespace)
RQ_zip = RedisQueue(key + '_ZIP', namespace)
RQ_loggy = RedisQueue(key + '_loggy', namespace)
RQ_final = RedisQueue('FinalQueue', namespace)
RQ_recipe = RedisQueue(key + '_recipe', namespace)
RQ_error = RedisQueue(upc_error_queue, namespace)
RHash = RedisHash(key + '_info')
RHerror = RedisHash(key + '_error')
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({'MAP': '1'})
if int(RQ_file.QueueSize()) > 0 and RQ_lock.available('MAP'):
jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(),
RQ_work.getQueueName())
# Setup system logging
basename = os.path.splitext(os.path.basename(jobFile))[0]
logger = logging.getLogger(key + '.' + basename)
logger.setLevel(logging.INFO)
logFileHandle = logging.FileHandler(pds_log + '/Service.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting MAP Processing')
# File Naming
infile = os.path.join(work_dir, \
os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub')
outfile = os.path.join(
work_dir,
os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub')
# Recipe Stuff
status = 'success'
recipe_string = RQ_recipe.QueueGet()
no_extension_inputfile = os.path.join(
work_dir,
os.path.splitext(os.path.basename(jobFile))[0])
processes = generate_processes(
jobFile,
recipe_string,
logger,
no_extension_inputfile=no_extension_inputfile)
failing_command, error = process(processes, work_dir, logger)
process_log = generate_log_json(processes, basename, failing_command,
error)
if failing_command:
status = 'error'
if status == 'success':
final_file_list = []
img_format = RHash.Format()
# If final output format is ISIS3 or PDS, will use an ISIS program to create
if img_format == 'ISIS3' or img_format == 'PDS':
last_output = list(processes.items())[-1][-1]['to']
last_output = last_output.split('+')[0]
if img_format == 'ISIS3':
finalfile = os.path.join(work_dir,
RHash.getMAPname() + '.cub')
else:
finalfile = os.path.join(work_dir,
RHash.getMAPname() + '.img')
# Else will use GDAL, so set extension and define possible ancillary files
else:
if img_format == 'GeoTiff-BigTiff' or img_format == 'GTiff':
fileext = 'tif'
elif img_format == 'GeoJPEG-2000':
fileext = 'jp2'
elif img_format == 'JPEG':
fileext = 'jpg'
elif img_format == 'PNG':
fileext = 'png'
elif img_format == 'GIF':
fileext = 'gif'
last_output = list(processes.items())[-1][-1]['dest']
finalfile = os.path.join(work_dir,
RHash.getMAPname() + '.' + fileext)
# Possible ancillary files
last_output_msk = last_output + '.msk'
last_output_aux = last_output + '.aux.xml'
if os.path.isfile(last_output_msk):
finalfile_msk = os.path.join(
work_dir,
RHash.getMAPname() + '.' + fileext + '.msk')
shutil.move(last_output_msk, finalfile_msk)
final_file_list.append(finalfile_msk)
if os.path.isfile(last_output_aux):
finalfile_aux = os.path.join(
work_dir,
RHash.getMAPname() + '.' + fileext + '.aux.xml')
shutil.move(last_output_aux, finalfile_aux)
final_file_list.append(finalfile_aux)
shutil.move(last_output, finalfile)
final_file_list.append(finalfile)
if RHash.getStatus() != 'ERROR':
RHash.Status('SUCCESS')
# Loop over the list of final output files and add them to RQ_zip
for item in final_file_list:
try:
RQ_zip.QueueAdd(item)
logger.info('File Added to ZIP Queue')
except:
logger.error('File NOT Added to ZIP Queue')
try:
RQ_loggy.QueueAdd(process_log)
logger.info('JSON Added to Loggy Queue')
except Exception as e:
logger.error(f'JSON NOT Added to Loggy Queue with error: {e}')
RQ_work.QueueRemove(jobFile)
elif status == 'error':
RHash.Status('ERROR')
logger.error(f'Process {failing_command} :: Error')
logger.error(error)
error_string = f'Error Executing {failing_command}' \
f'Standard Error: {error}'
RHerror.addError(basename, error_string)
RQ_error.QueueAdd(
f'Process {failing_command} failed for {jobFile}')
if os.path.isfile(infile):
os.remove(infile)
if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0:
try:
RQ_final.QueueAdd(key)
logger.info('Key %s Added to Final Queue: Success', key)
logger.info('Job Complete')
except:
logger.error('Key NOT Added to Final Queue')
else:
logger.warning('Queues Not Empty: filequeue = %s work queue = %s',
str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args):
log_level = user_args.log_level
namespace = user_args.namespace
PDSinfoDICT = json.load(open(pds_info, 'r'))
# Set up logging
logger = logging.getLogger('DI_Process')
level = logging.getLevelName(log_level)
logger.setLevel(level)
logFileHandle = logging.FileHandler(pds_log + 'DI.log')
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s, %(message)s')
logFileHandle.setFormatter(formatter)
logger.addHandler(logFileHandle)
logger.info('Starting DI Process')
try:
Session, engine = db_connect(pds_db)
session = Session()
logger.info('DataBase Connecton: Success')
except Exception as e:
logger.error('DataBase Connection Error: %s', str(e))
return 1
RQ = RedisQueue('DI_ReadyQueue', namespace)
RQ_work = RedisQueue('DI_WorkQueue', namespace)
RQ_error = RedisQueue(upc_error_queue, namespace)
RQ_lock = RedisLock(lock_obj)
RQ_lock.add({RQ.id_name: '1'})
index = 0
logger.info("DI Queue: %s", RQ.id_name)
while int(RQ.QueueSize()) > 0 and RQ_lock.available(RQ.id_name):
item = RQ.Qfile2Qwork(RQ.getQueueName(), RQ_work.getQueueName())
inputfile = literal_eval(item)[0]
archive = literal_eval(item)[1]
logger.debug("%s - %s", inputfile, archive)
subfile = inputfile.replace(PDSinfoDICT[archive]['path'], '')
try:
Qelement = session.query(Files).filter(
Files.filename == subfile).one()
except Exception as e:
logger.warning('Filename query failed for inputfile %s: %s',
inputfile, str(e))
continue
archive_path = PDSinfoDICT[archive]['path']
if os.path.isfile(inputfile):
f_hash = hashlib.md5()
with open(inputfile, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
f_hash.update(chunk)
checksum = f_hash.hexdigest()
Qelement.di_pass = checksum == Qelement.checksum
if not Qelement.di_pass:
RQ_error.QueueAdd(
f'File {inputfile} checksum {checksum} does not match the database entry checksum {Qelement.checksum}'
)
logger.warning(
'File %s checksum %s does not match the database entry checksum %s',
inputfile, checksum, Qelement.checksum)
Qelement.di_date = datetime.datetime.now(
pytz.utc).strftime("%Y-%m-%d %H:%M:%S")
session.flush()
RQ_work.QueueRemove(item)
index = index + 1
if index > 250:
session.commit()
logger.info('Session Commit for 250 Records: Success')
index = 0
else:
RQ_error.QueueAdd(
f'Unable to locate or access {inputfile} during DI processing')
logger.warning('File %s Not Found', inputfile)
try:
session.commit()
logger.info("End Commit DI process to Database: Success")
index = 1
except Exception as e:
logger.warning("Unable to commit changes to database\n\n%s", e)
session.rollback()
# Close connection to database
session.close()
engine.dispose()
if RQ.QueueSize() == 0 and RQ_work.QueueSize() == 0:
logger.info("Process Complete All Queues Empty")
elif RQ.QueueSize() == 0 and RQ_work.QueueSize() != 0:
logger.warning("Process Done Work Queue NOT Empty Contains %s Files",
str(RQ_work.QueueSize()))
