def main(user_args): key = user_args.key namespace = user_args.namespace if namespace is None: namespace = default_namespace workarea = scratch + key + '/' RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_work = RedisQueue(key + '_WorkQueue', namespace) RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_final = RedisQueue('FinalQueue', namespace) RQ_recipe = RedisQueue(key + '_recipe', namespace) RHash = RedisHash(key + '_info') RHerror = RedisHash(key + '_error') RQ_lock = RedisLock(lock_obj) RQ_lock.add({'MAP': '1'}) if int(RQ_file.QueueSize()) == 0: print("No Files Found in Redis Queue") elif RQ_lock.available('MAP'): jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(), RQ_work.getQueueName()) # Setup system logging basename = os.path.splitext(os.path.basename(jobFile))[0] logger = logging.getLogger(key + '.' + basename) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + '/Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting MAP Processing') loggyOBJ = Loggy(basename) # File Naming infile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub' outfile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub' # Recipe Stuff status = 'success' for element in RQ_recipe.RecipeGet(): if status == 'error': break elif status == 'success': processOBJ = Process() process = processOBJ.JSON2Process(element) if 'gdal_translate' not in processOBJ.getProcessName(): if 'cubeatt-band' in processOBJ.getProcessName(): if '+' in jobFile: processOBJ.updateParameter('from_', jobFile) processOBJ.updateParameter('to', outfile) processOBJ.ChangeProcess('cubeatt') else: continue elif 'map2map' in processOBJ.getProcessName(): if '+' in jobFile: processOBJ.updateParameter('from_', infile) else: processOBJ.updateParameter('from_', jobFile) processOBJ.updateParameter('to', outfile) elif 'cubeatt-bit' in processOBJ.getProcessName(): if RHash.OutBit() == 'unsignedbyte': temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254' elif RHash.OutBit() == 'signedword': temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', temp_outfile) processOBJ.ChangeProcess('cubeatt') elif 'isis2pds' in processOBJ.getProcessName(): # finalfile = infile.replace('.input.cub', '_final.img') finalfile = workarea + RHash.getMAPname() + '.img' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', finalfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) print(processOBJ.getProcess()) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) subloggyOBJ = SubLoggy(k) try: func(**v) logger.info('Process %s :: Success', k) subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if os.path.isfile(outfile): os.rename(outfile, infile) status = 'success' except ProcessError as e: logger.error('Process %s :: Error', k) logger.error(e) status = 'error' eSTR = 'Error Executing ' + k + \ ' Standard Error: ' + str(e) RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) RQ_work.QueueRemove(jobFile) break else: GDALcmd = "" for process, v, in processOBJ.getProcess().items(): subloggyOBJ = SubLoggy(process) GDALcmd += process for dict_key, value in v.items(): GDALcmd += ' ' + dict_key + ' ' + value img_format = RHash.Format() if img_format == 'GeoTiff-BigTiff': fileext = 'tif' elif img_format == 'GeoJPEG-2000': fileext = 'jp2' elif img_format == 'JPEG': fileext = 'jpg' elif img_format == 'PNG': fileext = 'png' elif img_format == 'GIF': fileext = 'gif' logGDALcmd = GDALcmd + ' ' + basename + '.input.cub ' + RHash.getMAPname( ) + '.' + fileext finalfile = workarea + RHash.getMAPname() + '.' + fileext GDALcmd += ' ' + infile + ' ' + finalfile print(GDALcmd) try: subprocess.call(GDALcmd, shell=True) logger.info('Process GDAL translate :: Success') status = 'success' subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'www.gdal.org/gdal_translate.html') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) os.remove(infile) except OSError as e: logger.error('Process GDAL translate :: Error') logger.error(e) status = 'error' RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], 'Process GDAL translate :: Error') subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') subloggyOBJ.errorOut(e) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if status == 'success': if RHash.Format() == 'ISIS3': finalfile = workarea + RHash.getMAPname() + '.cub' shutil.move(infile, finalfile) if RHash.getStatus() != 'ERROR': RHash.Status('SUCCESS') try: RQ_zip.QueueAdd(finalfile) logger.info('File Added to ZIP Queue') except: logger.error('File NOT Added to ZIP Queue') try: RQ_loggy.QueueAdd(loggyOBJ.Loggy2json()) logger.info('JSON Added to Loggy Queue') except: logger.error('JSON NOT Added to Loggy Queue') RQ_work.QueueRemove(jobFile) elif status == 'error': RHash.Status('ERROR') if os.path.isfile(infile): os.remove(infile) if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0: try: RQ_final.QueueAdd(key) logger.info('Key %s Added to Final Queue: Success', key) logger.info('Job Complete') except: logger.error('Key NOT Added to Final Queue') else: logger.warning('Queues Not Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args): log_level = user_args.log_level logger = logging.getLogger('Browse_Process') level = logging.getLevelName(log_level) logger.setLevel(level) logFileHandle = logging.FileHandler(pds_log + 'Process.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) RQ_main = RedisQueue('Browse_ReadyQueue') RQ_error = RedisQueue(upc_error_queue) RQ_lock = RedisLock(lock_obj) RQ_lock.add({RQ_main.id_name: '1'}) PDSinfoDICT = json.load(open(pds_info, 'r')) pds_session_maker, pds_engine = db_connect(pds_db) pds_session = pds_session_maker() upc_session_maker, upc_engine = db_connect(upc_db) upc_session = upc_session_maker() tid = get_tid('fullimageurl', upc_session) while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): item = literal_eval(RQ_main.QueueGet()) inputfile = item[0] fid = item[1] archive = item[2] if os.path.isfile(inputfile): logger.info('Starting Process: %s', inputfile) finalpath = makedir(inputfile) recipeOBJ = Recipe() recipeOBJ.addMissionJson(archive, 'reduced') infile = workarea + os.path.splitext( os.path.basename(inputfile))[0] + '.Binput.cub' outfile = workarea + os.path.splitext( os.path.basename(inputfile))[0] + '.Boutput.cub' status = 'success' for item in recipeOBJ.getProcesses(): if status == 'error': logger.error("Error processing %s", inputfile) break elif status == 'success': processOBJ = Process() processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe()) if '2isis' in item: processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) elif item == 'spiceinit': processOBJ.updateParameter('from_', infile) elif item == 'cubeatt': label = pvl.load(infile) bands = PDSinfoDICT[archive]['bandorder'] query_bands = label['IsisCube']['BandBin'][ PDSinfoDICT[archive]['bandbinQuery']] # Create a set from the list / single value try: query_band_set = set(query_bands) except: query_band_set = set([query_bands]) # Iterate through 'bands' and grab the first value that is present in the # set defined by 'bandbinquery' -- if not present, default to 1 exband = next( (band for band in bands if band in query_band_set), 1) band_infile = infile + '+' + str(exband) processOBJ.updateParameter('from_', band_infile) processOBJ.updateParameter('to', outfile) elif item == 'ctxevenodd': label = pvl.load(infile) SS = label['IsisCube']['Instrument']['SpatialSumming'] if SS != 1: break else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif item == 'reduce': label = pvl.load(infile) Nline = label['IsisCube']['Core']['Dimensions'][ 'Lines'] Nsample = label['IsisCube']['Core']['Dimensions'][ 'Samples'] Nline = int(Nline) Nsample = int(Nsample) Sfactor = scaleFactor(Nline, Nsample, recip_json) processOBJ.updateParameter('lscale', Sfactor) processOBJ.updateParameter('sscale', Sfactor) processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif item == 'isis2std': final_outfile = finalpath + '/' + os.path.splitext( os.path.basename(inputfile))[0] + '.browse.jpg' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', final_outfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) try: func(**v) logger.info('Process %s :: Success', k) if os.path.isfile(outfile): if '.cub' in outfile: os.rename(outfile, infile) status = 'success' if '2isis' in item: isisSerial = getISISid(infile) except ProcessError as e: print(e) logger.error('Process %s :: Error', k) status = 'error' if status == 'success': DB_addURL(upc_session, isisSerial, final_outfile, tid) os.remove(infile) logger.info('Browse Process Success: %s', inputfile) AddProcessDB(pds_session, fid, 't') else: RQ_error.QueueAdd( f'Unable to locate or access {inputfile} during browse processing' ) logger.error('File %s Not Found', inputfile) upc_session.close() pds_session.close() upc_engine.dispose() pds_engine.dispose()
def main(key, namespace=None): key = user_args.key namespace = user_args.namespace print(namespace) if namespace is None: namespace = default_namespace workarea = scratch + key + '/' RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_work = RedisQueue(key + '_WorkQueue', namespace) RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_final = RedisQueue('FinalQueue', namespace) RQ_recipe = RedisQueue(key + '_recipe', namespace) RHash = RedisHash(key + '_info') RHerror = RedisHash(key + '_error') RQ_lock = RedisLock(lock_obj) RQ_lock.add({'POW': '1'}) if int(RQ_file.QueueSize()) == 0: print("No Files Found in Redis Queue") elif RQ_lock.available('POW'): print(RQ_file.getQueueName()) jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(), RQ_work.getQueueName()) # Setup system logging basename = os.path.splitext(os.path.basename(jobFile))[0] logger = logging.getLogger(key + '.' + basename) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + '/Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting POW Processing') # set up loggy loggyOBJ = Loggy(basename) # File Naming if '+' in jobFile: bandSplit = jobFile.split('+') inputFile = bandSplit[0] else: inputFile = jobFile infile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub' outfile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub' status = 'success' for element in RQ_recipe.RecipeGet(): if status == 'error': break elif status == 'success': processOBJ = Process() process = processOBJ.JSON2Process(element) if 'gdal_translate' not in processOBJ.getProcessName(): print(processOBJ.getProcessName()) if '2isis' in processOBJ.getProcessName(): processOBJ.updateParameter('from_', inputFile) processOBJ.updateParameter('to', outfile) elif 'cubeatt-band' in processOBJ.getProcessName(): if '+' in jobFile: infileB = infile + '+' + bandSplit[1] processOBJ.updateParameter('from_', infileB) processOBJ.updateParameter('to', outfile) processOBJ.ChangeProcess('cubeatt') else: continue elif 'cubeatt-bit' in processOBJ.getProcessName(): if RHash.OutBit() == 'unsignedbyte': temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254' elif RHash.OutBit() == 'signedword': temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', temp_outfile) processOBJ.ChangeProcess('cubeatt') elif 'spice' in processOBJ.getProcessName(): processOBJ.updateParameter('from_', infile) elif 'ctxevenodd' in processOBJ.getProcessName(): label = pvl.load(infile) SS = label['IsisCube']['Instrument']['SpatialSumming'] print(SS) if SS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'mocevenodd' in processOBJ.getProcessName(): label = pvl.load(infile) CTS = label['IsisCube']['Instrument'][ 'CrosstrackSumming'] print(CTS) if CTS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'mocnoise50' in processOBJ.getProcessName(): label = pvl.load(infile) CTS = label['IsisCube']['Instrument'][ 'CrosstrackSumming'] if CTS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'cam2map' in processOBJ.getProcessName(): processOBJ.updateParameter('from', infile) processOBJ.updateParameter('to', outfile) if RHash.getGRtype() == 'smart' or RHash.getGRtype( ) == 'fill': subloggyOBJ = SubLoggy('cam2map') camrangeOUT = workarea + basename + '_camrange.txt' isis.camrange(from_=infile, to=camrangeOUT) cam = pvl.load(camrangeOUT) if cam['UniversalGroundRange']['MaximumLatitude'] < float(RHash.getMinLat()) or \ cam['UniversalGroundRange']['MinimumLatitude'] > float(RHash.getMaxLat()) or \ cam['UniversalGroundRange']['MaximumLongitude'] < float(RHash.getMinLon()) or \ cam['UniversalGroundRange']['MinimumLongitude'] > float(RHash.getMaxLon()): status = 'error' eSTR = "Error Ground Range Outside Extent Range" RHerror.addError( os.path.splitext( os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) break elif RHash.getGRtype() == 'smart': if cam['UniversalGroundRange'][ 'MinimumLatitude'] > float( RHash.getMinLat()): minlat = cam['UniversalGroundRange'][ 'MinimumLatitude'] else: minlat = RHash.getMinLat() if cam['UniversalGroundRange'][ 'MaximumLatitude'] < float( RHash.getMaxLat()): maxlat = cam['UniversalGroundRange'][ 'MaximumLatitude'] else: maxlat = RHash.getMaxLat() if cam['UniversalGroundRange'][ 'MinimumLongitude'] > float( RHash.getMinLon()): minlon = cam['UniversalGroundRange'][ 'MinimumLongitude'] else: minlon = RHash.getMinLon() if cam['UniversalGroundRange'][ 'MaximumLongitude'] < float( RHash.getMaxLon()): maxlon = cam['UniversalGroundRange'][ 'MaximumLongitude'] else: maxlon = RHash.getMaxLon() elif RHash.getGRtype() == 'fill': minlat = RHash.getMinLat() maxlat = RHash.getMaxLat() minlon = RHash.getMinLon() maxlon = RHash.getMaxLon() processOBJ.AddParameter('minlat', minlat) processOBJ.AddParameter('maxlat', maxlat) processOBJ.AddParameter('minlon', minlon) processOBJ.AddParameter('maxlon', maxlon) #os.remove(camrangeOUT) elif 'isis2pds' in processOBJ.getProcessName(): finalfile = infile.replace('.input.cub', '_final.img') processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', finalfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) print(processOBJ.getProcess()) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) subloggyOBJ = SubLoggy(k) try: func(**v) logger.info('Process %s :: Success', k) subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if os.path.isfile(outfile): os.rename(outfile, infile) status = 'success' except ProcessError as e: logger.error('Process %s :: Error', k) logger.error(e) status = 'error' eSTR = 'Error Executing ' + k + \ ' Standard Error: ' + str(e) RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) else: GDALcmd = "" for process, v, in processOBJ.getProcess().items(): subloggyOBJ = SubLoggy(process) GDALcmd += process for dict_key, value in v.items(): GDALcmd += ' ' + dict_key + ' ' + value frmt = RHash.Format() if frmt == 'GeoTiff-BigTiff': fileext = 'tif' elif frmt == 'GeoJPEG-2000': fileext = 'jp2' elif frmt == 'JPEG': fileext = 'jpg' elif frmt == 'PNG': fileext = 'png' elif frmt == 'GIF': fileext = 'gif' logGDALcmd = GDALcmd + ' ' + basename + \ '.input.cub ' + basename + '_final.' + fileext finalfile = infile.replace('.input.cub', '_final.' + fileext) GDALcmd += ' ' + infile + ' ' + finalfile print(GDALcmd) result = subprocess.call(GDALcmd, shell=True) if result == 0: logger.info('Process GDAL translate :: Success') status = 'success' subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) #os.remove(infile) else: errmsg = 'Error Executing GDAL translate: Error' logger.error(errmsg) status = 'error' RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], errmsg) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') subloggyOBJ.errorOut('Process GDAL translate :: Error') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if status == 'success': if RHash.Format() == 'ISIS3': finalfile = infile.replace('.input.cub', '_final.cub') shutil.move(infile, finalfile) if RHash.getStatus() != 'ERROR': RHash.Status('SUCCESS') try: RQ_zip.QueueAdd(finalfile) logger.info('File Added to ZIP Queue') except: logger.error('File NOT Added to ZIP Queue') elif status == 'error': RHash.Status('ERROR') if os.path.isfile(infile): #os.remove(infile) pass try: RQ_loggy.QueueAdd(loggyOBJ.Loggy2json()) RQ_work.QueueRemove(jobFile) logger.info('JSON Added to Loggy Queue') except: logger.error('JSON NOT Added to Loggy Queue') if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0: try: RQ_final.QueueAdd(key) logger.info('Key %s Added to Final Queue: Success', key) logger.info( 'Both Queues Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize())) logger.info('JOB Complete') except: logger.error('Key NOT Added to Final Queue') elif RQ_file.QueueSize() == 0 and RQ_work.QueueSize() != 0: logger.warning( 'Work Queue Not Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args): key = user_args.key norun = user_args.norun namespace = user_args.namespace if namespace is None: namespace = default_namespace # Set up logging logger = logging.getLogger(key) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + 'Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) RQ_lock = RedisLock(lock_obj) RQ_lock.add({'Services': '1'}) if not RQ_lock.available('Services'): exit() # Connect to database and access 'jobs' table DBQO = PDS_DBquery('JOBS') if key is None: # If no key is specified, grab the first key try: key = DBQO.jobKey() try: key = key.decode('utf-8') except: pass # If the queue is empty, it'll throw a type error. except TypeError: logger.debug('No keys found in clusterjobs database') exit(1) try: # Set the 'queued' column to current time i.e. prep for processing DBQO.setJobsQueued(key) except KeyError as e: logger.error('%s', e) exit(1) logger.info('Starting Process') xmlOBJ = jobXML(DBQO.jobXML4Key(key)) # Make directory if it doesn't exist directory = scratch + key if not os.path.exists(directory): os.makedirs(directory) logger.info('Working Area: %s', directory) # Set up Redis Hash for ground range RedisH = RedisHash(key + '_info') RedisH.RemoveAll() RedisErrorH = RedisHash(key + '_error') RedisErrorH.RemoveAll() RedisH_DICT = {} RedisH_DICT['service'] = xmlOBJ.getProcess() RedisH_DICT['fileformat'] = xmlOBJ.getOutFormat() RedisH_DICT['outbit'] = xmlOBJ.getOutBit() if xmlOBJ.getRangeType() is not None: RedisH_DICT['grtype'] = xmlOBJ.getRangeType() RedisH_DICT['minlat'] = xmlOBJ.getMinLat() RedisH_DICT['maxlat'] = xmlOBJ.getMaxLat() RedisH_DICT['minlon'] = xmlOBJ.getMinLon() RedisH_DICT['maxlon'] = xmlOBJ.getMaxLon() if RedisH.IsInHash('service'): pass else: RedisH.AddHash(RedisH_DICT) if RedisH.IsInHash('service'): logger.info('Redis info Hash: Success') else: logger.error('Redis info Hash Not Found') # End ground range RQ_recipe = RedisQueue(key + '_recipe', namespace) RQ_recipe.RemoveAll() RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_file.RemoveAll() RQ_WorkQueue = RedisQueue(key + '_WorkQueue', namespace) RQ_WorkQueue.RemoveAll() RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_loggy.RemoveAll() RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_zip.RemoveAll() if xmlOBJ.getProcess() == 'POW': fileList = xmlOBJ.getFileListWB() elif xmlOBJ.getProcess() == 'MAP2': fileList = xmlOBJ.getMFileListWB() for List_file in fileList: # Input and output file naming and path stuff if xmlOBJ.getProcess() == 'POW': if xmlOBJ.getInst() == 'THEMIS_IR': Input_file = List_file.replace('odtie1_', 'odtir1_') Input_file = Input_file.replace('xxedr', 'xxrdr') Input_file = Input_file.replace('EDR.QUB', 'RDR.QUB') Input_file = Input_file.replace( 'http://pdsimage.wr.usgs.gov/Missions/', archive_base) elif xmlOBJ.getInst() == 'ISSNA': Input_file = List_file.replace('.IMG', '.LBL') Input_file = Input_file.replace( 'http://pdsimage.wr.usgs.gov/Missions/', archive_base) elif xmlOBJ.getInst() == 'ISSWA': Input_file = List_file.replace('.IMG', '.LBL') Input_file = Input_file.replace( 'http://pdsimage.wr.usgs.gov/Missions/', archive_base) elif xmlOBJ.getInst() == 'SOLID STATE IMAGING SYSTEM': Input_file = List_file.replace('.img', '.lbl') Input_file = Input_file.replace( 'http://pdsimage.wr.usgs.gov/Missions/', archive_base) else: Input_file = List_file.replace( 'http://pdsimage.wr.usgs.gov/Missions/', archive_base) elif xmlOBJ.getProcess() == 'MAP2': Input_file = List_file.replace('file://pds_san', '/pds_san') if '+' in Input_file: tempsplit = Input_file.split('+') tempFile = tempsplit[0] else: tempFile = Input_file label = pvl.load(tempFile) # Output final file naming Tbasename = os.path.splitext(os.path.basename(tempFile))[0] splitBase = Tbasename.split('_') labP = xmlOBJ.getProjection() if labP == 'INPUT': lab_proj = label['IsisCube']['Mapping']['ProjectionName'][0:4] else: lab_proj = labP[0:4] if xmlOBJ.getClat() is None or xmlOBJ.getClon() is None: basefinal = splitBase[0] + splitBase[1] + \ splitBase[2] + '_MAP2_' + lab_proj.upper() else: lab_clat = float(xmlOBJ.getClat()) if lab_clat >= 0: labH = 'N' elif lab_clat < 0: labH = 'S' lab_clon = float(xmlOBJ.getClon()) basefinal = splitBase[0] + splitBase[1] + splitBase[ 2] + '_MAP2_' + str(lab_clat) + labH + str( lab_clon) + '_' + lab_proj.upper() RedisH.MAPname(basefinal) try: RQ_file.QueueAdd(Input_file) logger.info('File %s Added to Redis Queue', Input_file) except Exception as e: logger.warn('File %s NOT Added to Redis Queue', Input_file) print('Redis Queue Error', e) RedisH.FileCount(RQ_file.QueueSize()) logger.info('Count of Files Queue: %s', str(RQ_file.QueueSize())) # Map Template Stuff logger.info('Making Map File') mapOBJ = MakeMap() if xmlOBJ.getProcess() == 'MAP2' and xmlOBJ.getProjection() == 'INPUT': proj = label['IsisCube']['Mapping']['ProjectionName'] mapOBJ.Projection(proj) else: mapOBJ.Projection(xmlOBJ.getProjection()) if xmlOBJ.getClon() is not None: mapOBJ.CLon(float(xmlOBJ.getClon())) if xmlOBJ.getClat() is not None: mapOBJ.CLat(float(xmlOBJ.getClat())) if xmlOBJ.getFirstParallel() is not None: mapOBJ.FirstParallel(float(xmlOBJ.getFirstParallel())) if xmlOBJ.getSecondParallel() is not None: mapOBJ.SecondParallel(float(xmlOBJ.getSecondParallel())) if xmlOBJ.getResolution() is not None: mapOBJ.PixelRes(float(xmlOBJ.getResolution())) if xmlOBJ.getTargetName() is not None: mapOBJ.Target(xmlOBJ.getTargetName()) if xmlOBJ.getERadius() is not None: mapOBJ.ERadius(float(xmlOBJ.getERadius())) if xmlOBJ.getPRadius() is not None: mapOBJ.PRadius(float(xmlOBJ.getPRadius())) if xmlOBJ.getLatType() is not None: mapOBJ.LatType(xmlOBJ.getLatType()) if xmlOBJ.getLonDirection() is not None: mapOBJ.LonDirection(xmlOBJ.getLonDirection()) if xmlOBJ.getLonDomain() is not None: mapOBJ.LonDomain(int(xmlOBJ.getLonDomain())) if xmlOBJ.getProcess() == 'MAP2': if xmlOBJ.getMinLat() is not None: mapOBJ.MinLat(float(xmlOBJ.getMinLat())) if xmlOBJ.getMaxLat() is not None: mapOBJ.MaxLat(float(xmlOBJ.getMaxLat())) if xmlOBJ.getMinLon() is not None: mapOBJ.MinLon(float(xmlOBJ.getMinLon())) if xmlOBJ.getMaxLon() is not None: mapOBJ.MaxLon(float(xmlOBJ.getMaxLon())) mapOBJ.Map2pvl() MAPfile = directory + "/" + key + '.map' mapOBJ.Map2File(MAPfile) try: f = open(MAPfile) f.close logger.info('Map File Creation: Success') except IOError as e: logger.error('Map File %s Not Found', MAPfile) # ** End Map Template Stuff ** logger.info('Building Recipe') recipeOBJ = Recipe() if xmlOBJ.getProcess() == 'POW': recipeOBJ.AddJsonFile(recipe_base + xmlOBJ.getCleanName() + '.json', "pow") elif xmlOBJ.getProcess() == 'MAP2': recipeOBJ.AddJsonFile(recipe_base + "map2_process.json", "map") # Test for stretch and add to recipe # if MAP2 and 8 or 16 bit run stretch to set range if xmlOBJ.getOutBit() == 'input': testBitType = str(label['IsisCube']['Core']['Pixels']['Type']).upper() else: testBitType = xmlOBJ.getOutBit().upper() if xmlOBJ.getProcess() == 'MAP2' and xmlOBJ.STR_Type() is None: if str(label['IsisCube']['Core']['Pixels']['Type']).upper( ) != xmlOBJ.getOutBit().upper() and str( label['IsisCube']['Core']['Pixels']['Type']).upper() != 'REAL': if str(label['IsisCube']['Core']['Pixels'] ['Type']).upper() == 'SIGNEDWORD': strpairs = '0:-32765 0:-32765 100:32765 100:32765' elif str(label['IsisCube']['Core']['Pixels'] ['Type']).upper() == 'UNSIGNEDBYTE': strpairs = '0:1 0:1 100:254 100:254' STRprocessOBJ = Process() STRprocessOBJ.newProcess('stretch') STRprocessOBJ.AddParameter('from_', 'value') STRprocessOBJ.AddParameter('to', 'value') STRprocessOBJ.AddParameter('usepercentages', 'yes') STRprocessOBJ.AddParameter('pairs', strpairs) recipeOBJ.AddProcess(STRprocessOBJ.getProcess()) strType = xmlOBJ.STR_Type() if strType == 'StretchPercent' and xmlOBJ.STR_PercentMin( ) is not None and xmlOBJ.STR_PercentMax( ) is not None and testBitType != 'REAL': if float(xmlOBJ.STR_PercentMin()) != 0 and float( xmlOBJ.STR_PercentMax()) != 100: if testBitType == 'UNSIGNEDBYTE': strpairs = '0:1 ' + xmlOBJ.STR_PercentMin() + ':1 ' + \ xmlOBJ.STR_PercentMax() + ':254 100:254' elif testBitType == 'SIGNEDWORD': strpairs = '0:-32765 ' + xmlOBJ.STR_PercentMin() + ':-32765 ' + \ xmlOBJ.STR_PercentMax() + ':32765 100:32765' STRprocessOBJ = Process() STRprocessOBJ.newProcess('stretch') STRprocessOBJ.AddParameter('from_', 'value') STRprocessOBJ.AddParameter('to', 'value') STRprocessOBJ.AddParameter('usepercentages', 'yes') STRprocessOBJ.AddParameter('pairs', strpairs) recipeOBJ.AddProcess(STRprocessOBJ.getProcess()) elif strType == 'GaussStretch': STRprocessOBJ = Process() STRprocessOBJ.newProcess('gaussstretch') STRprocessOBJ.AddParameter('from_', 'value') STRprocessOBJ.AddParameter('to', 'value') STRprocessOBJ.AddParameter('gsigma', xmlOBJ.STR_GaussSigma()) recipeOBJ.AddProcess(STRprocessOBJ.getProcess()) elif strType == 'HistogramEqualization': STRprocessOBJ = Process() STRprocessOBJ.newProcess('histeq') STRprocessOBJ.AddParameter('from_', 'value') STRprocessOBJ.AddParameter('to', 'value') if xmlOBJ.STR_PercentMin() is None: STRprocessOBJ.AddParameter('minper', '0') else: STRprocessOBJ.AddParameter('minper', xmlOBJ.STR_PercentMin()) if xmlOBJ.STR_PercentMax() is None: STRprocessOBJ.AddParameter('maxper', '100') else: STRprocessOBJ.AddParameter('maxper', xmlOBJ.STR_PercentMax()) recipeOBJ.AddProcess(STRprocessOBJ.getProcess()) elif strType == 'SigmaStretch': STRprocessOBJ = Process() STRprocessOBJ.newProcess('sigmastretch') STRprocessOBJ.AddParameter('from_', 'value') STRprocessOBJ.AddParameter('to', 'value') STRprocessOBJ.AddParameter('variance', xmlOBJ.STR_SigmaVariance()) recipeOBJ.AddProcess(STRprocessOBJ.getProcess()) # Test for output bit type and add to recipe if xmlOBJ.getProcess() == 'POW': if xmlOBJ.getOutBit().upper() == 'UNSIGNEDBYTE' or xmlOBJ.getOutBit( ).upper() == 'SIGNEDWORD': CAprocessOBJ = Process() CAprocessOBJ.newProcess('cubeatt-bit') CAprocessOBJ.AddParameter('from_', 'value') CAprocessOBJ.AddParameter('to', 'value') recipeOBJ.AddProcess(CAprocessOBJ.getProcess()) elif xmlOBJ.getProcess() == 'MAP2': if xmlOBJ.getOutBit().upper() != 'INPUT': if xmlOBJ.getOutBit().upper( ) == 'UNSIGNEDBYTE' or xmlOBJ.getOutBit().upper() == 'SIGNEDWORD': if str(label['IsisCube']['Core']['Pixels'] ['Type']).upper() != xmlOBJ.getOutBit().upper(): CAprocessOBJ = Process() CAprocessOBJ.newProcess('cubeatt-bit') CAprocessOBJ.AddParameter('from_', 'value') CAprocessOBJ.AddParameter('to', 'value') recipeOBJ.AddProcess(CAprocessOBJ.getProcess()) # Add Grid(MAP2) if xmlOBJ.getGridInterval() is not None: GprocessOBJ = Process() GprocessOBJ.newProcess('grid') GprocessOBJ.AddParameter('from_', 'value') GprocessOBJ.AddParameter('to', 'value') GprocessOBJ.AddParameter('latinc', xmlOBJ.getGridInterval()) GprocessOBJ.AddParameter('loninc', xmlOBJ.getGridInterval()) GprocessOBJ.AddParameter('outline', 'yes') GprocessOBJ.AddParameter('boundary', 'yes') GprocessOBJ.AddParameter('linewidth', '3') recipeOBJ.AddProcess(GprocessOBJ.getProcess()) # OUTPUT FORMAT # Test for GDAL and add to recipe Oformat = xmlOBJ.getOutFormat() if Oformat == 'GeoTiff-BigTiff' or Oformat == 'GeoJPEG-2000' or Oformat == 'JPEG' or Oformat == 'PNG': if Oformat == 'GeoJPEG-2000': Oformat = 'JP2KAK' if Oformat == 'GeoTiff-BigTiff': Oformat = 'GTiff' GDALprocessOBJ = Process() GDALprocessOBJ.newProcess('gdal_translate') if xmlOBJ.getOutBit() != 'input': GDALprocessOBJ.AddParameter( '-ot', GDALprocessOBJ.GDAL_OBit(xmlOBJ.getOutBit())) GDALprocessOBJ.AddParameter('-of', Oformat) if Oformat == 'GTiff' or Oformat == 'JP2KAK' or Oformat == 'JPEG': GDALprocessOBJ.AddParameter('-co', GDALprocessOBJ.GDAL_Creation(Oformat)) recipeOBJ.AddProcess(GDALprocessOBJ.getProcess()) # set up pds2isis and add to recipe elif Oformat == 'PDS': pdsProcessOBJ = Process() pdsProcessOBJ.newProcess('isis2pds') pdsProcessOBJ.AddParameter('from_', 'value') pdsProcessOBJ.AddParameter('to', 'value') if xmlOBJ.getOutBit() == 'unsignedbyte': pdsProcessOBJ.AddParameter('bittype', '8bit') elif xmlOBJ.getOutBit() == 'signedword': pdsProcessOBJ.AddParameter('bittype', 's16bit') recipeOBJ.AddProcess(pdsProcessOBJ.getProcess()) for item in recipeOBJ.getProcesses(): processOBJ = Process() processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe()) if item == 'cam2map': processOBJ.updateParameter('map', MAPfile) if xmlOBJ.getResolution() is None: processOBJ.updateParameter('pixres', 'CAMERA') else: processOBJ.updateParameter('pixres', 'MAP') if xmlOBJ.getRangeType() is None: processOBJ.updateParameter('defaultrange', 'MINIMIZE') elif xmlOBJ.getRangeType() == 'smart' or xmlOBJ.getRangeType( ) == 'fill': processOBJ.updateParameter('defaultrange', 'CAMERA') processOBJ.AddParameter('trim', 'YES') elif item == 'map2map': processOBJ.updateParameter('map', MAPfile) if xmlOBJ.getResolution() is None: processOBJ.updateParameter('pixres', 'FROM') else: processOBJ.updateParameter('pixres', 'MAP') if xmlOBJ.OutputGeometry() is not None: processOBJ.updateParameter('defaultrange', 'MAP') processOBJ.AddParameter('trim', 'YES') else: processOBJ.updateParameter('defaultrange', 'FROM') processJSON = processOBJ.Process2JSON() try: RQ_recipe.QueueAdd(processJSON) logger.info('Recipe Element Added to Redis: %s : Success', item) except Exception as e: logger.warn('Recipe Element NOT Added to Redis: %s', item) # HPC job stuff logger.info('HPC Cluster job Submission Starting') jobOBJ = HPCjob() jobOBJ.setJobName(key + '_Service') jobOBJ.setStdOut(slurm_log + key + '_%A_%a.out') jobOBJ.setStdError(slurm_log + key + '_%A_%a.err') jobOBJ.setWallClock('24:00:00') jobOBJ.setMemory('24576') jobOBJ.setPartition('pds') JAsize = RQ_file.QueueSize() jobOBJ.setJobArray(JAsize) logger.info('Job Array Size : %s', str(JAsize)) # @TODO replace with source activate <env> #jobOBJ.addPath('/usgs/apps/anaconda/bin') # Whether or not we use the default namespace, this guarantees that the POW/MAP queues will match the namespace # used in the job manager. if xmlOBJ.getProcess() == 'POW': cmd = cmd_dir + "pow_process.py -k {} -n {}".format(key, namespace) elif xmlOBJ.getProcess() == 'MAP2': cmd = cmd_dir + "map_process.py -k {} -n {}".format(key, namespace) logger.info('HPC Command: %s', cmd) jobOBJ.setCommand(cmd) SBfile = directory + '/' + key + '.sbatch' jobOBJ.MakeJobFile(SBfile) try: sb = open(SBfile) sb.close logger.info('SBATCH File Creation: Success') except IOError as e: logger.error('SBATCH File %s Not Found', SBfile) if norun: logger.info('No-run mode, will not submit HPC job.') else: try: jobOBJ.Run() logger.info('Job Submission to HPC: Success') DBQO.setJobsStarted(key) except IOError as e: logger.error('Jobs NOT Submitted to HPC')
def generate_isis_processes(inputfile, archive, logger): logger.info('Starting Process: %s', inputfile) recipeOBJ = Recipe() recipeOBJ.addMissionJson(archive, 'upc') infile = os.path.splitext(inputfile)[0] + '.UPCinput.cub' logger.debug("Beginning processing on %s\n", inputfile) outfile = os.path.splitext(inputfile)[0] + '.UPCoutput.cub' caminfoOUT = os.path.splitext(inputfile)[0] + '_caminfo.pvl' processes = [] # Iterate through each process listed in the recipe for item in recipeOBJ.getProcesses(): # If any of the processes failed, discontinue processing processOBJ = Process() processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe()) # Handle processing based on string description. if '2isis' in item: processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) elif item == 'thmproc': processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) thmproc_odd = str(workarea) + str(os.path.splitext( os.path.basename(inputfile))[0]) + '.UPCoutput.raw.odd.cub' thmproc_even = str(workarea) + str( os.path.splitext(os.path.basename( inputfile))[0]) + '.UPCoutput.raw.even.cub' elif item == 'handmos': processOBJ.updateParameter('from_', thmproc_even) processOBJ.updateParameter('mosaic', thmproc_odd) elif item == 'spiceinit': processOBJ.updateParameter('from_', infile) elif item == 'cubeatt': band_infile = infile + '+' + str(1) processOBJ.updateParameter('from_', band_infile) processOBJ.updateParameter('to', outfile) elif item == 'footprintinit': processOBJ.updateParameter('from_', infile) elif item == 'caminfo': processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', caminfoOUT) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) processes.append(processOBJ) pwd = os.getcwd() return processes, infile, caminfoOUT, pwd