def main(): # Connect to database - ignore engine information pds_session, pds_engine = db_connect(pds_db) # Connect to database - ignore engine information session, upc_engine = db_connect(upc_db) # ***************** Set up logging ***************** logger = logging.getLogger('UPC_Process') logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + 'Process.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) PDSinfoDICT = json.load(open(pds_info, 'r')) # Redis Queue Objects RQ_main = RedisQueue('UPC_ReadyQueue') logger.info("UPC Processing Queue: %s", RQ_main.id_name) RQ_lock = RedisLock(lock_obj) # If the queue isn't registered, add it and set it to "running" RQ_lock.add({RQ_main.id_name: '1'}) proc_date_tid = get_tid('processdate', session) err_type_tid = get_tid('errortype', session) err_msg_tid = get_tid('errormessage', session) err_flag_tid = get_tid('error', session) isis_footprint_tid = get_tid('isisfootprint', session) isis_centroid_tid = get_tid('isiscentroid', session) start_time_tid = get_tid('starttime', session) stop_time_tid = get_tid('stoptime', session) checksum_tid = get_tid('checksum', session) # while there are items in the redis queue while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): # get a file from the queue item = literal_eval(RQ_main.QueueGet().decode("utf-8")) inputfile = item[0] fid = item[1] archive = item[2] #inputfile = (RQ_main.QueueGet()).decode('utf-8') if os.path.isfile(inputfile): pass else: print("{} is not a file\n".format(inputfile)) if os.path.isfile(inputfile): logger.info('Starting Process: %s', inputfile) # @TODO refactor this logic. We're using an object to find a path, returning it, # then passing it back to the object so that the object can use it. recipeOBJ = Recipe() recipe_json = recipeOBJ.getRecipeJSON(archive) #recipe_json = recipeOBJ.getRecipeJSON(getMission(str(inputfile))) recipeOBJ.AddJsonFile(recipe_json, 'upc') infile = workarea + os.path.splitext( str(os.path.basename(inputfile)))[0] + '.UPCinput.cub' outfile = workarea + os.path.splitext( str(os.path.basename(inputfile)))[0] + '.UPCoutput.cub' caminfoOUT = workarea + os.path.splitext( str(os.path.basename(inputfile)))[0] + '_caminfo.pvl' EDRsource = inputfile.replace( '/pds_san/PDS_Archive/', 'https://pdsimage.wr.ugs.gov/Missions/') status = 'success' # Iterate through each process listed in the recipe for item in recipeOBJ.getProcesses(): # If any of the processes failed, discontinue processing if status.lower() == 'error': break elif status.lower() == 'success': processOBJ = Process() processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe()) # Handle processing based on string description. if '2isis' in item: processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) elif item == 'thmproc': processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) thmproc_odd = str(workarea) + str( os.path.splitext(os.path.basename(inputfile)) [0]) + '.UPCoutput.raw.odd.cub' thmproc_even = str(workarea) + str( os.path.splitext(os.path.basename(inputfile)) [0]) + '.UPCoutput.raw.even.cub' elif item == 'handmos': processOBJ.updateParameter('from_', thmproc_even) processOBJ.updateParameter('mosaic', thmproc_odd) elif item == 'spiceinit': processOBJ.updateParameter('from_', infile) elif item == 'cubeatt': band_infile = infile + '+' + str(1) processOBJ.updateParameter('from_', band_infile) processOBJ.updateParameter('to', outfile) elif item == 'footprintinit': processOBJ.updateParameter('from_', infile) elif item == 'caminfo': processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', caminfoOUT) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) pwd = os.getcwd() # iterate through functions listed in process obj for k, v in processOBJ.getProcess().items(): # load a function into func func = getattr(isis, k) try: os.chdir(workarea) # execute function func(**v) os.chdir(pwd) if item == 'handmos': if os.path.isfile(thmproc_odd): os.rename(thmproc_odd, infile) else: if os.path.isfile(outfile): os.rename(outfile, infile) status = 'success' if '2isis' in item: label = pvl.load(infile) infile_bandlist = label['IsisCube']['BandBin'][ PDSinfoDICT[archive]['bandbinQuery']] infile_centerlist = label['IsisCube'][ 'BandBin']['Center'] elif item == 'thmproc': pass elif item == 'handmos': label = pvl.load(infile) infile_bandlist = label['IsisCube']['BandBin'][ PDSinfoDICT[archive]['bandbinQuery']] infile_centerlist = label['IsisCube'][ 'BandBin']['Center'] except ProcessError as e: print(e) status = 'error' processError = item # keyword definitions keywordsOBJ = None if status.lower() == 'success': try: keywordsOBJ = UPCkeywords(caminfoOUT) except: with open(caminfoOUT, 'r') as f: filedata = f.read() filedata = filedata.replace(';', '-').replace('&', '-') filedata = re.sub(r'\-\s+', r'', filedata, flags=re.M) with open(caminfoOUT, 'w') as f: f.write(filedata) keywordsOBJ = UPCkeywords(caminfoOUT) target_Qobj = session.query(upc_models.Targets).filter( upc_models.Targets.targetname == keywordsOBJ.getKeyword( 'TargetName').upper()).first() instrument_Qobj = session.query(upc_models.Instruments).filter( upc_models.Instruments.instrument == keywordsOBJ.getKeyword('InstrumentId')).first() if session.query(upc_models.DataFiles).filter( upc_models.DataFiles.isisid == keywordsOBJ.getKeyword( 'IsisId')).first() is None: test_input = upc_models.DataFiles( isisid=keywordsOBJ.getKeyword('IsisId'), productid=keywordsOBJ.getKeyword('ProductId'), edr_source=EDRsource, edr_detached_label='', instrumentid=instrument_Qobj.instrumentid, targetid=target_Qobj.targetid) session.merge(test_input) session.commit() Qobj = session.query(upc_models.DataFiles).filter( upc_models.DataFiles.isisid == keywordsOBJ.getKeyword( 'IsisId')).first() UPCid = Qobj.upcid print(UPCid) # block to add band information to meta_bands if isinstance(infile_bandlist, list): index = 0 while index < len(infile_bandlist): B_DBinput = upc_models.MetaBands( upcid=UPCid, filter=str(infile_bandlist[index]), centerwave=infile_centerlist[index]) session.merge(B_DBinput) index = index + 1 else: try: # If infile_centerlist is in "Units" format, grab the value f_centerlist = float(infile_centerlist[0]) except TypeError: f_centerlist = float(infile_centerlist) B_DBinput = upc_models.MetaBands(upcid=UPCid, filter=infile_bandlist, centerwave=f_centerlist) session.merge(B_DBinput) session.commit() # Block to add common keywords testjson = json.load(open(keyword_def, 'r')) for element_1 in testjson['instrument']['COMMON']: keyvalue = "" keytype = testjson['instrument']['COMMON'][element_1][ 'type'] keyword = testjson['instrument']['COMMON'][element_1][ 'keyword'] keyword_Qobj = session.query(upc_models.Keywords).filter( and_(upc_models.Keywords.typename == element_1, upc_models.Keywords.instrumentid == 1)).first() if keyword_Qobj is None: continue else: keyvalue = keywordsOBJ.getKeyword(keyword) if keyvalue is None: continue keyvalue = db2py(keytype, keyvalue) try: DBinput = upc_models.create_table( keytype, upcid=UPCid, typeid=keyword_Qobj.typeid, value=keyvalue) except Exception as e: logger.warn("Unable to enter %s into table\n\n%s", keytype, e) continue session.merge(DBinput) try: session.flush() except: logger.warn("Unable to flush database connection") session.commit() for element_1 in testjson['instrument'][archive]: keyvalue = "" keytype = testjson['instrument'][archive][element_1][ 'type'] keyword = testjson['instrument'][archive][element_1][ 'keyword'] keyword_Qobj = session.query(upc_models.Keywords).filter( and_( upc_models.Keywords.typename == element_1, upc_models.Keywords.instrumentid.in_( (1, instrument_Qobj.instrumentid)))).first() if keyword_Qobj is None: continue else: keyvalue = keywordsOBJ.getKeyword(keyword) if keyvalue is None: logger.debug("Keyword %s not found", keyword) continue keyvalue = db2py(keytype, keyvalue) try: DBinput = upc_models.create_table( keytype, upcid=UPCid, typeid=keyword_Qobj.typeid, value=keyvalue) except Exception as e: logger.warn("Unable to enter %s into database\n\n%s", keytype, e) continue session.merge(DBinput) try: session.flush() except: logger.warn("Unable to flush database connection") session.commit() # geometry stuff G_centroid = 'point ({} {})'.format( str(keywordsOBJ.getKeyword('CentroidLongitude')), str(keywordsOBJ.getKeyword('CentroidLatitude'))) G_keyword_Qobj = session.query( upc_models.Keywords.typeid).filter( upc_models.Keywords.typename == 'isiscentroid').first() G_footprint_Qobj = session.query( upc_models.Keywords.typeid).filter( upc_models.Keywords.typename == 'isisfootprint').first() G_footprint = keywordsOBJ.getKeyword('GisFootprint') G_DBinput = upc_models.MetaGeometry(upcid=UPCid, typeid=G_keyword_Qobj, value=G_centroid) session.merge(G_DBinput) G_DBinput = upc_models.MetaGeometry(upcid=UPCid, typeid=G_footprint_Qobj, value=G_footprint) session.merge(G_DBinput) session.flush() session.commit() f_hash = hashlib.md5() with open(inputfile, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): f_hash.update(chunk) checksum = f_hash.hexdigest() DBinput = upc_models.MetaString(upcid=UPCid, typeid=checksum_tid, value=checksum) session.merge(DBinput) DBinput = upc_models.MetaBoolean(upcid=UPCid, typeid=err_flag_tid, value=False) session.merge(DBinput) session.commit() AddProcessDB(pds_session, fid, True) os.remove(infile) os.remove(caminfoOUT) elif status.lower() == 'error': try: label = pvl.load(infile) except Exception as e: logger.info('%s', e) continue date = datetime.datetime.now( pytz.utc).strftime("%Y-%m-%d %H:%M:%S") if '2isis' in processError or processError == 'thmproc': if session.query(upc_models.DataFiles).filter( upc_models.DataFiles.edr_source == EDRsource.decode("utf-8")).first() is None: target_Qobj = session.query(upc_models.Targets).filter( upc_models.Targets.targetname == str( label['IsisCube']['Instrument'] ['TargetName']).upper()).first() instrument_Qobj = session.query( upc_models.Instruments).filter( upc_models.Instruments.instrument == str( label['IsisCube']['Instrument'] ['InstrumentId'])).first() error1_input = upc_models.DataFiles( isisid='1', edr_source=EDRsource) session.merge(error1_input) session.commit() EQ1obj = session.query(upc_models.DataFiles).filter( upc_models.DataFiles.edr_source == EDRsource).first() UPCid = EQ1obj.upcid errorMSG = 'Error running {} on file {}'.format( processError, inputfile) DBinput = MetaTime(upcid=UPCid, typeid=proc_date_tid, value=date) session.merge(DBinput) DBinput = MetaString(upcid=UPCid, typeid=err_type_tid, value=processError) session.merge(DBinput) DBinput = MetaString(upcid=UPCid, typeid=err_msg_tid, value=errorMSG) session.merge(DBinput) DBinput = MetaBoolean(upcid=UPCid, typeid=err_flag_tid, value=True) session.merge(DBinput) DBinput = MetaGeometry(upcid=UPCid, typeid=isis_footprint_tid, value='POINT(361 0)') session.merge(DBinput) DBinput = MetaGeometry(upcid=UPCid, typeid=isis_centroid_tid, value='POINT(361 0)') session.merge(DBinput) session.commit() else: try: label = pvl.load(infile) except Exception as e: logger.warn('%s', e) continue isisSerial = getISISid(infile) if session.query(upc_models.DataFiles).filter( upc_models.DataFiles.isisid == isisSerial).first() is None: target_Qobj = session.query(upc_models.Targets).filter( upc_models.Targets.targetname == str( label['IsisCube']['Instrument'] ['TargetName']).upper()).first() instrument_Qobj = session.query( upc_models.Instruments).filter( upc_models.Instruments.instrument == str( label['IsisCube']['Instrument'] ['InstrumentId'])).first() if target_Qobj is None or instrument_Qobj is None: continue error2_input = upc_models.DataFiles( isisid=isisSerial, productid=label['IsisCube']['Archive'] ['ProductId'], edr_source=EDRsource, instrumentid=instrument_Qobj.instrumentid, targetid=target_Qobj.targetid) session.merge(error2_input) session.commit() try: EQ2obj = session.query(upc_models.DataFiles).filter( upc_models.DataFiles.isisid == isisSerial).first() UPCid = EQ2obj.upcid errorMSG = 'Error running {} on file {}'.format( processError, inputfile) DBinput = MetaTime(upcid=UPCid, typeid=proc_date_tid, value=date) session.merge(DBinput) DBinput = MetaString(upcid=UPCid, typeid=err_type_tid, value=processError) session.merge(DBinput) DBinput = MetaString(upcid=UPCid, typeid=err_msg_tid, value=errorMSG) session.merge(DBinput) DBinput = MetaBoolean(upcid=UPCid, typeid=err_flag_tid, value=True) session.merge(DBinput) DBinput = MetaGeometry(upcid=UPCid, typeid=isis_footprint_tid, value='POINT(361 0)') session.merge(DBinput) DBinput = MetaGeometry(upcid=UPCid, typeid=isis_centroid_tid, value='POINT(361 0)') session.merge(DBinput) except: pass try: v = label['IsisCube']['Instrument']['StartTime'] except KeyError: v = None except: continue try: DBinput = MetaTime(upcid=UPCid, typeid=start_time_tid, value=v) session.merge(DBinput) except: continue try: v = label['IsisCube']['Instrument']['StopTime'] except KeyError: v = None DBinput = MetaTime(upcid=UPCid, typeid=stop_time_tid, value=v) session.merge(DBinput) session.commit() AddProcessDB(pds_session, fid, False) os.remove(infile) # Disconnect from db sessions pds_session.close() session.close() # Disconnect from the engines pds_engine.dispose() upc_engine.dispose() logger.info("UPC processing exited successfully")
def main(): args = Args() args.parse_args() override = args.override logger = logging.getLogger('Ingest_Process') level = logging.getLevelName(args.log_level) logger.setLevel(level) logFileHandle = logging.FileHandler(pds_log + 'Ingest.log') print("Log File: {}Ingest.log".format(pds_log)) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info("Starting Ingest Process") PDSinfoDICT = json.load(open(pds_info, 'r')) RQ_main = RedisQueue('Ingest_ReadyQueue') RQ_lock = RedisLock(lock_obj) RQ_lock.add({RQ_main.id_name: '1'}) RQ_work = RedisQueue('Ingest_WorkQueue') RQ_upc = RedisQueue('UPC_ReadyQueue') RQ_thumb = RedisQueue('Thumbnail_ReadyQueue') RQ_browse = RedisQueue('Browse_ReadyQueue') logger.info("UPC Queue: %s", RQ_upc.id_name) logger.info("Thumbnail Queue: %s", RQ_thumb.id_name) logger.info("Browse Queue: %s", RQ_browse.id_name) try: session, engine = db_connect(pds_db) logger.info('DataBase Connecton: Success') except: logger.error('DataBase Connection: Error') return 1 index = 1 while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): item = literal_eval(RQ_main.QueueGet().decode("utf-8")) inputfile = item[0] archive = item[1] RQ_work.QueueAdd(inputfile) subfile = inputfile.replace(PDSinfoDICT[archive]['path'], '') # Calculate checksum in chunks of 4096 f_hash = hashlib.md5() with open(inputfile, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): f_hash.update(chunk) filechecksum = f_hash.hexdigest() QOBJ = session.query(Files).filter_by(filename=subfile).first() runflag = False if QOBJ is None or filechecksum != QOBJ.checksum: runflag = True if runflag or override: date = datetime.datetime.now( pytz.utc).strftime("%Y-%m-%d %H:%M:%S") fileURL = inputfile.replace(archive_base, web_base) # If all upc requirements are in 'inputfile,' flag for upc upcflag = all(x in inputfile for x in PDSinfoDICT[archive]['upc_reqs']) filesize = os.path.getsize(inputfile) try: # If we found an existing file and want to overwrite the data if QOBJ is not None and override: ingest_entry = QOBJ # If the file was not found, create a new entry else: ingest_entry = Files() ingest_entry.archiveid = PDSinfoDICT[archive]['archiveid'] ingest_entry.filename = subfile ingest_entry.entry_date = date ingest_entry.checksum = filechecksum ingest_entry.upc_required = upcflag ingest_entry.validation_required = True ingest_entry.header_only = False ingest_entry.release_date = date ingest_entry.file_url = fileURL ingest_entry.file_size = filesize ingest_entry.di_pass = True ingest_entry.di_date = date session.merge(ingest_entry) session.flush() if upcflag: RQ_upc.QueueAdd((inputfile, ingest_entry.fileid, archive)) RQ_thumb.QueueAdd( (inputfile, ingest_entry.fileid, archive)) RQ_browse.QueueAdd( (inputfile, ingest_entry.fileid, archive)) #RQ_pilotB.QueueAdd((inputfile,ingest_entry.fileid, archive)) RQ_work.QueueRemove(inputfile) index = index + 1 except Exception as e: logger.error("Error During File Insert %s : %s", str(subfile), str(e)) elif not runflag and not override: RQ_work.QueueRemove(inputfile) logger.warn( "Not running ingest: file %s already present" " in database and no override flag supplied", inputfile) if index >= 250: try: session.commit() logger.info("Commit 250 files to Database: Success") index = 1 except Exception as e: session.rollback() logger.warn("Unable to commit to database: %s", str(e)) else: logger.info("No Files Found in Ingest Queue") try: session.commit() logger.info("Commit to Database: Success") except Exception as e: logger.error("Unable to commit to database: %s", str(e)) session.rollback() # Close connection to database session.close() engine.dispose() if RQ_main.QueueSize() == 0 and RQ_work.QueueSize() == 0: logger.info("Process Complete All Queues Empty") elif RQ_main.QueueSize() == 0 and RQ_work.QueueSize() != 0: logger.warning("Process Done Work Queue NOT Empty Contains %s Files", str(RQ_work.QueueSize())) logger.info("Ingest Complete")
def main(): args = Args() args.parse_args() key = args.key namespace = args.namespace if namespace is None: namespace is default_namespace workarea = scratch + args.key + '/' RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_work = RedisQueue(key + '_WorkQueue', namespace) RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_final = RedisQueue('FinalQueue', namespace) RHash = RedisHash(key + '_info') RHerror = RedisHash(key + '_error') RQ_lock = RedisLock(lock_obj) RQ_lock.add({'MAP':'1'}) if int(RQ_file.QueueSize()) == 0 and RQ_lock.available('MAP'): print("No Files Found in Redis Queue") else: jobFile = RQ_file.Qfile2Qwork( RQ_file.getQueueName(), RQ_work.getQueueName()).decode('utf-8') # Setup system logging basename = os.path.splitext(os.path.basename(jobFile))[0] logger = logging.getLogger(key + '.' + basename) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + '/Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting MAP Processing') loggyOBJ = Loggy(basename) # File Naming infile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub' outfile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub' # Recipe Stuff RQ_recipe = RedisQueue(key + '_recipe') status = 'success' for element in RQ_recipe.RecipeGet(): if status == 'error': break elif status == 'success': processOBJ = Process() process = processOBJ.JSON2Process(element) if 'gdal_translate' not in processOBJ.getProcessName(): if 'cubeatt-band' in processOBJ.getProcessName(): if '+' in jobFile: processOBJ.updateParameter('from_', jobFile) processOBJ.updateParameter('to', outfile) processOBJ.ChangeProcess('cubeatt') else: continue elif 'map2map' in processOBJ.getProcessName(): if '+' in jobFile: processOBJ.updateParameter('from_', infile) else: processOBJ.updateParameter('from_', jobFile) processOBJ.updateParameter('to', outfile) elif 'cubeatt-bit' in processOBJ.getProcessName(): if RHash.OutBit().decode('utf-8') == 'unsignedbyte': temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254' elif RHash.OutBit().decode('utf-8') == 'signedword': temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', temp_outfile) processOBJ.ChangeProcess('cubeatt') elif 'isis2pds' in processOBJ.getProcessName(): # finalfile = infile.replace('.input.cub', '_final.img') finalfile = workarea + RHash.getMAPname().decode('utf-8') + '.img' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', finalfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) print(processOBJ.getProcess()) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) subloggyOBJ = SubLoggy(k) try: func(**v) logger.info('Process %s :: Success', k) subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if os.path.isfile(outfile): os.rename(outfile, infile) status = 'success' except ProcessError as e: logger.error('Process %s :: Error', k) logger.error(e) status = 'error' eSTR = 'Error Executing ' + k + \ ' Standard Error: ' + str(e) RHerror.addError(os.path.splitext( os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) else: GDALcmd = "" for process, v, in processOBJ.getProcess().items(): subloggyOBJ = SubLoggy(process) GDALcmd += process for key, value in v.items(): GDALcmd += ' ' + key + ' ' + value img_format = RHash.Format().decode('utf-8') if img_format == 'GeoTiff-BigTiff': fileext = 'tif' elif img_format == 'GeoJPEG-2000': fileext = 'jp2' elif img_format == 'JPEG': fileext = 'jpg' elif img_format == 'PNG': fileext = 'png' elif img_format == 'GIF': fileext = 'gif' logGDALcmd = GDALcmd + ' ' + basename + '.input.cub ' + RHash.getMAPname().decode('utf-8') + '.' + fileext finalfile = workarea + RHash.getMAPname().decode('utf-8') + '.' + fileext GDALcmd += ' ' + infile + ' ' + finalfile print(GDALcmd) try: subprocess.call(GDALcmd, shell=True) logger.info('Process GDAL translate :: Success') status = 'success' subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'www.gdal.org/gdal_translate.html') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) os.remove(infile) except OSError as e: logger.error('Process GDAL translate :: Error') logger.error(e) status = 'error' RHerror.addError(os.path.splitext(os.path.basename(jobFile))[0], 'Process GDAL translate :: Error') subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') subloggyOBJ.errorOut(e) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if status == 'success': if RHash.Format().decode('utf-8') == 'ISIS3': finalfile = workarea + RHash.getMAPname().decode('utf-8') + '.cub' shutil.move(infile, finalfile) if RHash.getStatus() != b'ERROR': RHash.Status('SUCCESS') try: RQ_zip.QueueAdd(finalfile) logger.info('File Added to ZIP Queue') except: logger.error('File NOT Added to ZIP Queue') try: RQ_loggy.QueueAdd(loggyOBJ.Loggy2json()) logger.info('JSON Added to Loggy Queue') except: logger.error('JSON NOT Added to Loggy Queue') RQ_work.QueueRemove(jobFile) elif status == 'error': RHash.Status('ERROR') if os.path.isfile(infile): os.remove(infile) if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0: try: RQ_final.QueueAdd(key) logger.info('Key %s Added to Final Queue: Success', key) logger.info('Job Complete') except: logger.error('Key NOT Added to Final Queue') else: logger.warning('Queues Not Empty: filequeue = %s work queue = %s', str( RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(): # pdb.set_trace() PDSinfoDICT = json.load(open(pds_info, 'r')) # ********* Set up logging ************* logger = logging.getLogger('DI_Process') logger.setLevel(logging.INFO) #logFileHandle = logging.FileHandler('/usgs/cdev/PDS/logs/DI.log') logFileHandle = logging.FileHandler(pds_log + 'DI.log') #logFileHandle = logging.FileHandler('/home/arsanders/PDS-Pipelines/logs/DI.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting DI Process') try: session, engine = db_connect(pds_db) logger.info('DataBase Connecton: Success') except: logger.error('DataBase Connection: Error') RQ = RedisQueue('DI_ReadyQueue') RQ_lock = RedisLock(lock_obj) RQ_lock.add({RQ.id_name: '1'}) index = 0 while int(RQ.QueueSize()) > 0 and RQ_lock.available(RQ.id_name): item = literal_eval(RQ.QueueGet().decode("utf-8")) inputfile = item[0] archive = item[1] try: Qelement = session.query(Files).filter( Files.filename == inputfile).one() except: logger.error('Query for File: %s', inputfile) archive_path = PDSinfoDICT[archive]['path'] cpfile = archive_path + Qelement.filename if os.path.isfile(cpfile): f_hash = hashlib.md5() with open(cpfile, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): f_hash.update(chunk) checksum = f_hash.hexdigest() if checksum == Qelement.checksum: Qelement.di_pass = True else: Qelement.di_pass = False Qelement.di_date = datetime.datetime.now( pytz.utc).strftime("%Y-%m-%d %H:%M:%S") session.flush() index = index + 1 if index > 50: session.commit() logger.info('Session Commit for 50 Records: Success') index = 0 else: logger.error('File %s Not Found', cpfile) try: session.commit() logger.info("End Commit DI process to Database: Success") index = 1 except: session.rollback() # Close connection to database session.close() engine.dispose()
def main(): PDSinfoDICT = json.load(open(pds_info, 'r')) args = Args() args.parse_args() # Set up logging logger = logging.getLogger('DI_Process') level = logging.getLevelName(args.log_level) logger.setLevel(level) logFileHandle = logging.FileHandler(pds_log + 'DI.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting DI Process') try: session, engine = db_connect(pds_db) logger.info('DataBase Connecton: Success') except Exception as e: logger.error('DataBase Connection Error: %s', str(e)) return 1 RQ = RedisQueue('DI_ReadyQueue') RQ_lock = RedisLock(lock_obj) RQ_lock.add({RQ.id_name: '1'}) index = 0 logger.info("DI Queue: %s", RQ.id_name) while int(RQ.QueueSize()) > 0 and RQ_lock.available(RQ.id_name): item = literal_eval(RQ.QueueGet().decode("utf-8")) inputfile = item[0] archive = item[1] try: Qelement = session.query(Files).filter( Files.filename == inputfile).one() except Exception as e: logger.warn('Filename query failed for inputfile %s: %s', inputfile, str(e)) continue archive_path = PDSinfoDICT[archive]['path'] cpfile = archive_path + Qelement.filename if os.path.isfile(cpfile): f_hash = hashlib.md5() with open(cpfile, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): f_hash.update(chunk) checksum = f_hash.hexdigest() Qelement.di_pass = checksum == Qelement.checksum Qelement.di_date = datetime.datetime.now( pytz.utc).strftime("%Y-%m-%d %H:%M:%S") session.flush() index = index + 1 if index > 50: session.commit() logger.info('Session Commit for 50 Records: Success') index = 0 else: logger.warn('File %s Not Found', cpfile) try: session.commit() logger.info("End Commit DI process to Database: Success") index = 1 except Exception as e: logger.warn("Unable to commit changes to database\n\n%s", e) session.rollback() # Close connection to database session.close() engine.dispose()
def main(): args = Args() args.parse_args() key = args.key namespace = args.namespace if namespace is None: namespace is default_namespace workarea = scratch + key + '/' RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_work = RedisQueue(key + '_WorkQueue', namespace) RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_final = RedisQueue('FinalQueue', namespace) RHash = RedisHash(key + '_info') RHerror = RedisHash(key + '_error') RQ_lock = RedisLock(lock_obj) RQ_lock.add({'POW': '1'}) if int(RQ_file.QueueSize()) == 0 and RQ_lock.available('POW'): print("No Files Found in Redis Queue") else: print(RQ_file.getQueueName()) jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(), RQ_work.getQueueName()).decode('utf-8') # Setup system logging basename = os.path.splitext(os.path.basename(jobFile))[0] logger = logging.getLogger(key + '.' + basename) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + '/Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting POW Processing') # set up loggy loggyOBJ = Loggy(basename) # File Naming if '+' in jobFile: bandSplit = jobFile.split('+') inputFile = bandSplit[0] else: inputFile = jobFile infile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub' outfile = workarea + \ os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub' RQ_recipe = RedisQueue(key + '_recipe') status = 'success' for element in RQ_recipe.RecipeGet(): if status == 'error': break elif status == 'success': processOBJ = Process() process = processOBJ.JSON2Process(element) if 'gdal_translate' not in processOBJ.getProcessName(): print(processOBJ.getProcessName()) if '2isis' in processOBJ.getProcessName(): processOBJ.updateParameter('from_', inputFile) processOBJ.updateParameter('to', outfile) elif 'cubeatt-band' in processOBJ.getProcessName(): if '+' in jobFile: infileB = infile + '+' + bandSplit[1] processOBJ.updateParameter('from_', infileB) processOBJ.updateParameter('to', outfile) processOBJ.ChangeProcess('cubeatt') else: continue elif 'cubeatt-bit' in processOBJ.getProcessName(): if RHash.OutBit() == b'unsignedbyte': temp_outfile = outfile + '+lsb+tile+attached+unsignedbyte+1:254' elif RHash.OutBit() == b'signedword': temp_outfile = outfile + '+lsb+tile+attached+signedword+-32765:32765' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', temp_outfile) processOBJ.ChangeProcess('cubeatt') elif 'spice' in processOBJ.getProcessName(): processOBJ.updateParameter('from_', infile) elif 'ctxevenodd' in processOBJ.getProcessName(): label = pvl.load(infile) SS = label['IsisCube']['Instrument']['SpatialSumming'] print(SS) if SS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'mocevenodd' in processOBJ.getProcessName(): label = pvl.load(infile) CTS = label['IsisCube']['Instrument'][ 'CrosstrackSumming'] print(CTS) if CTS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'mocnoise50' in processOBJ.getProcessName(): label = pvl.load(infile) CTS = label['IsisCube']['Instrument'][ 'CrosstrackSumming'] if CTS != 1: continue else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif 'cam2map' in processOBJ.getProcessName(): processOBJ.updateParameter('from', infile) processOBJ.updateParameter('to', outfile) if RHash.getGRtype() == 'smart' or RHash.getGRtype( ) == 'fill': subloggyOBJ = SubLoggy('cam2map') camrangeOUT = workarea + basename + '_camrange.txt' isis.camrange(from_=infile, to=camrangeOUT) cam = pvl.load(camrangeOUT) if cam['UniversalGroundRange']['MaximumLatitude'] < float(RHash.getMinLat()) or \ cam['UniversalGroundRange']['MinimumLatitude'] > float(RHash.getMaxLat()) or \ cam['UniversalGroundRange']['MaximumLongitude'] < float(RHash.getMinLon()) or \ cam['UniversalGroundRange']['MinimumLongitude'] > float(RHash.getMaxLon()): status = 'error' eSTR = "Error Ground Range Outside Extent Range" RHerror.addError( os.path.splitext( os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) break elif RHash.getGRtype() == 'smart': if cam['UniversalGroundRange'][ 'MinimumLatitude'] > float( RHash.getMinLat()): minlat = cam['UniversalGroundRange'][ 'MinimumLatitude'] else: minlat = RHash.getMinLat() if cam['UniversalGroundRange'][ 'MaximumLatitude'] < float( RHash.getMaxLat()): maxlat = cam['UniversalGroundRange'][ 'MaximumLatitude'] else: maxlat = RHash.getMaxLat() if cam['UniversalGroundRange'][ 'MinimumLongitude'] > float( RHash.getMinLon()): minlon = cam['UniversalGroundRange'][ 'MinimumLongitude'] else: minlon = RHash.getMinLon() if cam['UniversalGroundRange'][ 'MaximumLongitude'] < float( RHash.getMaxLon()): maxlon = cam['UniversalGroundRange'][ 'MaximumLongitude'] else: maxlon = RHash.getMaxLon() elif RHash.getGRtype() == 'fill': minlat = RHash.getMinLat() maxlat = RHash.getMaxLat() minlon = RHash.getMinLon() maxlon = RHash.getMaxLon() processOBJ.AddParameter('minlat', minlat) processOBJ.AddParameter('maxlat', maxlat) processOBJ.AddParameter('minlon', minlon) processOBJ.AddParameter('maxlon', maxlon) #os.remove(camrangeOUT) elif 'isis2pds' in processOBJ.getProcessName(): finalfile = infile.replace('.input.cub', '_final.img') processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', finalfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) print(processOBJ.getProcess()) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) subloggyOBJ = SubLoggy(k) try: func(**v) logger.info('Process %s :: Success', k) subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if os.path.isfile(outfile): os.rename(outfile, infile) status = 'success' except ProcessError as e: logger.error('Process %s :: Error', k) logger.error(e) status = 'error' eSTR = 'Error Executing ' + k + \ ' Standard Error: ' + str(e) RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], eSTR) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(processOBJ.LogCommandline()) subloggyOBJ.setHelpLink(processOBJ.LogHelpLink()) subloggyOBJ.errorOut(eSTR) loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) else: GDALcmd = "" for process, v, in processOBJ.getProcess().items(): subloggyOBJ = SubLoggy(process) GDALcmd += process for key, value in v.items(): GDALcmd += ' ' + key + ' ' + value frmt = RHash.Format().decode('utf-8') if frmt == 'GeoTiff-BigTiff': fileext = 'tif' elif frmt == 'GeoJPEG-2000': fileext = 'jp2' elif frmt == 'JPEG': fileext = 'jpg' elif frmt == 'PNG': fileext = 'png' elif frmt == 'GIF': fileext = 'gif' logGDALcmd = GDALcmd + ' ' + basename + \ '.input.cub ' + basename + '_final.' + fileext finalfile = infile.replace('.input.cub', '_final.' + fileext) GDALcmd += ' ' + infile + ' ' + finalfile print(GDALcmd) result = subprocess.call(GDALcmd, shell=True) if result == 0: logger.info('Process GDAL translate :: Success') status = 'success' subloggyOBJ.setStatus('SUCCESS') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) #os.remove(infile) else: errmsg = 'Error Executing GDAL translate: Error' logger.error(errmsg) status = 'error' RHerror.addError( os.path.splitext(os.path.basename(jobFile))[0], errmsg) subloggyOBJ.setStatus('ERROR') subloggyOBJ.setCommand(logGDALcmd) subloggyOBJ.setHelpLink( 'http://www.gdal.org/gdal_translate.html') subloggyOBJ.errorOut('Process GDAL translate :: Error') loggyOBJ.AddProcess(subloggyOBJ.getSLprocess()) if status == 'success': if RHash.Format() == 'ISIS3': finalfile = infile.replace('.input.cub', '_final.cub') shutil.move(infile, finalfile) if RHash.getStatus() != 'ERROR': RHash.Status('SUCCESS') try: RQ_zip.QueueAdd(finalfile) logger.info('File Added to ZIP Queue') except: logger.error('File NOT Added to ZIP Queue') elif status == 'error': RHash.Status('ERROR') if os.path.isfile(infile): #os.remove(infile) pass try: RQ_loggy.QueueAdd(loggyOBJ.Loggy2json()) RQ_work.QueueRemove(jobFile) logger.info('JSON Added to Loggy Queue') except: logger.error('JSON NOT Added to Loggy Queue') if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0: try: RQ_final.QueueAdd(key) logger.info('Key %s Added to Final Queue: Success', key) logger.info( 'Both Queues Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize())) logger.info('JOB Complete') except: logger.error('Key NOT Added to Final Queue') elif RQ_file.QueueSize() == 0 and RQ_work.QueueSize() != 0: logger.warning( 'Work Queue Not Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(): # pdb.set_trace() args = Args() args.parse_args() logger = logging.getLogger('Browse_Process') level = logging.getLevelName(args.log_level) logger.setLevel(level) logFileHandle = logging.FileHandler(pds_log + 'Process.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) RQ_main = RedisQueue('Browse_ReadyQueue') RQ_lock = RedisLock(lock_obj) RQ_lock.add({RQ_main.id_name: '1'}) PDSinfoDICT = json.load(open(pds_info, 'r')) pds_session, pds_engine = db_connect(pds_db) upc_session, upc_engine = db_connect(upc_db) tid = get_tid('fullimageurl', upc_session) while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): item = literal_eval(RQ_main.QueueGet().decode("utf-8")) inputfile = item[0] fid = item[1] archive = item[2] if os.path.isfile(inputfile): logger.info('Starting Process: %s', inputfile) finalpath = makedir(inputfile) recipeOBJ = Recipe() recip_json = recipeOBJ.getRecipeJSON(archive) recipeOBJ.AddJsonFile(recip_json, 'reduced') infile = workarea + os.path.splitext( os.path.basename(inputfile))[0] + '.Binput.cub' outfile = workarea + os.path.splitext( os.path.basename(inputfile))[0] + '.Boutput.cub' status = 'success' for item in recipeOBJ.getProcesses(): if status == 'error': logger.error("Error processing %s", inputfile) break elif status == 'success': processOBJ = Process() processOBJ.ProcessFromRecipe(item, recipeOBJ.getRecipe()) if '2isis' in item: processOBJ.updateParameter('from_', inputfile) processOBJ.updateParameter('to', outfile) elif item == 'spiceinit': processOBJ.updateParameter('from_', infile) elif item == 'cubeatt': label = pvl.load(infile) bands = PDSinfoDICT[archive]['bandorder'] query_bands = label['IsisCube']['BandBin'][ PDSinfoDICT[archive]['bandbinQuery']] # Create a set from the list / single value try: query_band_set = set(query_bands) except: query_band_set = set([query_bands]) # Iterate through 'bands' and grab the first value that is present in the # set defined by 'bandbinquery' -- if not present, default to 1 exband = next( (band for band in bands if band in query_band_set), 1) band_infile = infile + '+' + str(exband) processOBJ.updateParameter('from_', band_infile) processOBJ.updateParameter('to', outfile) elif item == 'ctxevenodd': label = pvl.load(infile) SS = label['IsisCube']['Instrument']['SpatialSumming'] if SS != 1: break else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif item == 'reduce': label = pvl.load(infile) Nline = label['IsisCube']['Core']['Dimensions'][ 'Lines'] Nsample = label['IsisCube']['Core']['Dimensions'][ 'Samples'] Nline = int(Nline) Nsample = int(Nsample) Sfactor = scaleFactor(Nline, Nsample, recip_json) processOBJ.updateParameter('lscale', Sfactor) processOBJ.updateParameter('sscale', Sfactor) processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) elif item == 'isis2std': final_outfile = finalpath + '/' + os.path.splitext( os.path.basename(inputfile))[0] + '.browse.jpg' processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', final_outfile) else: processOBJ.updateParameter('from_', infile) processOBJ.updateParameter('to', outfile) for k, v in processOBJ.getProcess().items(): func = getattr(isis, k) try: func(**v) logger.info('Process %s :: Success', k) if os.path.isfile(outfile): if '.cub' in outfile: os.rename(outfile, infile) status = 'success' if '2isis' in item: isisSerial = getISISid(infile) except ProcessError as e: print(e) logger.error('Process %s :: Error', k) status = 'error' if status == 'success': DB_addURL(upc_session, isisSerial, final_outfile, tid) os.remove(infile) logger.info('Browse Process Success: %s', inputfile) AddProcessDB(pds_session, fid, 't') else: logger.error('File %s Not Found', inputfile) upc_session.close() pds_session.close() upc_engine.dispose() pds_engine.dispose()