def test_bad_process_isis(): logger = logging.getLogger('UPC_Process') processes = {'isis.spiceinit': {'from_': '/Path/to/some/cube.cub'}} failing_command, error = process(processes, '/', logger) assert failing_command == 'spiceinit' assert error != ''
def main(user_args): key = user_args.key namespace = user_args.namespace if namespace is None: namespace = default_namespace work_dir = os.path.join(workarea, key) RQ_file = RedisQueue(key + '_FileQueue', namespace) RQ_work = RedisQueue(key + '_WorkQueue', namespace) RQ_zip = RedisQueue(key + '_ZIP', namespace) RQ_loggy = RedisQueue(key + '_loggy', namespace) RQ_final = RedisQueue('FinalQueue', namespace) RQ_recipe = RedisQueue(key + '_recipe', namespace) RQ_error = RedisQueue(upc_error_queue, namespace) RHash = RedisHash(key + '_info') RHerror = RedisHash(key + '_error') RQ_lock = RedisLock(lock_obj) RQ_lock.add({'MAP': '1'}) if int(RQ_file.QueueSize()) > 0 and RQ_lock.available('MAP'): jobFile = RQ_file.Qfile2Qwork(RQ_file.getQueueName(), RQ_work.getQueueName()) # Setup system logging basename = os.path.splitext(os.path.basename(jobFile))[0] logger = logging.getLogger(key + '.' + basename) logger.setLevel(logging.INFO) logFileHandle = logging.FileHandler(pds_log + '/Service.log') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s, %(message)s') logFileHandle.setFormatter(formatter) logger.addHandler(logFileHandle) logger.info('Starting MAP Processing') # File Naming infile = os.path.join(work_dir, \ os.path.splitext(os.path.basename(jobFile))[0] + '.input.cub') outfile = os.path.join( work_dir, os.path.splitext(os.path.basename(jobFile))[0] + '.output.cub') # Recipe Stuff status = 'success' recipe_string = RQ_recipe.QueueGet() no_extension_inputfile = os.path.join( work_dir, os.path.splitext(os.path.basename(jobFile))[0]) processes = generate_processes( jobFile, recipe_string, logger, no_extension_inputfile=no_extension_inputfile) failing_command, error = process(processes, work_dir, logger) process_log = generate_log_json(processes, basename, failing_command, error) if failing_command: status = 'error' if status == 'success': final_file_list = [] img_format = RHash.Format() # If final output format is ISIS3 or PDS, will use an ISIS program to create if img_format == 'ISIS3' or img_format == 'PDS': last_output = list(processes.items())[-1][-1]['to'] last_output = last_output.split('+')[0] if img_format == 'ISIS3': finalfile = os.path.join(work_dir, RHash.getMAPname() + '.cub') else: finalfile = os.path.join(work_dir, RHash.getMAPname() + '.img') # Else will use GDAL, so set extension and define possible ancillary files else: if img_format == 'GeoTiff-BigTiff' or img_format == 'GTiff': fileext = 'tif' elif img_format == 'GeoJPEG-2000': fileext = 'jp2' elif img_format == 'JPEG': fileext = 'jpg' elif img_format == 'PNG': fileext = 'png' elif img_format == 'GIF': fileext = 'gif' last_output = list(processes.items())[-1][-1]['dest'] finalfile = os.path.join(work_dir, RHash.getMAPname() + '.' + fileext) # Possible ancillary files last_output_msk = last_output + '.msk' last_output_aux = last_output + '.aux.xml' if os.path.isfile(last_output_msk): finalfile_msk = os.path.join( work_dir, RHash.getMAPname() + '.' + fileext + '.msk') shutil.move(last_output_msk, finalfile_msk) final_file_list.append(finalfile_msk) if os.path.isfile(last_output_aux): finalfile_aux = os.path.join( work_dir, RHash.getMAPname() + '.' + fileext + '.aux.xml') shutil.move(last_output_aux, finalfile_aux) final_file_list.append(finalfile_aux) shutil.move(last_output, finalfile) final_file_list.append(finalfile) if RHash.getStatus() != 'ERROR': RHash.Status('SUCCESS') # Loop over the list of final output files and add them to RQ_zip for item in final_file_list: try: RQ_zip.QueueAdd(item) logger.info('File Added to ZIP Queue') except: logger.error('File NOT Added to ZIP Queue') try: RQ_loggy.QueueAdd(process_log) logger.info('JSON Added to Loggy Queue') except Exception as e: logger.error(f'JSON NOT Added to Loggy Queue with error: {e}') RQ_work.QueueRemove(jobFile) elif status == 'error': RHash.Status('ERROR') logger.error(f'Process {failing_command} :: Error') logger.error(error) error_string = f'Error Executing {failing_command}' \ f'Standard Error: {error}' RHerror.addError(basename, error_string) RQ_error.QueueAdd( f'Process {failing_command} failed for {jobFile}') if os.path.isfile(infile): os.remove(infile) if RQ_file.QueueSize() == 0 and RQ_work.QueueSize() == 0: try: RQ_final.QueueAdd(key) logger.info('Key %s Added to Final Queue: Success', key) logger.info('Job Complete') except: logger.error('Key NOT Added to Final Queue') else: logger.warning('Queues Not Empty: filequeue = %s work queue = %s', str(RQ_file.QueueSize()), str(RQ_work.QueueSize()))
def main(user_args): proc = user_args.proc derived = user_args.derived log_level = user_args.log_level namespace = user_args.namespace try: slurm_job_id = os.environ['SLURM_ARRAY_JOB_ID'] slurm_array_id = os.environ['SLURM_ARRAY_TASK_ID'] except: slurm_job_id = '' slurm_array_id = '' inputfile = '' context = {'job_id': slurm_job_id, 'array_id':slurm_array_id, 'inputfile': inputfile} logger = logging.getLogger('UPC_Process') level = logging.getLevelName(log_level) logger.setLevel(level) log_file_handle = logging.FileHandler(pds_log + 'Process.log') formatter = logging.Formatter( '%(asctime)s - %(job_id)s - %(array_id)s - %(inputfile)s - %(name)s - %(levelname)s, %(message)s') log_file_handle.setFormatter(formatter) logger.addHandler(log_file_handle) logger = logging.LoggerAdapter(logger, context) # Redis Queue Objects RQ_main = RedisQueue('UPC_ReadyQueue', namespace) RQ_error = RedisQueue(upc_error_queue, namespace) RQ_work = RedisQueue('UPC_WorkQueue', namespace) RQ_update = RedisQueue('UPC_UpdateQueue', namespace) logger.info("UPC Processing Queue: %s", RQ_main.id_name) RQ_lock = RedisLock(lock_obj) # If the queue isn't registered, add it and set it to "running" RQ_lock.add({RQ_main.id_name: '1'}) # while there are items in the redis queue while int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): # get a file from the queue item = RQ_main.Qfile2Qwork(RQ_main.getQueueName(), RQ_work.getQueueName()) inputfile = literal_eval(item)[0] archive = literal_eval(item)[1] if not os.path.isfile(inputfile): RQ_error.QueueAdd(f'Unable to locate or access {inputfile} during UPC processing') logger.debug("%s is not a file\n", inputfile) exit() # Update the logger context to include inputfile context['inputfile'] = inputfile recipe_file = recipe_base + "/" + archive + '.json' no_extension_inputfile = os.path.splitext(inputfile)[0] cam_info_file = no_extension_inputfile + '_caminfo.pvl' footprint_file = no_extension_inputfile + '_footprint.json' catlab_output = no_extension_inputfile + '_catlab.pvl' if proc: with open(recipe_file) as fp: upc_json = json.load(fp)['upc'] recipe_string = json.dumps(upc_json['recipe']) processes = generate_processes(inputfile, recipe_string, logger, no_extension_inputfile=no_extension_inputfile, catlab_output=catlab_output, cam_info_file=cam_info_file, footprint_file=footprint_file) failing_command, _ = process(processes, workarea, logger) if derived: if os.path.isfile(inputfile): recipe_file = recipe_base + "/" + archive + '.json' with open(recipe_file) as fp: recipe = json.load(fp, object_pairs_hook=parse_pairs)['reduced'] recipe_string = json.dumps(recipe['recipe']) logger.info('Starting Process: %s', inputfile) work_dir = os.path.dirname(inputfile) derived_product = os.path.join(work_dir, os.path.splitext(os.path.basename(inputfile))[0]) no_extension_inputfile = os.path.splitext(inputfile)[0] processes = generate_processes(inputfile, recipe_string, logger, no_extension_inputfile=no_extension_inputfile, derived_product=derived_product) failing_command, _ = process(processes, workarea, logger) if failing_command: logger.warn(logger.error('%s Processing Error: %s', inputfile, failing_command)) if proc: RQ_update.QueueAdd((inputfile, archive, failing_command, 'upc')) elif derived: RQ_update.QueueAdd((inputfile, archive, failing_command, 'derived')) RQ_work.QueueRemove(item) logger.info("UPC processing exited")
def main(user_args): upc_session_maker, upc_engine = db_connect(upc_db) pds_session_maker, pds_engine = db_connect(pds_db) persist = user_args.persist log_level = user_args.log_level try: slurm_job_id = os.environ['SLURM_ARRAY_JOB_ID'] slurm_array_id = os.environ['SLURM_ARRAY_TASK_ID'] except: slurm_job_id = '' slurm_array_id = '' inputfile = '' context = { 'job_id': slurm_job_id, 'array_id': slurm_array_id, 'inputfile': inputfile } logger = logging.getLogger('UPC_Process') level = logging.getLevelName(log_level) logger.setLevel(level) log_file_handle = logging.FileHandler(pds_log + 'Process.log') formatter = logging.Formatter( '%(asctime)s - %(job_id)s - %(array_id)s - %(inputfile)s - %(name)s - %(levelname)s, %(message)s' ) log_file_handle.setFormatter(formatter) logger.addHandler(log_file_handle) logger = logging.LoggerAdapter(logger, context) # ***************** Set up logging ***************** # Redis Queue Objects RQ_main = RedisQueue('UPC_ReadyQueue') logger.info("UPC Processing Queue: %s", RQ_main.id_name) RQ_error = RedisQueue(upc_error_queue) RQ_lock = RedisLock(lock_obj) # If the queue isn't registered, add it and set it to "running" RQ_lock.add({RQ_main.id_name: '1'}) # if there are items in the redis queue if int(RQ_main.QueueSize()) > 0 and RQ_lock.available(RQ_main.id_name): # get a file from the queue item = literal_eval(RQ_main.QueueGet()) inputfile = item[0] fid = item[1] archive = item[2] if not os.path.isfile(inputfile): RQ_error.QueueAdd( f'Unable to locate or access {inputfile} during UPC processing' ) logger.debug("%s is not a file\n", inputfile) exit() # Build URL for edr_source based on archive path from PDSinfo.json PDSinfoDICT = json.load(open(pds_info, 'r')) archive_path = PDSinfoDICT[archive]['path'] orig_file = inputfile.replace(workarea, archive_path) edr_source = orig_file.replace(archive_base, web_base) # Update the logger context to include inputfile context['inputfile'] = inputfile recipe_file = recipe_base + "/" + archive + '.json' with open(recipe_file) as fp: upc_json = json.load(fp)['upc'] recipe_string = json.dumps(upc_json['recipe']) # Attempt to get the optional search_term_mapping for the upc # process try: search_term_mapping = upc_json['search_term_mapping'] except KeyError: search_term_mapping = {} no_extension_inputfile = os.path.splitext(inputfile)[0] cam_info_file = no_extension_inputfile + '_caminfo.pvl' footprint_file = no_extension_inputfile + '_footprint.json' processes = generate_processes( inputfile, recipe_string, logger, no_extension_inputfile=no_extension_inputfile, cam_info_file=cam_info_file, footprint_file=footprint_file) failing_command, _ = process(processes, workarea, logger) pds_label = pvl.load(inputfile) ######## Generate DataFiles Record ######## upc_id = create_datafiles_record(pds_label, edr_source, no_extension_inputfile + '.cub', upc_session_maker) ######## Generate SearchTerms Record ######## create_search_terms_record(pds_label, cam_info_file, upc_id, no_extension_inputfile + '.cub', footprint_file, search_term_mapping, upc_session_maker) ######## Generate JsonKeywords Record ######## create_json_keywords_record(cam_info_file, upc_id, inputfile, failing_command, upc_session_maker, logger) try: pds_session = pds_session_maker() pds_session.flush() except: logger.debug("Unable to flush database connection") add_process_db(pds_session, fid, True) pds_session.close() if not persist: # Remove all files file from the workarea except for the copied # source file file_prefix = os.path.splitext(inputfile)[0] workarea_files = glob(file_prefix + '*') os.remove(os.path.join(workarea, 'print.prt')) for file in workarea_files: os.remove(file) # Disconnect from the engines pds_engine.dispose() upc_engine.dispose() logger.info("UPC processing exited")