def hyp3_process(cfg, n): try: log.info('Processing hello_world') if not cfg['skip_processing']: log.info(f'Process starting at {datetime.now()}') launch_dir = os.getcwd() os.chdir(cfg['workdir']) hyp3proclib.process(cfg, 'proc_ci', ["--hello-world"]) os.chdir(launch_dir) else: log.info('Processing skipped!') cfg['log'] += "(debug mode)" cfg['success'] = True hyp3proclib.update_completed_time(cfg) product_dir = os.path.join(cfg['workdir'], 'PRODUCT') if not os.path.isdir(product_dir): log.info(f'PRODUCT directory not found: {product_dir}') log.error('Processing failed') raise Exception('Processing failed: PRODUCT directory not found') # TODO: final product cleanup and upload to HyP3 DB except Exception as e: log.exception('ci processing failed!') log.exception('Notifying user') hyp3proclib.failure(cfg, str(e)) hyp3proclib.file_system.cleanup_workdir(cfg) log.info('ci done')
def add_instance_record(cfg, conn): log.debug('Adding instance record') instance_record = {} if cfg['proc_node_type'] != 'CLOUD': instance_record['instance_id'] = socket.gethostname() instance_record['instance_type'] = 'on-prem' else: instance_record['instance_id'] = get_instance_id() instance_record['instance_type'] = get_instance_type() instance_record['local_queue_id'] = cfg['id'] cfg['instance_record'] = instance_record try: instance_record_sql = ''' insert into instance_records (instance_id, local_queue_id, start_time, instance_type) values (%(instance_id)s, %(local_queue_id)s, current_timestamp, %(instance_type)s) ''' query_database(conn=conn, query=instance_record_sql, params=instance_record, commit=True) except Exception: log.exception("Instance record could not be inserted") else: log.info("Instance record of instance %s and job %s inserted", instance_record['instance_id'], instance_record['local_queue_id'])
def get_db_connection(s, tries=0): connection_string =\ "host='" + get_config(s, 'host') + "' " + \ "dbname='" + get_config(s, 'db') + "' " + \ "user='******'user') + "' " + \ "password='******'pass') + "'" log.info("Connected to db: {0}".format(get_config(s, 'host'))) try: conn = psycopg2.connect(connection_string) except Exception as e: if (tries > 4): log.exception('DB connection problem: ' + str(e)) raise else: log.warning("Problem connecting to DB: " + str(e)) log.info("Retrying in {0} seconds...".format(30 * (tries + 1))) time.sleep(30 * (tries + 1)) return get_db_connection(s, tries=tries + 1) return conn
def update_instance_record(cfg, conn): if 'instance_record' in cfg: instance_record = cfg['instance_record'] try: instance_record_sql = 'update instance_records set end_time=current_timestamp where (instance_id=%(instance_id)s and local_queue_id=%(local_queue_id)s);' query_database(conn=conn, query=instance_record_sql, params=instance_record, commit=True) except Exception: log.exception( "Instance record for instance %s and job %s could not be updated with job completion time", instance_record['instance_id'], instance_record['local_queue_id']) else: log.info( "Instance record for instance %s and job %s had end_time updated with job completion time", instance_record['instance_id'], instance_record['local_queue_id']) else: log.debug('No instance record found to update')
def process_insar(cfg, n): try: log.info('Processing ISCE InSAR pair "{0}" for "{1}"'.format(cfg['sub_name'], cfg['username'])) g1, g2 = earlier_granule_first(cfg['granule'], cfg['other_granules'][0]) list_file = 'list.csv' write_list_file(os.path.join(cfg['workdir'], list_file), g1, g2) d1 = g1[17:25] d2 = g2[17:25] delta = (datetime.datetime.strptime(d2, '%Y%m%d')-datetime.datetime.strptime(d1, '%Y%m%d')).days ifm_dir = d1 + '_' + d2 cfg['ifm'] = ifm_dir log.debug('IFM dir is: ' + ifm_dir) sd1 = d1[0:4]+'-'+d1[4:6]+'-'+d1[6:8] sd2 = d2[0:4]+'-'+d2[4:6]+'-'+d2[6:8] cfg["email_text"] = "This is a {0}-day InSAR pair from {1} to {2}.".format(delta, sd1, sd2) subswath = get_extra_arg(cfg, "subswath", "0") if subswath == "0": process(cfg, 'procAllS1StackISCE.py', ["-90", "90", "-180", "180", "-f", list_file, "-d"]) else: process(cfg, 'procS1StackISCE.py', ["-f", list_file, "-d", "-s", subswath]) subdir = os.path.join(cfg['workdir'], 'PRODUCT') if not os.path.isdir(subdir): log.info('PRODUCT directory not found: ' + subdir) log.error('Processing failed') raise Exception("Processing failed: PRODUCT directory not found") else: looks = get_looks(subdir) out_name = build_output_name_pair(g1, g2, cfg['workdir'], looks + "-iw" + subswath + cfg['suffix']) log.info('Output name: ' + out_name) out_path = os.path.join(cfg['workdir'], out_name) zip_file = out_path + '.zip' if os.path.isdir(out_path): shutil.rmtree(out_path) if os.path.isfile(zip_file): os.unlink(zip_file) cfg['out_path'] = out_path # clip_tiffs_to_roi(cfg, conn, product) log.debug('Renaming '+subdir+' to '+out_path) os.rename(subdir, out_path) find_browses(cfg, out_path) cfg['attachment'] = find_phase_png(out_path) add_esa_citation(g1, out_path) zip_dir(out_path, zip_file) cfg['final_product_size'] = [os.stat(zip_file).st_size, ] cfg['original_product_size'] = 0 with get_db_connection('hyp3-db') as conn: record_metrics(cfg, conn) upload_product(zip_file, cfg, conn) success(conn, cfg) except Exception as e: log.exception('Processing failed') log.info('Notifying user') failure(cfg, str(e)) cleanup_workdir(cfg) log.info('Done')