def update_item_old(table_name: str, key: str, attr_name: str, attr_value, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) key_json = {'id': key} update = 'SET ' + attr_name + ' = :new_value, modified = :m' expression_json = {':new_value': attr_value, ':m': str(now_with_tz())} logger.info('dynamodb update', extra={ 'table_name': table_name, 'key': key, 'attr_name': attr_name, 'attr_value': attr_value, 'correlation_id': correlation_id }) response = table.update_item(Key=key_json, UpdateExpression=update, ExpressionAttributeValues=expression_json) except Exception as ex: raise ex
def process_task_signup(notification): logger = get_logger() correlation_id = new_correlation_id() try: # get basic data out of notification signup_details = notification['details'] user_task_id = signup_details['id'] # get additional data that hubspot needs from database extra_data = get_task_signup_data_for_crm(user_task_id, correlation_id) # put it all together for dispatch to HubSpot signup_details.update(extra_data) signup_details['signup_event_type'] = 'Sign-up' # check here that we have a hubspot id if signup_details['crm_id'] is None: errorjson = {'user_task_id': user_task_id, 'correlation_id': str(correlation_id)} raise DetailedValueError('user does not have crm_id', errorjson) else: post_task_signup_to_crm(signup_details, correlation_id) mark_notification_processed(notification, correlation_id) except Exception as ex: error_message = str(ex) mark_notification_failure(notification, error_message, correlation_id)
def scan(table_name: str, filter_attr_name: str = None, filter_attr_values=None, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) # accept string but make it into a list for later processing if isinstance(filter_attr_values, str): filter_attr_values = [filter_attr_values] logger.info('dynamodb scan', extra={ 'table_name': table_name, 'filter_attr_name': filter_attr_name, 'filter_attr_value': str(filter_attr_values), 'correlation_id': correlation_id }) if filter_attr_name is None: response = table.scan() else: filter_expr = Attr(filter_attr_name).eq(filter_attr_values[0]) for value in filter_attr_values[1:]: filter_expr = filter_expr | Attr(filter_attr_name).eq(value) response = table.scan(FilterExpression=filter_expr) items = response['Items'] logger.info('dynamodb scan result', extra={ 'count': str(len(items)), 'correlation_id': correlation_id }) return items except Exception as ex: raise ex
def process_user_registration(notification): logger = get_logger() correlation_id = new_correlation_id() try: notification_id = notification['id'] details = notification['details'] user_id = details['id'] logger.info('process_user_registration: post to hubspot', extra={'notification_id': str(notification_id), 'user_id': str(user_id), 'email': details['email'], 'correlation_id': str(correlation_id)}) hubspot_id, is_new = post_new_user_to_crm(details, correlation_id) logger.info('process_user_registration: hubspot details', extra={'notification_id': str(notification_id), 'hubspot_id': str(hubspot_id), 'isNew': str(is_new), 'correlation_id': str(correlation_id)}) if hubspot_id == -1: errorjson = {'user_id': user_id, 'correlation_id': str(correlation_id)} raise DetailedValueError('could not find user in HubSpot', errorjson) user_jsonpatch = [ {'op': 'replace', 'path': '/crm_id', 'value': str(hubspot_id)}, ] patch_user(user_id, user_jsonpatch, now_with_tz(), correlation_id) mark_notification_processed(notification, correlation_id) except Exception as ex: error_message = str(ex) mark_notification_failure(notification, error_message, correlation_id)
def main(): # Argument parser parser = argparse.ArgumentParser() parser.add_argument("product", help="Venus product (ZIP)") parser.add_argument("coordinates", help="coordinate file") parser.add_argument("-e", "--extent", type=int, \ help="extent of the square ROI (meters), defaults to 100", default=100) parser.add_argument("-v", "--verbose", help="Set verbosity to DEBUG level", action="store_true", default=False) args = parser.parse_args() # Create the logger logger = utl.get_logger('roistats', args.verbose) # Create a Venus product object vns_product = common.Product.Product_zip_venus(args.product, logger) # Create an roi collection roi_collection = common.Roi.Roi_collection(args.coordinates, args.extent, logger) # Compute statistics for a given product list_stats = roi_collection.compute_stats_all_bands(vns_product, logger, stdout=True, withAOT=True, withVAP=True) sys.exit(0)
def process_notifications(event, context): logger = get_logger() notifications = get_notifications(NotificationAttributes.STATUS.value, [NotificationStatus.NEW.value, NotificationStatus.RETRYING.value]) logger.info('process_notifications', extra = {'count': str(len(notifications))}) # note that we need to process all registrations first, then do task signups (otherwise we might try to process a signup for someone not yet registered) signup_notifications = [] login_notifications = [] for notification in notifications: notification_type = notification['type'] if notification_type == NotificationType.USER_REGISTRATION.value: process_user_registration(notification) elif notification_type == NotificationType.TASK_SIGNUP.value: # add to list for later processing signup_notifications.append(notification) elif notification_type == NotificationType.USER_LOGIN.value: # add to list for later processing login_notifications.append(notification) else: # todo details raise Exception for signup_notification in signup_notifications: process_task_signup(signup_notification) for login_notification in login_notifications: process_user_login(login_notification)
def scan_old(table_name: str, filter_attr_name: str = None, filter_attr_value=None, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) logger.info('dynamodb scan', extra={ 'table_name': table_name, 'filter_attr_name': filter_attr_name, 'filter_attr_value': filter_attr_value, 'correlation_id': correlation_id }) if filter_attr_name is None: response = table.scan() else: response = table.scan( FilterExpression=Attr(filter_attr_name).eq(filter_attr_value)) items = response['Items'] logger.info('dynamodb scan result', extra={ 'count': str(len(items)), 'correlation_id': correlation_id }) return items except Exception as ex: raise ex
def process_user_login(notification): logger = get_logger() correlation_id = new_correlation_id() try: # get basic data out of notification login_details = notification['details'] posting_result = post_user_login_to_crm(login_details, correlation_id) marking_result = mark_notification_processed(notification, correlation_id) return posting_result, marking_result except Exception as ex: error_message = str(ex) mark_notification_failure(notification, error_message, correlation_id)
def update_item(table_name: str, key: str, name_value_pairs: dict, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) key_json = {'id': key} update_expr = 'SET #modified = :m ' values_expr = {':m': str(now_with_tz())} attr_names_expr = { '#modified': 'modified' } # not strictly necessary, but allows easy addition of names later param_count = 1 for name, value in name_value_pairs.items(): param_name = ':p' + str(param_count) map_name = None if name == 'status': # todo generalise this to other reserved words, and ensure it only catches whole words if '.' in name: map_name = name.split('.')[0] attr_name = '#a' + str(param_count) attr_names_expr[attr_name] = 'status' else: attr_name = name if map_name is not None: attr_name = map_name + '.' + attr_name update_expr += ', ' + attr_name + ' = ' + param_name values_expr[param_name] = str(value) param_count += 1 logger.info('dynamodb update', extra={ 'table_name': table_name, 'key': key, 'update_expr': update_expr, 'values_expr': values_expr, 'correlation_id': correlation_id }) response = table.update_item( Key=key_json, UpdateExpression=update_expr, ExpressionAttributeValues=values_expr, ExpressionAttributeNames=attr_names_expr, ) return response except Exception as ex: raise ex
def delete_item(table_name: str, key: str, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) key_json = {'id': key} logger.info('dynamodb delete', extra={ 'table_name': table_name, 'key': key, 'correlation_id': correlation_id }) response = table.delete_item(Key=key_json) except Exception as err: raise err
def delete_all(table_name: str, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) items = scan(table_name) for item in items: key = item['id'] key_json = {'id': key} logger.info('dynamodb delete_all', extra={ 'table_name': table_name, 'key': key, 'correlation_id': correlation_id }) table.delete_item(Key=key_json) except Exception as err: raise err
def dateformattest(event, context): logger = get_logger() logger.info('dateformattest', extra=event) try: test_json = json.loads(event['body']) logger.info('body:', extra=test_json) date_string = test_json['date'] date_string = date_string[:19] # strip timezone and milliseconds format_string = test_json['format'] logger.info('dateformattest', extra={'date_string': date_string, 'format_string': format_string}) datetime_obj = datetime.strptime(date_string, format_string) created_timestamp = int(datetime_obj.timestamp() * 1000) response = {"statusCode": 200, "body": json.dumps({"created_timestamp": str(created_timestamp)})} return response except: logger.error(sys.exc_info()[0])
def get_item(table_name: str, key: str, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) key_json = {'id': key} logger.info('dynamodb get', extra={ 'table_name': table_name, 'key': key, 'correlation_id': correlation_id }) response = table.get_item(Key=key_json) if 'Item' in response: return response['Item'] else: # not found return None except Exception as ex: raise ex
def run_tasks(calling_func, val, lock): from common.utilities import get_logger logger = get_logger(fname='/backup/rabbit.log.out') try: parameters = pika.ConnectionParameters(**CONNECTION_PARAMETERS) factory = PikaFactory(parameters, 0, calling_func, logger) for i in range(total_tasks): reactor.connectTCP(parameters.host, parameters.port, factory) logger.info(' [*] Waiting for messages. To exit press CTRL+C') looping_call = task.LoopingCall(factory.stop_tasks, val, lock) looping_call.start(5) reactor.run() except: logger.exception("Error") if reactor.running: reactor.stop() logger.info("Process Finished - %s" % os.getpid())
def put_item(table_name: str, key, item_type: str, item_details, item: dict, update_allowed=False, correlation_id=new_correlation_id()): try: logger = get_logger() table = get_table(table_name) item['id'] = str(key) item['type'] = item_type item['details'] = item_details now = str(now_with_tz()) item['created'] = now item['modified'] = now logger.info('dynamodb put', extra={ 'table_name': table_name, 'item': item, 'correlation_id': correlation_id }) if update_allowed: response = table.put_item(Item=item) else: response = table.put_item( Item=item, ConditionExpression='attribute_not_exists(id)') except ClientError as ex: error_code = ex.response['Error']['Code'] errorjson = { 'error_code': error_code, 'table_name': table_name, 'item_type': item_type, 'id': str(key), 'correlation_id': correlation_id } raise DuplicateInsertError('item already exists', errorjson)
import os import sys import time import multiprocessing from common.SharedFibonacci import Manager from common.utilities import get_basic_logger, get_logger workers = 4 total_process = workers + 1 # logger = get_basic_logger() logger = get_logger(fname='/backup/rabbit.log.out') def launch_process(calling_func, val, lock): logger.info("Inside Launch Process with PId - %s" % os.getpid()) from AsyncPika.task_factory import run_tasks run_tasks(calling_func, val, lock) def handle_error(e): logger.exception(e) def terminate_handler(val): logger.info("PID - %s Checking for stop.txt" % os.getpid()) while val.value == 0: # logger.info("stop instruction -> "+ str(os.path.isfile("stop.txt"))) if os.path.isfile("stop.txt"): val.value = 1 break
def main(): # Argument parser parser = argparse.ArgumentParser() parser.add_argument("list", help="List of paths of collection") parser.add_argument("--band", help="Specific band in acix band definition", type=int, required=True) parser.add_argument( "--samples", help="Reflectance sampling, defaults to 100 (ie. 0.01)", type=int, default=100) parser.add_argument( "-s", "--save", help="Write location results as npy instead of stacking in memory", action="store_true", default=False) parser.add_argument("-v", "--verbose", help="Set verbosity to DEBUG level", action="store_true", default=False) parser.add_argument("--negative", help="Save sr lt 0", action="store_true", default=False) parser.add_argument("--keepall", help="Keep sr <= 0 but cloudfree", action="store_true", default=False) parser.add_argument("--stack", help="Stack all sites in one file", action="store_true", default=False) args = parser.parse_args() # <maja_band>, <hdf_band>, <resolution>, <ref_samples>, <maja_samples> bdef_acix = (["band02", "SRE_B2.", "R1", []], ["band03", "SRE_B3.", "R1", []], ["band04", "SRE_B4.", "R1", []], ["band05", "SRE_B5.", "R2", []], ["band06", "SRE_B6.", "R2", []], ["band07", "SRE_B7.", "R2", []], ["band08", "SRE_B8.", "R1", []], ["band8a", "SRE_B8A.", "R2", []], ["band11", "SRE_B11.", "R2", []], ["band12", "SRE_B12.", "R2", []]) band_id = args.band # Create the logger logger = utl.get_logger('acix_validate_' + bdef_acix[band_id][0], args.verbose) if (band_id < 0 or band_id > len(bdef_acix)): logger.error("Band ID out of range with value %i" % band_id) sys.exit(3) # vector containers for stacked data v_stacked_valid_ref = np.zeros((0)) v_stacked_valid_maja = np.zeros((0)) v_stacked_lt0_ref = np.zeros((0)) v_stacked_lt0_maja = np.zeros((0)) v_stacked_keep_all_ref = np.zeros((0)) v_stacked_keep_all_maja = np.zeros((0)) match_count = 0 len_check = 0 f = open(args.list, 'r') paths_list = f.read().splitlines() for p in paths_list: paths = p.split(',') location_name = paths[0].split('/')[-1] # vector containers for location specific data v_local_valid_ref = np.zeros((0)) v_local_valid_maja = np.zeros((0)) v_local_lt0_ref = np.zeros((0)) v_local_lt0_maja = np.zeros((0)) acix_vermote_collection = clc.Collection(paths[0], logger) acix_maja_collection = clc.Collection(paths[1], logger) compare = cmp.Comparison(acix_vermote_collection, acix_maja_collection, logger) for match in compare.matching_products: logger.info("One-by-one for %s between %s and %s" % (match[0], match[1], match[2])) p_ref = prd.Product_hdf_acix(match[1], logger) p_maja = prd.Product_dir_maja(match[2], logger) try: b_ref = p_ref.get_band(p_ref.find_band(bdef_acix[band_id][0]), scalef=p_ref.sre_scalef) m_ref_qa = p_ref.get_band(p_ref.find_band("refqa")) b_maja = p_maja.get_band(p_maja.find_band( bdef_acix[band_id][1]), scalef=p_maja.sre_scalef) clm = p_maja.get_band( p_maja.find_band("CLM_" + bdef_acix[band_id][2])) edg = p_maja.get_band( p_maja.find_band("EDG_" + bdef_acix[band_id][2])) m_maja_qa, ratio = p_maja.get_mask(clm, edg, stats=True) del clm del edg # default filter : any cloudfree flaged both by ref and maja and sr >= 0 is_valid = np.where((b_ref > 0) & (b_maja > 0) & (m_ref_qa == 1) & (m_maja_qa == 1)) if args.negative: # get sr < 0 though flaged cloudfree is_cloudfree_but_negative = np.where((m_maja_qa == 1) & (m_ref_qa == 1) & ((b_maja < 0) | (b_ref < 0))) if args.keepall: # get sr < 0 though flaged cloudfree is_cloudfree_keep_all = np.where((m_maja_qa == 1) & (m_ref_qa == 1)) # stack local valid values for all timestamp matches v_local_valid_ref = np.append(v_local_valid_ref, (b_ref[is_valid])) v_local_valid_maja = np.append(v_local_valid_maja, (b_maja[is_valid])) if args.negative: # stack local cloudfree negative sr for all timestamp matches v_local_lt0_ref = np.append( v_local_lt0_ref, (b_ref[is_cloudfree_but_negative])) v_local_lt0_maja = np.append( v_local_lt0_maja, (b_maja[is_cloudfree_but_negative])) if args.keepall: v_local_keep_all_ref = np.append( v_local_lt0_ref, (b_ref[is_cloudfree_keep_all])) v_local_keep_all_maja = np.append( v_local_lt0_maja, (b_maja[is_cloudfree_keep_all])) match_count += 1 len_check += len(b_ref[is_valid]) if args.stack: # if all locations have to be stacked in one single vector v_stacked_valid_ref = np.append(v_stacked_valid_ref, v_local_valid_ref) v_stacked_valid_maja = np.append(v_stacked_valid_maja, v_local_valid_maja) if args.negative: v_stacked_lt0_ref = np.append(v_stacked_lt0_ref, v_local_lt0_ref) v_stacked_lt0_maja = np.append(v_stacked_lt0_maja, v_local_lt0_maja) if args.keepall: v_stacked_keep_all_ref = np.append( v_stacked_keep_all_ref, v_local_keep_all_ref) v_stacked_keep_all_maja = np.append( v_stacked_keep_all_maja, v_local_keep_all_maja) else: # save local vectors in one compressed file if not args.keepall: np.savez_compressed( location_name + "_valid_" + bdef_acix[band_id][0], [ v_local_valid_ref.astype('float32'), v_local_valid_maja.astype('float32') ]) if args.negative: np.savez_compressed( location_name + "_sr_lt_0_" + bdef_acix[band_id][0], [ v_local_lt0_ref.astype('float32'), v_local_lt0_maja.astype('float32') ]) if args.keepall: np.savez_compressed( location_name + "_keep_all_" + bdef_acix[band_id][0], [ v_local_keep_all_ref.astype('float32'), v_local_keep_all_maja.astype('float32') ]) except TypeError as err: logger.warning( "Had to skip comparison for %s because of unexpected product dimension (see previous error)" % (match[0])) if args.stack: # if --stack, save stacked vector in one single compressed file np.savez_compressed("Stacked_valid_" + bdef_acix[band_id][0], [ v_stacked_valid_ref.astype('float32'), v_stacked_valid_maja.astype('float32') ]) if args.negative: np.savez_compressed("Stacked_sr_lt_0_" + bdef_acix[band_id][0], [ v_stacked_lt0_ref.astype('float32'), v_stacked_lt0_maja.astype('float32') ]) if args.keepall: np.savez_compressed("Stacked_sr_keep_all_" + bdef_acix[band_id][0], [ v_stacked_keep_all_ref.astype('float32'), v_stacked_keep_all_maja.astype('float32') ]) if len_check == len(v_stacked_valid_ref) and len_check == len( v_stacked_valid_maja): logger.info("Saved %i samples to %s.npy" % (len_check, location_name)) else: logger.error( "Inconsistent sample len between len_check=%i and len(v_stacked_valid_ref)=%i" % (len_check, len(v_stacked_valid_ref))) sys.exit(0)
def main(): # Argument parser parser = argparse.ArgumentParser() parser.add_argument("list", help="List of paths of collection") parser.add_argument("--saveto", help="subdirectory to save figs to", type=str) parser.add_argument("--hist", help="Display quicklooks with histograms", action="store_true", default=False) parser.add_argument( "--keepall", help="Display quicklooks with histograms with keep_all", action="store_true", default=False) parser.add_argument("-v", "--verbose", help="Set verbosity to DEBUG level", action="store_true", default=False) args = parser.parse_args() gain_true = 3. gain_false = 2.5 if args.keepall: l_stats = [] # Create the logger logger = utl.get_logger('Diag', args.verbose) bdef_acix = (["band02", "SRE_B2.", "R1", []], ["band03", "SRE_B3.", "R1", []], ["band04", "SRE_B4.", "R1", []], ["band05", "SRE_B5.", "R2", []], ["band06", "SRE_B6.", "R2", []], ["band07", "SRE_B7.", "R2", []], ["band08", "SRE_B8.", "R1", []], ["band8a", "SRE_B8A.", "R2", []], ["band11", "SRE_B11.", "R2", []], ["band12", "SRE_B12.", "R2", []]) f = open(args.list, 'r') paths_list = f.read().splitlines() for p in paths_list: paths = p.split(',') location_name = paths[0].split('/')[-1] acix_vermote_collection = clc.Collection(paths[0], logger) acix_maja_collection = clc.Collection(paths[1], logger) compare = cmp.Comparison(acix_vermote_collection, acix_maja_collection, logger) for match in compare.matching_products: logger.info("One-by-one for %s between %s and %s" % (match[0], match[1], match[2])) p_ref = prd.Product_hdf_acix(match[1], logger) p_maja = prd.Product_dir_maja(match[2], logger) timestamp = match[0] try: b_ref_b2 = p_ref.get_band(p_ref.find_band(bdef_acix[0][0]), scalef=p_ref.sre_scalef) b_ref_b3 = p_ref.get_band(p_ref.find_band(bdef_acix[1][0]), scalef=p_ref.sre_scalef) b_ref_b4 = p_ref.get_band(p_ref.find_band(bdef_acix[2][0]), scalef=p_ref.sre_scalef) b_ref_b8 = p_ref.get_band(p_ref.find_band(bdef_acix[6][0]), scalef=p_ref.sre_scalef) m_ref_qa = p_ref.get_band(p_ref.find_band("refqa")) b_maja_b2 = p_maja.get_band(p_maja.find_band(bdef_acix[0][1]), scalef=p_maja.sre_scalef) b_maja_b3 = p_maja.get_band(p_maja.find_band(bdef_acix[1][1]), scalef=p_maja.sre_scalef) b_maja_b4 = p_maja.get_band(p_maja.find_band(bdef_acix[2][1]), scalef=p_maja.sre_scalef) b_maja_b8 = p_maja.get_band(p_maja.find_band(bdef_acix[6][1]), scalef=p_maja.sre_scalef) b_maja_aot = p_maja.get_band(p_maja.find_band("ATB_R1"), layer=1, scalef=p_maja.aot_scalef) b_maja_vap = p_maja.get_band(p_maja.find_band("ATB_R1"), layer=0, scalef=p_maja.vap_scalef) clm = p_maja.get_band( p_maja.find_band("CLM_" + bdef_acix[0][2])) edg = p_maja.get_band( p_maja.find_band("EDG_" + bdef_acix[0][2])) m_maja_qa, ratio = p_maja.get_mask(clm, edg, stats=True) if args.hist: fig, axs = pl.subplots(nrows=3, ncols=3, figsize=[12, 12]) fig.suptitle(location_name + ' ' + timestamp[0:4] + '-' + timestamp[4:6] + '-' + timestamp[6:8], fontsize=16) cset_true = axs[0, 0].imshow(np.dstack( (b_maja_b4 * gain_true, b_maja_b3 * gain_true, b_maja_b2 * gain_true)), interpolation='none', aspect='equal') axs[0, 0].set_title("Maja quicklook (B4, B3, B2)") cset_maja_cloud_contour = axs[0, 0].contour(m_maja_qa) # axs[0, 0].clabel(cset_maja_cloud_contour, inline=1, fontsize=10) cset_maja_qa = axs[1, 0].imshow(m_maja_qa, interpolation='none', aspect='equal', vmin=0, vmax=1, cmap='gray') axs[1, 0].set_title("Maja CLM & EDG (valid=1)") divider = make_axes_locatable(axs[1, 0]) cax = divider.append_axes("right", size="5%", pad=0.05) pl.colorbar(cset_maja_qa, cax=cax) # , orientation='horizontal') # cset_ref_qa = axs[2, 0].imshow(m_ref_qa, interpolation='none', aspect='equal', vmin=0, vmax=1, cmap='gray') # axs[2, 0].set_title("Reference QA (valid=1)") # divider = make_axes_locatable(axs[2, 0]) # cax = divider.append_axes("right", size="5%", pad=0.05) # pl.colorbar(cset_ref_qa, cax=cax) # , orientation='horizontal') cset_true = axs[2, 0].imshow(np.dstack( (b_ref_b4 * gain_true, b_ref_b3 * gain_true, b_ref_b2 * gain_true)), interpolation='none', aspect='equal') axs[2, 0].set_title("Ref quicklook (B4, B3, B2)") axs[2, 0].contour(m_ref_qa) # cset_false = axs[0, 1].imshow(np.dstack((b_maja_b8*gain_false, b_maja_b3*gain_false, b_maja_b2*gain_false)), interpolation='none', aspect='equal') # axs[0, 1].set_title("%s %s (B8,B3,B2)" % (location_name, timestamp)) cset_maja_vap = axs[0, 1].imshow(b_maja_vap, interpolation='none', aspect='equal', cmap='RdBu') axs[0, 1].set_title("Maja VAP $(g.cm^{-2})$") divider = make_axes_locatable(axs[0, 1]) cax = divider.append_axes("right", size="5%", pad=0.05) pl.colorbar(cset_maja_vap, cax=cax, format='%4.2f') # , orientation='horizontal') # cset_maja_vap = axs[0, 1].imshow(np.dstack((b_maja_b4*gain_true, b_maja_b3*gain_true, b_maja_b2*gain_true)), interpolation='none', aspect='equal') # axs[0, 1].set_title("Maja VAP") # divider = make_axes_locatable(axs[0, 1]) # cax = divider.append_axes("right", size="5%", pad=0.05) # pl.colorbar(cset_maja_vap, cax=cax) # , orientation='horizontal') # cset_maja_vap_contour = axs[0, 1].contour(b_maja_vap) # axs[0, 1].clabel(cset_maja_vap_contour, inline=1, fontsize=10) cset_maja_aot = axs[0, 2].imshow(b_maja_aot, cmap='Wistia') axs[0, 2].imshow(np.dstack( (b_maja_b4 * gain_true, b_maja_b3 * gain_true, b_maja_b2 * gain_true)), interpolation='none', aspect='equal') axs[0, 2].set_title("Maja AOT $(-)$") divider = make_axes_locatable(axs[0, 2]) cax = divider.append_axes("right", size="5%", pad=0.05) pl.colorbar(cset_maja_aot, cax=cax, format='%4.2f') # , orientation='horizontal') cset_maja_aot_contour = axs[0, 2].contour(b_maja_aot, cmap='Wistia') axs[0, 2].clabel(cset_maja_aot_contour, inline=1, fontsize=10) # B2 if args.keepall: is_valid = np.where((m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = -0.1 max_sr = 0.7 is_log = False filter_label = "(QA=1)" b_ref_b2_is_valid_count = len( b_ref_b2[is_valid].flatten()) search = np.where(b_ref_b2[is_valid] < 0) b_ref_b2_is_valid_and_lt0_count = len( b_ref_b2[is_valid][search].flatten()) search = np.where(b_maja_b2[is_valid] < 0) b_maja_b2_is_valid_and_lt0_count = len( b_maja_b2[is_valid][search].flatten()) else: is_valid = np.where((b_ref_b2 > 0) & (b_maja_b2 > 0) & (m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = 0 max_sr = 1 is_log = False filter_label = "(QA=1 & sr>0)" axs[1, 1].hist(b_ref_b2[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Ref', range=(min_sr, max_sr), density=False) axs[1, 1].hist(b_maja_b2[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Maja', range=(min_sr, max_sr), density=False) axs[1, 1].set_title("B2 " + filter_label + " RMSE=%8.6f" % utl.rmse(b_ref_b2[is_valid].flatten(), b_maja_b2[is_valid].flatten())) axs[1, 1].legend() # B3 if args.keepall: is_valid = np.where((m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = -0.1 max_sr = 0.7 is_log = False filter_label = "(QA=1)" b_ref_b3_is_valid_count = len( b_ref_b3[is_valid].flatten()) search = np.where(b_ref_b3[is_valid] < 0) b_ref_b3_is_valid_and_lt0_count = len( b_ref_b3[is_valid][search].flatten()) search = np.where(b_maja_b3[is_valid] < 0) b_maja_b3_is_valid_and_lt0_count = len( b_maja_b3[is_valid][search].flatten()) else: is_valid = np.where((b_ref_b3 > 0) & (b_maja_b3 > 0) & (m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = 0 max_sr = 1 is_log = False filter_label = "(QA=1 & sr>0)" axs[1, 2].hist(b_ref_b3[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Ref', range=(min_sr, max_sr)) axs[1, 2].hist(b_maja_b3[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Maja', range=(min_sr, max_sr)) axs[1, 2].set_title("B3 " + filter_label + " RMSE=%8.6f" % utl.rmse(b_ref_b3[is_valid].flatten(), b_maja_b3[is_valid].flatten())) axs[1, 2].legend() # B4 if args.keepall: is_valid = np.where((m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = -0.1 max_sr = 0.7 is_log = False filter_label = "(QA=1)" b_ref_b4_is_valid_count = len( b_ref_b4[is_valid].flatten()) search = np.where(b_ref_b4[is_valid] < 0) b_ref_b4_is_valid_and_lt0_count = len( b_ref_b4[is_valid][search].flatten()) search = np.where(b_maja_b4[is_valid] < 0) b_maja_b4_is_valid_and_lt0_count = len( b_maja_b4[is_valid][search].flatten()) else: is_valid = np.where((b_ref_b4 > 0) & (b_maja_b4 > 0) & (m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = 0 max_sr = 1 is_log = False filter_label = "(QA=1 & sr>0)" axs[2, 1].hist(b_ref_b4[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Ref', range=(min_sr, max_sr)) axs[2, 1].hist(b_maja_b4[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Maja', range=(min_sr, max_sr)) axs[2, 1].set_title("B4 " + filter_label + " RMSE=%8.6f" % utl.rmse(b_ref_b4[is_valid].flatten(), b_maja_b4[is_valid].flatten())) axs[2, 1].legend() # B8 if args.keepall: is_valid = np.where((m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = -0.1 max_sr = 0.7 is_log = False filter_label = "(QA=1)" b_ref_b8_is_valid_count = len( b_ref_b8[is_valid].flatten()) search = np.where(b_ref_b8[is_valid] < 0) b_ref_b8_is_valid_and_lt0_count = len( b_ref_b8[is_valid][search].flatten()) search = np.where(b_maja_b8[is_valid] < 0) b_maja_b8_is_valid_and_lt0_count = len( b_maja_b8[is_valid][search].flatten()) else: is_valid = np.where((b_ref_b8 > 0) & (b_maja_b8 > 0) & (m_ref_qa == 1) & (m_maja_qa == 1)) min_sr = 0 max_sr = 1 is_log = False filter_label = "(QA=1 & sr>0)" axs[2, 2].hist(b_ref_b8[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Ref', range=(min_sr, max_sr)) axs[2, 2].hist(b_maja_b8[is_valid].flatten(), bins=200, histtype='step', log=is_log, label='Maja', range=(min_sr, max_sr)) axs[2, 2].set_title("B8 " + filter_label + " RMSE=%8.6f" % utl.rmse(b_ref_b8[is_valid].flatten(), b_maja_b8[is_valid].flatten())) axs[2, 2].legend() fig.tight_layout() fig.subplots_adjust(top=0.88) pl.savefig(location_name + '_' + timestamp + '_All_quicklooks.png') pl.close('all') if args.keepall: l_stats = l_stats + [[ location_name + timestamp, b_ref_b2_is_valid_count, b_ref_b3_is_valid_count, b_ref_b4_is_valid_count, b_ref_b8_is_valid_count, b_ref_b2_is_valid_and_lt0_count, b_ref_b3_is_valid_and_lt0_count, b_ref_b4_is_valid_and_lt0_count, b_ref_b8_is_valid_and_lt0_count, b_maja_b2_is_valid_and_lt0_count, b_maja_b3_is_valid_and_lt0_count, b_maja_b4_is_valid_and_lt0_count, b_maja_b8_is_valid_and_lt0_count ]] else: try: fig, axs = pl.subplots(figsize=[12, 12]) pl.title(location_name + ' ' + timestamp[0:4] + '-' + timestamp[4:6] + '-' + timestamp[6:8], fontsize=16) axs.imshow(np.dstack( (b_maja_b4 * gain_true, b_maja_b3 * gain_true, b_maja_b2 * gain_true)), interpolation='none', aspect='equal') axs.set_title("MAJA Quicklook (B4, B3, B2)") axs.contour(m_maja_qa) pl.savefig(location_name + '_' + timestamp + '_Maja_quicklooks.png') pl.close('all') fig, axs = pl.subplots(figsize=[12, 12]) pl.title(location_name + ' ' + timestamp[0:4] + '-' + timestamp[4:6] + '-' + timestamp[6:8], fontsize=16) axs.imshow(np.dstack( (b_ref_b4 * gain_true, b_ref_b3 * gain_true, b_ref_b2 * gain_true)), interpolation='none', aspect='equal') axs.set_title("REF Quicklook (B4, B3, B2)") axs.contour(m_ref_qa) pl.savefig(location_name + '_' + timestamp + '_Ref_quicklooks.png') pl.close('all') except: print(sys.exc_info()) except: e = sys.exc_info() logger.error(e) b_common_stats_dataset_count = 0 b_ref_stats_dataset_with_any_sr_lt0_count = 0 b_ref_stats_dataset_with_atmost_025prc_sr_lt0_count = 0 b_ref_stats_dataset_with_atmost_05prc_sr_lt0_count = 0 b_ref_stats_dataset_with_atmost_10prc_sr_lt0_count = 0 b_ref_stats_dataset_with_morethan_10prc_sr_lt0_count = 0 b_maja_stats_dataset_with_any_sr_lt0_count = 0 b_maja_stats_dataset_with_atmost_025prc_sr_lt0_count = 0 b_maja_stats_dataset_with_atmost_05prc_sr_lt0_count = 0 b_maja_stats_dataset_with_atmost_10prc_sr_lt0_count = 0 b_maja_stats_dataset_with_morethan_10prc_sr_lt0_count = 0 for l in range(len(l_stats)): b_common_stats_dataset_count += 1 if (l_stats[l][5] > 0) or (l_stats[l][6] > 0) or ( l_stats[l][7] > 0) or (l_stats[l][8] > 0): b_ref_stats_dataset_with_any_sr_lt0_count += 1 if max(l_stats[l][5], l_stats[l][6], l_stats[l][7], l_stats[l][8]) <= 20250: logger.info( "STATS REF_LT025: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][5], l_stats[l][1], l_stats[l][6], l_stats[l][2], l_stats[l][7], l_stats[l][3], l_stats[l][8], l_stats[l][4])) b_ref_stats_dataset_with_atmost_025prc_sr_lt0_count += 1 elif max(l_stats[l][5], l_stats[l][6], l_stats[l][7], l_stats[l][8]) <= 40500: logger.info( "STATS REF_LT05: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][5], l_stats[l][1], l_stats[l][6], l_stats[l][2], l_stats[l][7], l_stats[l][3], l_stats[l][8], l_stats[l][4])) b_ref_stats_dataset_with_atmost_05prc_sr_lt0_count += 1 elif max(l_stats[l][5], l_stats[l][6], l_stats[l][7], l_stats[l][8]) <= 81000: b_ref_stats_dataset_with_atmost_10prc_sr_lt0_count += 1 logger.info( "STATS REF_LT10: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][5], l_stats[l][1], l_stats[l][6], l_stats[l][2], l_stats[l][7], l_stats[l][3], l_stats[l][8], l_stats[l][4])) else: b_ref_stats_dataset_with_morethan_10prc_sr_lt0_count += 1 logger.info( "STATS REF_GT10: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][5], l_stats[l][1], l_stats[l][6], l_stats[l][2], l_stats[l][7], l_stats[l][3], l_stats[l][8], l_stats[l][4])) if (l_stats[l][9] > 0) or (l_stats[l][10] > 0) or ( l_stats[l][11] > 0) or (l_stats[l][12] > 0): b_maja_stats_dataset_with_any_sr_lt0_count += 1 if max(l_stats[l][9], l_stats[l][10], l_stats[l][11], l_stats[l][12]) <= 20250: logger.info( "STATS MAJA_LT025: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][9], l_stats[l][1], l_stats[l][10], l_stats[l][2], l_stats[l][11], l_stats[l][3], l_stats[l][12], l_stats[l][4])) b_maja_stats_dataset_with_atmost_025prc_sr_lt0_count += 1 elif max(l_stats[l][9], l_stats[l][10], l_stats[l][11], l_stats[l][12]) <= 40500: logger.info( "STATS LT05: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][9], l_stats[l][1], l_stats[l][10], l_stats[l][2], l_stats[l][11], l_stats[l][3], l_stats[l][12], l_stats[l][4])) b_maja_stats_dataset_with_atmost_05prc_sr_lt0_count += 1 elif max(l_stats[l][9], l_stats[l][10], l_stats[l][11], l_stats[l][12]) <= 81000: b_maja_stats_dataset_with_atmost_10prc_sr_lt0_count += 1 logger.info( "STATS LT10: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][9], l_stats[l][1], l_stats[l][10], l_stats[l][2], l_stats[l][11], l_stats[l][3], l_stats[l][12], l_stats[l][4])) else: b_maja_stats_dataset_with_morethan_10prc_sr_lt0_count += 1 logger.info( "STATS GT10: %s: b2_ratio=%i/%i, b3_ratio=%i/%i, b4_ratio=%i/%i, b8_ratio=%i/%i" % (l_stats[l][0], l_stats[l][9], l_stats[l][1], l_stats[l][10], l_stats[l][2], l_stats[l][11], l_stats[l][3], l_stats[l][12], l_stats[l][4])) logger.info("STATS: Tested %i location and timestamps" % b_common_stats_dataset_count) logger.info("STATS: Found %i datasets with sr_ref < 0" % b_ref_stats_dataset_with_any_sr_lt0_count) logger.info("STATS: %i datasets with at most 2.5%% of sr_ref < 0" % b_ref_stats_dataset_with_atmost_025prc_sr_lt0_count) logger.info("STATS: %i datasets with 2.5%% to 5%% of sr_ref < 0" % b_ref_stats_dataset_with_atmost_05prc_sr_lt0_count) logger.info("STATS: %i datasets with 5%% to 10%% of sr_ref < 0" % b_ref_stats_dataset_with_atmost_10prc_sr_lt0_count) logger.info("STATS: %i datasets with more than 10%% of sr_ref < 0" % b_ref_stats_dataset_with_morethan_10prc_sr_lt0_count) logger.info("STATS: Found %i datasets with sr_maja < 0" % b_maja_stats_dataset_with_any_sr_lt0_count) logger.info("STATS: %i datasets with at most 2.5%% of sr_maja < 0" % b_maja_stats_dataset_with_atmost_025prc_sr_lt0_count) logger.info("STATS: %i datasets with 2.5%% to 5%% of sr_maja < 0" % b_maja_stats_dataset_with_atmost_05prc_sr_lt0_count) logger.info("STATS: %i datasets with 5%% to 10%% of sr_maja < 0" % b_maja_stats_dataset_with_atmost_10prc_sr_lt0_count) logger.info("STATS: %i datasets with more than 10%% of sr_maja < 0" % b_maja_stats_dataset_with_morethan_10prc_sr_lt0_count) print(b_common_stats_dataset_count, b_ref_stats_dataset_with_any_sr_lt0_count, b_ref_stats_dataset_with_atmost_10prc_sr_lt0_count) print(l_stats) sys.exit(0)