def compile_assets_rd(): try: asset_rds = {} with current_app.test_request_context(): print "[+] Starting asset reference designators cache reset..." cache = Cache(config={'CACHE_TYPE': 'redis', 'CACHE_REDIS_DB': 0}) cache.init_app(current_app) try: asset_rds, _ = _compile_asset_rds() except Exception as err: message = 'Error processing _compile_asset_rds: ', err.message current_app.logger.warning(message) if asset_rds: cache.set('asset_rds', asset_rds, timeout=CACHE_TIMEOUT) print "[+] Asset reference designators cache reset..." else: print "[-] Error in cache update" except Exception as err: message = 'compile_asset_rds exception: %s' % err.message current_app.logger.warning(message) raise Exception(message)
def _compile_bad_assets(data): """ Process list of 'bad' asset dictionaries from uframe; return list of bad assets. transform into (ooi-ui-services) list of asset dictionaries. Keys in row: [u'purchaseAndDeliveryInfo', u'assetId', u'lastModifiedTimestamp', u'physicalInfo', u'manufactureInfo', u'dataSource', u'remoteDocuments', u'assetInfo', u'@class', u'metaData'] Route: http://localhost:4000/uframe/assets?augmented=false """ bad_data = [] bad_data_ids = [] info = False # detect missing vocab items when unable to create display name(s) feedback = False # (development/debug) display messages while processing each asset vocab_failures = [] dict_asset_ids = {} try: cached = cache.get('asset_rds') if cached: dict_asset_ids = cached if not cached or not isinstance(cached, dict): # If no asset_rds cached, then fetch and cache asset_rds = {} try: asset_rds, _ = _compile_asset_rds() except Exception as err: message = 'Error processing _compile_asset_rds: ', err.message current_app.logger.warning(message) if asset_rds: cache.set('asset_rds', asset_rds, timeout=CACHE_TIMEOUT) else: message = 'Error in asset_rds cache update.' current_app.logger.warning(message) dict_asset_ids = asset_rds except Exception as err: message = 'Error compiling asset_rds: %s' % err.message current_app.logger.info(message) raise Exception(message) # Process uframe list of asset dictionaries (data) valid_asset_classes = ['.InstrumentAssetRecord', '.NodeAssetRecord', '.AssetRecord'] for row in data: ref_des = '' lat = "" lon = "" latest_deployment = None has_deployment_event = False deployment_number = "" try: # Get asset_id, if not asset_id then continue row['augmented'] = False asset_id = None if 'assetId' in row: row['id'] = row.pop('assetId') asset_id = row['id'] if asset_id is None: bad_data.append(row) continue if not asset_id: bad_data.append(row) continue row['asset_class'] = row.pop('@class') row['events'] = associate_events(row['id']) if len(row['events']) == 0: row['events'] = [] row['tense'] = None # If ref_des not provided in row, use dictionary lookup. if asset_id in dict_asset_ids: ref_des = dict_asset_ids[asset_id] # Process metadata (Note: row['metaData'] is often an empty list (especially for instruments). # -- Process when row['metaData'] is None (add metaData if possible) if row['metaData'] is None: if ref_des: row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}] else: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # -- Process when row['metaData'] is not None elif row['metaData'] is not None: # If metaData provided is empty and ref_des is available manually add metaData; no ref_des then continue if not row['metaData']: # Manually add 'Ref Des' value to row using ref_des. if ref_des: row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}] # No 'metaData' and no ref_des, continue. else: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # Process 'metaData' provided else: for meta_data in row['metaData']: if meta_data['key'] == 'Latitude': lat = meta_data['value'] coord = convert_lat_lon(lat, "") meta_data['value'] = coord[0] if meta_data['key'] == 'Longitude': lon = meta_data['value'] coord = convert_lat_lon("", lon) meta_data['value'] = coord[1] if meta_data['key'] == 'Deployment Number': deployment_number = meta_data['value'] # If key 'Ref Des' has a value, use it, otherwise use ref_des value. if meta_data['key'] == 'Ref Des': if meta_data['value']: ref_des = meta_data['value'] meta_data['value'] = ref_des # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # If no reference designator available, but have asset id, continue. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not ref_des or ref_des is None: # If reference designator not provided, use lookup; if still no ref_des, continue if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # Set row values with reference designator row['ref_des'] = ref_des row['Ref Des'] = ref_des if feedback: print '\n debug ---------------- (%r) ref_des: *%s*' % (asset_id, ref_des) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Get asset class based on reference designator # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not row['asset_class'] or row['asset_class'] is None: if len(ref_des) == 27: row['asset_class'] = '.InstrumentAssetRecord' elif len(ref_des) == 14: row['asset_class'] = '.NodeAssetRecord' elif len(ref_des) == 8: row['asset_class'] = '.AssetRecord' else: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue else: # Log asset class is unknown. asset_class = row['asset_class'] if asset_class not in valid_asset_classes: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue if deployment_number is not None: row['deployment_number'] = deployment_number # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Process events # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - for events in row['events']: if events['eventClass'] == '.DeploymentEvent': has_deployment_event = True if events['tense'] == 'PRESENT': row['tense'] = events['tense'] else: row['tense'] = 'PAST' if latest_deployment is None and\ events['locationLonLat'] is not None and\ len(events['locationLonLat']) == 2: latest_deployment = events['startDate'] lat = events['locationLonLat'][1] lon = events['locationLonLat'][0] if events['locationLonLat'] is not None and\ latest_deployment is not None and\ len(events['locationLonLat']) == 2 and\ events['startDate'] > latest_deployment: latest_deployment = events['startDate'] lat = events['locationLonLat'][1] lon = events['locationLonLat'][0] row['hasDeploymentEvent'] = has_deployment_event row['coordinates'] = convert_lat_lon(lat, lon) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Populate assetInfo dictionary # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not row['assetInfo']: row['assetInfo'] = { 'name': '', 'type': '', 'owner': '', 'description': '' } # Populate assetInfo type if row['asset_class'] == '.InstrumentAssetRecord': row['assetInfo']['type'] = 'Sensor' elif row['asset_class'] == '.NodeAssetRecord': row['assetInfo']['type'] = 'Mooring' elif row['asset_class'] == '.AssetRecord': if len(ref_des) == 27: row['assetInfo']['type'] = 'Sensor' elif len(ref_des) == 14: row['assetInfo']['type'] = 'Platform' elif len(ref_des) == 8: row['assetInfo']['type'] = 'Mooring' else: if info: message = 'Asset id %d, type .AssetRecord, has malformed a reference designator (%s)' % \ (asset_id, ref_des) current_app.logger.info(message) row['assetInfo']['type'] = 'Unknown' else: if info: message = 'Note ----- Unknown asset_class (%s), set to \'Unknown\'. ' % row['assetInfo']['type'] current_app.logger.info(message) row['assetInfo']['type'] = 'Unknown' try: # Verify all necessary attributes are available, if not create and set to empty. if 'name' not in row['assetInfo']: row['assetInfo']['name'] = '' if 'longName' not in row['assetInfo']: row['assetInfo']['longName'] = '' if 'array' not in row['assetInfo']: row['assetInfo']['array'] = '' if 'assembly' not in row['assetInfo']: row['assetInfo']['assembly'] = '' # Populate assetInfo - name and long name; if failure to get display name, use ref_des, log failure. name = get_dn_by_rd(ref_des) if name is None: if ref_des not in vocab_failures: vocab_failures.append(ref_des) name = ref_des longName = get_ldn_by_rd(ref_des) if longName is None: if ref_des not in vocab_failures: vocab_failures.append(ref_des) longName = ref_des row['assetInfo']['name'] = name row['assetInfo']['longName'] = longName # Populate assetInfo - array and assembly if len(ref_des) >= 8: row['assetInfo']['array'] = get_dn_by_rd(ref_des[:2]) if len(ref_des) >= 14: row['assetInfo']['assembly'] = get_dn_by_rd(ref_des[:14]) except Exception: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue except Exception: continue """ print '\n debug -- len(bad_data): ', len(bad_data) print '\n debug -- len(bad_data_ids): ', len(bad_data_ids) """ return bad_data
def _compile_assets(data): """ Process list of asset dictionaries from uframe; transform into (ooi-ui-services) list of asset dictionaries. Keys in row: [u'purchaseAndDeliveryInfo', u'assetId', u'lastModifiedTimestamp', u'physicalInfo', u'manufactureInfo', u'dataSource', u'remoteDocuments', u'assetInfo', u'@class', u'metaData'] Sample Event: {'eventId': 83, 'startDate': -62135769600000, 'endDate': None, 'locationLonLat': [], 'notes': 0, 'tense': u'PRESENT', 'eventClass': u'.TagEvent'} Metadata: [{u'type': u'java.lang.String', u'key': u'Anchor Launch Date', u'value': u'20-Apr-14'}, {u'type': u'java.lang.String', u'key': u'Water Depth', u'value': u'0'}, {u'type': u'java.lang.String', u'key': u'Anchor Launch Time', u'value': u'18:26'}, {u'type': u'java.lang.String', u'key': u'Ref Des', u'value': u'CE05MOAS-GL319'}, {u'type': u'java.lang.String', u'key': u'Cruise Number', u'value': u'Oceanus'}, {u'type': u'java.lang.String', u'key': u'Latitude', u'value': u"44\xb042.979' N"}, {u'type': u'java.lang.String', u'key': u'Deployment Number', u'value': u'1'}, {u'type': u'java.lang.String', u'key': u'Recover Date', u'value': u'28-May-14'}, {u'type': u'java.lang.String', u'key': u'Longitude', u'value': u"124\xb032.0615' W"}] """ info = False # Log missing vocab items when unable to create display name(s) feedback = False # Display information as assets are processed new_data = [] bad_data = [] bad_data_ids = [] vocab_failures = [] # Vocabulary failures identified during asset processing are written to log. dict_asset_ids = {} depth = None try: update_asset_rds_cache = False cached = cache.get('asset_rds') if cached: dict_asset_ids = cached if not cached or not isinstance(cached, dict): # If no asset_rds cached, then fetch and cache asset_rds = {} try: asset_rds, _ = _compile_asset_rds() except Exception as err: message = 'Error processing _compile_asset_rds: ', err.message current_app.logger.warning(message) if asset_rds: cache.set('asset_rds', asset_rds, timeout=CACHE_TIMEOUT) dict_asset_ids = asset_rds except Exception as err: message = 'Error compiling asset_rds: %s' % err.message current_app.logger.info(message) raise Exception(message) # Process uframe list of asset dictionaries (data) print '\n Compiling assets...' valid_asset_classes = ['.InstrumentAssetRecord', '.NodeAssetRecord', '.AssetRecord'] for row in data: ref_des = '' lat = "" lon = "" latest_deployment = None has_deployment_event = False deployment_number = "" try: # Get asset_id, if not asset_id then continue row['augmented'] = False asset_id = None if 'assetId' in row: row['id'] = row.pop('assetId') asset_id = row['id'] if asset_id is None: bad_data.append(row) continue if not asset_id: bad_data.append(row) continue row['asset_class'] = row.pop('@class') row['events'] = associate_events(row['id']) if len(row['events']) == 0: row['events'] = [] row['tense'] = None # If ref_des not provided in row, use dictionary lookup. if asset_id in dict_asset_ids: ref_des = dict_asset_ids[asset_id] # Process metadata (Note: row['metaData'] is often an empty list (especially for instruments). # -- Process when row['metaData'] is None (add metaData if possible) if row['metaData'] is None: if ref_des: row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}] else: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # -- Process when row['metaData'] is not None elif row['metaData'] is not None: # If metaData provided is empty and ref_des is available manually add metaData; no ref_des then continue if not row['metaData']: # Manually add 'Ref Des' value to row using ref_des. if ref_des: row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}] # No 'metaData' and no ref_des, continue. else: if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # Process 'metaData' provided else: for meta_data in row['metaData']: if meta_data['key'] == 'Latitude': lat = meta_data['value'] coord = convert_lat_lon(lat, "") meta_data['value'] = coord[0] if meta_data['key'] == 'Longitude': lon = meta_data['value'] coord = convert_lat_lon("", lon) meta_data['value'] = coord[1] if meta_data['key'] == 'Deployment Number': deployment_number = meta_data['value'] if meta_data['key'] == 'Water Depth': depth = meta_data['value'] # If key 'Ref Des' has a value, use it, otherwise use ref_des value. if meta_data['key'] == 'Ref Des': if meta_data['value']: ref_des = meta_data['value'] meta_data['value'] = ref_des # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # If no reference designator available, but have asset id, continue. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not ref_des or ref_des is None: # If reference designator not provided, use lookup; if still no ref_des, continue if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # Set row values with reference designator row['ref_des'] = ref_des row['Ref Des'] = ref_des if feedback: print '\n debug ---------------- (%r) ref_des: *%s*' % (asset_id, ref_des) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Get asset class based on reference designator # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not row['asset_class'] or row['asset_class'] is None: if len(ref_des) == 27: row['asset_class'] = '.InstrumentAssetRecord' elif len(ref_des) == 14: row['asset_class'] = '.NodeAssetRecord' elif len(ref_des) == 8: row['asset_class'] = '.AssetRecord' else: message = 'ref_des is malformed (%s), unable to determine asset_class.' % row['asset_class'] print '\n INFO: ', message current_app.logger.info(message) if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue else: # Log asset class as unknown. asset_class = row['asset_class'] if asset_class not in valid_asset_classes: if info: message = 'Reference designator (%s) has an asset class value (%s) not one of: %s' % \ (ref_des, asset_class, valid_asset_classes) print '\n INFO: ', message current_app.logger.info(message) if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue if depth is not None: row['depth'] = depth if deployment_number is not None: row['deployment_number'] = deployment_number # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Process events # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - for events in row['events']: if events['eventClass'] == '.DeploymentEvent': has_deployment_event = True if events['tense'] == 'PRESENT': row['tense'] = events['tense'] else: row['tense'] = 'PAST' if latest_deployment is None and\ events['locationLonLat'] is not None and\ len(events['locationLonLat']) == 2: latest_deployment = events['startDate'] lat = events['locationLonLat'][1] lon = events['locationLonLat'][0] if events['locationLonLat'] is not None and\ latest_deployment is not None and\ len(events['locationLonLat']) == 2 and\ events['startDate'] > latest_deployment: latest_deployment = events['startDate'] lat = events['locationLonLat'][1] lon = events['locationLonLat'][0] row['hasDeploymentEvent'] = has_deployment_event row['coordinates'] = convert_lat_lon(lat, lon) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Populate assetInfo dictionary # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not row['assetInfo']: row['assetInfo'] = { 'name': '', 'type': '', 'owner': '', 'description': '' } # Populate assetInfo type if row['asset_class'] == '.InstrumentAssetRecord': row['assetInfo']['type'] = 'Sensor' elif row['asset_class'] == '.NodeAssetRecord': row['assetInfo']['type'] = 'Mooring' elif row['asset_class'] == '.AssetRecord': if len(ref_des) == 27: row['assetInfo']['type'] = 'Sensor' elif len(ref_des) == 14: row['assetInfo']['type'] = 'Platform' elif len(ref_des) == 8: row['assetInfo']['type'] = 'Mooring' else: if info: message = 'Asset id %d, type .AssetRecord, has malformed a reference designator (%s)' % \ (asset_id, ref_des) print '\n INFO: ', message current_app.logger.info(message) row['assetInfo']['type'] = 'Unknown' else: if info: message = 'Note ----- Unknown asset_class (%s), set to \'Unknown\'. ' % row['assetInfo']['type'] print '\n INFO: ', message current_app.logger.info(message) row['assetInfo']['type'] = 'Unknown' try: # Verify all necessary attributes are available, if not create and set to empty. if 'name' not in row['assetInfo']: row['assetInfo']['name'] = '' if 'longName' not in row['assetInfo']: row['assetInfo']['longName'] = '' if 'array' not in row['assetInfo']: row['assetInfo']['array'] = '' if 'assembly' not in row['assetInfo']: row['assetInfo']['assembly'] = '' # Populate assetInfo - name, if failure to get display name, use ref_des, log failure. name = get_dn_by_rd(ref_des) if name is None: if ref_des not in vocab_failures: vocab_failures.append(ref_des) if info: message = 'Vocab Note ----- reference designator (%s) failed to get get_dn_by_rd' % ref_des current_app.logger.info(message) name = ref_des # Populate assetInfo - long name, if failure to get long name then use ref_des, log failure. longName = get_ldn_by_rd(ref_des) if longName is None: if ref_des not in vocab_failures: vocab_failures.append(ref_des) if info: message = 'Vocab Note ----- reference designator (%s) failed to get get_ldn_by_rd' % ref_des current_app.logger.info(message) longName = ref_des row['assetInfo']['name'] = name row['assetInfo']['longName'] = longName # Populate assetInfo - array and assembly if len(ref_des) >= 8: row['assetInfo']['array'] = get_dn_by_rd(ref_des[:2]) if len(ref_des) >= 14: row['assetInfo']['assembly'] = get_dn_by_rd(ref_des[:14]) except Exception as err: # asset info error current_app.logger.info('asset info error' + str(err.message)) if asset_id not in bad_data_ids: bad_data_ids.append(asset_id) bad_data.append(row) continue # Add new row to output dictionary if asset_id and ref_des: row['augmented'] = True new_data.append(row) # if new item for dictionary of asset ids, add id with value of reference designator if asset_id not in dict_asset_ids: dict_asset_ids[asset_id] = ref_des update_asset_rds_cache = True except Exception as err: current_app.logger.info(str(err)) continue if dict_asset_ids: if update_asset_rds_cache: cache.set('asset_rds', dict_asset_ids, timeout=CACHE_TIMEOUT) # Log vocabulary failures (occur when creating display names) if vocab_failures: vocab_failures.sort() message = 'These reference designator(s) are not defined, causing display name failures(%d): %s' \ % (len(vocab_failures), vocab_failures) current_app.logger.info(message) # Update cache for bad_asset_list bad_assets_cached = cache.get('bad_asset_list') if bad_assets_cached: cache.delete('bad_asset_list') cache.set('bad_asset_list', bad_data, timeout=CACHE_TIMEOUT) else: cache.set('bad_asset_list', bad_data, timeout=CACHE_TIMEOUT) print '\n Completed compiling assets...' return new_data, dict_asset_ids