def _compile_bad_assets(data):
""" Process list of 'bad' asset dictionaries from uframe; return list of bad assets.
transform into (ooi-ui-services) list of asset dictionaries.
Keys in row:
[u'purchaseAndDeliveryInfo', u'assetId', u'lastModifiedTimestamp', u'physicalInfo', u'manufactureInfo',
u'dataSource', u'remoteDocuments', u'assetInfo', u'@class', u'metaData']
Route: http://localhost:4000/uframe/assets?augmented=false
"""
bad_data = []
bad_data_ids = []
info = False # detect missing vocab items when unable to create display name(s)
feedback = False # (development/debug) display messages while processing each asset
vocab_failures = []
dict_asset_ids = {}
try:
cached = cache.get('asset_rds')
if cached:
dict_asset_ids = cached
if not cached or not isinstance(cached, dict):
# If no asset_rds cached, then fetch and cache
asset_rds = {}
try:
asset_rds, _ = _compile_asset_rds()
except Exception as err:
message = 'Error processing _compile_asset_rds: ', err.message
current_app.logger.warning(message)
if asset_rds:
cache.set('asset_rds', asset_rds, timeout=CACHE_TIMEOUT)
else:
message = 'Error in asset_rds cache update.'
current_app.logger.warning(message)
dict_asset_ids = asset_rds
except Exception as err:
message = 'Error compiling asset_rds: %s' % err.message
current_app.logger.info(message)
raise Exception(message)
# Process uframe list of asset dictionaries (data)
valid_asset_classes = ['.InstrumentAssetRecord', '.NodeAssetRecord', '.AssetRecord']
for row in data:
ref_des = ''
lat = ""
lon = ""
latest_deployment = None
has_deployment_event = False
deployment_number = ""
try:
# Get asset_id, if not asset_id then continue
row['augmented'] = False
asset_id = None
if 'assetId' in row:
row['id'] = row.pop('assetId')
asset_id = row['id']
if asset_id is None:
bad_data.append(row)
continue
if not asset_id:
bad_data.append(row)
continue
row['asset_class'] = row.pop('@class')
row['events'] = associate_events(row['id'])
if len(row['events']) == 0:
row['events'] = []
row['tense'] = None
# If ref_des not provided in row, use dictionary lookup.
if asset_id in dict_asset_ids:
ref_des = dict_asset_ids[asset_id]
# Process metadata (Note: row['metaData'] is often an empty list (especially for instruments).
# -- Process when row['metaData'] is None (add metaData if possible)
if row['metaData'] is None:
if ref_des:
row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}]
else:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# -- Process when row['metaData'] is not None
elif row['metaData'] is not None:
# If metaData provided is empty and ref_des is available manually add metaData; no ref_des then continue
if not row['metaData']:
# Manually add 'Ref Des' value to row using ref_des.
if ref_des:
row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}]
# No 'metaData' and no ref_des, continue.
else:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# Process 'metaData' provided
else:
for meta_data in row['metaData']:
if meta_data['key'] == 'Latitude':
lat = meta_data['value']
coord = convert_lat_lon(lat, "")
meta_data['value'] = coord[0]
if meta_data['key'] == 'Longitude':
lon = meta_data['value']
coord = convert_lat_lon("", lon)
meta_data['value'] = coord[1]
if meta_data['key'] == 'Deployment Number':
deployment_number = meta_data['value']
# If key 'Ref Des' has a value, use it, otherwise use ref_des value.
if meta_data['key'] == 'Ref Des':
if meta_data['value']:
ref_des = meta_data['value']
meta_data['value'] = ref_des
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# If no reference designator available, but have asset id, continue.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not ref_des or ref_des is None:
# If reference designator not provided, use lookup; if still no ref_des, continue
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# Set row values with reference designator
row['ref_des'] = ref_des
row['Ref Des'] = ref_des
if feedback: print '\n debug ---------------- (%r) ref_des: *%s*' % (asset_id, ref_des)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Get asset class based on reference designator
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not row['asset_class'] or row['asset_class'] is None:
if len(ref_des) == 27:
row['asset_class'] = '.InstrumentAssetRecord'
elif len(ref_des) == 14:
row['asset_class'] = '.NodeAssetRecord'
elif len(ref_des) == 8:
row['asset_class'] = '.AssetRecord'
else:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
else:
# Log asset class is unknown.
asset_class = row['asset_class']
if asset_class not in valid_asset_classes:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
if deployment_number is not None:
row['deployment_number'] = deployment_number
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Process events
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for events in row['events']:
if events['eventClass'] == '.DeploymentEvent':
has_deployment_event = True
if events['tense'] == 'PRESENT':
row['tense'] = events['tense']
else:
row['tense'] = 'PAST'
if latest_deployment is None and\
events['locationLonLat'] is not None and\
len(events['locationLonLat']) == 2:
latest_deployment = events['startDate']
lat = events['locationLonLat'][1]
lon = events['locationLonLat'][0]
if events['locationLonLat'] is not None and\
latest_deployment is not None and\
len(events['locationLonLat']) == 2 and\
events['startDate'] > latest_deployment:
latest_deployment = events['startDate']
lat = events['locationLonLat'][1]
lon = events['locationLonLat'][0]
row['hasDeploymentEvent'] = has_deployment_event
row['coordinates'] = convert_lat_lon(lat, lon)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Populate assetInfo dictionary
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not row['assetInfo']:
row['assetInfo'] = {
'name': '',
'type': '',
'owner': '',
'description': ''
}
# Populate assetInfo type
if row['asset_class'] == '.InstrumentAssetRecord':
row['assetInfo']['type'] = 'Sensor'
elif row['asset_class'] == '.NodeAssetRecord':
row['assetInfo']['type'] = 'Mooring'
elif row['asset_class'] == '.AssetRecord':
if len(ref_des) == 27:
row['assetInfo']['type'] = 'Sensor'
elif len(ref_des) == 14:
row['assetInfo']['type'] = 'Platform'
elif len(ref_des) == 8:
row['assetInfo']['type'] = 'Mooring'
else:
if info:
message = 'Asset id %d, type .AssetRecord, has malformed a reference designator (%s)' % \
(asset_id, ref_des)
current_app.logger.info(message)
row['assetInfo']['type'] = 'Unknown'
else:
if info:
message = 'Note ----- Unknown asset_class (%s), set to \'Unknown\'. ' % row['assetInfo']['type']
current_app.logger.info(message)
row['assetInfo']['type'] = 'Unknown'
try:
# Verify all necessary attributes are available, if not create and set to empty.
if 'name' not in row['assetInfo']:
row['assetInfo']['name'] = ''
if 'longName' not in row['assetInfo']:
row['assetInfo']['longName'] = ''
if 'array' not in row['assetInfo']:
row['assetInfo']['array'] = ''
if 'assembly' not in row['assetInfo']:
row['assetInfo']['assembly'] = ''
# Populate assetInfo - name and long name; if failure to get display name, use ref_des, log failure.
name = get_dn_by_rd(ref_des)
if name is None:
if ref_des not in vocab_failures:
vocab_failures.append(ref_des)
name = ref_des
longName = get_ldn_by_rd(ref_des)
if longName is None:
if ref_des not in vocab_failures:
vocab_failures.append(ref_des)
longName = ref_des
row['assetInfo']['name'] = name
row['assetInfo']['longName'] = longName
# Populate assetInfo - array and assembly
if len(ref_des) >= 8:
row['assetInfo']['array'] = get_dn_by_rd(ref_des[:2])
if len(ref_des) >= 14:
row['assetInfo']['assembly'] = get_dn_by_rd(ref_des[:14])
except Exception:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
except Exception:
continue
"""
print '\n debug -- len(bad_data): ', len(bad_data)
print '\n debug -- len(bad_data_ids): ', len(bad_data_ids)
"""
return bad_data
def _compile_assets(data):
""" Process list of asset dictionaries from uframe; transform into (ooi-ui-services) list of asset dictionaries.
Keys in row:
[u'purchaseAndDeliveryInfo', u'assetId', u'lastModifiedTimestamp', u'physicalInfo', u'manufactureInfo',
u'dataSource', u'remoteDocuments', u'assetInfo', u'@class', u'metaData']
Sample Event:
{'eventId': 83, 'startDate': -62135769600000, 'endDate': None, 'locationLonLat': [], 'notes': 0,
'tense': u'PRESENT', 'eventClass': u'.TagEvent'}
Metadata:
[{u'type': u'java.lang.String', u'key': u'Anchor Launch Date', u'value': u'20-Apr-14'},
{u'type': u'java.lang.String', u'key': u'Water Depth', u'value': u'0'},
{u'type': u'java.lang.String', u'key': u'Anchor Launch Time', u'value': u'18:26'},
{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': u'CE05MOAS-GL319'},
{u'type': u'java.lang.String', u'key': u'Cruise Number', u'value': u'Oceanus'},
{u'type': u'java.lang.String', u'key': u'Latitude', u'value': u"44\xb042.979' N"},
{u'type': u'java.lang.String', u'key': u'Deployment Number', u'value': u'1'},
{u'type': u'java.lang.String', u'key': u'Recover Date', u'value': u'28-May-14'},
{u'type': u'java.lang.String', u'key': u'Longitude', u'value': u"124\xb032.0615' W"}]
"""
info = False # Log missing vocab items when unable to create display name(s)
feedback = False # Display information as assets are processed
new_data = []
bad_data = []
bad_data_ids = []
vocab_failures = [] # Vocabulary failures identified during asset processing are written to log.
dict_asset_ids = {}
depth = None
try:
update_asset_rds_cache = False
cached = cache.get('asset_rds')
if cached:
dict_asset_ids = cached
if not cached or not isinstance(cached, dict):
# If no asset_rds cached, then fetch and cache
asset_rds = {}
try:
asset_rds, _ = _compile_asset_rds()
except Exception as err:
message = 'Error processing _compile_asset_rds: ', err.message
current_app.logger.warning(message)
if asset_rds:
cache.set('asset_rds', asset_rds, timeout=CACHE_TIMEOUT)
dict_asset_ids = asset_rds
except Exception as err:
message = 'Error compiling asset_rds: %s' % err.message
current_app.logger.info(message)
raise Exception(message)
# Process uframe list of asset dictionaries (data)
print '\n Compiling assets...'
valid_asset_classes = ['.InstrumentAssetRecord', '.NodeAssetRecord', '.AssetRecord']
for row in data:
ref_des = ''
lat = ""
lon = ""
latest_deployment = None
has_deployment_event = False
deployment_number = ""
try:
# Get asset_id, if not asset_id then continue
row['augmented'] = False
asset_id = None
if 'assetId' in row:
row['id'] = row.pop('assetId')
asset_id = row['id']
if asset_id is None:
bad_data.append(row)
continue
if not asset_id:
bad_data.append(row)
continue
row['asset_class'] = row.pop('@class')
row['events'] = associate_events(row['id'])
if len(row['events']) == 0:
row['events'] = []
row['tense'] = None
# If ref_des not provided in row, use dictionary lookup.
if asset_id in dict_asset_ids:
ref_des = dict_asset_ids[asset_id]
# Process metadata (Note: row['metaData'] is often an empty list (especially for instruments).
# -- Process when row['metaData'] is None (add metaData if possible)
if row['metaData'] is None:
if ref_des:
row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}]
else:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# -- Process when row['metaData'] is not None
elif row['metaData'] is not None:
# If metaData provided is empty and ref_des is available manually add metaData; no ref_des then continue
if not row['metaData']:
# Manually add 'Ref Des' value to row using ref_des.
if ref_des:
row['metaData'] = [{u'type': u'java.lang.String', u'key': u'Ref Des', u'value': ref_des}]
# No 'metaData' and no ref_des, continue.
else:
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# Process 'metaData' provided
else:
for meta_data in row['metaData']:
if meta_data['key'] == 'Latitude':
lat = meta_data['value']
coord = convert_lat_lon(lat, "")
meta_data['value'] = coord[0]
if meta_data['key'] == 'Longitude':
lon = meta_data['value']
coord = convert_lat_lon("", lon)
meta_data['value'] = coord[1]
if meta_data['key'] == 'Deployment Number':
deployment_number = meta_data['value']
if meta_data['key'] == 'Water Depth':
depth = meta_data['value']
# If key 'Ref Des' has a value, use it, otherwise use ref_des value.
if meta_data['key'] == 'Ref Des':
if meta_data['value']:
ref_des = meta_data['value']
meta_data['value'] = ref_des
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# If no reference designator available, but have asset id, continue.
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not ref_des or ref_des is None:
# If reference designator not provided, use lookup; if still no ref_des, continue
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# Set row values with reference designator
row['ref_des'] = ref_des
row['Ref Des'] = ref_des
if feedback: print '\n debug ---------------- (%r) ref_des: *%s*' % (asset_id, ref_des)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Get asset class based on reference designator
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not row['asset_class'] or row['asset_class'] is None:
if len(ref_des) == 27:
row['asset_class'] = '.InstrumentAssetRecord'
elif len(ref_des) == 14:
row['asset_class'] = '.NodeAssetRecord'
elif len(ref_des) == 8:
row['asset_class'] = '.AssetRecord'
else:
message = 'ref_des is malformed (%s), unable to determine asset_class.' % row['asset_class']
print '\n INFO: ', message
current_app.logger.info(message)
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
else:
# Log asset class as unknown.
asset_class = row['asset_class']
if asset_class not in valid_asset_classes:
if info:
message = 'Reference designator (%s) has an asset class value (%s) not one of: %s' % \
(ref_des, asset_class, valid_asset_classes)
print '\n INFO: ', message
current_app.logger.info(message)
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
if depth is not None:
row['depth'] = depth
if deployment_number is not None:
row['deployment_number'] = deployment_number
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Process events
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for events in row['events']:
if events['eventClass'] == '.DeploymentEvent':
has_deployment_event = True
if events['tense'] == 'PRESENT':
row['tense'] = events['tense']
else:
row['tense'] = 'PAST'
if latest_deployment is None and\
events['locationLonLat'] is not None and\
len(events['locationLonLat']) == 2:
latest_deployment = events['startDate']
lat = events['locationLonLat'][1]
lon = events['locationLonLat'][0]
if events['locationLonLat'] is not None and\
latest_deployment is not None and\
len(events['locationLonLat']) == 2 and\
events['startDate'] > latest_deployment:
latest_deployment = events['startDate']
lat = events['locationLonLat'][1]
lon = events['locationLonLat'][0]
row['hasDeploymentEvent'] = has_deployment_event
row['coordinates'] = convert_lat_lon(lat, lon)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Populate assetInfo dictionary
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if not row['assetInfo']:
row['assetInfo'] = {
'name': '',
'type': '',
'owner': '',
'description': ''
}
# Populate assetInfo type
if row['asset_class'] == '.InstrumentAssetRecord':
row['assetInfo']['type'] = 'Sensor'
elif row['asset_class'] == '.NodeAssetRecord':
row['assetInfo']['type'] = 'Mooring'
elif row['asset_class'] == '.AssetRecord':
if len(ref_des) == 27:
row['assetInfo']['type'] = 'Sensor'
elif len(ref_des) == 14:
row['assetInfo']['type'] = 'Platform'
elif len(ref_des) == 8:
row['assetInfo']['type'] = 'Mooring'
else:
if info:
message = 'Asset id %d, type .AssetRecord, has malformed a reference designator (%s)' % \
(asset_id, ref_des)
print '\n INFO: ', message
current_app.logger.info(message)
row['assetInfo']['type'] = 'Unknown'
else:
if info:
message = 'Note ----- Unknown asset_class (%s), set to \'Unknown\'. ' % row['assetInfo']['type']
print '\n INFO: ', message
current_app.logger.info(message)
row['assetInfo']['type'] = 'Unknown'
try:
# Verify all necessary attributes are available, if not create and set to empty.
if 'name' not in row['assetInfo']:
row['assetInfo']['name'] = ''
if 'longName' not in row['assetInfo']:
row['assetInfo']['longName'] = ''
if 'array' not in row['assetInfo']:
row['assetInfo']['array'] = ''
if 'assembly' not in row['assetInfo']:
row['assetInfo']['assembly'] = ''
# Populate assetInfo - name, if failure to get display name, use ref_des, log failure.
name = get_dn_by_rd(ref_des)
if name is None:
if ref_des not in vocab_failures:
vocab_failures.append(ref_des)
if info:
message = 'Vocab Note ----- reference designator (%s) failed to get get_dn_by_rd' % ref_des
current_app.logger.info(message)
name = ref_des
# Populate assetInfo - long name, if failure to get long name then use ref_des, log failure.
longName = get_ldn_by_rd(ref_des)
if longName is None:
if ref_des not in vocab_failures:
vocab_failures.append(ref_des)
if info:
message = 'Vocab Note ----- reference designator (%s) failed to get get_ldn_by_rd' % ref_des
current_app.logger.info(message)
longName = ref_des
row['assetInfo']['name'] = name
row['assetInfo']['longName'] = longName
# Populate assetInfo - array and assembly
if len(ref_des) >= 8:
row['assetInfo']['array'] = get_dn_by_rd(ref_des[:2])
if len(ref_des) >= 14:
row['assetInfo']['assembly'] = get_dn_by_rd(ref_des[:14])
except Exception as err:
# asset info error
current_app.logger.info('asset info error' + str(err.message))
if asset_id not in bad_data_ids:
bad_data_ids.append(asset_id)
bad_data.append(row)
continue
# Add new row to output dictionary
if asset_id and ref_des:
row['augmented'] = True
new_data.append(row)
# if new item for dictionary of asset ids, add id with value of reference designator
if asset_id not in dict_asset_ids:
dict_asset_ids[asset_id] = ref_des
update_asset_rds_cache = True
except Exception as err:
current_app.logger.info(str(err))
continue
if dict_asset_ids:
if update_asset_rds_cache:
cache.set('asset_rds', dict_asset_ids, timeout=CACHE_TIMEOUT)
# Log vocabulary failures (occur when creating display names)
if vocab_failures:
vocab_failures.sort()
message = 'These reference designator(s) are not defined, causing display name failures(%d): %s' \
% (len(vocab_failures), vocab_failures)
current_app.logger.info(message)
# Update cache for bad_asset_list
bad_assets_cached = cache.get('bad_asset_list')
if bad_assets_cached:
cache.delete('bad_asset_list')
cache.set('bad_asset_list', bad_data, timeout=CACHE_TIMEOUT)
else:
cache.set('bad_asset_list', bad_data, timeout=CACHE_TIMEOUT)
print '\n Completed compiling assets...'
return new_data, dict_asset_ids
