def incoming(self, invocation):
#log.debug("ValidateInterceptor.incoming: %s", invocation)
if self.enabled:
payload = invocation.message
# If payload is IonObject, convert from dict to object for processing
if "format" in invocation.headers and isinstance(payload, dict):
clzz = invocation.headers["format"]
if is_ion_object(clzz):
payload = IonObject(clzz, payload)
#log.debug("Payload, pre-validate: %s", payload)
# IonObject _validate will throw AttributeError on validation failure.
# Raise corresponding BadRequest exception into message stack.
# Ideally the validator should pass on problems, but for now just log
# any errors and keep going, since logging and seeing invalid situations are better
# than skipping validation altogether.
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
obj._validate()
return obj
try:
walk(payload, validate_ionobj)
except AttributeError as e:
if invocation.headers.has_key("raise-exception") and invocation.headers['raise-exception']:
log.warn('message failed validation: %s\nheaders %s\npayload %s', e.message, invocation.headers, payload)
raise BadRequest(e.message)
else:
log.warn('message failed validation, but allowing it anyway: %s\nheaders %s\npayload %s', e.message, invocation.headers, payload)
return invocation
def outgoing(self, invocation):
# Set validate flag in header if IonObject(s) found in message
log.debug("ValidateInterceptor.outgoing: %s", invocation)
if self.enabled:
payload = invocation.message
log.debug("Payload, pre-validate: %s", payload)
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
invocation.headers["validate"] = True
return obj
walk(payload, validate_ionobj)
return invocation
def outgoing(self, invocation):
# Set validate flag in header if IonObject(s) found in message
log.debug("ValidateInterceptor.outgoing: %s", invocation)
if self.enabled:
payload = invocation.message
log.debug("Payload, pre-validate: %s", payload)
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
invocation.headers["validate"] = True
return obj
walk(payload, validate_ionobj)
return invocation
def incoming(self, invocation):
log.debug("ValidateInterceptor.incoming: %s", invocation)
if self.enabled:
payload = invocation.message
log.debug("Payload, pre-validate: %s", payload)
# IonObject _validate will throw AttributeError on validation failure.
# Raise corresponding BadRequest exception into message stack.
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
obj._validate()
return obj
try:
walk(payload, validate_ionobj)
except AttributeError as e:
raise BadRequest(e.message)
return invocation
def incoming(self, invocation):
log.debug("ValidateInterceptor.incoming: %s", invocation)
if self.enabled:
payload = invocation.message
log.debug("Payload, pre-validate: %s", payload)
# IonObject _validate will throw AttributeError on validation failure.
# Raise corresponding BadRequest exception into message stack.
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
obj._validate()
return obj
try:
walk(payload, validate_ionobj)
except AttributeError as e:
raise BadRequest(e.message)
return invocation
def incoming(self, invocation):
if self.enabled:
payload = invocation.message
# If payload is IonObject, convert from dict to object for processing
if "format" in invocation.headers and isinstance(payload, dict):
clzz = invocation.headers["format"]
if is_ion_object(clzz):
payload = IonObject(clzz, payload)
#log.debug("Payload, pre-validate: %s", payload)
# IonObject _validate will throw AttributeError on validation failure.
# Raise corresponding BadRequest exception into message stack.
# Ideally the validator should pass on problems, but for now just log
# any errors and keep going, since logging and seeing invalid situations are better
# than skipping validation altogether.
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
obj._validate(validate_objects=False)
return obj
try:
walk(payload, validate_ionobj)
except AttributeError as e:
raise_hdr = invocation.headers.get('raise-exception', None)
if (self.raise_exception
and raise_hdr is not False) or invocation.headers.get(
'raise-exception', None):
log.warn(
'message failed validation: %s\nheaders %s\npayload %s',
e.message, invocation.headers, payload)
raise BadRequest(e.message)
else:
log.warn(
'message failed validation, but allowing it anyway: %s\nheaders %s\npayload %s',
e.message, invocation.headers, payload)
return invocation
def incoming(self, invocation):
log.debug("ValidateInterceptor.incoming: %s", invocation)
if self.enabled:
payload = invocation.message
# If payload is IonObject, convert from dict to object for processing
if "format" in invocation.headers and isinstance(payload, dict):
clzz = invocation.headers["format"]
if is_ion_object(clzz):
payload = IonObject(clzz, payload)
log.debug("Payload, pre-validate: %s", payload)
# IonObject _validate will throw AttributeError on validation failure.
# Raise corresponding BadRequest exception into message stack.
def validate_ionobj(obj):
if isinstance(obj, IonObjectBase):
obj._validate()
return obj
try:
walk(payload, validate_ionobj)
except AttributeError as e:
import traceback
import sys
(exc_type, exc_value, exc_traceback) = sys.exc_info()
tb_list = traceback.extract_tb(sys.exc_info()[2])
tb_list = traceback.format_list(tb_list)
tb_output = ""
for elt in tb_list:
tb_output += elt
log.debug("Object validation failed. %s" % e.message)
log.debug("Traceback: %s" % str(tb_output))
raise BadRequest(e.message)
return invocation
def test_walk(self):
test_obj = None
with time_it("create"):
test_obj = create_test_object(
3, 40, do_list=True, uvals=True,
ukeys=True) # 30 for fast, 50 for slower
with time_it("deepcopy"):
o1 = copy.deepcopy(test_obj)
import simplejson, json
with time_it("simplejson.dumps"):
oj = simplejson.dumps(test_obj)
log.info(" len(json): %s", len(oj))
with time_it("json.dumps"):
oj = json.dumps(test_obj)
with time_it("simplejson.loads"):
o2 = simplejson.loads(oj)
with time_it("json.loads"):
o2 = json.loads(oj)
def unicode_to_utf8(value):
if isinstance(value, unicode):
value = str(value.encode('utf8'))
return value
from pyon.core.object import walk
with time_it("pyon.core.object.walk / unicode"):
o3 = walk(test_obj, unicode_to_utf8)
with time_it("walk1 / unicode"):
o4 = walk1(test_obj, unicode_to_utf8)
from pyon.util.containers import recursive_encode
o2 = simplejson.loads(oj)
with time_it("pyon.util.containers.recursive_utf8encode"):
recursive_encode(o2)
o2 = simplejson.loads(oj)
with time_it("recursive_utf8encode1"):
recursive_encode1(o2)
def test_walk(self):
test_obj = None
with time_it("create"):
test_obj = create_test_object(3, 40, do_list=True, uvals=True, ukeys=True) # 30 for fast, 50 for slower
with time_it("deepcopy"):
o1 = copy.deepcopy(test_obj)
import simplejson, json
with time_it("simplejson.dumps"):
oj = simplejson.dumps(test_obj)
log.info(" len(json): %s", len(oj))
with time_it("json.dumps"):
oj = json.dumps(test_obj)
with time_it("simplejson.loads"):
o2 = simplejson.loads(oj)
with time_it("json.loads"):
o2 = json.loads(oj)
def unicode_to_utf8(value):
if isinstance(value, unicode):
value = str(value.encode('utf8'))
return value
from pyon.core.object import walk
with time_it("pyon.core.object.walk / unicode"):
o3 = walk(test_obj, unicode_to_utf8)
with time_it("walk1 / unicode"):
o4 = walk1(test_obj, unicode_to_utf8)
from pyon.util.containers import recursive_encode
o2 = simplejson.loads(oj)
with time_it("pyon.util.containers.recursive_utf8encode"):
recursive_encode(o2)
o2 = simplejson.loads(oj)
with time_it("recursive_utf8encode1"):
recursive_encode1(o2)
def incoming(self, invocation):
log.debug("CodecInterceptor.incoming: %s", invocation)
payload = invocation.message
log.debug("Payload, pre-transform: %s", payload)
# Horrible, hacky workaround for msgpack issue
# See http://jira.msgpack.org/browse/MSGPACK-15
def convert_tuples_to_lists(obj):
if isinstance(obj, tuple):
res = list(obj)
return res
return obj
payload = walk(payload, convert_tuples_to_lists)
invocation.message = self._io_deserializer.deserialize(payload)
log.debug("Payload, post-transform: %s", invocation.message)
return invocation
def incoming(self, invocation):
log.debug("CodecInterceptor.incoming: %s", invocation)
payload = invocation.message
log.debug("Payload, pre-transform: %s", payload)
# Horrible, hacky workaround for msgpack issue
# See http://jira.msgpack.org/browse/MSGPACK-15
#@todo replace this with use_list in msgpack.unpackb !!!
def convert_tuples_to_lists(obj):
if isinstance(obj, tuple):
res = list(obj)
return res
return obj
payload = walk(payload, convert_tuples_to_lists)
invocation.message = self._io_deserializer.deserialize(payload)
log.debug("Payload, post-transform: %s", invocation.message)
return invocation
def map_data_product(self, data_product):
ds = {} # Catalog Dataset
ds['dataset_id'] = 'data' + data_product._id
ds['url'] = self.pydap_url + data_product._id
ds['face_page'] = self.ux_url + 'DataProduct/face/' + data_product._id
ds['title'] = data_product.name
ds['summary'] = data_product.description or data_product.name
ds['attrs'] = {}
metadata_attrs = [
i for i in self.catalog_metadata if i not in
['name', 'synonyms', 'iso_topic_category', 'reference_urls']
]
for attr in metadata_attrs:
self.slam(ds['attrs'], data_product, attr)
if data_product.synonyms:
ds['attrs']['synonyms'] = ','.join(data_product.synonyms)
if data_product.iso_topic_category:
ds['attrs']['iso_topic_category'] = ','.join(
data_product.iso_topic_category)
if data_product.reference_urls:
ds['attrs']['reference_urls'] = '\n'.join(
data_product.reference_urls)
# Grab the parameters
stream_def_id = self.resource_registry.find_objects(
data_product._id, PRED.hasStreamDefinition, id_only=True)[0][0]
pdict_id = self.resource_registry.find_objects(
stream_def_id, PRED.hasParameterDictionary, id_only=True)[0][0]
parameter_contexts, _ = self.resource_registry.find_objects(
pdict_id, PRED.hasParameterContext, id_only=False)
ds['vars'] = []
for param in parameter_contexts:
# Handle the placeholder variables
if re.match(r'.*_[a-z0-9]{32}', param.name):
continue # Let's not do this
var = {}
var['name'] = param.name
var['attrs'] = {}
attrs = var['attrs']
attrs['units'] = param.units or '1'
attrs['ioos_category'] = self.get_ioos_category(
param.name, attrs['units'])
attrs['long_name'] = param.display_name
if param.standard_name:
attrs['standard_name'] = param.standard_name
if 'seconds' in attrs['units'] and 'since' in attrs['units']:
attrs['time_precision'] = '1970-01-01T00:00:00.000Z'
if param.ooi_short_name:
sname = param.ooi_short_name
sname = re.sub('[\t\n ]+', ' ', sname)
attrs['ooi_short_name'] = sname
m = re.match(r'[A-Z0-9]{7}', sname)
if m:
reference_url = 'https://confluence.oceanobservatories.org/display/instruments/' + m.group(
)
attrs['references'] = reference_url
if 'L2' in param.ooi_short_name:
attrs['data_product_level'] = 'L2'
attrs['source'] = 'level 2 calibrated sensor observation'
elif 'L1' in param.ooi_short_name:
attrs['data_product_level'] = 'L1'
attrs['source'] = 'level 1 calibrated sensor observation'
elif 'L0' in param.ooi_short_name:
attrs['data_product_level'] = 'L0'
attrs['source'] = 'level 0 calibrated sensor observation'
elif 'QC' in param.ooi_short_name:
attrs['data_product_level'] = 'QC'
elif param.parameter_type != 'function':
if attrs['units'] == 'counts':
attrs['data_product_level'] = 'L0'
attrs['source'] = 'sensor observation'
elif 'seconds' in attrs['units'] and 'since' in attrs['units']:
attrs['data_product_level'] = 'axis'
elif var['name'].lower() in ('time', 'lat', 'lon', 'latitude',
'longitude'):
attrs['data_product_level'] = 'axis'
else:
attrs['data_product_level'] = 'unknown'
if param.reference_urls:
attrs['instrument_type'] = '\n'.join(param.reference_urls)
if param.parameter_type == 'function':
parameter_function = self.resource_registry.read(
param.parameter_function_id)
if parameter_function.function_type == PFT.PYTHON:
attrs['function_module'] = parameter_function.owner or ''
attrs['function_name'] = parameter_function.function or ''
if attrs['function_module'].startswith('ion_functions'):
s = attrs['function_module']
url = s.replace('.', '/') + '.py'
url = 'https://github.com/ooici/ion-functions/blob/master/' + url
attrs['function_url'] = url
elif parameter_function.egg_uri:
attrs['function_url'] = parameter_function.egg_uri
elif parameter_function.function_type == PFT.NUMEXPR:
attrs['function_name'] = parameter_function.name
attrs['expression'] = parameter_function.function
for k, v in param.additional_metadata.iteritems():
if re.match(r'[a-z][a-zA-Z0-9_]+', k):
attrs[k] = v
ds['vars'].append(var)
ds = walk(ds, self.xml_escape)
return ds
def test_walk(self):
t1 = time.time()
test_obj = create_test_object(3, 40, do_list=True, uvals=True, ukeys=True) # 30 for fast, 50 for slower
t2 = time.time()
log.info("Time create: %s", (t2-t1))
t1 = time.time()
o1 = copy.deepcopy(test_obj)
t2 = time.time()
log.info("Time deepcopy: %s", (t2-t1))
import simplejson
t1 = time.time()
oj = simplejson.dumps(test_obj)
t2 = time.time()
log.info("Time simplejson.dumps: %s", (t2-t1))
log.info(" len(json): %s", len(oj))
import json
t1 = time.time()
oj = json.dumps(test_obj)
t2 = time.time()
log.info("Time json.dumps: %s", (t2-t1))
t1 = time.time()
o2 = simplejson.loads(oj)
t2 = time.time()
log.info("Time simplejson.loads: %s", (t2-t1))
t1 = time.time()
o2 = json.loads(oj)
t2 = time.time()
log.info("Time json.loads: %s", (t2-t1))
def unicode_to_utf8(value):
if isinstance(value, unicode):
value = str(value.encode('utf8'))
return value
from pyon.core.object import walk
t1 = time.time()
o3 = walk(test_obj, unicode_to_utf8)
t2 = time.time()
log.info("Time pyon.core.object.walk / unicode: %s", (t2-t1))
t1 = time.time()
o4 = walk1(test_obj, unicode_to_utf8)
t2 = time.time()
log.info("Time walk1 / unicode: %s", (t2-t1))
from pyon.util.containers import recursive_encode
o2 = simplejson.loads(oj)
t1 = time.time()
recursive_encode(o2)
t2 = time.time()
log.info("Time pyon.util.containers.recursive_utf8encode: %s", (t2-t1))
o2 = simplejson.loads(oj)
t1 = time.time()
recursive_encode1(o2)
t2 = time.time()
log.info("Time recursive_utf8encode1: %s", (t2-t1))
def map_data_product(self, data_product):
ds = {} # Catalog Dataset
ds['dataset_id'] = 'data' + data_product._id
ds['url'] = self.pydap_url + data_product._id
ds['face_page'] = self.ux_url + 'DataProduct/face/' + data_product._id
ds['title'] = data_product.name
ds['summary'] = data_product.description or data_product.name
ds['attrs'] = {}
metadata_attrs = [ i for i in self.catalog_metadata if i not in ['name', 'synonyms', 'iso_topic_category', 'reference_urls'] ]
for attr in metadata_attrs:
self.slam(ds['attrs'], data_product, attr)
if data_product.synonyms:
ds['attrs']['synonyms'] = ','.join(data_product.synonyms)
if data_product.iso_topic_category:
ds['attrs']['iso_topic_category'] = ','.join(data_product.iso_topic_category)
if data_product.reference_urls:
ds['attrs']['reference_urls'] = '\n'.join(data_product.reference_urls)
# Grab the parameters
stream_def_id = self.resource_registry.find_objects(data_product._id, PRED.hasStreamDefinition, id_only=True)[0][0]
pdict_id = self.resource_registry.find_objects(stream_def_id, PRED.hasParameterDictionary, id_only=True)[0][0]
parameter_contexts, _ = self.resource_registry.find_objects(pdict_id, PRED.hasParameterContext, id_only=False)
ds['vars'] = []
for param in parameter_contexts:
# Handle the placeholder variables
if re.match(r'.*_[a-z0-9]{32}', param.name):
continue # Let's not do this
var = {}
var['name'] = param.name
var['attrs'] = {}
attrs = var['attrs']
attrs['units'] = param.units or '1'
attrs['ioos_category'] = self.get_ioos_category(param.name, attrs['units'])
attrs['long_name'] = param.display_name
if param.standard_name:
attrs['standard_name'] = param.standard_name
if 'seconds' in attrs['units'] and 'since' in attrs['units']:
attrs['time_precision'] = '1970-01-01T00:00:00.000Z'
if param.ooi_short_name:
sname = param.ooi_short_name
sname = re.sub('[\t\n ]+', ' ', sname)
attrs['ooi_short_name'] = sname
m = re.match(r'[A-Z0-9]{7}', sname)
if m:
reference_url = 'https://confluence.oceanobservatories.org/display/instruments/' + m.group()
attrs['references'] = reference_url
if 'L2' in param.ooi_short_name:
attrs['data_product_level'] = 'L2'
attrs['source'] = 'level 2 calibrated sensor observation'
elif 'L1' in param.ooi_short_name:
attrs['data_product_level'] = 'L1'
attrs['source'] = 'level 1 calibrated sensor observation'
elif 'L0' in param.ooi_short_name:
attrs['data_product_level'] = 'L0'
attrs['source'] = 'level 0 calibrated sensor observation'
elif 'QC' in param.ooi_short_name:
attrs['data_product_level'] = 'QC'
elif param.parameter_type != 'function':
if attrs['units'] == 'counts':
attrs['data_product_level'] = 'L0'
attrs['source'] = 'sensor observation'
elif 'seconds' in attrs['units'] and 'since' in attrs['units']:
attrs['data_product_level'] = 'axis'
elif var['name'].lower() in ('time', 'lat', 'lon', 'latitude', 'longitude'):
attrs['data_product_level'] = 'axis'
else:
attrs['data_product_level'] = 'unknown'
if param.reference_urls:
attrs['instrument_type'] = '\n'.join(param.reference_urls)
if param.parameter_type == 'function':
parameter_function = self.resource_registry.read(param.parameter_function_id)
if parameter_function.function_type == PFT.PYTHON:
attrs['function_module'] = parameter_function.owner or ''
attrs['function_name'] = parameter_function.function or ''
if attrs['function_module'].startswith('ion_functions'):
s = attrs['function_module']
url = s.replace('.','/') + '.py'
url = 'https://github.com/ooici/ion-functions/blob/master/' + url
attrs['function_url'] = url
elif parameter_function.egg_uri:
attrs['function_url'] = parameter_function.egg_uri
elif parameter_function.function_type == PFT.NUMEXPR:
attrs['function_name'] = parameter_function.name
attrs['expression'] = parameter_function.function
for k,v in param.additional_metadata:
if re.match(r'[a-z][a-zA-Z0-9_]+', k):
attrs[k] = v
ds['vars'].append(var)
ds = walk(ds, self.xml_escape)
return ds
def recursive_encoding(value):
value = walk(value, unicode_to_utf8, 'key_value')
return value