def getSmapMessage(self, nodeid, attrid, reading):
"""
This grabs the metadata description from the json smap manifest and performs
all the inheritance and transformations to create valid sMAP metadata for the
source. It then adds a new timeseries to this source (if one does not already exist)
and constructs an attached actuator if the source needs one.
Returns the path and the constructed sMAP object.
This method is idempotent in regards to construction actuators and timeseries
"""
obj = self.smapManifest.get(nodeid,None).copy()
if not obj:
return {}
inherit = obj.pop('INHERIT') if 'INHERIT' in obj else []
for section in inherit:
obj = update(obj, self.smapManifest.get(section, {}))
attrs = obj.pop('ATTRS') if 'ATTRS' in obj else {}
upd = attrs.get(attrid, {}).copy()
path = str(upd.pop('Path'))
actuator = upd.pop('ACTUATOR') if 'ACTUATOR' in upd else ''
obj = update(obj, upd)
print path, self.addedTimeseries
if path not in self.addedTimeseries:
self.addedTimeseries.add(path)
print 'GETTING MD for', path
svcid, desc = self.getDescription(attrid)
print desc
ts = self.add_timeseries(path, desc['format'][0][1], data_type='double')
if actuator == 'binary':
ts.add_actuator(OnOffActuator(stormIP=PREFIX+nodeid, svcID=svcid, attrID=attrid, archiver=self.archiver))
print dict(util.buildkv('', obj))
self.set_metadata(path, dict(util.buildkv('', obj)))
return path, {path: obj}
def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = {u'Path': path}
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
if not (util.is_string(name) and util.is_string(val)):
raise SmapException(
'Invalid metadata pair: "%s" -> "%s"' %
(str(name), str(val)), 400)
tags[name] = val
query = "UPDATE stream SET metadata = metadata || %s " \
" WHERE uuid = %s "
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query, (
tags,
ts['uuid'],
))
logging.getLogger('stats').info("Metadata insert took %0.6fs" %
(time.time() - tic))
def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = ["hstore('Path', %s)" % escape_string(path)]
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
name, val = escape_string(name), escape_string(str(val))
if not (util.is_string(name) and util.is_string(val)):
raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name),
str(val)),
400)
tags.append("hstore(%s, %s)" % (name, val))
query = "UPDATE stream SET metadata = metadata || " + " || ".join(tags) + \
" WHERE uuid = %s" % escape_string(ts['uuid'])
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query)
logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def start_processing(self, data):
"""data: a list with two elements: the first is the metadata,
and the second is the stream information we will need to fetch
the actual data"""
# save the metadata and streamids for loading
opmeta = data[0][1]
opmeta = map(lambda x: dict(util.buildkv("", x)), opmeta)
if not len(opmeta):
self.consumer.write(json.dumps([]))
self.consumer.unregisterProducer()
self.consumer.finish()
return
# sort the streamids to be in the same order as the operator inputs
meta_uid_order = dict(zip(map(operator.itemgetter("uuid"), opmeta), xrange(0, len(opmeta))))
self.streamids = data[1][1]
self.streamids.sort(key=lambda elt: meta_uid_order[elt[0]])
# use a heuristic for how much data we want to load at once...
self.chunk_length = (3600 * 24 * self.DATA_DAYS) / len(self.streamids)
if self.chunk_length < 300:
self.chunk_length = 300
# build the operator
if self.group and len(self.group):
self.op = grouping.GroupByTagOperator(opmeta, self.group, self.op)
else:
self.op = self.op(opmeta)
for o in self.op.outputs:
if not "Metadata/Extra/Operator" in o:
o["Metadata/Extra/Operator"] = str(self.op)
self.resumeProducing()
def start_processing(self, data):
"""data: a list with two elements: the first is the metadata,
and the second is the stream information we will need to fetch
the actual data"""
# save the metadata and streamids for loading
opmeta = data[0][1]
opmeta = map(lambda x: dict(util.buildkv('', x)), opmeta)
if not len(opmeta):
self.consumer.write([])
self.consumer.unregisterProducer()
self.consumer.finish()
return
# sort the streamids to be in the same order as the operator inputs
meta_uid_order = dict(
zip(map(operator.itemgetter('uuid'), opmeta),
xrange(0, len(opmeta))))
self.streamids = data[1][1]
self.streamids.sort(key=lambda elt: meta_uid_order[elt[0]])
# use a heuristic for how much data we want to load at once...
self.chunk_length = (3600 * 24 * self.DATA_DAYS) / len(self.streamids)
if self.chunk_length < 300:
self.chunk_length = 300
# build the operator
if self.group and len(self.group):
self.op = grouping.GroupByTagOperator(opmeta, self.group, self.op)
else:
self.op = self.op(opmeta)
for o in self.op.outputs:
if not 'Metadata/Extra/Operator' in o:
o['Metadata/Extra/Operator'] = str(self.op)
self.resumeProducing()
def tags(self, where, tags='*', nest=False, asdict=False):
"""Look up tags associated with a specific query body"""
log.msg(where)
tags = self.query('select %s where %s' % (tags, where))
if not nest:
tags = map(lambda t: dict(util.buildkv('', t)), tags)
if asdict:
tags = dict([(x['uuid'], x) for x in tags])
return tags
def tags(self, where, tags='*', nest=False, asdict=False):
"""Look up tags associated with a specific query body"""
log.msg(where)
tags = self.query('select %s where %s' % (tags, where))
if not nest:
tags = map(lambda t: dict(util.buildkv('', t)), tags)
if asdict:
tags = dict([(x['uuid'], x) for x in tags])
return tags
def json2capnp(jsonobj):
"""
Expecting a sMAP report json object where the toplevel keys are paths
and the toplevel values are the usual sMAP objects, e.g.
{
"/fast": {
"Contents": [
"sensor0"
]
},
"/fast/sensor0": {
"Properties": {
"ReadingType": "long",
...
},
"Metadata": {
"Site": "Test Site",
...
},
"Readings": [[9182731928374, 30]],
"uuid": "b86df176-6b40-5d58-8f29-3b85f5cfbf1e"
}
}
"""
messages = []
for path, contents in jsonobj.iteritems():
msg = smap_capnp.SmapMessage.new_message()
msg.path = path
msg.uuid = bytes(contents.get('uuid'))
if contents.get('Contents'):
msg_contents = msg.init('contents', len(contents.get('Contents')))
for i, item in enumerate(contents.get('Contents')):
msg_contents[i] = item
if contents.get('Readings'):
msg_readings = msg.init('readings', len(contents.get('Readings')))
for i, item in enumerate(contents.get('Readings')):
msg_readings[i] = smap_capnp.SmapMessage.Reading.new_message(
time=item[0], data=item[1])
if contents.get('Properties'):
msg_properties = msg.init('properties',
len(contents.get('Properties')))
for i, kv in enumerate(contents.get('Properties').iteritems()):
msg_properties[i] = smap_capnp.SmapMessage.Pair.new_message(
key=kv[0], value=kv[1])
if contents.get('Metadata'):
md = buildkv('', contents.get('Metadata'))
msg_metadata = msg.init('metadata', len(md))
for i, kv in enumerate(md):
msg_metadata[i] = smap_capnp.SmapMessage.Pair.new_message(
key=kv[0], value=kv[1])
messages.append(msg)
return messages
def getSmapMessage(self, nodeid, attrid, reading):
"""
This grabs the metadata description from the json smap manifest and performs
all the inheritance and transformations to create valid sMAP metadata for the
source. It then adds a new timeseries to this source (if one does not already exist)
and constructs an attached actuator if the source needs one.
Returns the path and the constructed sMAP object.
This method is idempotent in regards to construction actuators and timeseries
"""
if nodeid not in self.smapManifest: return '', {}
obj = self.smapManifest.get(nodeid, None).copy()
if not obj:
return '', {}
inherit = obj.pop('INHERIT') if 'INHERIT' in obj else []
for section in inherit:
obj = update(obj, self.smapManifest.get(section, {}))
attrs = obj.pop('ATTRS') if 'ATTRS' in obj else {}
upd = attrs.get(attrid, {}).copy()
if 'Path' not in upd: return '', {}
path = str(upd.pop('Path'))
actuator = upd.pop('ACTUATOR') if 'ACTUATOR' in upd else ''
obj = update(obj, upd)
print path, self.addedTimeseries
if path and path not in self.addedTimeseries:
self.addedTimeseries.add(path)
print 'GETTING MD for', path
svcid, desc = self.getDescription(attrid)
print desc
ts = self.add_timeseries(path,
desc['format'][0][1],
data_type='double')
if actuator == 'binary':
ts.add_actuator(
OnOffActuator(stormIP=PREFIX + nodeid,
svcID=svcid,
attrID=attrid,
archiver=self.archiver))
print dict(util.buildkv('', obj))
self.set_metadata(path, dict(util.buildkv('', obj)))
return path, {path: obj}
def start_processing(self, data):
"""data: a list with two elements: the first is the metadata,
and the second is the stream information we will need to fetch
the actual data"""
# save the metadata and streamids for loading
opmeta = data[0][1]
opmeta = map(lambda x: dict(util.buildkv('', x)), opmeta)
if not len(opmeta):
self.consumer.write([])
self.consumer.unregisterProducer()
self.consumer.finish()
return
# sort the streamids to be in the same order as the operator inputs
meta_uid_order = dict(zip(map(operator.itemgetter('uuid'), opmeta),
xrange(0, len(opmeta))))
self.streamids = data[1][1]
self.streamids.sort(key=lambda elt: meta_uid_order[elt[0]])
# use a heuristic for how much data we want to load at once...
self.chunk_length = (3600 * 24 * self.DATA_DAYS) / len(self.streamids)
if self.chunk_length < 300:
self.chunk_length = 300
# build the operator
if self.group and len(self.group):
self.op = grouping.GroupByTagOperator(opmeta,
self.group,
self.op)
else:
self.op = self.op(opmeta)
for o in self.op.outputs:
if not 'Metadata/Extra/Operator' in o:
o['Metadata/Extra/Operator'] = str(self.op)
self.sketch = None
if settings.conf['features']['sketches']:
# ask the operator expression for a sketch name that describe
# it -- eg, ('mean', 300). If that's in our list of supported
# sketches (by the backend), we can nullify that operator and
# fetch the sketch instead.
sketch = self.op.sketch()
print "INITAL SKETCH POTENTIAL", sketch
if sketch and sketch[1] in SUPPORTED_SKETCHES:
node, self.sketch = sketch
node.nullify() # convert the sketch ast node to a no-op
name, size = self.sketch
# try to load things in readingdb-friendly 10000-point chunks
self.chunk_length = size * 9990
print "Final sketch:", self.sketch
self.resumeProducing()
def start_processing(self, data):
"""data: a list with two elements: the first is the metadata,
and the second is the stream information we will need to fetch
the actual data"""
# save the metadata and streamids for loading
opmeta = data[0][1]
opmeta = map(lambda x: dict(util.buildkv('', x)), opmeta)
if not len(opmeta):
self.consumer.write([])
self.consumer.unregisterProducer()
self.consumer.finish()
return
# sort the streamids to be in the same order as the operator inputs
meta_uid_order = dict(
zip(map(operator.itemgetter('uuid'), opmeta),
xrange(0, len(opmeta))))
self.streamids = data[1][1]
self.streamids.sort(key=lambda elt: meta_uid_order[elt[0]])
# use a heuristic for how much data we want to load at once...
self.chunk_length = (3600 * 24 * self.DATA_DAYS) / len(self.streamids)
if self.chunk_length < 300:
self.chunk_length = 300
# build the operator
if self.group and len(self.group):
self.op = grouping.GroupByTagOperator(opmeta, self.group, self.op)
else:
self.op = self.op(opmeta)
for o in self.op.outputs:
if not 'Metadata/Extra/Operator' in o:
o['Metadata/Extra/Operator'] = str(self.op)
self.sketch = None
if settings.conf['features']['sketches']:
# ask the operator expression for a sketch name that describe
# it -- eg, ('mean', 300). If that's in our list of supported
# sketches (by the backend), we can nullify that operator and
# fetch the sketch instead.
sketch = self.op.sketch()
if sketch and sketch[1] in SUPPORTED_SKETCHES:
node, self.sketch = sketch
node.nullify() # convert the sketch ast node to a no-op
name, size = self.sketch
# try to load things in readingdb-friendly 10000-point chunks
self.chunk_length = size * 9990
print "Final sketch:", self.sketch
self.resumeProducing()
def json2capnp(jsonobj):
"""
Expecting a sMAP report json object where the toplevel keys are paths
and the toplevel values are the usual sMAP objects, e.g.
{
"/fast": {
"Contents": [
"sensor0"
]
},
"/fast/sensor0": {
"Properties": {
"ReadingType": "long",
...
},
"Metadata": {
"Site": "Test Site",
...
},
"Readings": [[9182731928374, 30]],
"uuid": "b86df176-6b40-5d58-8f29-3b85f5cfbf1e"
}
}
"""
messages = []
for path, contents in jsonobj.iteritems():
msg = smap_capnp.SmapMessage.new_message()
msg.path = path
msg.uuid = bytes(contents.get('uuid'))
if contents.get('Contents'):
msg_contents = msg.init('contents', len(contents.get('Contents')))
for i,item in enumerate(contents.get('Contents')):
msg_contents[i] = item
if contents.get('Readings'):
msg_readings = msg.init('readings', len(contents.get('Readings')))
for i, item in enumerate(contents.get('Readings')):
msg_readings[i] = smap_capnp.SmapMessage.Reading.new_message(time= item[0], data= item[1])
if contents.get('Properties'):
msg_properties = msg.init('properties', len(contents.get('Properties')))
for i, kv in enumerate(contents.get('Properties').iteritems()):
msg_properties[i] = smap_capnp.SmapMessage.Pair.new_message(key = kv[0], value = kv[1])
if contents.get('Metadata'):
md = buildkv('',contents.get('Metadata'))
msg_metadata = msg.init('metadata', len(md))
for i, kv in enumerate(md):
msg_metadata[i] = smap_capnp.SmapMessage.Pair.new_message(key = kv[0], value = kv[1])
messages.append(msg)
return messages
def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = {u'Path': path}
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
if not (util.is_string(name) and util.is_string(val)):
raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name),
str(val)),
400)
tags[name] = val
query = "UPDATE stream SET metadata = metadata || %s " \
" WHERE uuid = %s "
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query, (tags, ts['uuid'],))
logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def setup(self, opts):
self.interval = float(opts.get('interval', 60))
self.set_metadata('/',
{'Instrument/SamplingPeriod': str(self.interval)})
self.add_collection('/actuators')
publish_address = opts.get('publish_address', default_publish_address)
PublishMixin._setup(self, publish_address)
self.interface = self.get_interface(opts)
self.all_path_depth, self.all_path_breadth = self.get_paths_for_point(
'/' + DRIVER_TOPIC_ALL)
self.meta_data = {}
c = self.get_collection('/')
#Flatten the meta data dict
kv = buildkv('', c['Metadata'])
self.meta_data[c.path] = dict(kv)
for point in self.interface.get_register_names():
register = self.interface.get_register_by_name(point)
if register.register_type == 'bit':
data_type = 'long'
else:
if register.python_type is int:
data_type = 'long'
elif register.python_type is float:
data_type = 'double'
else:
raise ValueError(
'sMAP currently only supports int and float based data types.'
)
ts = self.add_timeseries('/' + point,
register.units,
data_type=data_type,
description=register.description)
pd = ts['Properties']
ts_type = pd['ReadingType']
if ts_type == 'double':
ts_type = 'float'
elif ts_type == 'long':
ts_type = 'integer'
self.meta_data[point] = {
'units': pd['UnitofMeasure'],
'type': ts_type,
'tz': pd['Timezone']
}
for register in self.interface.get_registers_by_type('bit', False):
point = register.point_name
actuator_point = '/actuators/' + point
print 'Setting up actuator point:', actuator_point
a = self.add_actuator(actuator_point,
register.units,
InterfaceBitActuator,
setup={
'point_name': point,
'interface': self.interface
}) #, read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is None:
print("ERROR: Failed to read " + actuator_point +
" interface returned None")
else:
self.add(actuator_point, value)
for register in self.interface.get_registers_by_type('byte', False):
point = register.point_name
actuator_point = '/actuators/' + point
print 'Setting up actuator point:', actuator_point
if register.python_type is int:
act_class, data_type = (InterfaceIntActuator, 'long')
elif register.python_type is float:
act_class, data_type = (InterfaceFloatActuator, 'double')
else:
raise ValueError(
'sMAP currently only supports int and float based data types.'
)
a = self.add_actuator(
actuator_point,
register.units,
act_class,
setup={
'point_name': point,
'interface': self.interface
},
data_type=data_type) #, read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is not None:
self.add(actuator_point, value)
else:
print("ERROR: Failed to read " + actuator_point +
" interface returned None")
def make_dns_meta(inst):
root = inst.lookup("/")
m = {"uuid": str(inst.root_uuid)}
if root and "Metadata" in root:
m.update(dict(buildkv("Metadata", root["Metadata"])))
return m
def setup(self, opts):
self.interval = float(opts.get("interval", 60))
self.set_metadata("/", {"Instrument/SamplingPeriod": str(self.interval)})
self.add_collection("/actuators")
publish_address = opts.get("publish_address", default_publish_address)
PublishMixin._setup(self, publish_address)
self.interface = self.get_interface(opts)
self.all_path_depth, self.all_path_breadth = self.get_paths_for_point("/" + DRIVER_TOPIC_ALL)
self.meta_data = {}
c = self.get_collection("/")
# Flatten the meta data dict
kv = buildkv("", c["Metadata"])
self.meta_data[c.path] = dict(kv)
for point in self.interface.get_register_names():
register = self.interface.get_register_by_name(point)
if register.register_type == "bit":
data_type = "long"
else:
if register.python_type is int:
data_type = "long"
elif register.python_type is float:
data_type = "double"
else:
raise ValueError("sMAP currently only supports int and float based data types.")
ts = self.add_timeseries("/" + point, register.units, data_type=data_type, description=register.description)
pd = ts["Properties"]
ts_type = pd["ReadingType"]
if ts_type == "double":
ts_type = "float"
elif ts_type == "long":
ts_type = "integer"
self.meta_data[point] = {"units": pd["UnitofMeasure"], "type": ts_type, "tz": pd["Timezone"]}
for register in self.interface.get_registers_by_type("bit", False):
point = register.point_name
actuator_point = "/actuators/" + point
print "Setting up actuator point:", actuator_point
a = self.add_actuator(
actuator_point,
register.units,
InterfaceBitActuator,
setup={"point_name": point, "interface": self.interface},
) # , read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is None:
print ("ERROR: Failed to read " + actuator_point + " interface returned None")
else:
self.add(actuator_point, value)
for register in self.interface.get_registers_by_type("byte", False):
point = register.point_name
actuator_point = "/actuators/" + point
print "Setting up actuator point:", actuator_point
if register.python_type is int:
act_class, data_type = (InterfaceIntActuator, "long")
elif register.python_type is float:
act_class, data_type = (InterfaceFloatActuator, "double")
else:
raise ValueError("sMAP currently only supports int and float based data types.")
a = self.add_actuator(
actuator_point,
register.units,
act_class,
setup={"point_name": point, "interface": self.interface},
data_type=data_type,
) # , read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is not None:
self.add(actuator_point, value)
else:
print ("ERROR: Failed to read " + actuator_point + " interface returned None")
def setup(self, opts):
self.interval = float(opts.get('interval',60))
self.set_metadata('/', {'Instrument/SamplingPeriod' : str(self.interval)})
self.add_collection('/actuators')
publish_address = opts.get('publish_address', default_publish_address)
PublishMixin._setup(self, publish_address)
self.interface = self.get_interface(opts)
self.all_path_depth, self.all_path_breadth = self.get_paths_for_point('/'+DRIVER_TOPIC_ALL)
self.meta_data = {}
c = self.get_collection('/')
#Flatten the meta data dict
kv = buildkv('', c['Metadata'])
self.meta_data[c.path] = dict(kv)
for point in self.interface.get_register_names():
register = self.interface.get_register_by_name(point)
if register.register_type == 'bit':
data_type = 'long'
else:
if register.python_type is int:
data_type = 'long'
elif register.python_type is float:
data_type = 'double'
else:
raise ValueError('sMAP currently only supports int and float based data types.')
ts = self.add_timeseries('/'+point, register.units, data_type=data_type, description=register.description)
pd = ts['Properties']
ts_type = pd['ReadingType']
if ts_type == 'double':
ts_type = 'float'
elif ts_type == 'long':
ts_type = 'integer'
self.meta_data[point] = {'units': pd['UnitofMeasure'],
'type': ts_type,
'tz': pd['Timezone']}
for register in self.interface.get_registers_by_type('bit',False):
point = register.point_name
actuator_point = '/actuators/'+point
print 'Setting up actuator point:', actuator_point
a = self.add_actuator(actuator_point, register.units, InterfaceBitActuator,
setup={'point_name':point, 'interface': self.interface}) #, read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is None:
print("ERROR: Failed to read " + actuator_point + " interface returned None")
else:
self.add(actuator_point, value)
for register in self.interface.get_registers_by_type('byte',False):
point = register.point_name
actuator_point = '/actuators/'+point
print 'Setting up actuator point:', actuator_point
if register.python_type is int:
act_class, data_type = (InterfaceIntActuator, 'long')
elif register.python_type is float:
act_class, data_type = (InterfaceFloatActuator, 'double')
else:
raise ValueError('sMAP currently only supports int and float based data types.')
a = self.add_actuator(actuator_point, register.units, act_class,
setup={'point_name':point, 'interface': self.interface},
data_type=data_type) #, read_limit=1.0, write_limit=1.0)
value = self.interface.get_point_sync(point)
if value is not None:
self.add(actuator_point, value)
else:
print("ERROR: Failed to read " + actuator_point + " interface returned None")
def make_dns_meta(inst):
root = inst.lookup("/")
m = {"uuid": str(inst.root_uuid)}
if root and "Metadata" in root:
m.update(dict(buildkv("Metadata", root["Metadata"])))
return m