def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = ["hstore('Path', %s)" % escape_string(path)]
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
name, val = escape_string(name), escape_string(str(val))
if not (util.is_string(name) and util.is_string(val)):
raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name),
str(val)),
400)
tags.append("hstore(%s, %s)" % (name, val))
query = "UPDATE stream SET metadata = metadata || " + " || ".join(tags) + \
" WHERE uuid = %s" % escape_string(ts['uuid'])
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query)
logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = {u'Path': path}
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
if not (util.is_string(name) and util.is_string(val)):
raise SmapException(
'Invalid metadata pair: "%s" -> "%s"' %
(str(name), str(val)), 400)
tags[name] = val
query = "UPDATE stream SET metadata = metadata || %s " \
" WHERE uuid = %s "
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query, (
tags,
ts['uuid'],
))
logging.getLogger('stats').info("Metadata insert took %0.6fs" %
(time.time() - tic))
def find_conversion(self, stream):
unit = stream['Properties/UnitofMeasure']
if self.oldname == unit:
return (self.newname, self.extra_op)
for pat, converter in StandardizeUnitsOperator.units.iteritems():
if util.is_string(pat):
if unit == pat: return converter
elif pat.match(unit):
return converter
return (unit, 1)
def find_conversion(self, stream):
unit = stream['Properties/UnitofMeasure']
if self.oldname == unit:
return (self.newname, self.extra_op)
for pat, converter in StandardizeUnitsOperator.units.iteritems():
if util.is_string(pat):
if unit == pat: return converter
elif pat.match(unit):
return converter
return (unit, 1)
def set_metadata(self, path, *metadata):
if len(metadata) > 1:
metadata = dict([metadata])
else: metadata = metadata[0]
for v in metadata.itervalues():
if not util.is_string(v):
raise SmapException("set_metadata: values must be strings!")
o = self.lookup(path)
o.set_metadata(metadata)
def run_query(opts, q):
try:
query = {}
if opts.key:
query['key'] = opts.key.split(',')
if opts.private:
query['private'] = ['']
if opts.noop:
query['noop'] = 1
qurl = opts.url + '?' + urllib.urlencode(query, doseq=True)
print qurl
if opts.verbose > 0:
print >>sys.stderr, qurl
print >>sys.stderr, q
if q.strip().startswith("plot"):
q = q.strip()[5:]
doplot = True
elif q.strip().startswith("gp"):
add_gnuplot(q.strip()[3:])
else:
doplot = False
tic = time.time()
total = 0
fp = urllib2.urlopen(qurl, data=q, timeout=opts.timeout)
obj = client.parser(fp.read())
if util.is_string(obj):
# make newlines come out right -- it's been json'ed
print json.loads(obj)
elif doplot:
plot(obj)
else:
if opts.dates:
stringify_times(obj)
json.dump(sorted(obj), sys.stdout, sort_keys=True, indent=2)
# json
# while True:
# obj = fp.readline()
# if not obj: break
# obj = json.loads(obj)
# print # add a newline
# toc = time.time()
# print >>sys.stderr, "%i (%i ms)" % (len(obj), int((toc - tic) * 1000))
# total += len(obj)
toc = time.time()
print >>sys.stderr, "%i (%i ms)" % (len(obj), int((toc - tic) * 1000))
except urllib2.HTTPError, e:
print >>sys.stderr, "ERROR:", e.read()
def set_metadata(self, path, *metadata):
if len(metadata) > 1:
metadata = dict([metadata])
else: metadata = metadata[0]
for v in metadata.itervalues():
if not util.is_string(v):
raise SmapException("set_metadata: values must be strings!")
o = self.lookup(path)
o.set_metadata(metadata)
def add(self, subid, ids, obj):
"""Set the metadata for a Timeseries object
"""
tic = time.time()
for path, ts in obj.iteritems():
if not util.is_string(path):
raise Exception("Invalid path: " + path)
tags = {u'Path': path}
for name, val in util.buildkv('', ts):
if name == 'Readings' or name == 'uuid': continue
if not (util.is_string(name) and util.is_string(val)):
raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name),
str(val)),
400)
tags[name] = val
query = "UPDATE stream SET metadata = metadata || %s " \
" WHERE uuid = %s "
# skip path updates if no other metadata
if len(tags) == 1:
continue
yield self.db.runOperation(query, (tags, ts['uuid'],))
logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def runquery(self, db, s, run=True, verbose=False):
logging.getLogger("queries.aql").info(s)
ext, q = self.parse(s)
if is_string(ext):
return defer.succeed(ext)
elif not isinstance(q, list):
q = [None, q]
ext = [None, ext]
logging.getLogger("queries.sql").info(q[1])
if verbose:
print q[1]
if not run: return defer.succeed([])
deferreds = []
for ext_, q_ in zip(ext[1:], q[1:]):
def print_time(result, start):
logging.getLogger('stats').info("Query took %0.6fs" %
(time.time() - start))
settings.metrics.timing('query_time', time.time() - start)
return result
if not ext_:
d = db.runOperation(q_)
d.addCallback(print_time, time.time())
d.addCallback(lambda _: [])
else:
d = db.runQuery(q_)
d.addCallback(print_time, time.time())
d.addCallback(ext_)
deferreds.append(d)
if len(deferreds) > 1:
d = defer.DeferredList(deferreds)
if ext[0]:
d.addCallback(ext[0])
else:
d = deferreds[0]
return d
def runquery(self, db, s, run=True, verbose=False):
logging.getLogger("queries.aql").info(s)
ext, q = self.parse(s)
if is_string(ext):
return defer.succeed(ext)
elif not isinstance(q, list):
q = [None, q]
ext = [None, ext]
logging.getLogger("queries.sql").info(q[1])
if verbose:
print q[1]
if not run: return defer.succeed([])
deferreds = []
for ext_, q_ in zip(ext[1:], q[1:]):
def print_time(result, start):
logging.getLogger('stats').info("Query took %0.6fs" % (time.time() - start))
settings.metrics.timing('query_time', time.time() - start)
return result
if not ext_:
d = db.runOperation(q_)
d.addCallback(print_time, time.time())
d.addCallback(lambda _: [])
else:
d = db.runQuery(q_)
d.addCallback(print_time, time.time())
d.addCallback(ext_)
deferreds.append(d)
if len(deferreds) > 1:
d = defer.DeferredList(deferreds)
if ext[0]:
d.addCallback(ext[0])
else:
d = deferreds[0]
return d
def lookup(self, id, pred=None):
"""Retrieve an object in the resource hierarchy by path or uuid. If
*id* is a string not starting with ``/``, it will be passed to the
:py:class:`uuid.UUID` constructor; otherwise it will be treated as a
pathname. *pred* is an optional predicate which can be used to test
the result.
"""
if util.is_string(id):
path = util.split_path(id)
if len(path) > 0 and path[-1][0] == "+":
return self._lookup_r(util.join_path(path[:-1]), pred=pred)
else:
obj = self.OBJS_PATH.get(util.join_path(path), None)
elif isinstance(id, uuid.UUID):
return self.OBJS_UUID.get(id, None)
else:
obj = None
if not pred or pred(obj):
return obj
else: return None
def lookup(self, id, pred=None):
"""Retrieve an object in the resource hierarchy by path or uuid. If
*id* is a string not starting with ``/``, it will be passed to the
:py:class:`uuid.UUID` constructor; otherwise it will be treated as a
pathname. *pred* is an optional predicate which can be used to test
the result.
"""
if util.is_string(id):
path = util.split_path(id)
if len(path) > 0 and path[-1][0] == "+":
return self._lookup_r(util.join_path(path[:-1]), pred=pred)
else:
obj = self.OBJS_PATH.get(util.join_path(path), None)
elif isinstance(id, uuid.UUID):
return self.OBJS_UUID.get(id, None)
else:
obj = None
if not pred or pred(obj):
return obj
else: return None
def make_inclusive(range):
if util.is_string(range):
if range == 'inclusive': range = (True, True)
elif range == 'inc-exc': range = (True, False)
else: raise core.SmapException("Unsupported range: " + range)
return range
def make_inclusive(range):
if util.is_string(range):
if range == 'inclusive': range = (True, True)
elif range == 'inc-exc': range = (True, False)
else: raise core.SmapException("Unsupported range: " + range)
return range
