def add(self, subid, ids, obj): """Set the metadata for a Timeseries object """ tic = time.time() for path, ts in obj.iteritems(): if not util.is_string(path): raise Exception("Invalid path: " + path) tags = ["hstore('Path', %s)" % escape_string(path)] for name, val in util.buildkv('', ts): if name == 'Readings' or name == 'uuid': continue name, val = escape_string(name), escape_string(str(val)) if not (util.is_string(name) and util.is_string(val)): raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name), str(val)), 400) tags.append("hstore(%s, %s)" % (name, val)) query = "UPDATE stream SET metadata = metadata || " + " || ".join(tags) + \ " WHERE uuid = %s" % escape_string(ts['uuid']) # skip path updates if no other metadata if len(tags) == 1: continue yield self.db.runOperation(query) logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def add(self, subid, ids, obj): """Set the metadata for a Timeseries object """ tic = time.time() for path, ts in obj.iteritems(): if not util.is_string(path): raise Exception("Invalid path: " + path) tags = {u'Path': path} for name, val in util.buildkv('', ts): if name == 'Readings' or name == 'uuid': continue if not (util.is_string(name) and util.is_string(val)): raise SmapException( 'Invalid metadata pair: "%s" -> "%s"' % (str(name), str(val)), 400) tags[name] = val query = "UPDATE stream SET metadata = metadata || %s " \ " WHERE uuid = %s " # skip path updates if no other metadata if len(tags) == 1: continue yield self.db.runOperation(query, ( tags, ts['uuid'], )) logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def find_conversion(self, stream): unit = stream['Properties/UnitofMeasure'] if self.oldname == unit: return (self.newname, self.extra_op) for pat, converter in StandardizeUnitsOperator.units.iteritems(): if util.is_string(pat): if unit == pat: return converter elif pat.match(unit): return converter return (unit, 1)
def set_metadata(self, path, *metadata): if len(metadata) > 1: metadata = dict([metadata]) else: metadata = metadata[0] for v in metadata.itervalues(): if not util.is_string(v): raise SmapException("set_metadata: values must be strings!") o = self.lookup(path) o.set_metadata(metadata)
def run_query(opts, q): try: query = {} if opts.key: query['key'] = opts.key.split(',') if opts.private: query['private'] = [''] if opts.noop: query['noop'] = 1 qurl = opts.url + '?' + urllib.urlencode(query, doseq=True) print qurl if opts.verbose > 0: print >>sys.stderr, qurl print >>sys.stderr, q if q.strip().startswith("plot"): q = q.strip()[5:] doplot = True elif q.strip().startswith("gp"): add_gnuplot(q.strip()[3:]) else: doplot = False tic = time.time() total = 0 fp = urllib2.urlopen(qurl, data=q, timeout=opts.timeout) obj = client.parser(fp.read()) if util.is_string(obj): # make newlines come out right -- it's been json'ed print json.loads(obj) elif doplot: plot(obj) else: if opts.dates: stringify_times(obj) json.dump(sorted(obj), sys.stdout, sort_keys=True, indent=2) # json # while True: # obj = fp.readline() # if not obj: break # obj = json.loads(obj) # print # add a newline # toc = time.time() # print >>sys.stderr, "%i (%i ms)" % (len(obj), int((toc - tic) * 1000)) # total += len(obj) toc = time.time() print >>sys.stderr, "%i (%i ms)" % (len(obj), int((toc - tic) * 1000)) except urllib2.HTTPError, e: print >>sys.stderr, "ERROR:", e.read()
def add(self, subid, ids, obj): """Set the metadata for a Timeseries object """ tic = time.time() for path, ts in obj.iteritems(): if not util.is_string(path): raise Exception("Invalid path: " + path) tags = {u'Path': path} for name, val in util.buildkv('', ts): if name == 'Readings' or name == 'uuid': continue if not (util.is_string(name) and util.is_string(val)): raise SmapException('Invalid metadata pair: "%s" -> "%s"' % (str(name), str(val)), 400) tags[name] = val query = "UPDATE stream SET metadata = metadata || %s " \ " WHERE uuid = %s " # skip path updates if no other metadata if len(tags) == 1: continue yield self.db.runOperation(query, (tags, ts['uuid'],)) logging.getLogger('stats').info("Metadata insert took %0.6fs" % (time.time() - tic))
def runquery(self, db, s, run=True, verbose=False): logging.getLogger("queries.aql").info(s) ext, q = self.parse(s) if is_string(ext): return defer.succeed(ext) elif not isinstance(q, list): q = [None, q] ext = [None, ext] logging.getLogger("queries.sql").info(q[1]) if verbose: print q[1] if not run: return defer.succeed([]) deferreds = [] for ext_, q_ in zip(ext[1:], q[1:]): def print_time(result, start): logging.getLogger('stats').info("Query took %0.6fs" % (time.time() - start)) settings.metrics.timing('query_time', time.time() - start) return result if not ext_: d = db.runOperation(q_) d.addCallback(print_time, time.time()) d.addCallback(lambda _: []) else: d = db.runQuery(q_) d.addCallback(print_time, time.time()) d.addCallback(ext_) deferreds.append(d) if len(deferreds) > 1: d = defer.DeferredList(deferreds) if ext[0]: d.addCallback(ext[0]) else: d = deferreds[0] return d
def lookup(self, id, pred=None): """Retrieve an object in the resource hierarchy by path or uuid. If *id* is a string not starting with ``/``, it will be passed to the :py:class:`uuid.UUID` constructor; otherwise it will be treated as a pathname. *pred* is an optional predicate which can be used to test the result. """ if util.is_string(id): path = util.split_path(id) if len(path) > 0 and path[-1][0] == "+": return self._lookup_r(util.join_path(path[:-1]), pred=pred) else: obj = self.OBJS_PATH.get(util.join_path(path), None) elif isinstance(id, uuid.UUID): return self.OBJS_UUID.get(id, None) else: obj = None if not pred or pred(obj): return obj else: return None
def make_inclusive(range): if util.is_string(range): if range == 'inclusive': range = (True, True) elif range == 'inc-exc': range = (True, False) else: raise core.SmapException("Unsupported range: " + range) return range