def column_desc(self, req, relpath, ds=None, column=None, **args):
backend = self.App.backendForDataset(ds)
key = "%s:%s:@@desc.json" % (ds, column)
data = backend.get_json([key]).get(key)
if data is None:
return Response("Column %s %s not found" % (ds, column),
status=400)
if not isinstance(data, (str, unicode)):
data = json.dumps(data)
return Response(data,
cache_expires=self.MetadataTTL,
content_type="text/json")
def stripes(self,
req,
relpath,
ds=None,
rgid=None,
columns=None,
compressed="no",
**args):
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
columns = columns.split(",")
keys = ["%s:%s:%s.bin" % (ds, column, rgid) for column in columns]
data = backend.get_data(keys)
data_out = []
for k, val in data.items():
cn = k.split(":")[1]
if val is not None:
if compressed:
val = zlib.compress(val, 1)
data_out.append((cn, len(val), val))
header = " ".join(("%s:%d" % (cn, n) for cn, n, v in data_out)) + "\n"
def data_iterator(header, data):
yield header
for cn, n, v in data:
yield v
return Response(app_iter=data_iterator(header, data_out),
cache_expires=self.DataTTL)
def rginfo(self, req, relpath, ds=None, rgid=None, rgids=None, **args):
backend = self.App.backendForDataset(ds)
if rgids:
rgids = rgids.split(",")
lst = []
for r in rgids:
words = r.split(":", 1)
if len(words) == 1:
lst.append(int(r))
else:
lst += range(int(words[0]), int(words[1]) + 1)
rgids = lst
else:
rgids = [int(rgid)]
keys = ["%s:@@rginfo:%s.json" % (ds, rgid) for rgid in rgids]
data = backend.get_json(keys)
out = []
for rgid in rgids:
k = "%s:@@rginfo:%s.json" % (ds, rgid)
info = data.get(k)
if info is not None:
if isinstance(info, (str, unicode)):
info = json.loads(info)
out.append(info)
return Response(json.dumps(out),
content_type="text/json",
cache_expires=self.MetadataTTL)
def allRGInfos(self, ds):
backend = self.App.backendForDataset(ds)
rginfos = backend.RGInfos(ds)
return Response(app_iter=self.batchLines(
self.streamListAsJSON((rginfo for dataset, rginfo in rginfos))),
content_type="text/json",
cache_expires=self.MetadataTTL)
def stripes_sizes(self,
req,
relpath,
ds=None,
rgids=None,
columns=None,
**args):
if req.body:
params = json.loads(req.body)
ds = params["ds"]
rgids = params["rgids"]
columns = params["columns"]
else:
rgids = rgids.split(",")
columns = columns.split(",")
backend = self.App.backendForDataset(ds)
out = {}
for cn in columns:
keys = ["%s:%s:%s.bin" % (ds, cn, rgid) for rgid in rgids]
data = backend.get_data(keys)
lst = []
for k in keys:
rgid = int(k.split(".bin", 1)[0].split(":")[2])
lst.append([rgid, len(data[k])])
out[cn] = lst
out = json.dumps(out)
exp = 0 if req.body else self.DataTTL
return Response(out, cache_expires=exp, content_type="text/json")
def columns(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
columns = backend.columns(ds)
if ds is not None:
columns = [cn for _, cn in columns]
return Response(json.dumps(sorted(columns)),
content_type="text/json",
cache_expires=self.MetadataTTL)
def data(self,
req,
relpath,
ds=None,
key=None,
compressed="no",
json="no",
**args):
ds = urllib.unquote(ds or "")
backend = self.App.backendForDataset(ds or self.App.defaultBucket())
key = urllib.unquote(key)
json = json == "yes"
dbkey = standalone_data_key(ds, key, json=json)
data = backend.get_data([dbkey]).get(dbkey)
if data is None:
return Response("Not found", status=400)
if compressed:
data = zlib.compress(data, 1)
return Response(data, cache_expires=self.DataTTL)
def stripe(self,
req,
relpath,
ds=None,
rgid=None,
column=None,
compressed="no",
**args):
#
# stripe?ds=<dataset>&rgid=<rgid>&column=<column>
#
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
key = "%s:%s:%s.bin" % (ds, column, rgid)
data = backend.get_data([key]).get(key)
if data is None:
return Response("Stripe %s %s %s not found" % (ds, column, rgid),
status=400)
if compressed:
data = zlib.compress(data, 1)
return Response(data, cache_expires=self.DataTTL)
def column_descs(self, req, relpath, ds=None, columns=None, **args):
backend = self.App.backendForDataset(ds)
columns = columns.split(",")
keys = ["%s:%s:@@desc.json" % (ds, column) for column in columns]
data = backend.get_json(keys)
out = {}
for key, desc in data.items():
if isinstance(desc, (str, unicode)):
desc = json.loads(desc)
column = key.split(":")[1]
out[column] = desc
return Response(json.dumps(out),
content_type="text/json",
cache_expires=self.MetadataTTL)
def stripe_meta(self,
req,
relpath,
ds=None,
rgid=None,
column=None,
compressed="no",
**args):
#
# stripe?ds=<dataset>&rgid=<rgid>&column=<column>
#
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
key = "%s:%s:%s.bin" % (ds, column, rgid)
data = backend.get_data([key]).get(key)
if data is None:
return Response("Stripe %s %s %s not found" % (ds, column, rgid),
status=400)
if compressed:
data = zlib.compress(data, 1)
return Response(json.dumps({"length": len(data)}),
cache_expires=self.MetadataTTL,
content_type="text/json")
def tagged_events(self,
req,
relpath,
ds=None,
event_id_range=None,
conditions=None,
**args):
wheres = []
tables = []
event_id_range = map(int, event_id_range.split(":"))
for i, c in enumerate(conditions.split(",")):
if ":" in c:
words = c.split(":")
tag = words[0]
op = {
"eq": "=",
"ne": "!=",
"lt": "<",
"le": "<=",
"gt": ">",
"ge": ">="
}[words[1]]
value = words[2]
wheres.append("""
t%d.name='%s' and t%d.value %s '%s'
and t%d.dataset = '%s'
and t%d.event_id = tags.event_id""" %
(i, tag, i, op, value, i, ds, i))
else:
wheres.append("""t%d.name='%s'
and t%d.dataset = '%s'
and t%d.event_id = tags.event_id""" % (i, c, i, ds, i))
tables.append("tags t%d" % (i, ))
conn = self.App.tagsDatabase()
c = conn.cursor()
sql = """select distinct tags.event_id from tags, %s
where tags.dataset = '%s'
and tags.event_id >= %d and tags.event_id < %d
and %s order by tags.event_id""" % (
",".join(tables), ds, event_id_range[0], event_id_range[1],
" and ".join(wheres))
c.execute(sql)
lst = [tup[0] for tup in c.fetchall()]
return Response(json.dumps(lst),
content_type="text/json",
cache_expires=self.MetadataTTL)
def update_stripes(self, req, relpath, ds=None, rgid=None, **args):
data = req.body
header, data = data.split("\n", 1)
l = len(data)
i = 0
stripes = {}
for w in header.split():
column, size = w.split(":")
size = int(size)
stripe = data[i:i + size]
i += size
assert i <= l # i can be == l only after last stripe
key = "%s:%s:%s.bin" % (ds, column, rgid)
stripes[key] = stripe
backend = self.App.backendForDataset(ds)
backend.put_data(stripes)
return Response("OK")
def dataset_info(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
if backend is None:
return Response("Dataset %s is not found" % (ds, ), status=400)
ne = 0
ng = 0
files = set()
for rginfo in backend.RGInfos(ds):
ne += rginfo["NEvents"]
ng += 1
for s in rginfo["Segments"]:
files.add(s["FileName"])
schema = backend.schema(ds)
branches = [(bname, sorted(bdict.items()))
for bname, bdict in schema["branches"].items()]
return self.render_to_response(
"dataset_info.html",
ds=ds,
nfiles=len(files),
nevents=ne,
ngroups=ng,
attributes=schema["attributes"].items(),
branches=branches,
)
def hello(self, req, relpath, **args):
return Response("hello")
def dataset_list(self, req, relpath, **args):
lst = sorted(Config.datasets())
return Response(json.dumps(lst), content_type="text/json")
def dataset_schema(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
schema = backend.schema(ds)
return Response(json.dumps(schema), content_type="text/json")
def rgids(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
rgids = backend.RGIDs(ds)
return Response(json.dumps(sorted(rgids)),
content_type="text/json",
cache_expires=self.MetadataTTL)