def upload_stripes(self, req, relpath, ds=None, rgid=None, **args):
token = req.headers.get("X-Authorization-Token")
if not token:
return Response("Unauthorized", status=403)
authorized, extra = self.App.authorizeToken(token, "upload_stripes")
if not authorized:
return Response("Unauthorized", status=403)
payload = extra
username = extra["identity"]
data = req.body
header, data = data.split("\n", 1)
l = len(data)
i = 0
stripes = {}
for w in header.split():
column, size = w.split(":")
size = int(size)
stripe = data[i:i + size]
i += size
assert i <= l # i can be == l only after last stripe
key = "%s:%s:%s.bin" % (ds, column, rgid)
stripes[key] = stripe
backend = self.App.backendForDataset(ds)
backend.put_data(stripes)
return Response("OK")
def dataset_info(self, req, relpath, ds=None, **args):
self.App.scanDatasets()
backend = self.App.backendForDataset(ds)
if backend is None:
return Response("Dataset %s is not found" % (ds, ), status=404)
try:
schema = backend.schema(ds)
except StripedNotFoundException:
return Response("Dataset %s is not found" % (ds, ), status=404)
ne = 0
ng = 0
files = set()
for rginfo in backend.RGInfos(ds):
ne += rginfo["NEvents"]
ng += 1
for s in rginfo["Segments"]:
files.add(s["FileName"])
branches = [(bname, sorted(bdict.items()))
for bname, bdict in schema["branches"].items()]
return self.render_to_response("dataset_info.html",
ds=ds,
nfiles=len(files),
nevents=ne,
ngroups=ng,
attributes=schema["attributes"].items(),
branches=branches)
def add_data(self, req, relpath, key=None, ds=None, **args):
if req.environ.get("REQUEST_METHOD") != "POST":
return Response("Bad request", status=402)
ok, resp = self.authenticate(req, "add_data")
if not ok: return resp
backend = self.App.backendForDataset(ds)
backend.putStandaloneData({key: bytes(req.body)}, dataset_name=ds)
return Response("OK")
def dataset_schema(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
if not backend:
return Response("Dataset %s is not found" % (ds, ), status=404)
try:
schema = backend.schema(ds)
except StripedNotFoundException:
return Response("Dataset %s is not found" % (ds, ), status=404)
return Response(json.dumps(schema),
content_type="text/json",
cache_expires=self.MetadataTTL)
def column_desc(self, req, relpath, ds=None, column=None, **args):
backend = self.App.backendForDataset(ds)
key = "%s:%s:@@desc.json" % (ds, column)
data = backend.get_json([key]).get(key)
if data is None:
return Response("Column %s %s not found" % (ds, column),
status=400)
if not isinstance(data, (str, unicode)):
data = json.dumps(data)
return Response(data,
cache_expires=self.MetadataTTL,
content_type="text/json")
def authenticate(self, req, role):
body = req.body
#print folder.Name
ok, header = digest_server(role, req.environ, self.App.authorizeUser)
if not ok:
resp = Response("Authorization required", status=401)
if header:
#print "Header: %s %s" % (type(header), header)
resp.headers['WWW-Authenticate'] = header
return False, resp
user = header
return True, user
def ___validate(self, request, relpath, token=None, **args):
tup = self.App.validate(token)
if not tup:
return Response("400 Validation failed", status=400)
else:
return Response(json.dumps({
"token": token,
"expiration": tup[0],
"username": tup[1],
"identity": tup[2]
}),
content_type="text/josn")
def data_get(self, request, relpath, **args):
dfarm_client = self.App.DiskFarmClient
info, err = dfarm_client.getInfo(relpath)
if not info:
return Response(err, status=400)
if info.Type != 'f':
return Response("Not a file", status=403)
resp = Response(body_file=dfarm_client.open(info, 'r', tmo=10))
if info.Size is not None:
resp.headers["Content-Length"] = "%d" % (info.Size, )
return resp
def allRGInfos(self, ds):
backend = self.App.backendForDataset(ds)
rginfos = backend.RGInfos(ds)
return Response(app_iter=self.batchLines(
self.streamListAsJSON((rginfo for dataset, rginfo in rginfos))),
content_type="text/json",
cache_expires=self.MetadataTTL)
def stripes_sizes(self,
req,
relpath,
ds=None,
rgids=None,
columns=None,
**args):
if req.body:
params = json.loads(req.body)
ds = params["ds"]
rgids = params["rgids"]
columns = params["columns"]
else:
rgids = rgids.split(",")
columns = columns.split(",")
backend = self.App.backendForDataset(ds)
out = {}
for cn in columns:
keys = ["%s:%s:%s.bin" % (ds, cn, rgid) for rgid in rgids]
data = backend.get_data(keys)
lst = []
for k in keys:
rgid = int(k.split(".bin", 1)[0].split(":")[2])
lst.append([rgid, len(data[k])])
out[cn] = lst
out = json.dumps(out)
exp = 0 if req.body else self.DataTTL
return Response(out, cache_expires=exp, content_type="text/json")
def stripes(self,
req,
relpath,
ds=None,
rgid=None,
columns=None,
compressed="no",
**args):
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
columns = columns.split(",")
keys = ["%s:%s:%s.bin" % (ds, column, rgid) for column in columns]
data = backend.get_data(keys)
data_out = []
for k, val in data.items():
cn = k.split(":")[1]
#print type(cn), cn
if val is not None:
if compressed:
val = zlib.compress(val, 1)
data_out.append((cn, len(val), val))
header = " ".join(("%s:%d" % (cn, n) for cn, n, v in data_out)) + "\n"
def data_iterator(header, data):
yield bytes(header)
for cn, n, v in data:
yield bytes(v)
return Response(app_iter=data_iterator(header, data_out),
cache_expires=self.DataTTL)
def rginfo(self, req, relpath, ds=None, rgid=None, rgids=None, **args):
backend = self.App.backendForDataset(ds)
if rgids:
rgids = rgids.split(",")
lst = []
for r in rgids:
words = r.split(":", 1)
if len(words) == 1:
lst.append(int(r))
else:
lst += range(int(words[0]), int(words[1]) + 1)
rgids = lst
else:
rgids = [int(rgid)]
keys = ["%s:@@rginfo:%s.json" % (ds, rgid) for rgid in rgids]
data = backend.get_json(keys)
out = []
for rgid in rgids:
k = "%s:@@rginfo:%s.json" % (ds, rgid)
info = data.get(k)
if info is not None:
if isinstance(info, (str, unicode)):
info = json.loads(info)
out.append(info)
return Response(json.dumps(out),
content_type="text/json",
cache_expires=self.MetadataTTL)
def session_info(self, request, relpath):
sid = self.session.session_id
items = self.session.items()
return Response(app_iter=[
"Session id = %s\nData:\n" % (sid, ),
] + ["%s: %s\n" % item for item in items],
content_type="text/plain")
def jobs_json(self, request, relpath, **args):
queued, running, history = self.App.jobs()
data = {
"queued": [{
"jid": j.JID,
"username": j.JobDescription.Username,
"dataset": j.JobDescription.DatasetName,
"created": j.Created,
"fraction": j.JobDescription.Fraction
} for j in queued],
"running": [{
"jid": j.JID,
"username": j.JobDescription.Username,
"dataset": j.JobDescription.DatasetName,
"created": j.Created,
"started": j.Started,
"fraction": j.JobDescription.Fraction
} for j in running],
"history": [{
"jid": j.JID,
"username": j.JobDescription.Username,
"dataset": j.JobDescription.DatasetName,
"created": j.Created,
"started": j.Started,
"ended": j.Ended,
"fraction": j.JobDescription.Fraction
} for j in history]
}
return Response(json.dumps(data, indent=4, sort_keys=True),
content_type="text/json")
def token(self, req, relpath, role=None, ds=None, **args):
#
# Dataset (ds) is not use for now
#
if isinstance(role, unicode):
role = role.encode("ascii", "ignore")
if isinstance(ds, unicode):
ds = ds.encode("ascii", "ignore")
ok, resp = self.authenticate(req, role)
#print "authenticate:", ok, type(resp), resp
if not ok:
return resp
user = resp
token = self.App.tokenForUserRole(user, role)
if not token:
return Response("Unauthorized", status=403)
return Response(token, content_type="text/plain")
def columns(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
columns = backend.columns(ds)
if ds is not None:
columns = [cn for _, cn in columns]
return Response(json.dumps(sorted(columns)),
content_type="text/json",
cache_expires=self.MetadataTTL)
def hello(self, request, relpath, t=3):
c = self.App.Count
t = int(t)
time.sleep(t)
assert self.App.Count == c
self.App.Count += 1
return Response("Request count is now %d\n" % (self.App.Count, ),
content_type="text/plain")
def __init__(self, *params):
WPHandler.__init__(self, *params)
self.addHandler("robots1.txt",
RobotsHandler(*params)) # as external handler
self.addHandler("robots2.txt", self.robots) # as method
self.addHandler("robots3.txt",
(robots_response, "text/plain")) # as text
self.addHandler(
"robots.txt", # as Response object
Response(robots_response, content_type="text/plain"))
def stripe(self,
req,
relpath,
ds=None,
rgid=None,
column=None,
compressed="no",
**args):
#
# stripe?ds=<dataset>&rgid=<rgid>&column=<column>
#
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
key = stripe_key(ds, column, rgid)
data = backend.get_data([key]).get(key)
if data is None:
return Response("Stripe %s %s %s not found" % (ds, column, rgid),
status=400)
if compressed:
data = zlib.compress(data, 1)
return Response(data, cache_expires=self.DataTTL)
def data(self, req, relpath, ds=None, key=None, compressed="no", **args):
as_json = args.get("json", "no") == "yes"
if ds is not None:
ds = urllib.unquote(ds or "")
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
if backend is None:
return Response("Can not find bucket", status=400)
key = urllib.unquote(key)
data = backend.getStandaloneData([key],
dataset_name=ds,
as_json=as_json)[key]
if data is None:
return Response("Not found", status=400)
if as_json:
data = json.dumps(data)
return Response(data,
cache_expires=self.DataTTL,
content_type="text/json")
if compressed:
data = zlib.compress(data, 1)
return Response(data, cache_expires=self.DataTTL)
def stripe_meta(self,
req,
relpath,
ds=None,
rgid=None,
column=None,
compressed="no",
**args):
#
# stripe?ds=<dataset>&rgid=<rgid>&column=<column>
#
compressed = compressed == "yes"
backend = self.App.backendForDataset(ds)
key = "%s:%s:%s.bin" % (ds, column, rgid)
data = backend.get_data([key]).get(key)
if data is None:
return Response("Stripe %s %s %s not found" % (ds, column, rgid),
status=400)
if compressed:
data = zlib.compress(data, 1)
return Response(json.dumps({"length": len(data)}),
cache_expires=self.MetadataTTL,
content_type="text/json")
def column_descs(self, req, relpath, ds=None, columns=None, **args):
backend = self.App.backendForDataset(ds)
columns = columns.split(",")
keys = ["%s:%s:@@desc.json" % (ds, column) for column in columns]
data = backend.get_json(keys)
out = {}
for key, desc in data.items():
if isinstance(desc, (str, unicode)):
desc = json.loads(desc)
column = key.split(":")[1]
out[column] = desc
return Response(json.dumps(out),
content_type="text/json",
cache_expires=self.MetadataTTL)
def tagged_events(self,
req,
relpath,
ds=None,
event_id_range=None,
conditions=None,
**args):
wheres = []
tables = []
event_id_range = map(int, event_id_range.split(":"))
for i, c in enumerate(conditions.split(",")):
if ":" in c:
words = c.split(":")
tag = words[0]
op = {
"eq": "=",
"ne": "!=",
"lt": "<",
"le": "<=",
"gt": ">",
"ge": ">="
}[words[1]]
value = words[2]
wheres.append("""
t%d.name='%s' and t%d.value %s '%s'
and t%d.dataset = '%s'
and t%d.event_id = tags.event_id""" %
(i, tag, i, op, value, i, ds, i))
else:
wheres.append("""t%d.name='%s'
and t%d.dataset = '%s'
and t%d.event_id = tags.event_id""" % (i, c, i, ds, i))
tables.append("tags t%d" % (i, ))
conn = self.App.tagsDatabase()
c = conn.cursor()
sql = """select distinct tags.event_id from tags, %s
where tags.dataset = '%s'
and tags.event_id >= %d and tags.event_id < %d
and %s order by tags.event_id""" % (
",".join(tables), ds, event_id_range[0], event_id_range[1],
" and ".join(wheres))
c.execute(sql)
lst = [tup[0] for tup in c.fetchall()]
return Response(json.dumps(lst),
content_type="text/json",
cache_expires=self.MetadataTTL)
def token_digest(self, request, relpath, **args):
ok, header = rfc2617.digest_server("run", request.environ,
self.App.get_password)
if not ok:
if header:
resp = Response("Authorization required", status=401)
resp.headers['WWW-Authenticate'] = header
else:
self.log("authentication failed for user %s %s" % (username, ))
resp = Response("Authentication or authorization failed",
status=403,
content_type="text/plain")
return resp
username = header
token = self.App.authorize(username)
if not token:
self.log("authorization failed for user %s %s" % (username, ))
return Response("Authentication or authorization failed",
status=403,
content_type="text/plain")
return Response(token, content_type="text/plain")
def workers_json(self, request, relpath, tags=None, **args):
tags = tags.split(",") if tags else None
lst = self.App.workers(tags)
out = json.dumps(lst)
return Response(out, content_type="text/json")
def token(self, request, relpath, method="digest", **args):
if method == "digest":
return self.token_digest(request, relpath, **args)
else:
return Response("Invalid authentication method", status=400)
def job_server_address(self, request, relpath, **args):
return Response(json.dumps(
[self.App.JobServer.Host, self.App.JobServer.Port]),
content_type="text/json")
def hello(self, req, relpath, **args):
return Response("hello")
def time_response(self, request, relpath):
return Response(time.ctime()+"\n", content_type="text/plain")
def rgids(self, req, relpath, ds=None, **args):
backend = self.App.backendForDataset(ds)
rgids = backend.RGIDs(ds)
return Response(json.dumps(sorted(rgids)),
content_type="text/json",
cache_expires=self.MetadataTTL)