def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', 'adocs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# underscore may come from ajax call via jQuery
validate_str('_', param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', 'adocs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# underscore may come from ajax call via jQuery
validate_str('_', param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
out = []
year, month, _ = today()
hdir = '%s/%s/%s' % (self.hdir, year, month)
fname = file_name(hdir, spec, self.compress)
data = hdfs.load(fname)
bytes_reader = io.BytesIO(data)
if self.compress:
# use gzip'ed reader and pass to it BytesIO as file object
gzip_reader = gzip.GzipFile(fileobj=bytes_reader)
decoder = avro.io.BinaryDecoder(gzip_reader)
else:
# use non-compressed reader
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(self.schema)
while True:
try:
rec = reader.read(decoder)
out.append(rec)
except:
break
# close gzip stream if necessary
if self.compress:
gzip_reader.close()
# close bytes stream
bytes_reader.close()
return out
return self.empty_data
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
out = []
year, month, _ = today()
hdir = '%s/%s/%s' % (self.hdir, year, month)
fname = file_name(hdir, spec, self.compress)
data = hdfs.load(fname)
bytes_reader = io.BytesIO(data)
if self.compress:
# use gzip'ed reader and pass to it BytesIO as file object
gzip_reader = gzip.GzipFile(fileobj=bytes_reader)
decoder = avro.io.BinaryDecoder(gzip_reader)
else:
# use non-compressed reader
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(self.schema)
while True:
try:
rec = reader.read(decoder)
out.append(rec)
except:
break
# close gzip stream if necessary
if self.compress:
gzip_reader.close()
# close bytes stream
bytes_reader.close()
return out
return self.empty_data
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
fname = '%s/%s.gz' % (self.uri, spec)
data = json.load(open_file(fname))
if isinstance(data, list):
for rec in data:
self.check(rec)
return data
self.check(data)
return [data]
return self.empty_data
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
fname = '%s/%s.gz' % (self.uri, spec)
data = json.load(open_file(fname))
if isinstance(data, list):
for rec in data:
self.check(rec)
return data
self.check(data)
return [data]
return self.empty_data
def read(self, spec, fields=None):
"Read API for LTS"
try:
if not spec:
spec = {}
if isinstance(spec, list):
spec = {'wmaid': {'$in': spec}}
return self.read_from_storage(spec) # list of wmaids
elif PAT_UID.match(str(spec)):
return self.read_from_storage([spec]) # one wmaid
else:
return self.submit(spec, fields)
except Exception as exp:
raise ReadError(str(exp))
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
# Check for `performance` endpoint, documented in
# https://github.com/knly/WMArchiveAggregation
if len(param.args) == 1 and param.args[0] == 'performance':
safe.args.append(param.args[0])
param.args.remove(param.args[0])
# Validate arguments
validate_strlist('metrics[]', param, safe, re.compile(r'^[a-zA-Z.]+'))
validate_strlist('axes[]', param, safe, re.compile(r'^[a-zA-Z_]+'))
validate_strlist('suggestions[]', param, safe, re.compile(r'^[a-zA-Z]+'))
date_pattern = PAT_YYYYMMDD
validate_str('start_date', param, safe, date_pattern, optional=True)
validate_str('end_date', param, safe, date_pattern, optional=True)
validate_rx('workflow', param, safe, optional=True)
validate_rx('task', param, safe, optional=True)
validate_rx('host', param, safe, optional=True)
validate_rx('site', param, safe, optional=True)
validate_rx('jobtype', param, safe, optional=True)
validate_rx('jobstate', param, safe, optional=True)
validate_rx('acquisitionEra', param, safe, optional=True)
validate_rx('exitCode', param, safe, optional=True)
validate_rx('exitStep', param, safe, optional=True)
validate_str('_', param, safe, PAT_INFO, optional=True)
return True
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
def read(self, spec, fields=None):
"Read API for LTS"
try:
if not spec:
spec = {}
if isinstance(spec, list):
spec = {'wmaid': {'$in': spec}}
return self.read_from_storage(spec) # list of wmaids
elif PAT_UID.match(str(spec)):
return self.read_from_storage([spec]) # one wmaid
else:
return self.submit(spec, fields)
except Exception as exp:
raise ReadError(str(exp))
def find(self, spec, fields):
"""
Find records in MongoDB storage for provided spec, returns generator
over MongoDB collection
"""
if not spec:
spec = {}
if isinstance(spec, list):
spec = {'wmaid': {'$in': spec}}
return self.jobs.find(spec)
elif PAT_UID.match(str(spec)):
spec = {'wmaid': spec}
return self.jobs.find(spec)
if fields:
return self.coll.find(spec, fields)
return self.coll.find(spec)
def find(self, spec, fields):
"""
Find records in MongoDB storage for provided spec, returns generator
over MongoDB collection
"""
if not spec:
spec = {}
if isinstance(spec, list):
spec = {'wmaid': {'$in': spec}}
return self.jobs.find(spec)
elif PAT_UID.match(str(spec)):
spec = {'wmaid': spec}
return self.jobs.find(spec)
if fields:
return self.coll.find(spec, fields)
return self.coll.find(spec)
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
out = []
fname = file_name(self.hdir, spec)
with open_file(fname) as istream:
reader = DataFileReader(istream, DatumReader())
for data in reader:
if isinstance(data, list):
for rec in data:
self.check(rec)
return data
self.check(data)
out.append(data)
return out
return self.empty_data
def _read(self, spec, fields=None):
"Internal read API"
if PAT_UID.match(str(spec)): # requested to read concrete file
out = []
fname = file_name(self.hdir, spec)
with open_file(fname) as istream:
reader = DataFileReader(istream, DatumReader())
for data in reader:
if isinstance(data, list):
for rec in data:
self.check(rec)
return data
self.check(data)
out.append(data)
return out
return self.empty_data
def validate(self, apiobj, method, api, param, safe):
"""
Validate request input data.
Has to be implemented, otherwise the service fails to start.
If it's not implemented correctly (e.g. just pass), the arguments
are not passed in the method at all.
"""
if method == 'GET':
# Check for `performance` endpoint, documented in
# https://github.com/knly/WMArchiveAggregation
if len(param.args) == 1 and param.args[0] == 'performance':
safe.args.append(param.args[0])
param.args.remove(param.args[0])
# Validate arguments
validate_strlist('metrics[]', param, safe,
re.compile(r'^[a-zA-Z.]+'))
validate_strlist('axes[]', param, safe,
re.compile(r'^[a-zA-Z_]+'))
validate_strlist('suggestions[]', param, safe,
re.compile(r'^[a-zA-Z]+'))
date_pattern = PAT_YYYYMMDD
validate_str('start_date',
param,
safe,
date_pattern,
optional=True)
validate_str('end_date',
param,
safe,
date_pattern,
optional=True)
validate_rx('workflow', param, safe, optional=True)
validate_rx('task', param, safe, optional=True)
validate_rx('host', param, safe, optional=True)
validate_rx('site', param, safe, optional=True)
validate_rx('jobtype', param, safe, optional=True)
validate_rx('jobstate', param, safe, optional=True)
validate_rx('acquisitionEra', param, safe, optional=True)
validate_rx('exitCode', param, safe, optional=True)
validate_rx('exitStep', param, safe, optional=True)
validate_str('_', param, safe, PAT_INFO, optional=True)
return True
if 'query' in param.kwargs.keys():
validate_str('query', param, safe, PAT_QUERY, optional=True)
for key in ['status', 'jobs', '_']:
if key in param.kwargs.keys():
validate_str(key, param, safe, PAT_INFO, optional=True)
# test if user provided uid
if len(param.args) == 1 and PAT_UID.match(param.args[0]):
safe.args.append(param.args[0])
param.args.remove(param.args[0])
return True
elif method == 'POST':
if not param.args or not param.kwargs:
return False # this class does not need any parameters
return True
