def validate(self, apiobj, method, api, param, safe): """ Validate request input data. Has to be implemented, otherwise the service fails to start. If it's not implemented correctly (e.g. just pass), the arguments are not passed in the method at all. """ if method == 'GET': if 'query' in param.kwargs.keys(): validate_str('query', param, safe, PAT_QUERY, optional=True) for key in ['status', 'jobs', 'adocs', '_']: if key in param.kwargs.keys(): validate_str(key, param, safe, PAT_INFO, optional=True) # underscore may come from ajax call via jQuery validate_str('_', param, safe, PAT_INFO, optional=True) # test if user provided uid if len(param.args) == 1 and PAT_UID.match(param.args[0]): safe.args.append(param.args[0]) param.args.remove(param.args[0]) return True elif method == 'POST': if not param.args or not param.kwargs: return False # this class does not need any parameters return True
def _read(self, spec, fields=None): "Internal read API" if PAT_UID.match(str(spec)): # requested to read concrete file out = [] year, month, _ = today() hdir = '%s/%s/%s' % (self.hdir, year, month) fname = file_name(hdir, spec, self.compress) data = hdfs.load(fname) bytes_reader = io.BytesIO(data) if self.compress: # use gzip'ed reader and pass to it BytesIO as file object gzip_reader = gzip.GzipFile(fileobj=bytes_reader) decoder = avro.io.BinaryDecoder(gzip_reader) else: # use non-compressed reader decoder = avro.io.BinaryDecoder(bytes_reader) reader = avro.io.DatumReader(self.schema) while True: try: rec = reader.read(decoder) out.append(rec) except: break # close gzip stream if necessary if self.compress: gzip_reader.close() # close bytes stream bytes_reader.close() return out return self.empty_data
def _read(self, spec, fields=None): "Internal read API" if PAT_UID.match(str(spec)): # requested to read concrete file fname = '%s/%s.gz' % (self.uri, spec) data = json.load(open_file(fname)) if isinstance(data, list): for rec in data: self.check(rec) return data self.check(data) return [data] return self.empty_data
def read(self, spec, fields=None): "Read API for LTS" try: if not spec: spec = {} if isinstance(spec, list): spec = {'wmaid': {'$in': spec}} return self.read_from_storage(spec) # list of wmaids elif PAT_UID.match(str(spec)): return self.read_from_storage([spec]) # one wmaid else: return self.submit(spec, fields) except Exception as exp: raise ReadError(str(exp))
def validate(self, apiobj, method, api, param, safe): """ Validate request input data. Has to be implemented, otherwise the service fails to start. If it's not implemented correctly (e.g. just pass), the arguments are not passed in the method at all. """ if method == 'GET': # Check for `performance` endpoint, documented in # https://github.com/knly/WMArchiveAggregation if len(param.args) == 1 and param.args[0] == 'performance': safe.args.append(param.args[0]) param.args.remove(param.args[0]) # Validate arguments validate_strlist('metrics[]', param, safe, re.compile(r'^[a-zA-Z.]+')) validate_strlist('axes[]', param, safe, re.compile(r'^[a-zA-Z_]+')) validate_strlist('suggestions[]', param, safe, re.compile(r'^[a-zA-Z]+')) date_pattern = PAT_YYYYMMDD validate_str('start_date', param, safe, date_pattern, optional=True) validate_str('end_date', param, safe, date_pattern, optional=True) validate_rx('workflow', param, safe, optional=True) validate_rx('task', param, safe, optional=True) validate_rx('host', param, safe, optional=True) validate_rx('site', param, safe, optional=True) validate_rx('jobtype', param, safe, optional=True) validate_rx('jobstate', param, safe, optional=True) validate_rx('acquisitionEra', param, safe, optional=True) validate_rx('exitCode', param, safe, optional=True) validate_rx('exitStep', param, safe, optional=True) validate_str('_', param, safe, PAT_INFO, optional=True) return True if 'query' in param.kwargs.keys(): validate_str('query', param, safe, PAT_QUERY, optional=True) for key in ['status', 'jobs', '_']: if key in param.kwargs.keys(): validate_str(key, param, safe, PAT_INFO, optional=True) # test if user provided uid if len(param.args) == 1 and PAT_UID.match(param.args[0]): safe.args.append(param.args[0]) param.args.remove(param.args[0]) return True elif method == 'POST': if not param.args or not param.kwargs: return False # this class does not need any parameters return True
def find(self, spec, fields): """ Find records in MongoDB storage for provided spec, returns generator over MongoDB collection """ if not spec: spec = {} if isinstance(spec, list): spec = {'wmaid': {'$in': spec}} return self.jobs.find(spec) elif PAT_UID.match(str(spec)): spec = {'wmaid': spec} return self.jobs.find(spec) if fields: return self.coll.find(spec, fields) return self.coll.find(spec)
def _read(self, spec, fields=None): "Internal read API" if PAT_UID.match(str(spec)): # requested to read concrete file out = [] fname = file_name(self.hdir, spec) with open_file(fname) as istream: reader = DataFileReader(istream, DatumReader()) for data in reader: if isinstance(data, list): for rec in data: self.check(rec) return data self.check(data) out.append(data) return out return self.empty_data