def fetch_json(self, k, default=None, required=True, multiple=False): v = self.fetch(k=k, default=default, required=required, multiple=multiple) try: return json_deserializer(v) except ValueError: raise ValidationError("Invalid JSON: '{}'".format(k))
def load_cols(self, fn): self.lgg.info('Reading columns from ' + fn) with open(fn, 'rt', encoding='utf-8') as fh: data = fh.read() data = json_deserializer(data) for k, v in data.items(): setattr(self, k, v)
def load(self, index, doc_type, id_, source=None): """ Loads specified document. :param index: Index :param doc_type: Document type :param id_: ID :param source: Optionally restrict to some or all source fields, i.e. no meta data. If ``source`` is True, returns all source fields, else ``source`` must be an iterable of field names. :return: JSON """ params = {} if not source: url = '{base}/{index}/{doc_type}/{id}'.format( base=self.url, index=index, doc_type=doc_type, id=id_ ) elif source is True: url = '{base}/{index}/{doc_type}/{id}/_source'.format( base=self.url, index=index, doc_type=doc_type, id=id_ ) else: url = '{base}/{index}/{doc_type}/{id}'.format( base=self.url, index=index, doc_type=doc_type, id=id_ ) params = dict(_source=source.join(',')) r = requests.get(url, params=params) r.raise_for_status() return json_deserializer(r.text)
def delete_index(self, index): url = '{base}/{index}/'.format( base=self.url, index=index ) r = requests.delete(url) r.raise_for_status() return json_deserializer(r.text)
def create_index(self, index, rc=None): url = '{base}/{index}/'.format( base=self.url, index=index ) s = json_serializer(rc) if rc else None r = requests.put(url, data=s) r.raise_for_status() return json_deserializer(r.text)
def count(self): url = self.url + '/_count' q = { "query": { "match_all": {} } } r = requests.get(url, data=json_serializer(q)) r.raise_for_status() return json_deserializer(r.text)
def search(self, index, doc_type, q): url = '{base}/{index}/{doc_type}/_search'.format( base=self.url, index=index, doc_type=doc_type ) if isinstance(q, str): r = requests.get(url, params=dict(q=q)) else: r = requests.get(url, data=json_serializer(q)) r.raise_for_status() return json_deserializer(r.text)
def save(self, index, doc_type, data, id_=None, create=None): if create and not id_: raise ValueError('To force document creation, id must be given') if id_: url = '{base}/{index}/{doc_type}/{id}'.format( base=self.url, index=index, doc_type=doc_type, id=id_ ) if create: url += '/_create' s = json_serializer(data).encode('utf-8') r = requests.put(url, data=s) else: url = '{base}/{index}/{doc_type}/'.format( base=self.url, index=index, doc_type=doc_type ) s = json_serializer(data).encode('utf-8') r = requests.post(url, data=s) r.raise_for_status() return json_deserializer(r.text)
def filter(self): """ Fetches the filter expression from key ``fil`` (required, single). Key ``fil`` must contain the expression as a single JSON string. :return: Filter expression as a data structure. :raise ValidationError: If key is missing or invalid. """ # TODO TESTING!! try: fil = self.parent.inp['fil'] except KeyError: #raise ValidationError("Missing filter") return None try: fil = json_deserializer(fil) except ValueError: raise ValidationError("Invalid JSON") # Expect be [CONJ, TAIL] if len(fil) != 2: raise ValidationError("Invalid expression") aconj = self.allowed_conjunctions aops = self.allowed_operators acase = self.allowed_case_sensitivity aff = self.allowed_fields # (country='f' and plz >=1 and plz <=2) # or # (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4))) # # [or, [ # [and, [ # [country, =, 'f'], # [plz, >=, 1], # [plz, <=, 2] # ], # [and, [ # [country, =, 'd'], # [or, [ # [and, [ # [plz, >=, 5], # [plz, <=, 8] # ], # [and, [ # [plz, >, 2], # [plz, <, 4] # ] # ] # ] # ] def check(conj, tail): # check conjunction if conj not in aconj: raise ValidationError("Invalid conjunction: '{}'".format(conj)) # tail must be list if not isinstance(tail, list): raise ValidationError("Invalid tail: '{}'".format(tail)) # tail is itself CONJ + TAIL if len(tail) == 2 and tail[0] in aconj: check(tail[0], tail[1:]) # consider tail to be list of things else: for thing in tail: l = len(thing) # this thing is CONJ + TAIL if l == 2: check(thing[0], thing[1:]) # this thing is filter expression elif l == 4: fld, op, case, val = thing if not fld in aff: raise ValidationError( "Invalid field: '{}'".format(fld)) if not op in aops: raise ValidationError( "Invalid op: '{}'".format(op)) if not case in acase: raise ValidationError( "Invalid case: '{}'".format(case)) # this thing is garbage else: raise ValidationError( "Invalid thing: '{}' ({})".format( thing, type(thing))) # 1st level is always CONJ + TAIL check(fil[0], fil[1]) return fil
def filter(self): try: fil = self.parent.inp['fil'] except KeyError: raise ValidationError("Missing filter") try: fil = json_deserializer(fil) except ValueError: raise ValidationError("Invalid filter") # Expect be [CONJ, TAIL] if len(fil) != 2: raise ValidationError("Invalid filter") aconj = self.allowed_conjunctions aops = self.allowed_operators acase = self.allowed_case_sensitivity aff = self.allowed_fields # (country='f' and plz >=1 and plz <=2) # or # (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4))) # # [or, [ # [and, [ # [country, =, 'f'], # [plz, >=, 1], # [plz, <=, 2] # ], # [and, [ # [country, =, 'd'], # [or, [ # [and, [ # [plz, >=, 5], # [plz, <=, 8] # ], # [and, [ # [plz, >, 2], # [plz, <, 4] # ] # ] # ] # ] def check(conj, tail): # check conjunction if conj not in aconj: raise ValidationError("Invalid conjunction: '{}'".format(conj)) # tail must be list if not isinstance(tail, list): raise ValidationError("Invalid tail: '{}'".format(tail)) # tail is itself CONJ + TAIL if len(tail) == 2 and tail[0] in aconj: check(tail[0], tail[1:]) # consider tail to be list of things else: for thing in tail: l = len(thing) # this thing is CONJ + TAIL if l == 2: check(thing[0], thing[1:]) # this thing is filter expression elif l == 4: fld, op, case, val = thing if not fld in aff: raise ValidationError("Invalid field: '{}'".format(fld)) if not op in aops: raise ValidationError("Invalid op: '{}'".format(op)) if not case in acase: raise ValidationError("Invalid case: '{}'".format(case)) # this thing is garbage else: raise ValidationError("Invalid thing: '{}'".format(thing)) check(fil[0], fil[1:]) return fil
def filter(self): """ Fetches the filter expression from key ``fil`` (required, single). Key ``fil`` must contain the expression as a single JSON string. :return: Filter expression as a data structure. :raise ValidationError: If key is missing or invalid. """ # TODO TESTING!! try: fil = self.parent.inp['fil'] except KeyError: #raise ValidationError("Missing filter") return None try: fil = json_deserializer(fil) except ValueError: raise ValidationError("Invalid JSON") # Expect be [CONJ, TAIL] if len(fil) != 2: raise ValidationError("Invalid expression") aconj = self.allowed_conjunctions aops = self.allowed_operators acase = self.allowed_case_sensitivity aff = self.allowed_fields # (country='f' and plz >=1 and plz <=2) # or # (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4))) # # [or, [ # [and, [ # [country, =, 'f'], # [plz, >=, 1], # [plz, <=, 2] # ], # [and, [ # [country, =, 'd'], # [or, [ # [and, [ # [plz, >=, 5], # [plz, <=, 8] # ], # [and, [ # [plz, >, 2], # [plz, <, 4] # ] # ] # ] # ] def check(conj, tail): # check conjunction if conj not in aconj: raise ValidationError("Invalid conjunction: '{}'".format(conj)) # tail must be list if not isinstance(tail, list): raise ValidationError("Invalid tail: '{}'".format(tail)) # tail is itself CONJ + TAIL if len(tail) == 2 and tail[0] in aconj: check(tail[0], tail[1:]) # consider tail to be list of things else: for thing in tail: l = len(thing) # this thing is CONJ + TAIL if l == 2: check(thing[0], thing[1:]) # this thing is filter expression elif l == 4: fld, op, case, val = thing if not fld in aff: raise ValidationError("Invalid field: '{}'".format(fld)) if not op in aops: raise ValidationError("Invalid op: '{}'".format(op)) if not case in acase: raise ValidationError("Invalid case: '{}'".format(case)) # this thing is garbage else: raise ValidationError("Invalid thing: '{}' ({})".format(thing, type(thing))) # 1st level is always CONJ + TAIL check(fil[0], fil[1]) return fil
def hello(self): url = self.url r = requests.get(url) r.raise_for_status() return json_deserializer(r.text)