def fetch_json(self, k, default=None, required=True, multiple=False):
v = self.fetch(k=k, default=default, required=required,
multiple=multiple)
try:
return json_deserializer(v)
except ValueError:
raise ValidationError("Invalid JSON: '{}'".format(k))
def load_cols(self, fn):
self.lgg.info('Reading columns from ' + fn)
with open(fn, 'rt', encoding='utf-8') as fh:
data = fh.read()
data = json_deserializer(data)
for k, v in data.items():
setattr(self, k, v)
def load(self, index, doc_type, id_, source=None):
"""
Loads specified document.
:param index: Index
:param doc_type: Document type
:param id_: ID
:param source: Optionally restrict to some or all source fields, i.e. no
meta data. If ``source`` is True, returns all source fields, else
``source`` must be an iterable of field names.
:return: JSON
"""
params = {}
if not source:
url = '{base}/{index}/{doc_type}/{id}'.format(
base=self.url, index=index, doc_type=doc_type, id=id_
)
elif source is True:
url = '{base}/{index}/{doc_type}/{id}/_source'.format(
base=self.url, index=index, doc_type=doc_type, id=id_
)
else:
url = '{base}/{index}/{doc_type}/{id}'.format(
base=self.url, index=index, doc_type=doc_type, id=id_
)
params = dict(_source=source.join(','))
r = requests.get(url, params=params)
r.raise_for_status()
return json_deserializer(r.text)
def delete_index(self, index):
url = '{base}/{index}/'.format(
base=self.url, index=index
)
r = requests.delete(url)
r.raise_for_status()
return json_deserializer(r.text)
def load_cols(self, fn):
self.lgg.info('Reading columns from ' + fn)
with open(fn, 'rt', encoding='utf-8') as fh:
data = fh.read()
data = json_deserializer(data)
for k, v in data.items():
setattr(self, k, v)
def create_index(self, index, rc=None):
url = '{base}/{index}/'.format(
base=self.url, index=index
)
s = json_serializer(rc) if rc else None
r = requests.put(url, data=s)
r.raise_for_status()
return json_deserializer(r.text)
def fetch_json(self, k, default=None, required=True, multiple=False):
v = self.fetch(k=k,
default=default,
required=required,
multiple=multiple)
try:
return json_deserializer(v)
except ValueError:
raise ValidationError("Invalid JSON: '{}'".format(k))
def count(self):
url = self.url + '/_count'
q = {
"query": {
"match_all": {}
}
}
r = requests.get(url, data=json_serializer(q))
r.raise_for_status()
return json_deserializer(r.text)
def search(self, index, doc_type, q):
url = '{base}/{index}/{doc_type}/_search'.format(
base=self.url, index=index, doc_type=doc_type
)
if isinstance(q, str):
r = requests.get(url, params=dict(q=q))
else:
r = requests.get(url, data=json_serializer(q))
r.raise_for_status()
return json_deserializer(r.text)
def save(self, index, doc_type, data, id_=None, create=None):
if create and not id_:
raise ValueError('To force document creation, id must be given')
if id_:
url = '{base}/{index}/{doc_type}/{id}'.format(
base=self.url, index=index, doc_type=doc_type, id=id_
)
if create:
url += '/_create'
s = json_serializer(data).encode('utf-8')
r = requests.put(url, data=s)
else:
url = '{base}/{index}/{doc_type}/'.format(
base=self.url, index=index, doc_type=doc_type
)
s = json_serializer(data).encode('utf-8')
r = requests.post(url, data=s)
r.raise_for_status()
return json_deserializer(r.text)
def filter(self):
"""
Fetches the filter expression from key ``fil`` (required, single).
Key ``fil`` must contain the expression as a single JSON string.
:return: Filter expression as a data structure.
:raise ValidationError: If key is missing or invalid.
"""
# TODO TESTING!!
try:
fil = self.parent.inp['fil']
except KeyError:
#raise ValidationError("Missing filter")
return None
try:
fil = json_deserializer(fil)
except ValueError:
raise ValidationError("Invalid JSON")
# Expect be [CONJ, TAIL]
if len(fil) != 2:
raise ValidationError("Invalid expression")
aconj = self.allowed_conjunctions
aops = self.allowed_operators
acase = self.allowed_case_sensitivity
aff = self.allowed_fields
# (country='f' and plz >=1 and plz <=2)
# or
# (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4)))
#
# [or, [
# [and, [
# [country, =, 'f'],
# [plz, >=, 1],
# [plz, <=, 2]
# ],
# [and, [
# [country, =, 'd'],
# [or, [
# [and, [
# [plz, >=, 5],
# [plz, <=, 8]
# ],
# [and, [
# [plz, >, 2],
# [plz, <, 4]
# ]
# ]
# ]
# ]
def check(conj, tail):
# check conjunction
if conj not in aconj:
raise ValidationError("Invalid conjunction: '{}'".format(conj))
# tail must be list
if not isinstance(tail, list):
raise ValidationError("Invalid tail: '{}'".format(tail))
# tail is itself CONJ + TAIL
if len(tail) == 2 and tail[0] in aconj:
check(tail[0], tail[1:])
# consider tail to be list of things
else:
for thing in tail:
l = len(thing)
# this thing is CONJ + TAIL
if l == 2:
check(thing[0], thing[1:])
# this thing is filter expression
elif l == 4:
fld, op, case, val = thing
if not fld in aff:
raise ValidationError(
"Invalid field: '{}'".format(fld))
if not op in aops:
raise ValidationError(
"Invalid op: '{}'".format(op))
if not case in acase:
raise ValidationError(
"Invalid case: '{}'".format(case))
# this thing is garbage
else:
raise ValidationError(
"Invalid thing: '{}' ({})".format(
thing, type(thing)))
# 1st level is always CONJ + TAIL
check(fil[0], fil[1])
return fil
def filter(self):
try:
fil = self.parent.inp['fil']
except KeyError:
raise ValidationError("Missing filter")
try:
fil = json_deserializer(fil)
except ValueError:
raise ValidationError("Invalid filter")
# Expect be [CONJ, TAIL]
if len(fil) != 2:
raise ValidationError("Invalid filter")
aconj = self.allowed_conjunctions
aops = self.allowed_operators
acase = self.allowed_case_sensitivity
aff = self.allowed_fields
# (country='f' and plz >=1 and plz <=2)
# or
# (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4)))
#
# [or, [
# [and, [
# [country, =, 'f'],
# [plz, >=, 1],
# [plz, <=, 2]
# ],
# [and, [
# [country, =, 'd'],
# [or, [
# [and, [
# [plz, >=, 5],
# [plz, <=, 8]
# ],
# [and, [
# [plz, >, 2],
# [plz, <, 4]
# ]
# ]
# ]
# ]
def check(conj, tail):
# check conjunction
if conj not in aconj:
raise ValidationError("Invalid conjunction: '{}'".format(conj))
# tail must be list
if not isinstance(tail, list):
raise ValidationError("Invalid tail: '{}'".format(tail))
# tail is itself CONJ + TAIL
if len(tail) == 2 and tail[0] in aconj:
check(tail[0], tail[1:])
# consider tail to be list of things
else:
for thing in tail:
l = len(thing)
# this thing is CONJ + TAIL
if l == 2:
check(thing[0], thing[1:])
# this thing is filter expression
elif l == 4:
fld, op, case, val = thing
if not fld in aff:
raise ValidationError("Invalid field: '{}'".format(fld))
if not op in aops:
raise ValidationError("Invalid op: '{}'".format(op))
if not case in acase:
raise ValidationError("Invalid case: '{}'".format(case))
# this thing is garbage
else:
raise ValidationError("Invalid thing: '{}'".format(thing))
check(fil[0], fil[1:])
return fil
def filter(self):
"""
Fetches the filter expression from key ``fil`` (required, single).
Key ``fil`` must contain the expression as a single JSON string.
:return: Filter expression as a data structure.
:raise ValidationError: If key is missing or invalid.
"""
# TODO TESTING!!
try:
fil = self.parent.inp['fil']
except KeyError:
#raise ValidationError("Missing filter")
return None
try:
fil = json_deserializer(fil)
except ValueError:
raise ValidationError("Invalid JSON")
# Expect be [CONJ, TAIL]
if len(fil) != 2:
raise ValidationError("Invalid expression")
aconj = self.allowed_conjunctions
aops = self.allowed_operators
acase = self.allowed_case_sensitivity
aff = self.allowed_fields
# (country='f' and plz >=1 and plz <=2)
# or
# (country='d' and ((plz >=5 and plz <=8) or (plz > 2 and plz < 4)))
#
# [or, [
# [and, [
# [country, =, 'f'],
# [plz, >=, 1],
# [plz, <=, 2]
# ],
# [and, [
# [country, =, 'd'],
# [or, [
# [and, [
# [plz, >=, 5],
# [plz, <=, 8]
# ],
# [and, [
# [plz, >, 2],
# [plz, <, 4]
# ]
# ]
# ]
# ]
def check(conj, tail):
# check conjunction
if conj not in aconj:
raise ValidationError("Invalid conjunction: '{}'".format(conj))
# tail must be list
if not isinstance(tail, list):
raise ValidationError("Invalid tail: '{}'".format(tail))
# tail is itself CONJ + TAIL
if len(tail) == 2 and tail[0] in aconj:
check(tail[0], tail[1:])
# consider tail to be list of things
else:
for thing in tail:
l = len(thing)
# this thing is CONJ + TAIL
if l == 2:
check(thing[0], thing[1:])
# this thing is filter expression
elif l == 4:
fld, op, case, val = thing
if not fld in aff:
raise ValidationError("Invalid field: '{}'".format(fld))
if not op in aops:
raise ValidationError("Invalid op: '{}'".format(op))
if not case in acase:
raise ValidationError("Invalid case: '{}'".format(case))
# this thing is garbage
else:
raise ValidationError("Invalid thing: '{}' ({})".format(thing, type(thing)))
# 1st level is always CONJ + TAIL
check(fil[0], fil[1])
return fil
def hello(self):
url = self.url
r = requests.get(url)
r.raise_for_status()
return json_deserializer(r.text)
