def test_delete(self):
settings = self.utils.fill_container(
wrap({"data": [
{"a": 1, "b": 5},
{"a": 3, "b": 4},
{"a": 4, "b": 3},
{"a": 6, "b": 2},
{"a": 2}
]}),
typed=True
)
import jx_elasticsearch
container = jx_elasticsearch.new_instance(read_only=False, kwargs=self.utils._es_test_settings)
container.update({
"update": settings.alias,
"clear": ".",
"where": {"lt": {"a": 4}}
})
self.utils.send_queries({
"query": {
"from": settings.alias,
"sort":"a"
},
"expecting_list": {"data": [
{"a": 4, "b": 3},
{"a": 6, "b": 2}
]}
})
def test_new_field(self):
settings = self.utils.fill_container(
wrap({"data": [
{"a": 1, "b": 5},
{"a": 3, "b": 4},
{"a": 4, "b": 3},
{"a": 6, "b": 2},
{"a": 2}
]}),
typed=False
)
import jx_elasticsearch
container = jx_elasticsearch.new_instance(read_only=False, kwargs=self.utils._es_test_settings)
container.update({
"update": settings.alias,
"set": {"c": {"add": ["a", "b"]}}
})
self.utils.send_queries({
"query": {
"from": settings.alias,
"select": ["c", "a"]
},
"expecting_table": {
"header": ["a", "c"],
"data": [[1, 6], [3, 7], [4, 7], [6, 8], [2, NULL]]
}
})
def get_branches(hg, branches, kwargs=None):
# TRY ES
cluster = elasticsearch.Cluster(branches)
try:
es = cluster.get_index(kwargs=branches, read_only=False)
esq = jx_elasticsearch.new_instance(branches)
found_branches = esq.query({"from": "branches", "format": "list", "limit": 10000}).data
# IF IT IS TOO OLD, THEN PULL FROM HG
oldest = Date(MAX(found_branches.etl.timestamp))
if oldest == None or Date.now() - oldest > OLD_BRANCH:
found_branches = _get_branches_from_hg(hg)
es.extend({"id": b.name + " " + b.locale, "value": b} for b in found_branches)
es.flush()
try:
return UniqueIndex(["name", "locale"], data=found_branches, fail_on_dup=False)
except Exception as e:
Log.error("Bad branch in ES index", cause=e)
except Exception as e:
e = Except.wrap(e)
if "Can not find index " in e:
set_default(branches, {"schema": branches_schema})
es = cluster.get_or_create_index(branches)
es.add_alias()
return get_branches(kwargs)
Log.error("problem getting branches", cause=e)
def send_queries(self, subtest, places=6):
subtest = wrap(subtest)
try:
# EXECUTE QUERY
num_expectations = 0
for i, (k, v) in enumerate(subtest.items()):
if k in ["expecting", "expecting_error"]: # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS)
format = None
elif k.startswith("expecting_"): # WHAT FORMAT ARE WE REQUESTING
format = k[len("expecting_"):]
else:
continue
num_expectations += 1
expected = v
subtest.query.format = format
subtest.query.meta.testing = (num_expectations == 1) # MARK FIRST QUERY FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION
query = value2json(subtest.query).encode('utf8')
# EXECUTE QUERY
response = self.try_till_response(self.testing.query, data=query)
if k == "expecting_error":
if response.status_code != 200:
message = response.content.decode('utf8')
if v in message:
Log.note("PASS {{name|quote}} (expected error)", name=subtest.name)
continue
else:
Log.error("expecting {{expecting}} not {{error}}", expecting=v, error=message)
else:
Log.error("expecting a failure")
else:
if response.status_code != 200:
error(response)
result = json2value(response.all_content.decode('utf8'))
container = jx_elasticsearch.new_instance(self._es_test_settings)
query = QueryOp.wrap(subtest.query, container, container.namespace)
if is_many(expected.data) and len(result.data) != len(expected.data):
Log.error(
"expecting data (len={{rlen}}) to have length of {{elen}}",
rlen=len(result.data),
elen=len(expected.data)
)
compare_to_expected(query, result, expected, places)
Log.note("PASS {{name|quote}} (format={{format}})", name=subtest.name, format=format)
if num_expectations == 0:
Log.error(
"Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses",
name=subtest.name
)
except Exception as e:
Log.error("Failed test {{name|quote}}", name=subtest.name, cause=e)
def __init__(
self, host, index, type=DATA_TYPE, max_size=10, batch_size=10, kwargs=None
):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
es = Cluster(kwargs).get_or_create_index(
schema=json2value(convert.value2json(SCHEMA), leaves=True),
limit_replicas=True,
typed=False,
kwargs=kwargs,
)
es.add_alias(index)
es.set_refresh_interval(seconds=1)
self.queue = es.threaded_queue(max_size=max_size, batch_size=batch_size)
self.es = jx_elasticsearch.new_instance(es.settings)
def send_queries(self, subtest, places=6):
subtest = wrap(subtest)
try:
# EXECUTE QUERY
num_expectations = 0
for i, (k, v) in enumerate(subtest.items()):
if k.startswith("expecting_"): # WHAT FORMAT ARE WE REQUESTING
format = k[len("expecting_"):]
elif k == "expecting": # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS)
format = None
else:
continue
num_expectations += 1
expected = v
subtest.query.format = format
subtest.query.meta.testing = (
num_expectations == 1
) # MARK FIRST QUERY FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION
query = unicode2utf8(value2json(subtest.query))
# EXECUTE QUERY
response = self.try_till_response(self.testing.query,
data=query)
if response.status_code != 200:
error(response)
result = json2value(utf82unicode(response.all_content))
container = jx_elasticsearch.new_instance(
self._es_test_settings)
query = QueryOp.wrap(subtest.query, container,
container.namespace)
compare_to_expected(query, result, expected, places)
Log.note("PASS {{name|quote}} (format={{format}})",
name=subtest.name,
format=format)
if num_expectations == 0:
Log.error(
"Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses",
name=subtest.name)
except Exception as e:
Log.error("Failed test {{name|quote}}", {"name": subtest.name}, e)
def send_queries(self, subtest, places=6):
subtest = wrap(subtest)
try:
# EXECUTE QUERY
num_expectations = 0
for i, (k, v) in enumerate(subtest.items()):
if k.startswith("expecting_"): # WHAT FORMAT ARE WE REQUESTING
format = k[len("expecting_"):]
elif k == "expecting": # NO FORMAT REQUESTED (TO TEST DEFAULT FORMATS)
format = None
else:
continue
num_expectations += 1
expected = v
subtest.query.format = format
subtest.query.meta.testing = (
num_expectations == 1
) # MARK FIRST QUERY FOR TESTING SO FULL METADATA IS AVAILABLE BEFORE QUERY EXECUTION
query = subtest.query
# EXECUTE QUERY
try:
endpoint = jx_elasticsearch.new_instance(
self._es_test_settings)
result = endpoint.query(query)
if is_data(result):
result = wrap(result)
else:
result = result.__data__()
except Exception as e:
raise Log.error("Problem executing", cause=e)
compare_to_expected(query, result, expected, places)
Log.note("PASS {{name|quote}} (format={{format}})",
name=subtest.name,
format=format)
if num_expectations == 0:
Log.error(
"Expecting test {{name|quote}} to have property named 'expecting_*' for testing the various format clauses",
name=subtest.name)
except Exception as e:
Log.error("Failed test {{name|quote}}", {"name": subtest.name}, e)
def __init__(self, index):
self.es = FakeES({"host": "example.com", "index": "index"})
self.esq = jx_elasticsearch.new_instance(self.es)