def es_fieldop(es, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
FromES.query = {
"filtered": {
"query": {
"match_all": {}
},
"filter":
simplify_esfilter(jx_expression(query.where).to_esfilter())
}
}
FromES.size = coalesce(query.limit, 200000)
FromES.fields = DictList()
for s in select.value:
if s == "*":
FromES.fields = None
elif isinstance(s, list):
FromES.fields.extend(s)
elif isinstance(s, Mapping):
FromES.fields.extend(s.values())
else:
FromES.fields.append(s)
FromES.sort = [{
s.field: "asc" if s.sort >= 0 else "desc"
} for s in query.sort]
data = es09.util.post(es, FromES, query.limit)
T = data.hits.hits
matricies = {}
for s in select:
if s.value == "*":
matricies[s.name] = Matrix.wrap([t._source for t in T])
elif isinstance(s.value, Mapping):
# for k, v in s.value.items():
# matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T])
matricies[s.name] = Matrix.wrap([{
k: unwrap(t.fields).get(v, None)
for k, v in s.value.items()
} for t in T])
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([
tuple(unwrap(t.fields).get(ss, None) for ss in s.value)
for t in T
])
elif not s.value:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
else:
try:
matricies[s.name] = Matrix.wrap(
[unwrap(t.fields).get(s.value, None) for t in T])
except Exception, e:
Log.error("", e)
def format_cube(T, select, source):
matricies = {}
for s in select:
try:
if s.value == ".":
matricies[s.name] = Matrix.wrap(T.select(source))
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([tuple(unwraplist(t[source][ss]) for ss in s.value) for t in T])
else:
if source == "_source":
matricies[s.name] = Matrix.wrap([unwraplist(t[source][s.value]) for t in T])
elif isinstance(s.value, basestring): # fields
matricies[s.name] = Matrix.wrap([unwraplist(t[source].get(s.value)) for t in T])
else:
matricies[s.name] = Matrix.wrap([unwraplist(t[source].get(s.name)) for t in T])
except Exception, e:
Log.error("", e)
def es_fieldop(es, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
FromES.query = {
"filtered": {
"query": {
"match_all": {}
},
"filter": simplify_esfilter(query.where)
}
}
FromES.size = coalesce(query.limit, 200000)
FromES.fields = DictList()
for s in select.value:
if s == "*":
FromES.fields = None
elif isinstance(s, list):
FromES.fields.extend(s)
elif isinstance(s, Mapping):
FromES.fields.extend(s.values())
else:
FromES.fields.append(s)
FromES.sort = [{s.field: "asc" if s.sort >= 0 else "desc"} for s in query.sort]
data = es09.util.post(es, FromES, query.limit)
T = data.hits.hits
matricies = {}
for s in select:
if s.value == "*":
matricies[s.name] = Matrix.wrap([t._source for t in T])
elif isinstance(s.value, Mapping):
# for k, v in s.value.items():
# matricies[join_field(split_field(s.name)+[k])] = Matrix.wrap([unwrap(t.fields)[v] for t in T])
matricies[s.name] = Matrix.wrap([{k: unwrap(t.fields).get(v, None) for k, v in s.value.items()}for t in T])
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([tuple(unwrap(t.fields).get(ss, None) for ss in s.value) for t in T])
elif not s.value:
matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T])
else:
try:
matricies[s.name] = Matrix.wrap([unwrap(t.fields).get(s.value, None) for t in T])
except Exception, e:
Log.error("", e)
def __init__(self, select, edges, data, frum=None):
"""
data IS EXPECTED TO BE A dict TO MATRICES, BUT OTHER COLLECTIONS ARE
ALLOWED, USING THE select AND edges TO DESCRIBE THE data
"""
self.is_value = False if isinstance(select, list) else True
self.select = select
self.meta = Dict(format="cube") # PUT EXTRA MARKUP HERE
self.is_none = False
if not all(data.values()):
is_none = True
# ENSURE frum IS PROPER FORM
if isinstance(select, list):
if edges and OR(not isinstance(v, Matrix) for v in data.values()):
Log.error("Expecting data to be a dict with Matrix values")
if not edges:
if not data:
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix.ZERO}
self.edges = DictList.EMPTY
elif isinstance(data, Mapping):
# EXPECTING NO MORE THAN ONE rownum EDGE IN THE DATA
length = MAX([len(v) for v in data.values()])
if length >= 1:
self.edges = wrap([{"name": "rownum", "domain": {"type": "rownum"}}])
else:
self.edges = DictList.EMPTY
elif isinstance(data, list):
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix.wrap(data)}
self.edges = wrap(
[{"name": "rownum", "domain": {"type": "rownum", "min": 0, "max": len(data), "interval": 1}}]
)
elif isinstance(data, Matrix):
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: data}
else:
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix(value=data)}
self.edges = DictList.EMPTY
else:
self.edges = wrap(edges)
self.data = data
def __init__(self, select, edges, data, frum=None):
"""
data IS EXPECTED TO BE A dict TO MATRICES, BUT OTHER COLLECTIONS ARE
ALLOWED, USING THE select AND edges TO DESCRIBE THE data
"""
self.is_value = False if isinstance(select, list) else True
self.select = select
self.meta = Dict(format="cube") # PUT EXTRA MARKUP HERE
self.is_none = False
if not all(data.values()):
is_none = True
# ENSURE frum IS PROPER FORM
if isinstance(select, list):
if edges and OR(not isinstance(v, Matrix) for v in data.values()):
Log.error("Expecting data to be a dict with Matrix values")
if not edges:
if not data:
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix.ZERO}
self.edges = DictList.EMPTY
elif isinstance(data, Mapping):
# EXPECTING NO MORE THAN ONE rownum EDGE IN THE DATA
length = MAX([len(v) for v in data.values()])
if length >= 1:
self.edges = wrap([{"name": "rownum", "domain": {"type": "rownum"}}])
else:
self.edges = DictList.EMPTY
elif isinstance(data, list):
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix.wrap(data)}
self.edges = wrap([{"name": "rownum", "domain": {"type": "rownum", "min": 0, "max": len(data), "interval": 1}}])
elif isinstance(data, Matrix):
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: data}
else:
if isinstance(select, list):
Log.error("not expecting a list of records")
data = {select.name: Matrix(value=data)}
self.edges = DictList.EMPTY
else:
self.edges = wrap(edges)
self.data = data
def format_cube(T, select, source):
matricies = {}
for s in select:
try:
if s.value == ".":
matricies[s.name] = Matrix.wrap(T.select(source))
elif isinstance(s.value, list):
matricies[s.name] = Matrix.wrap([
tuple(unwraplist(t[source][ss]) for ss in s.value)
for t in T
])
else:
if source == "_source":
matricies[s.name] = Matrix.wrap(
[unwraplist(t[source][s.value]) for t in T])
elif isinstance(s.value, basestring): # fields
matricies[s.name] = Matrix.wrap(
[unwraplist(t[source].get(s.value)) for t in T])
else:
matricies[s.name] = Matrix.wrap(
[unwraplist(t[source].get(s.name)) for t in T])
except Exception, e:
Log.error("", e)
def es_setop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
isDeep = len(split_field(
query.frum.name)) > 1 # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
isComplex = OR([
s.value == None and s.aggregate not in ("count", "none")
for s in select
]) # CONVERTING esfilter DEFINED PARTS WILL REQUIRE SCRIPT
if not isDeep and not isComplex:
if len(select) == 1 and not select[0].value or select[0].value == "*":
FromES = wrap({
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter":
simplify_esfilter(
jx_expression(query.where).to_esfilter())
}
},
"sort": query.sort,
"size": 1
})
elif all(isinstance(v, Variable) for v in select.value):
FromES = wrap({
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": simplify_esfilter(query.where.to_esfilter())
}
},
"fields": select.value,
"sort": query.sort,
"size": coalesce(query.limit, 200000)
})
elif not isDeep:
simple_query = query.copy()
simple_query.where = TRUE_FILTER # THE FACET FILTER IS FASTER
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(simple_query),
"size": coalesce(simple_query.limit, 200000)
},
"facet_filter":
simplify_esfilter(jx_expression(query.where).to_esfilter())
}
else:
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(query),
"size": coalesce(query.limit, 200000)
},
"facet_filter":
simplify_esfilter(jx_expression(query.where).to_esfilter())
}
data = es09.util.post(es, FromES, query.limit)
if len(select) == 1 and not select[0].value or select[0].value == "*":
# SPECIAL CASE FOR SINGLE COUNT
cube = wrap(data).hits.hits._source
elif isinstance(select[0].value, Variable):
# SPECIAL CASE FOR SINGLE TERM
cube = wrap(data).hits.hits.fields
else:
data_list = unpack_terms(data.facets.mvel, select)
if not data_list:
cube = Cube(select, [], {s.name: Matrix.wrap([]) for s in select})
else:
output = zip(*data_list)
cube = Cube(
select, [],
{s.name: Matrix(list=output[i])
for i, s in enumerate(select)})
return Dict(meta={"esquery": FromES}, data=cube)
def es_setop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
isDeep = len(split_field(query.frum.name)) > 1 # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
isComplex = OR([s.value == None and s.aggregate not in ("count", "none") for s in select]) # CONVERTING esfilter DEFINED PARTS WILL REQUIRE SCRIPT
if not isDeep and not isComplex and len(select) == 1:
if not select[0].value:
FromES.query = {"filtered": {
"query": {"match_all": {}},
"filter": simplify_esfilter(query.where)
}}
FromES.size = 1 # PREVENT QUERY CHECKER FROM THROWING ERROR
elif isKeyword(select[0].value):
FromES.facets.mvel = {
"terms": {
"field": select[0].value,
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
if query.sort:
s = query.sort
if len(s) > 1:
Log.error("can not sort by more than one field")
s0 = s[0]
if s0.field != select[0].value:
Log.error("can not sort by anything other than count, or term")
FromES.facets.terms.order = "term" if s0.sort >= 0 else "reverse_term"
elif not isDeep:
simple_query = query.copy()
simple_query.where = TRUE_FILTER # THE FACET FILTER IS FASTER
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(simple_query),
"size": coalesce(simple_query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
else:
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(query),
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
data = es09.util.post(es, FromES, query.limit)
if len(select) == 1:
if not select[0].value:
# SPECIAL CASE FOR SINGLE COUNT
output = Matrix(value=data.hits.total)
cube = Cube(query.select, [], {select[0].name: output})
elif isKeyword(select[0].value):
# SPECIAL CASE FOR SINGLE TERM
T = data.facets.terms
output = Matrix.wrap([t.term for t in T])
cube = Cube(query.select, [], {select[0].name: output})
else:
data_list = unpack_terms(data.facets.mvel, select)
if not data_list:
cube = Cube(select, [], {s.name: Matrix.wrap([]) for s in select})
else:
output = zip(*data_list)
cube = Cube(select, [], {s.name: Matrix(list=output[i]) for i, s in enumerate(select)})
cube.frum = query
return cube
def es_setop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
isDeep = len(split_field(query.frum.name)) > 1 # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
isComplex = OR([s.value == None and s.aggregate not in ("count", "none") for s in select]) # CONVERTING esfilter DEFINED PARTS WILL REQUIRE SCRIPT
if not isDeep and not isComplex:
if len(select) == 1 and not select[0].value or select[0].value == "*":
FromES = wrap({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": simplify_esfilter(jx_expression(query.where).to_esfilter())
}},
"sort": query.sort,
"size": 1
})
elif all(isinstance(v, Variable) for v in select.value):
FromES = wrap({
"query": {"filtered": {
"query": {"match_all": {}},
"filter": simplify_esfilter(query.where.to_esfilter())
}},
"fields": select.value,
"sort": query.sort,
"size": coalesce(query.limit, 200000)
})
elif not isDeep:
simple_query = query.copy()
simple_query.where = TRUE_FILTER # THE FACET FILTER IS FASTER
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(simple_query),
"size": coalesce(simple_query.limit, 200000)
},
"facet_filter": simplify_esfilter(jx_expression(query.where).to_esfilter())
}
else:
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(query),
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(jx_expression(query.where).to_esfilter())
}
data = es09.util.post(es, FromES, query.limit)
if len(select) == 1 and not select[0].value or select[0].value == "*":
# SPECIAL CASE FOR SINGLE COUNT
cube = wrap(data).hits.hits._source
elif isinstance(select[0].value, Variable):
# SPECIAL CASE FOR SINGLE TERM
cube = wrap(data).hits.hits.fields
else:
data_list = unpack_terms(data.facets.mvel, select)
if not data_list:
cube = Cube(select, [], {s.name: Matrix.wrap([]) for s in select})
else:
output = zip(*data_list)
cube = Cube(select, [], {s.name: Matrix(list=output[i]) for i, s in enumerate(select)})
return Data(
meta={"esquery": FromES},
data=cube
)
def es_setop(es, mvel, query):
FromES = es09.util.build_es_query(query)
select = listwrap(query.select)
isDeep = len(split_field(
query.frum.name)) > 1 # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
isComplex = OR([
s.value == None and s.aggregate not in ("count", "none")
for s in select
]) # CONVERTING esfilter DEFINED PARTS WILL REQUIRE SCRIPT
if not isDeep and not isComplex and len(select) == 1:
if not select[0].value:
FromES.query = {
"filtered": {
"query": {
"match_all": {}
},
"filter": simplify_esfilter(query.where)
}
}
FromES.size = 1 # PREVENT QUERY CHECKER FROM THROWING ERROR
elif isKeyword(select[0].value):
FromES.facets.mvel = {
"terms": {
"field": select[0].value,
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
if query.sort:
s = query.sort
if len(s) > 1:
Log.error("can not sort by more than one field")
s0 = s[0]
if s0.field != select[0].value:
Log.error(
"can not sort by anything other than count, or term")
FromES.facets.terms.order = "term" if s0.sort >= 0 else "reverse_term"
elif not isDeep:
simple_query = query.copy()
simple_query.where = TRUE_FILTER # THE FACET FILTER IS FASTER
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(simple_query),
"size": coalesce(simple_query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
else:
FromES.facets.mvel = {
"terms": {
"script_field": mvel.code(query),
"size": coalesce(query.limit, 200000)
},
"facet_filter": simplify_esfilter(query.where)
}
data = es09.util.post(es, FromES, query.limit)
if len(select) == 1:
if not select[0].value:
# SPECIAL CASE FOR SINGLE COUNT
output = Matrix(value=data.hits.total)
cube = Cube(query.select, [], {select[0].name: output})
elif isKeyword(select[0].value):
# SPECIAL CASE FOR SINGLE TERM
T = data.facets.terms
output = Matrix.wrap([t.term for t in T])
cube = Cube(query.select, [], {select[0].name: output})
else:
data_list = unpack_terms(data.facets.mvel, select)
if not data_list:
cube = Cube(select, [], {s.name: Matrix.wrap([]) for s in select})
else:
output = zip(*data_list)
cube = Cube(
select, [],
{s.name: Matrix(list=output[i])
for i, s in enumerate(select)})
cube.frum = query
return cube
