def _normalize_select(self, select):
output = []
if select.value == ".":
for cname, cs in self.columns.items():
num_types = 0
for c in cs:
if c.type in ["nested", "object"]:
continue
if not num_types:
new_select = select.copy()
new_select.name = cname
new_select.value = Variable(c.es_column)
output.append(new_select)
elif num_types == 1:
new_select.value = CoalesceOp(
"coalesce",
[Variable(c.es_column), new_select.value])
else:
new_select.value.terms.append(Variable(c.es_column))
num_types += 1
elif select.value.endswith(".*"):
Log.error("not done")
else:
Log.error("not done")
return output
def _normalize_select_no_context(select, schema=None):
"""
SAME NORMALIZE, BUT NO SOURCE OF COLUMNS
"""
if not _Column:
_late_import()
if isinstance(select, basestring):
select = Data(value=select)
else:
select = wrap(select)
output = select.copy()
if not select.value:
output.name = coalesce(select.name, select.aggregate)
if output.name:
output.value = jx_expression(".")
else:
return output
elif isinstance(select.value, basestring):
if select.value.endswith(".*"):
output.name = coalesce(select.name, select.value[:-2],
select.aggregate)
output.value = LeavesOp("leaves", Variable(select.value[:-2]))
else:
if select.value == ".":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = jx_expression(select.value)
elif select.value == "*":
output.name = coalesce(select.name, select.aggregate, ".")
output.value = LeavesOp("leaves", Variable("."))
else:
output.name = coalesce(select.name, select.value,
select.aggregate)
output.value = jx_expression(select.value)
elif isinstance(select.value, (int, float)):
if not output.name:
output.name = unicode(select.value)
output.value = jx_expression(select.value)
else:
output.value = jx_expression(select.value)
if not output.name:
Log.error("expecting select to have a name: {{select}}", select=select)
if output.name.endswith(".*"):
Log.error("{{name|quote}} is invalid select", name=output.name)
output.aggregate = coalesce(canonical_aggregates[select.aggregate].name,
select.aggregate, "none")
output.default = coalesce(select.default,
canonical_aggregates[output.aggregate].default)
return output
def _normalize_select(self, select):
output = []
if select.value == ".":
for cname, cs in self.columns.items():
for c in cs:
if c.type in STRUCT:
continue
new_select = select.copy()
new_select.name = cname
new_select.value = Variable(cname)
output.append(new_select)
break
elif select.value.endswith(".*"):
Log.error("not done")
else:
Log.error("not done")
return output
def extract_rows(es, es_query, query):
is_list = isinstance(query.select, list)
selects = wrap([s.copy() for s in listwrap(query.select)])
new_select = FlatList()
schema = query.frum.schema
columns = schema.columns
leaf_columns = set(
c.names["."] for c in columns if c.type not in STRUCT and (
c.nested_path[0] == "." or c.es_column == c.nested_path[0]))
nested_columns = set(c.names["."] for c in columns
if c.nested_path[0] != ".")
i = 0
source = "fields"
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if isinstance(select.value, LeavesOp):
new_name_prefix = select.name + "\\." if select.name != "." else ""
term = select.value.term
if isinstance(term, Variable):
if term.var == ".":
es_query.fields = None
source = "_source"
for cname, cs in schema.lookup.items():
for c in cs:
if c.type not in STRUCT and c.es_column != "_id":
new_name = new_name_prefix + literal_field(
cname)
new_select.append({
"name":
new_name,
"value":
Variable(c.es_column),
"put": {
"name": new_name,
"index": i,
"child": "."
}
})
i += 1
else:
prefix = term.var + "."
prefix_length = len(prefix)
for cname, cs in schema.lookup.items():
if cname.startswith(prefix):
suffix = cname[prefix_length:]
for c in cs:
if c.type not in STRUCT:
if es_query.fields is not None:
es_query.fields.append(c.es_column)
new_name = new_name_prefix + literal_field(
suffix)
new_select.append({
"name":
new_name,
"value":
Variable(c.es_column),
"put": {
"name": new_name,
"index": i,
"child": "."
}
})
i += 1
elif isinstance(select.value, Variable):
if select.value.var == ".":
es_query.fields = None
source = "_source"
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
elif select.value.var == "_id":
new_select.append({
"name": select.name,
"value": select.value,
"pull": "_id",
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
elif select.value.var in nested_columns or [
c for c in nested_columns
if c.startswith(select.value.var + ".")
]:
es_query.fields = None
source = "_source"
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
else:
prefix = select.value.var + "."
prefix_length = len(prefix)
net_columns = [c for c in leaf_columns if c.startswith(prefix)]
if not net_columns:
# LEAF
if es_query.fields is not None:
es_query.fields.append(select.value.var)
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
else:
# LEAVES OF OBJECT
for cname, cs in schema.lookup.items():
if cname.startswith(prefix):
for c in cs:
if c.type not in STRUCT:
if es_query.fields is not None:
es_query.fields.append(c.es_column)
new_select.append({
"name":
select.name,
"value":
Variable(c.es_column),
"put": {
"name": select.name,
"index": i,
"child": cname[prefix_length:]
}
})
i += 1
else:
es_query.script_fields[literal_field(select.name)] = {
"script": select.value.to_ruby()
}
new_select.append({
"name": select.name,
"pull": "fields." + literal_field(select.name),
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
for n in new_select:
if n.pull:
continue
if source == "_source":
n.pull = concat_field("_source", n.value.var)
elif isinstance(n.value, Variable):
n.pull = "fields." + literal_field(n.value.var)
else:
Log.error("Do not know what to do")
with Timer("call to ES") as call_timer:
data = es09.util.post(es, es_query, query.limit)
T = data.hits.hits
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception as e:
Log.error("problem formatting", e)
def extract_rows(es, es_query, query):
is_list = isinstance(query.select, list)
selects = wrap([s.copy() for s in listwrap(query.select)])
new_select = DictList()
columns = query.frum.get_columns()
leaf_columns = set(c.name for c in columns if c.type not in STRUCT and (
c.nested_path[0] == "." or c.es_column == c.nested_path))
nested_columns = set(c.name for c in columns if len(c.nested_path) != 1)
i = 0
source = "fields"
for select in selects:
# IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
if isinstance(select.value, LeavesOp):
term = select.value.term
if isinstance(term, Variable):
if term.var == ".":
es_query.fields = None
source = "_source"
net_columns = leaf_columns - set(selects.name)
for n in net_columns:
new_select.append({
"name": n,
"value": Variable(n),
"put": {
"name": n,
"index": i,
"child": "."
}
})
i += 1
else:
parent = term.var + "."
prefix = len(parent)
for c in leaf_columns:
if c.startswith(parent):
if es_query.fields is not None:
es_query.fields.append(c)
new_select.append({
"name":
select.name + "." + c[prefix:],
"value":
Variable(c),
"put": {
"name": select.name + "." + c[prefix:],
"index": i,
"child": "."
}
})
i += 1
elif isinstance(select.value, Variable):
if select.value.var == ".":
es_query.fields = None
source = "_source"
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
elif select.value.var == "_id":
new_select.append({
"name": select.name,
"value": select.value,
"pull": "_id",
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
elif select.value.var in nested_columns or [
c for c in nested_columns
if c.startswith(select.value.var + ".")
]:
es_query.fields = None
source = "_source"
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
else:
parent = select.value.var + "."
prefix = len(parent)
net_columns = [c for c in leaf_columns if c.startswith(parent)]
if not net_columns:
# LEAF
if es_query.fields is not None:
es_query.fields.append(select.value.var)
new_select.append({
"name": select.name,
"value": select.value,
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
else:
# LEAVES OF OBJECT
for n in net_columns:
if es_query.fields is not None:
es_query.fields.append(n)
new_select.append({
"name": select.name,
"value": Variable(n),
"put": {
"name": select.name,
"index": i,
"child": n[prefix:]
}
})
i += 1
else:
es_query.script_fields[literal_field(select.name)] = {
"script": select.value.to_ruby()
}
new_select.append({
"name": select.name,
"pull": "fields." + literal_field(select.name),
"put": {
"name": select.name,
"index": i,
"child": "."
}
})
i += 1
for n in new_select:
if n.pull:
continue
if source == "_source":
n.pull = join_field(["_source"] + split_field(n.value.var))
elif isinstance(n.value, Variable):
n.pull = "fields." + literal_field(n.value.var)
else:
Log.error("Do not know what to do")
with Timer("call to ES") as call_timer:
data = es09.util.post(es, es_query, query.limit)
T = data.hits.hits
try:
formatter, groupby_formatter, mime_type = format_dispatch[query.format]
output = formatter(T, new_select, query)
output.meta.timing.es = call_timer.duration
output.meta.content_type = mime_type
output.meta.es_query = es_query
return output
except Exception, e:
Log.error("problem formatting", e)