def _select(template, data, fields, depth):
output = FL()
deep_path = []
deep_fields = UniqueIndex(["name"])
for d in data:
if isinstance(d, Data):
Log.error("programmer error, _select can not handle Data")
record = template.copy()
children = None
for f in fields:
index, c = _select_deep(d, f, depth, record)
children = c if children is None else children
if index:
path = f.value[0:index:]
if not deep_fields[f]:
deep_fields.add(f) # KEEP TRACK OF WHICH FIELDS NEED DEEPER SELECT
short = MIN([len(deep_path), len(path)])
if path[:short:] != deep_path[:short:]:
Log.error("Dangerous to select into more than one branch at time")
if len(deep_path) < len(path):
deep_path = path
if not children:
output.append(record)
else:
output.extend(_select(record, children, deep_fields, depth + 1))
return output
def parse_properties(parent_index_name, parent_name, esProperties):
"""
RETURN THE COLUMN DEFINITIONS IN THE GIVEN esProperties OBJECT
"""
from pyLibrary.queries.meta import Column
columns = FlatList()
for name, property in esProperties.items():
index_name = parent_index_name
column_name = join_field(split_field(parent_name) + [name])
if property.type == "nested" and property.properties:
# NESTED TYPE IS A NEW TYPE DEFINITION
# MARKUP CHILD COLUMNS WITH THE EXTRA DEPTH
self_columns = parse_properties(index_name, column_name, property.properties)
for c in self_columns:
c.nested_path = [column_name] + c.nested_path
columns.extend(self_columns)
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name,
es_column=column_name,
type="nested",
nested_path=ROOT_PATH
))
continue
if property.properties:
child_columns = parse_properties(index_name, column_name, property.properties)
columns.extend(child_columns)
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name,
es_column=column_name,
nested_path=ROOT_PATH,
type="source" if property.enabled == False else "object"
))
if property.dynamic:
continue
if not property.type:
continue
if property.type == "multi_field":
property.type = property.fields[name].type # PULL DEFAULT TYPE
for i, (n, p) in enumerate(property.fields.items()):
if n == name:
# DEFAULT
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name,
es_column=column_name,
nested_path=ROOT_PATH,
type=p.type
))
else:
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name + "\\." + n,
es_column=column_name + "\\." + n,
nested_path=ROOT_PATH,
type=p.type
))
continue
if property.type in ["string", "boolean", "integer", "date", "long", "double"]:
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name,
es_column=column_name,
nested_path=ROOT_PATH,
type=property.type
))
if property.index_name and name != property.index_name:
columns.append(Column(
table=index_name,
es_index=index_name,
es_column=column_name,
name=column_name,
nested_path=ROOT_PATH,
type=property.type
))
elif property.enabled == None or property.enabled == False:
columns.append(Column(
table=index_name,
es_index=index_name,
name=column_name,
es_column=column_name,
nested_path=ROOT_PATH,
type="source" if property.enabled==False else "object"
))
else:
Log.warning("unknown type {{type}} for property {{path}}", type=property.type, path=query_path)
return columns
