def doc_to_column(doc):
try:
doc = wrap(untyped(doc))
# I HAVE MANAGED TO MAKE MANY MISTAKES WRITING COLUMNS TO ES. HERE ARE THE FIXES
# FIX
if not doc.last_updated:
doc.last_updated = Date.now() - YEAR
# FIX
if doc.es_type == None:
if doc.jx_type == OBJECT:
doc.es_type = "object"
else:
Log.warning("{{doc}} has no es_type", doc=doc)
# FIX
doc.multi = 1001 if doc.es_type == "nested" else doc.multi
# FIX
doc.nested_path = tuple(listwrap(doc.nested_path))
if last(split_field(
doc.es_column)) == NESTED_TYPE and doc.es_type != "nested":
doc.es_type = "nested"
doc.jx_type = NESTED
doc.multi = 1001
doc.last_updated = Date.now()
# FIX
expected_nested_path = get_nested_path(doc.es_column)
if len(doc.nested_path) > 1 and doc.nested_path[-2] == '.':
doc.nested_path = doc.nested_path[:-1]
# FIX
if untype_path(doc.es_column) == doc.es_column:
if doc.nested_path != (".", ):
if doc.es_index in {"repo"}:
pass
else:
Log.note("not expected")
doc.nested_path = expected_nested_path
else:
if doc.nested_path != expected_nested_path:
doc.nested_path = expected_nested_path
# FIX
if last(split_field(doc.es_column)) == EXISTS_TYPE:
doc.jx_type = EXISTS
return Column(**doc)
except Exception:
doc.nested_path = ["."]
mark_as_deleted(Column(**doc))
return None
def _get_queue(self, row):
row = wrap(row)
if row.json:
row.value, row.json = json2value(row.json), None
timestamp = Date(self.rollover_field(row.value))
if timestamp == None:
return Null
elif timestamp < Date.today() - self.rollover_max:
return DATA_TOO_OLD
rounded_timestamp = timestamp.floor(self.rollover_interval)
with self.locker:
queue = self.known_queues.get(rounded_timestamp.unix)
if queue == None:
candidates = wrap(sort_using_key(
filter(
lambda r: re.match(
re.escape(self.settings.index) + r"\d\d\d\d\d\d\d\d_\d\d\d\d\d\d$",
r['index']
),
self.cluster.get_aliases()
),
key=lambda r: r['index']
))
best = None
for c in candidates:
c.date = unicode2Date(c.index[-15:], elasticsearch.INDEX_DATE_FORMAT)
if timestamp > c.date:
best = c
if not best or rounded_timestamp > best.date:
if rounded_timestamp < wrap(last(candidates)).date:
es = self.cluster.get_or_create_index(read_only=False, alias=best.alias, index=best.index, kwargs=self.settings)
else:
try:
es = self.cluster.create_index(create_timestamp=rounded_timestamp, kwargs=self.settings)
es.add_alias(self.settings.index)
except Exception as e:
e = Except.wrap(e)
if "IndexAlreadyExistsException" not in e:
Log.error("Problem creating index", cause=e)
return self._get_queue(row) # TRY AGAIN
else:
es = self.cluster.get_or_create_index(read_only=False, alias=best.alias, index=best.index, kwargs=self.settings)
def refresh(please_stop):
try:
es.set_refresh_interval(seconds=coalesce(Duration(self.settings.refresh_interval).seconds, 60 * 10), timeout=5)
except Exception:
Log.note("Could not set refresh interval for {{index}}", index=es.settings.index)
Thread.run("refresh", refresh).release()
self._delete_old_indexes(candidates)
threaded_queue = es.threaded_queue(max_size=self.settings.queue_size, batch_size=self.settings.batch_size, silent=True)
with self.locker:
queue = self.known_queues[rounded_timestamp.unix] = threaded_queue
return queue
def partial_eval(self):
term = self.term.partial_eval()
if is_op(self.term, LastOp):
return term
elif term.type != OBJECT and not term.many:
return term
elif term is NULL:
return term
elif is_literal(term):
return last(term)
else:
return self.lang[LastOp(term)]
def __init__(self, var):
"""
:param var: DOT DELIMITED PATH INTO A DOCUMENT
"""
Expression.__init__(self, None)
# if self.lang != self.__class_.lang:
# pass
self.var = get_property_name(var)
jx_type = inserter_type_to_json_type.get(last(split_field(var)))
if jx_type:
self.data_type = jx_type
def doc_to_column(doc):
try:
doc = wrap(untyped(doc))
if not doc.last_updated:
doc.last_updated = Date.now() - YEAR
if doc.es_type == None:
if doc.jx_type == OBJECT:
doc.es_type = "object"
else:
Log.warning("{{doc}} has no es_type", doc=doc)
doc.multi = 1001 if doc.es_type == "nested" else doc.multi
doc.nested_path = tuple(listwrap(doc.nested_path))
if last(split_field(
doc.es_column)) == NESTED_TYPE and doc.es_type != "nested":
doc.es_type = "nested"
doc.jx_type = NESTED
doc.multi = 1001
doc.last_updated = Date.now()
expected_nested_path = get_nested_path(doc.es_column)
if len(doc.nested_path) > 1 and doc.nested_path[-2] == '.':
doc.nested_path = doc.nested_path[:-1]
if untype_path(doc.es_column) == doc.es_column:
if doc.nested_path != (".", ):
if doc.es_index in {"repo"}:
pass
else:
Log.note("not expected")
doc.nested_path = expected_nested_path
else:
if doc.nested_path != expected_nested_path:
doc.nested_path = expected_nested_path
return Column(**doc)
except Exception:
doc.nested_path = ["."]
mark_as_deleted(Column(**doc))
return None
def __init__(self, var, type=None, multi=None):
"""
:param var: DOT DELIMITED PATH INTO A DOCUMENT
:param type: JSON TYPE, IF KNOWN
:param multi: NUMBER OF DISTINCT VALUES IN A SLOT
"""
Expression.__init__(self, None)
# if self.lang != self.__class_.lang:
# pass
self.var = get_property_name(var)
if type == None:
jx_type = inserter_type_to_json_type.get(last(split_field(var)))
if jx_type:
self.data_type = jx_type
else:
self.data_type = type
self._many = False
if multi and multi > 1:
self._many = True
def doc_to_column(doc):
now = Date.now()
try:
doc = to_data(untyped(doc))
# I HAVE MANAGED TO MAKE MANY MISTAKES WRITING COLUMNS TO ES. HERE ARE THE FIXES
# FIX
if not doc.last_updated:
doc.last_updated = Date.now() - YEAR
# FIX
if doc.es_type == None:
if doc.jx_type == OBJECT:
doc.es_type = "object"
else:
Log.warning("{{doc}} has no es_type", doc=doc)
# FIX
if doc.es_type == "nested":
doc.multi = 1001
if doc.multi == None:
doc.multi = 1
# FIX
if doc.es_column.endswith("." + NESTED_TYPE):
if doc.jx_type == OBJECT:
doc.jx_type = NESTED
doc.last_updated = now
if doc.es_type == "nested":
doc.es_type = "nested"
doc.last_updated = now
# FIX
doc.nested_path = tuple(listwrap(doc.nested_path))
if last(split_field(
doc.es_column)) == NESTED_TYPE and doc.es_type != "nested":
doc.es_type = "nested"
doc.jx_type = NESTED
doc.multi = 1001
doc.last_updated = now
# FIX
expected_nested_path = get_nested_path(doc.es_column)
if len(doc.nested_path) > 1 and doc.nested_path[-2] == '.':
doc.nested_path = doc.nested_path[:-1]
doc.last_updated = now
# FIX
if untype_path(doc.es_column) == doc.es_column:
if doc.nested_path != (".", ):
if doc.es_index in {"repo"}:
pass
else:
Log.note("not expected")
doc.nested_path = expected_nested_path
doc.last_updated = now
else:
if doc.nested_path != expected_nested_path:
doc.nested_path = expected_nested_path
doc.last_updated = now
# FIX
if last(split_field(doc.es_column)) == EXISTS_TYPE:
if doc.jx_type != EXISTS:
doc.jx_type = EXISTS
doc.last_updated = now
if doc.cardinality == None:
doc.cardinality = 1
doc.last_updated = now
# FIX
if doc.jx_type in STRUCT:
if doc.cardinality not in [0, 1]:
doc.cardinality = 1 # DO NOT KNOW IF EXISTS OR NOT
doc.last_updated = now
return Column(**doc)
except Exception as e:
try:
mark_as_deleted(Column(**doc), now)
except Exception:
pass
return None
def __init__(
self,
host,
index, # THE NAME OF THE SNOWFLAKE (IF WRITING)
alias=None, # THE NAME OF THE SNOWFLAKE (FOR READING)
type=None,
name=None, # THE FULL NAME OF THE TABLE (THE NESTED PATH INTO THE SNOWFLAKE)
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
Container.__init__(self)
if not container.config.default:
container.config.default = {
"type": "elasticsearch",
"settings": unwrap(kwargs)
}
self.edges = Data() # SET EARLY, SO OTHER PROCESSES CAN REQUEST IT
self.worker = None
self.settings = kwargs
self._namespace = ElasticsearchMetadata(kwargs=kwargs)
self.name = name = self._namespace._find_alias(
coalesce(alias, index, name))
if read_only:
self.es = elasticsearch.Alias(alias=name,
index=None,
kwargs=kwargs)
else:
self.es = elasticsearch.Cluster(kwargs=kwargs).get_index(
read_only=read_only, kwargs=kwargs)
self._ensure_max_result_window_set(name)
self.settings.type = self.es.settings.type
self.stats = QueryStats(self.es.cluster)
columns = self.snowflake.columns # ABSOLUTE COLUMNS
is_typed = any(c.es_column == EXISTS_TYPE for c in columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = is_typed
else:
if is_typed != typed:
Log.error(
"Expecting given typed {{typed}} to match {{is_typed}}",
typed=typed,
is_typed=is_typed)
self.typed = typed
if not typed:
# ADD EXISTENCE COLUMNS
all_paths = {'.': None} # MAP FROM path TO parent TO MAKE A TREE
def nested_path_of(v):
if v == '.':
return ('.', )
return (v, ) + nested_path_of(all_paths[v])
query_paths = sort_using_key(set(
step for path in self.snowflake.query_paths for step in path),
key=lambda p: len(split_field(p)))
for step in query_paths:
if step in all_paths:
continue
else:
best = '.'
for candidate in all_paths.keys():
if startswith_field(step, candidate):
if startswith_field(candidate, best):
best = candidate
all_paths[step] = best
for p in all_paths.keys():
nested_path = nested_path_of(p)
try:
self.namespace.meta.columns.add(
Column(name=p,
es_column=p,
es_index=self.name,
es_type=OBJECT,
jx_type=OBJECT,
nested_path=nested_path,
multi=1001 if last(split_field(p))
== NESTED_TYPE else None,
last_updated=Date.now()))
except Exception as e:
raise e
def get_revision(self,
revision,
locale=None,
get_diff=False,
get_moves=True,
after=None):
"""
EXPECTING INCOMPLETE revision OBJECT
RETURNS revision
"""
rev = revision.changeset.id
if not rev:
return Null
elif rev == "None":
return Null
elif revision.branch.name == None:
return Null
locale = coalesce(locale, revision.branch.locale, DEFAULT_LOCALE)
output = self._get_from_elasticsearch(revision,
locale=locale,
get_diff=get_diff,
get_moves=get_moves,
after=after)
if output:
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
DEBUG and Log.note(
"Got hg ({{branch}}, {{locale}}, {{revision}}) from ES",
branch=output.branch.name,
locale=locale,
revision=output.changeset.id,
)
if output.push.date >= Date.now() - MAX_TODO_AGE:
self.todo.add((output.branch, listwrap(output.parents), None))
self.todo.add((output.branch, listwrap(output.children), None))
if output.push.date:
return output
# RATE LIMIT CALLS TO HG (CACHE MISSES)
next_cache_miss = self.last_cache_miss + (
Random.float(WAIT_AFTER_CACHE_MISS * 2) * SECOND)
self.last_cache_miss = Date.now()
if next_cache_miss > self.last_cache_miss:
Log.note(
"delaying next hg call for {{seconds|round(decimal=1)}} seconds",
seconds=next_cache_miss - self.last_cache_miss,
)
Till(till=next_cache_miss.unix).wait()
found_revision = copy(revision)
if isinstance(found_revision.branch, (text, binary_type)):
lower_name = found_revision.branch.lower()
else:
lower_name = found_revision.branch.name.lower()
if not lower_name:
Log.error("Defective revision? {{rev|json}}",
rev=found_revision.branch)
b = found_revision.branch = self.branches[(lower_name, locale)]
if not b:
b = found_revision.branch = self.branches[(lower_name,
DEFAULT_LOCALE)]
if not b:
Log.warning(
"can not find branch ({{branch}}, {{locale}})",
branch=lower_name,
locale=locale,
)
return Null
if Date.now() - Date(b.etl.timestamp) > _hg_branches.OLD_BRANCH:
self.branches = _hg_branches.get_branches(kwargs=self.settings)
push = self._get_push(found_revision.branch,
found_revision.changeset.id)
id12 = found_revision.changeset.id[0:12]
url1 = found_revision.branch.url.rstrip(
"/") + "/json-info?node=" + id12
url2 = found_revision.branch.url.rstrip("/") + "/json-rev/" + id12
url3 = (found_revision.branch.url.rstrip("/") +
"/json-automationrelevance/" + id12)
with Explanation("get revision from {{url}}", url=url1, debug=DEBUG):
raw_rev2 = Null
automation_details = Null
try:
raw_rev1 = self._get_raw_json_info(url1, found_revision.branch)
raw_rev2 = self._get_raw_json_rev(url2, found_revision.branch)
automation_details = self._get_raw_json_rev(
url3, found_revision.branch)
except Exception as e:
if "Hg denies it exists" in e:
raw_rev1 = Data(node=revision.changeset.id)
else:
raise e
raw_rev3_changeset = first(r for r in automation_details.changesets
if r.node[:12] == id12)
if last(automation_details.changesets) != raw_rev3_changeset:
Log.note("interesting")
output = self._normalize_revision(
set_default(raw_rev1, raw_rev2, raw_rev3_changeset),
found_revision,
push,
get_diff,
get_moves,
)
if output.push.date >= Date.now() - MAX_TODO_AGE:
self.todo.add((output.branch, listwrap(output.parents), None))
self.todo.add((output.branch, listwrap(output.children), None))
self.todo.add((output.branch, listwrap(output.backsoutnodes),
output.push.date))
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
return output
def _get_from_hg(self,
revision,
locale=None,
get_diff=False,
get_moves=True):
# RATE LIMIT CALLS TO HG (CACHE MISSES)
next_cache_miss = self.last_cache_miss + (
Random.float(WAIT_AFTER_CACHE_MISS * 2) * SECOND)
self.last_cache_miss = Date.now()
if next_cache_miss > self.last_cache_miss:
Log.note(
"delaying next hg call for {{seconds|round(decimal=1)}} seconds",
seconds=next_cache_miss - self.last_cache_miss,
)
Till(till=next_cache_miss.unix).wait()
# CLEAN UP BRANCH NAME
found_revision = copy(revision)
if isinstance(found_revision.branch, (text, binary_type)):
lower_name = found_revision.branch.lower()
else:
lower_name = found_revision.branch.name.lower()
if not lower_name:
Log.error("Defective revision? {{rev|json}}",
rev=found_revision.branch)
b = found_revision.branch = self.branches[(lower_name, locale)]
if not b:
b = found_revision.branch = self.branches[(lower_name,
DEFAULT_LOCALE)]
if not b:
Log.warning(
"can not find branch ({{branch}}, {{locale}})",
branch=lower_name,
locale=locale,
)
return Null
# REFRESH BRANCHES, IF TOO OLD
if Date.now() - Date(b.etl.timestamp) > _hg_branches.OLD_BRANCH:
self.branches = _hg_branches.get_branches(kwargs=self.settings)
# FIND THE PUSH
push = self._get_push(found_revision.branch,
found_revision.changeset.id)
id12 = found_revision.changeset.id[0:12]
base_url = URL(found_revision.branch.url)
with Explanation("get revision from {{url}}",
url=base_url,
debug=DEBUG):
raw_rev2 = Null
automation_details = Null
try:
raw_rev1 = self._get_raw_json_info((base_url / "json-info") +
{"node": id12})
raw_rev2 = self._get_raw_json_rev(base_url / "json-rev" / id12)
automation_details = self._get_raw_json_rev(
base_url / "json-automationrelevance" / id12)
except Exception as e:
if "Hg denies it exists" in e:
raw_rev1 = Data(node=revision.changeset.id)
else:
raise e
raw_rev3_changeset = first(r for r in automation_details.changesets
if r.node[:12] == id12)
if last(automation_details.changesets) != raw_rev3_changeset:
Log.note("interesting")
output = self._normalize_revision(
set_default(raw_rev1, raw_rev2, raw_rev3_changeset),
found_revision,
push,
get_diff,
get_moves,
)
if output.push.date >= Date.now() - MAX_TODO_AGE:
self.todo.extend([
(output.branch, listwrap(output.parents), None),
(output.branch, listwrap(output.children), None),
(
output.branch,
listwrap(output.backsoutnodes),
output.push.date,
),
])
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
return output