def get_branches(hg, branches, kwargs=None):
# TRY ES
try:
es = elasticsearch.Cluster(kwargs=branches).get_index(kwargs=branches, read_only=False)
query = {
"query": {"match_all": {}},
"size": 10000
}
found_branches = es.search(query).hits.hits._source
# IF IT IS TOO OLD, THEN PULL FROM HG
oldest = Date(MAX(found_branches.etl.timestamp))
if oldest == None or Date.now() - oldest > OLD_BRANCH:
found_branches = _get_branches_from_hg(hg)
es.extend({"id": b.name + " " + b.locale, "value": b} for b in found_branches)
es.flush()
try:
return UniqueIndex(["name", "locale"], data=found_branches, fail_on_dup=False)
except Exception as e:
Log.error("Bad branch in ES index", cause=e)
except Exception as e:
if "Can not find index " in e:
set_default(branches, {"schema": branches_schema})
es = elasticsearch.Cluster(kwargs=branches).get_or_create_index(kwargs=branches)
es.add_alias()
return get_branches(kwargs=kwargs)
Log.error("problem getting branches", cause=e)
def __new__(cls, kwargs, *args, **_kwargs):
es_cluster = elasticsearch.Cluster(kwargs)
output = known_clusters.get(id(es_cluster))
if output is None:
output = object.__new__(cls)
known_clusters[id(es_cluster)] = output
return output
def __init__(self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
typed=None,
settings=None):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = settings
self.settings = settings
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index),
settings=settings)
else:
self._es = elasticsearch.Cluster(settings=settings).get_index(
read_only=read_only, settings=settings)
self.meta = FromESMetadata(settings=settings)
self.settings.type = self._es.settings.type
self.edges = Dict()
self.worker = None
if typed == None:
self._columns = self.get_columns(table_name=index)
# SWITCH ON TYPED MODE
self.typed = any(c.name in ("$value", "$object")
for c in self._columns)
else:
self.typed = typed
def get_branches(hg, branches, kwargs=None):
# TRY ES
cluster = elasticsearch.Cluster(branches)
try:
es = cluster.get_index(kwargs=branches, read_only=False)
esq = jx_elasticsearch.new_instance(branches)
found_branches = esq.query({"from": "branches", "format": "list", "limit": 10000}).data
# IF IT IS TOO OLD, THEN PULL FROM HG
oldest = Date(MAX(found_branches.etl.timestamp))
if oldest == None or Date.now() - oldest > OLD_BRANCH:
found_branches = _get_branches_from_hg(hg)
es.extend({"id": b.name + " " + b.locale, "value": b} for b in found_branches)
es.flush()
try:
return UniqueIndex(["name", "locale"], data=found_branches, fail_on_dup=False)
except Exception as e:
Log.error("Bad branch in ES index", cause=e)
except Exception as e:
e = Except.wrap(e)
if "Can not find index " in e:
set_default(branches, {"schema": branches_schema})
es = cluster.get_or_create_index(branches)
es.add_alias()
return get_branches(kwargs)
Log.error("problem getting branches", cause=e)
def __init__(
self,
rollover_field, # the FIELD with a timestamp to use for determining which index to push to
rollover_interval, # duration between roll-over to new index
rollover_max, # remove old indexes, do not add old records
schema, # es schema
queue_size=10000, # number of documents to queue in memory
batch_size=5000, # number of documents to push at once
typed=None, # indicate if we are expected typed json
kwargs=None # plus additional ES settings
):
if kwargs.tjson != None:
Log.error("not expected")
if typed == None:
Log.error("not expected")
schema.settings.index.max_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES
schema.settings.index.max_inner_result_window = 100000 # REQUIRED FOR ACTIVEDATA NESTED QUERIES
self.settings = kwargs
self.locker = Lock("lock for rollover_index")
self.rollover_field = jx.get(rollover_field)
self.rollover_interval = self.settings.rollover_interval = Duration(
rollover_interval)
self.rollover_max = self.settings.rollover_max = Duration(rollover_max)
self.known_queues = {} # MAP DATE TO INDEX
self.cluster = elasticsearch.Cluster(self.settings)
def __init__(self, host, index, alias=None, name=None, port=9200, kwargs=None):
global _elasticsearch
if hasattr(self, "settings"):
return
from pyLibrary.queries.containers.list_usingPythonList import ListContainer
from pyLibrary.env import elasticsearch as _elasticsearch
self.settings = kwargs
self.default_name = coalesce(name, alias, index)
self.default_es = _elasticsearch.Cluster(kwargs=kwargs)
self.todo = Queue("refresh metadata", max=100000, unique=True)
self.es_metadata = Null
self.last_es_metadata = Date.now()-OLD_METADATA
self.meta=Data()
table_columns = metadata_tables()
column_columns = metadata_columns()
self.meta.tables = ListContainer("meta.tables", [], wrap({c.names["."]: c for c in table_columns}))
self.meta.columns = ColumnList()
self.meta.columns.insert(column_columns)
self.meta.columns.insert(table_columns)
# TODO: fix monitor so it does not bring down ES
if ENABLE_META_SCAN:
self.worker = Thread.run("refresh metadata", self.monitor)
else:
self.worker = Thread.run("refresh metadata", self.not_monitor)
return
def __init__(self,
rollover_field,
rollover_interval,
rollover_max,
queue_size=10000,
batch_size=5000,
kwargs=None):
"""
:param rollover_field: the FIELD with a timestamp to use for determining which index to push to
:param rollover_interval: duration between roll-over to new index
:param rollover_max: remove old indexes, do not add old records
:param queue_size: number of documents to queue in memory
:param batch_size: number of documents to push at once
:param kwargs: plus additional ES settings
:return:
"""
self.settings = kwargs
self.locker = Lock("lock for rollover_index")
self.rollover_field = jx.get(rollover_field)
self.rollover_interval = self.settings.rollover_interval = Duration(
kwargs.rollover_interval)
self.rollover_max = self.settings.rollover_max = Duration(
kwargs.rollover_max)
self.known_queues = {} # MAP DATE TO INDEX
self.cluster = elasticsearch.Cluster(self.settings)
def setup(settings=None):
global config
try:
config = startup.read_settings(defs={
"name": ["--process_num", "--process"],
"help": "Additional port offset (for multiple Flask processes",
"type": int,
"dest": "process_num",
"default": 0,
"required": False
},
filename=settings)
constants.set(config.constants)
Log.start(config.debug)
if config.args.process_num and config.flask.port:
config.flask.port += config.args.process_num
# PIPE REQUEST LOGS TO ES DEBUG
if config.request_logs:
request_logger = elasticsearch.Cluster(
config.request_logs).get_or_create_index(config.request_logs)
active_data.request_log_queue = request_logger.threaded_queue(
max_size=2000)
# SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
containers.config.default = {
"type": "elasticsearch",
"settings": config.elasticsearch.copy()
}
# TURN ON /exit FOR WINDOWS DEBUGGING
if config.flask.debug or config.flask.allow_exit:
config.flask.allow_exit = None
Log.warning("ActiveData is in debug mode")
app.add_url_rule('/exit', 'exit', _exit)
# TRIGGER FIRST INSTANCE
FromESMetadata(config.elasticsearch)
if config.saved_queries:
setattr(save_query, "query_finder",
SaveQueries(config.saved_queries))
HeaderRewriterFix(app, remove_headers=['Date', 'Server'])
if config.flask.ssl_context:
if config.args.process_num:
Log.error(
"can not serve ssl and multiple Flask instances at once")
setup_ssl()
return app
except Exception, e:
Log.error(
"Serious problem with ActiveData service construction! Shutdown!",
cause=e)
def __init__(self,
host,
index,
alias=None,
name=None,
port=9200,
settings=None):
self.settings = settings
self.name = coalesce(name, alias, index)
self._es = elasticsearch.Cluster(settings=settings)
self.metadata = self._es.get_metadata()
self.columns = None
def __init__(self, settings, queue_size=10000):
self.settings = settings
self.queue_size = queue_size
self.indicies = {} # MAP DATE (AS UNIX TIMESTAMP) TO INDEX
es = elasticsearch.Cluster(
self.settings).get_or_create_index(settings=self.settings)
es.add_alias(self.settings.index)
es.set_refresh_interval(seconds=60 * 60)
self.queue = es.threaded_queue(max_size=self.queue_size,
batch_size=5000,
silent=False)
self.es = elasticsearch.Alias(alias=settings.index, settings=settings)
def main():
try:
config = startup.read_settings()
with startup.SingleInstance(flavor_id=config.args.filename):
constants.set(config.constants)
Log.start(config.debug)
please_stop = Signal("main stop signal")
coverage_index = elasticsearch.Cluster(config.source).get_index(settings=config.source)
config.destination.schema = coverage_index.get_schema()
coverage_summary_index = elasticsearch.Cluster(config.destination).get_or_create_index(read_only=False, settings=config.destination)
coverage_summary_index.add_alias(config.destination.index)
Thread.run(
"processing loop",
loop,
config.source,
coverage_summary_index,
config,
please_stop=please_stop
)
Thread.wait_for_shutdown_signal(please_stop)
except Exception, e:
Log.error("Problem with code coverage score calculation", cause=e)
def backfill(settings):
source = aws.s3.Bucket(settings=settings.source)
destination = elasticsearch.Cluster(
settings=settings.destination).get_or_create_index(
settings=settings.destination)
keep_trying = True
while keep_trying:
try:
all_keys = source.keys()
keep_trying = False
except Exception, e:
Log.warning("problem", e)
def __init__(self,
host,
index,
sql_file='metadata.sqlite',
alias=None,
name=None,
port=9200,
kwargs=None):
if hasattr(self, "settings"):
return
self.too_old = TOO_OLD
self.settings = kwargs
self.default_name = coalesce(name, alias, index)
self.es_cluster = elasticsearch.Cluster(kwargs=kwargs)
self.index_does_not_exist = set()
self.todo = Queue("refresh metadata", max=100000, unique=True)
self.index_to_alias = Relation_usingList()
self.es_metadata = Null
self.metadata_last_updated = Date.now() - OLD_METADATA
self.meta = Data()
self.meta.columns = ColumnList()
self.alias_to_query_paths = {
"meta.columns": [['.']],
"meta.tables": [['.']]
}
self.alias_last_updated = {
"meta.columns": Date.now(),
"meta.tables": Date.now()
}
table_columns = metadata_tables()
self.meta.tables = ListContainer(
"meta.tables",
[
# TableDesc("meta.columns", None, ".", Date.now()),
# TableDesc("meta.tables", None, ".", Date.now())
],
jx_base.Schema(".", table_columns))
self.meta.columns.extend(table_columns)
# TODO: fix monitor so it does not bring down ES
if ENABLE_META_SCAN:
self.worker = Thread.run("refresh metadata", self.monitor)
else:
self.worker = Thread.run("refresh metadata", self.not_monitor)
return
def get_branches(hg, branches, use_cache=True, settings=None):
if not settings.branches or not use_cache:
found_branches = _get_branches_from_hg(hg)
es = elasticsearch.Cluster(settings=branches).get_or_create_index(
settings=branches)
es.add_alias()
es.extend({
"id": b.name + " " + b.locale,
"value": b
} for b in found_branches)
es.flush()
return found_branches
# TRY ES
try:
es = elasticsearch.Cluster(settings=branches).get_index(
settings=branches)
query = {"query": {"match_all": {}}, "size": 20000}
docs = es.search(query).hits.hits._source
# IF IT IS TOO OLD, THEN PULL FROM HG
oldest = Date(Math.MAX(docs.etl.timestamp))
if Date.now() - oldest > OLD_BRANCH:
return get_branches(use_cache=False, settings=settings)
try:
return UniqueIndex(["name", "locale"],
data=docs,
fail_on_dup=False)
except Exception, e:
Log.error("Bad branch in ES index", cause=e)
except Exception, e:
if "Can not find index " in e:
return get_branches(use_cache=False, settings=settings)
Log.error("problem getting branches", cause=e)
def __init__(
self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
Container.__init__(self)
if not container.config.default:
container.config.default = {
"type": "elasticsearch",
"settings": unwrap(kwargs)
}
self.settings = kwargs
self.name = name = coalesce(name, alias, index)
if read_only:
self.es = elasticsearch.Alias(alias=coalesce(alias, index),
kwargs=kwargs)
else:
self.es = elasticsearch.Cluster(kwargs=kwargs).get_index(
read_only=read_only, kwargs=kwargs)
self._namespace = ElasticsearchMetadata(kwargs=kwargs)
self.settings.type = self.es.settings.type
self.edges = Data()
self.worker = None
columns = self._namespace.get_snowflake(
self.es.settings.alias).columns # ABSOLUTE COLUMNS
is_typed = any(c.es_column == EXISTS_TYPE for c in columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = is_typed
else:
if is_typed != typed:
Log.error(
"Expecting given typed {{typed}} to match {{is_typed}}",
typed=typed,
is_typed=is_typed)
self.typed = typed
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
branches = _get_branches_from_hg(settings.hg)
es = elasticsearch.Cluster(kwargs=settings.hg.branches).get_or_create_index(kwargs=settings.hg.branches)
es.add_alias()
es.extend({"id": b.name + " " + b.locale, "value": b} for b in branches)
Log.alert("DONE!")
except Exception as e:
Log.error("Problem with etl", e)
finally:
Log.stop()
def __init__(
self,
hg=None, # CONNECT TO hg
repo=None, # CONNECTION INFO FOR ES CACHE
branches=None, # CONNECTION INFO FOR ES CACHE
use_cache=False, # True IF WE WILL USE THE ES FOR DOWNLOADING BRANCHES
timeout=30 * SECOND,
kwargs=None):
if not _hg_branches:
_late_imports()
self.es_locker = Lock()
self.todo = mo_threads.Queue("todo for hg daemon",
max=DAEMON_QUEUE_SIZE)
self.settings = kwargs
self.timeout = Duration(timeout)
# VERIFY CONNECTIVITY
with Explanation("Test connect with hg"):
response = http.head(self.settings.hg.url)
if branches == None:
self.branches = _hg_branches.get_branches(kwargs=kwargs)
self.es = None
return
self.last_cache_miss = Date.now()
set_default(repo, {"schema": revision_schema})
self.es = elasticsearch.Cluster(kwargs=repo).get_or_create_index(
kwargs=repo)
def setup_es(please_stop):
with suppress_exception:
self.es.add_alias()
with suppress_exception:
self.es.set_refresh_interval(seconds=1)
Thread.run("setup_es", setup_es)
self.branches = _hg_branches.get_branches(kwargs=kwargs)
self.timeout = timeout
Thread.run("hg daemon", self._daemon)
def __init__(
self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
Container.__init__(self, None)
if not container.config.default:
container.config.default = {
"type": "elasticsearch",
"settings": unwrap(kwargs)
}
self.settings = kwargs
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index),
kwargs=kwargs)
else:
self._es = elasticsearch.Cluster(kwargs=kwargs).get_index(
read_only=read_only, kwargs=kwargs)
self.meta = FromESMetadata(kwargs=kwargs)
self.settings.type = self._es.settings.type
self.edges = Data()
self.worker = None
columns = self.meta.get_columns(
table_name=coalesce(name, alias, index))
self._schema = Schema(coalesce(name, alias, index), columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = any(
c.es_column.find("." + TYPE_PREFIX) != -1 for c in columns)
else:
self.typed = typed
def __init__(self, host, index, alias=None, name=None, port=9200, kwargs=None):
if hasattr(self, "settings"):
return
self.settings = kwargs
self.too_old = TOO_OLD
self.es_cluster = elasticsearch.Cluster(kwargs=kwargs)
self.index_does_not_exist = set()
self.todo = Queue("refresh metadata", max=100000, unique=True)
self.meta = Data()
self.meta.columns = ColumnList(self.es_cluster)
self.meta.columns.extend(META_TABLES_DESC.columns)
self.meta.tables = ListContainer(META_TABLES_NAME, [], jx_base.Schema(".", META_TABLES_DESC.columns))
self.meta.table.extend([META_COLUMNS_DESC, META_TABLES_DESC])
self.alias_to_query_paths = {}
for i, settings in self.es_cluster.get_metadata().indices.items():
if len(settings.aliases) == 0:
alias = i
elif len(settings.aliases) == 1:
alias = first(settings.aliases)
else:
Log.error("expecting only one alias per index")
desc = TableDesc(
name=alias,
url=None,
query_path=ROOT_PATH,
last_updated=Date.MIN,
columns=[]
)
self.meta.tables.add(desc)
self.alias_to_query_paths[alias] = [desc.query_path]
self.alias_to_query_paths[self._find_alias(alias)] = [desc.query_path]
# WE MUST PAUSE?
# TODO: fix monitor so it does not bring down ES
if ENABLE_META_SCAN:
self.worker = Thread.run("refresh metadata", self.monitor)
else:
self.worker = Thread.run("not refresh metadata", self.not_monitor)
return
def setup():
global config
config = startup.read_settings(
filename=os.environ.get('ACTIVEDATA_CONFIG'),
defs=[
{
"name": ["--process_num", "--process"],
"help": "Additional port offset (for multiple Flask processes",
"type": int,
"dest": "process_num",
"default": 0,
"required": False
}
]
)
constants.set(config.constants)
Log.start(config.debug)
# PIPE REQUEST LOGS TO ES DEBUG
if config.request_logs:
cluster = elasticsearch.Cluster(config.request_logs)
request_logger = cluster.get_or_create_index(config.request_logs)
active_data.request_log_queue = request_logger.threaded_queue(max_size=2000)
if config.dockerflow:
def backend_check():
http.get_json(config.elasticsearch.host + ":" + text_type(config.elasticsearch.port))
dockerflow(flask_app, backend_check)
# SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
container.config.default = {
"type": "elasticsearch",
"settings": config.elasticsearch.copy()
}
# TRIGGER FIRST INSTANCE
if config.saved_queries:
setattr(save_query, "query_finder", SaveQueries(config.saved_queries))
HeaderRewriterFix(flask_app, remove_headers=['Date', 'Server'])
def __init__(self,
hg,
rate_limit,
use_cache=True,
cache=None,
settings=None):
self.settings = settings
self.failure_classification = {
c.id: c.name
for c in http.get_json(FAILURE_CLASSIFICATION_URL)
}
self.repo = {c.id: c.name for c in http.get_json(REPO_URL)}
self.hg = hg
self.cache = elasticsearch.Cluster(cache).get_or_create_index(cache)
self.locker = Lock()
self.pending = {}
self.rate_locker = Lock()
self.request_times = [0] * rate_limit
self.request_pointer = 0
def __init__(self,
host,
index,
sql_file='metadata.sqlite',
alias=None,
name=None,
port=9200,
kwargs=None):
if hasattr(self, "settings"):
return
self.too_old = TOO_OLD
self.settings = kwargs
self.default_name = coalesce(name, alias, index)
self.default_es = elasticsearch.Cluster(kwargs=kwargs)
self.index_does_not_exist = set()
self.todo = Queue("refresh metadata", max=100000, unique=True)
self.es_metadata = Null
self.abs_columns = set()
self.last_es_metadata = Date.now() - OLD_METADATA
self.meta = Data()
table_columns = metadata_tables()
column_columns = metadata_columns()
self.meta.tables = ListContainer(
"meta.tables", [], wrap({c.names["."]: c
for c in table_columns}))
self.meta.columns = ColumnList()
self.meta.columns.insert(column_columns)
self.meta.columns.insert(table_columns)
# TODO: fix monitor so it does not bring down ES
if ENABLE_META_SCAN:
self.worker = Thread.run("refresh metadata", self.monitor)
else:
self.worker = Thread.run("refresh metadata", self.not_monitor)
return
def __init__(
self,
host,
index,
type=None,
alias=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
consistency="one", # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None
):
Container.__init__(self, None)
if not containers.config.default:
containers.config.default.settings = kwargs
self.settings = kwargs
self.name = coalesce(name, alias, index)
if read_only:
self._es = elasticsearch.Alias(alias=coalesce(alias, index), kwargs=kwargs)
else:
self._es = elasticsearch.Cluster(kwargs=kwargs).get_index(read_only=read_only, kwargs=kwargs)
self.meta = FromESMetadata(kwargs=kwargs)
self.settings.type = self._es.settings.type
self.edges = Data()
self.worker = None
columns = self.get_columns(table_name=name)
self._schema = Schema(columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = any(c.name in ("$value", "$object") for c in columns)
else:
self.typed = typed
def __init__(
self,
host,
index,
type=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None
):
Container.__init__(self)
if not container.config.default:
container.config.default = {
"type": "elasticsearch",
"settings": unwrap(kwargs)
}
self.settings = kwargs
self.name = name = coalesce(name, index)
if read_only:
self.es = elasticsearch.Alias(alias=index, kwargs=kwargs)
else:
self.es = elasticsearch.Cluster(kwargs=kwargs).get_index(read_only=read_only, kwargs=kwargs)
self._namespace = ElasticsearchMetadata(kwargs=kwargs)
self.settings.type = self.es.settings.type
self.edges = Data()
self.worker = None
columns = self.snowflake.columns # ABSOLUTE COLUMNS
is_typed = any(c.es_column == EXISTS_TYPE for c in columns)
if typed == None:
# SWITCH ON TYPED MODE
self.typed = is_typed
else:
if is_typed != typed:
Log.error("Expecting given typed {{typed}} to match {{is_typed}}", typed=typed, is_typed=is_typed)
self.typed = typed
if not typed:
# ADD EXISTENCE COLUMNS
all_paths = {".": None} # MAP FROM path TO parent TO MAKE A TREE
def nested_path_of(v):
if not v:
return []
else:
return [v] + nested_path_of(all_paths[v])
all = sort_using_key(set(step for path in self.snowflake.query_paths for step in path), key=lambda p: len(split_field(p)))
for step in sorted(all):
if step in all_paths:
continue
else:
best = '.'
for candidate in all_paths.keys():
if startswith_field(step, candidate):
if startswith_field(candidate, best):
best = candidate
all_paths[step] = best
for p in all_paths.keys():
nested_path = nested_path_of(all_paths[p])
if not nested_path:
nested_path = ['.']
self.namespace.meta.columns.add(Column(
name=p,
es_column=p,
es_index=self.name,
es_type=OBJECT,
jx_type=EXISTS,
nested_path=nested_path,
last_updated=Date.now()
))
def loop(source, coverage_summary_index, settings, please_stop):
try:
cluster = elasticsearch.Cluster(source)
aliases = cluster.get_aliases()
candidates = []
for pairs in aliases:
if pairs.alias == source.index:
candidates.append(pairs.index)
candidates = jx.sort(candidates, {".": "desc"})
for index_name in candidates:
coverage_index = elasticsearch.Index(index=index_name, read_only=False, settings=source)
push_date_filter = unicode2Date(coverage_index.settings.index[-15::], elasticsearch.INDEX_DATE_FORMAT)
while not please_stop:
# IDENTIFY NEW WORK
Log.note("Working on index {{index}}", index=index_name)
coverage_index.refresh()
todo = http.post_json(settings.url, json={
"from": "coverage",
"groupby": ["source.file.name", "build.revision12"],
"where": {"and": [
{"missing": "source.method.name"},
{"missing": "source.file.min_line_siblings"},
{"gte": {"repo.push.date": push_date_filter}}
]},
"format": "list",
"limit": coalesce(settings.batch_size, 100)
})
if not todo.data:
break
queue = Queue("pending source files to review")
queue.extend(todo.data[0:coalesce(settings.batch_size, 100):])
threads = [
Thread.run(
"processor" + unicode(i),
process_batch,
queue,
coverage_index,
coverage_summary_index,
settings,
please_stop=please_stop
)
for i in range(NUM_THREAD)
]
# ADD STOP MESSAGE
queue.add(Thread.STOP)
# WAIT FOR THEM TO COMPLETE
for t in threads:
t.join()
please_stop.go()
return
except Exception, e:
Log.warning("Problem processing", cause=e)
fuzzytestcase.assertAlmostEqual(e[0], settings)
return e[1]
except Exception, _:
pass
output = S3Bucket(settings)
sinks.append((settings, output))
return output
else:
with sinks_locker:
for e in sinks:
try:
fuzzytestcase.assertAlmostEqual(e[0], settings)
return e[1]
except Exception, _:
pass
output = elasticsearch.Cluster(settings).get_or_create_index(
settings)
if settings.use_daily is not False:
output = MultiDayIndex(settings)
else:
output = output.threaded_queue(max_size=2000, batch_size=1000)
setattr(output, "keys", lambda prefix: set())
sinks.append((settings, output))
return output
def main():
try:
settings = startup.read_settings(defs=[{
"name": ["--id"],
