def test_big_result_works(self):
result = http.post_json(global_settings.service_url,
data={
"from": "unittest",
"where": {
"and": [{
"gte": {
"run.timestamp":
Date.today() - DAY
}
}, {
"lt": {
"run.timestamp": Date.today()
}
}, {
"eq": {
"result.ok": False
}
}]
},
"format": "list",
"limit": 10000
})
if result.template:
result = Except.new_instance(result)
Log.error("problem with call", cause=result)
Log.note("Got {{num}} test failures", num=len(result.data))
def _get_queue(self, row):
row = wrap(row)
if row.json:
row.value, row.json = convert.json2value(row.json), None
timestamp = Date(self.rollover_field(wrap(row).value))
if timestamp == None or timestamp < Date.today() - self.rollover_max:
return Null
rounded_timestamp = timestamp.floor(self.rollover_interval)
queue = self.known_queues.get(rounded_timestamp.unix)
if queue == None:
candidates = jx.run({
"from": self.cluster.get_aliases(),
"where": {
"regex": {
"index":
self.settings.index + "\d\d\d\d\d\d\d\d_\d\d\d\d\d\d"
}
},
"sort": "index"
})
best = None
for c in candidates:
c = wrap(c)
c.date = unicode2Date(c.index[-15:],
elasticsearch.INDEX_DATE_FORMAT)
if timestamp > c.date:
best = c
if not best or rounded_timestamp > best.date:
if rounded_timestamp < wrap(candidates[-1]).date:
es = elasticsearch.Index(read_only=False,
alias=best.alias,
index=best.index,
settings=self.settings)
else:
try:
es = self.cluster.create_index(
create_timestamp=rounded_timestamp,
settings=self.settings)
es.add_alias(self.settings.index)
except Exception, e:
if "IndexAlreadyExistsException" not in e:
Log.error("Problem creating index", cause=e)
return self._get_queue(row) # TRY AGAIN
else:
es = elasticsearch.Index(read_only=False,
alias=best.alias,
index=best.index,
settings=self.settings)
with suppress_exception:
es.set_refresh_interval(seconds=60 * 10, timeout=5)
self._delete_old_indexes(candidates)
queue = self.known_queues[
rounded_timestamp.unix] = es.threaded_queue(
max_size=self.settings.queue_size,
batch_size=self.settings.batch_size,
silent=True)
def find_some_work(th):
# th.get_markup("fx-team", "036f62007472", "B8kS5IJ5Rom8l-kcSIRIlA")
# th.get_markup("mozilla-inbound", "971c1ee26cad", "fNuzNmZxS6m3i_p9jDh8iA")
# GET SOME TASKS
result = http.post_json(url="http://activedata.allizom.org/query",
data={
"from":
"task",
"select":
["build.branch", "build.revision", "task.id"],
"where": {
"and": [{
"gt": {
"task.run.start_time":
(Date.today() - DAY).unix
}
}, {
"exists": "build.revision"
}, {
"exists": "build.branch"
}]
},
"format":
"list"
})
# TRY TO GET THEM OUT OF OUR CACHE
for r in result.data:
Log.note("look for task {{task_id}}", task_id=r.task.id)
th.get_markup(r.build.branch, r.build.revision, r.task.id)
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
some_failures = http.post_json("http://activedata.allizom.org/query", data={
"from": "unittest",
"select": [
{"name": "branch", "value": "build.branch"},
{"name": "revision", "value": "build.revision12"},
{"name": "suite", "value": "run.suite"},
{"name": "chunk", "value": "run.chunk"},
{"name": "test", "value": "result.test"}
],
"where": {"and": [
{"eq": {"result.ok": False}},
{"gt": {"run.timestamp": Date.today() - WEEK}},
{"missing": "treeherder.job.note"}
]},
"format": "list",
"limit": 10
})
th = TreeHerder(settings={})
# th.get_job_classification("mozilla-inbound", "7380457b8ba0")
for f in some_failures.data:
th.get_job_classification(f.branch, f.revision)
except Exception, e:
Log.error("Problem with etl", e)
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
some_failures = http.post_json("http://activedata.allizom.org/query",
data={
"from":
"unittest",
"select": [{
"name": "branch",
"value": "build.branch"
}, {
"name":
"revision",
"value":
"build.revision12"
}, {
"name": "suite",
"value": "run.suite"
}, {
"name": "chunk",
"value": "run.chunk"
}, {
"name": "test",
"value": "result.test"
}],
"where": {
"and": [{
"eq": {
"result.ok": False
}
}, {
"gt": {
"run.timestamp":
Date.today() - WEEK
}
}, {
"missing":
"treeherder.job.note"
}]
},
"format":
"list",
"limit":
10
})
th = TreeHerder(settings={})
# th.get_job_classification("mozilla-inbound", "7380457b8ba0")
for f in some_failures.data:
th.get_job_classification(f.branch, f.revision)
except Exception, e:
Log.error("Problem with etl", e)
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today() - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception, e:
Log.warning("could not delete index {{index}}", index=c.index, cause=e)
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today(
) - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception, e:
Log.warning("could not delete index {{index}}",
index=c.index,
cause=e)
def _get_queue(self, row):
row = wrap(row)
if row.json:
row.value, row.json = convert.json2value(row.json), None
timestamp = Date(self.rollover_field(wrap(row).value))
if timestamp == None or timestamp < Date.today() - self.rollover_max:
return Null
rounded_timestamp = timestamp.floor(self.rollover_interval)
queue = self.known_queues.get(rounded_timestamp.unix)
if queue == None:
candidates = jx.run({
"from": self.cluster.get_aliases(),
"where": {"regex": {"index": self.settings.index + "\d\d\d\d\d\d\d\d_\d\d\d\d\d\d"}},
"sort": "index"
})
best = None
for c in candidates:
c = wrap(c)
c.date = unicode2Date(c.index[-15:], elasticsearch.INDEX_DATE_FORMAT)
if timestamp > c.date:
best = c
if not best or rounded_timestamp > best.date:
if rounded_timestamp < wrap(candidates[-1]).date:
es = elasticsearch.Index(read_only=False, alias=best.alias, index=best.index, settings=self.settings)
else:
try:
es = self.cluster.create_index(create_timestamp=rounded_timestamp, settings=self.settings)
es.add_alias(self.settings.index)
except Exception, e:
if "IndexAlreadyExistsException" not in e:
Log.error("Problem creating index", cause=e)
return self._get_queue(row) # TRY AGAIN
else:
es = elasticsearch.Index(read_only=False, alias=best.alias, index=best.index, settings=self.settings)
with suppress_exception:
es.set_refresh_interval(seconds=60 * 10, timeout=5)
self._delete_old_indexes(candidates)
queue = self.known_queues[rounded_timestamp.unix] = es.threaded_queue(max_size=self.settings.queue_size, batch_size=self.settings.batch_size, silent=True)
def find_some_work(th):
# th.get_markup("fx-team", "036f62007472", "B8kS5IJ5Rom8l-kcSIRIlA")
# th.get_markup("mozilla-inbound", "971c1ee26cad", "fNuzNmZxS6m3i_p9jDh8iA")
# GET SOME TASKS
result = http.post_json(url="http://activedata.allizom.org/query", data={
"from": "task",
"select": ["build.branch", "build.revision", "task.id"],
"where": {"and": [
{"gt": {"task.run.start_time": (Date.today() - DAY).unix}},
{"exists": "build.revision"},
{"exists": "build.branch"}
]},
"format": "list"
})
# TRY TO GET THEM OUT OF OUR CACHE
for r in result.data:
Log.note("look for task {{task_id}}", task_id=r.task.id)
th.get_markup(r.build.branch, r.build.revision, r.task.id)
def loop_all_days(destination, please_stop):
try:
today = Date.today()
# WHICH DAYS DO WE NEED TO CALCULATE
# ALL BUILD DATES WITH WITH ETL TIMESTAMP OF A WEEK AGO
# ALL BUILD DATES THAT HAVE NOT BEEN PROCESSED YET
build_dates = http.post_json(config.source.url, json={
"from": "unittest",
"edges": [
{
"name": "date",
"value": "build.date",
"allowNulls": False,
"domain": {
"type": "time",
"min": "today-week",
"max": "eod",
"interval": "day"
}
}
],
"where": {"gte": {"etl.timestamp": (today - WEEK).unix}},
"sort": {"value": "build.date", "sort": -1},
"limit": 14,
"format": "list"
})
build_dates.data = jx.sort(build_dates.data, {"value": "date", "sort": -1})
for d in build_dates.data:
if please_stop:
return
agg(Date(d.date), destination, please_stop=please_stop)
finally:
please_stop.go()
def test_end_of_month(self):
self.assertAlmostEqual(parse("today|month+month").unix, Date.today().floor(MONTH).add(MONTH).unix)
def test_last_year(self):
self.assertAlmostEqual(parse("today-12month").unix, (Date.today() - MONTH * 12).unix)
def test_beginning_of_month(self):
self.assertAlmostEqual(parse("today|month").unix, Date.today().floor(MONTH).unix)
def test_next_week(self):
self.assertAlmostEqual(parse("today+7day").unix, (Date.today() + DAY * 7).unix)
def test_week_before(self):
self.assertAlmostEqual(parse("today-2week").unix, (Date.today() - WEEK * 2).unix)
def test_yesterday(self):
self.assertAlmostEqual(parse("today-day").unix, (Date.today() - DAY).unix)
def test_last_week(self):
self.assertAlmostEqual(parse("today-7day").unix, (Date.today() - DAY * 7).unix)
from pyLibrary import convert
from pyLibrary.debugs import constants, startup
from pyLibrary.debugs.logs import Log
from pyLibrary.dot import Dict, set_default
from pyLibrary.env import http, big_data
from pyLibrary.env.files import File
from pyLibrary.queries.unique_index import UniqueIndex
from pyLibrary.times.dates import Date
from pyLibrary.times.durations import WEEK
RECENT = Date.today() - 1 * WEEK
blacklist = [
'automation.py',
'remoteautomation.py',
'Shutdown',
'undefined',
'Main app process exited normally',
'Traceback (most recent call last):',
'Return code: 0',
'Return code: 1',
'Return code: 2',
'Return code: 9',
'Return code: 10',
'Exiting 1',
'Exiting 9',
'CrashingThread(void *)',
'libSystem.B.dylib + 0xd7a',
'linux-gate.so + 0x424',
'TypeError: content is null',
self._delete_old_indexes(candidates)
queue = self.known_queues[rounded_timestamp.unix] = es.threaded_queue(max_size=self.settings.queue_size, batch_size=self.settings.batch_size, silent=True)
return queue
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today() - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception, e:
Log.warning("could not delete index {{index}}", index=c.index, cause=e)
for t, q in list(self.known_queues.items()):
if unix2Date(t) + self.rollover_interval < Date.today() - self.rollover_max:
del self.known_queues[t]
pass
# ADD keys() SO ETL LOOP CAN FIND WHAT'S GETTING REPLACED
def keys(self, prefix=None):
path = jx.reverse(etl2path(key2etl(prefix)))
result = self.es.search({
"fields": ["_id"],
"query": {
"filtered": {
"query": {"match_all": {}},
"filter": {"and": [{"term": {"etl" + (".source" * i) + ".id": v}} for i, v in enumerate(path)]}
}
def today():
return int((Date.today() - ZERO_DATE).floor(DAY) / DAY)
def test_overload(self):
today = Date.today()
result = replace_vars('"{{today|week}}" "{{var}}"', {"today": 1000, "var": 20})
expect = '"'+unicode(today.floor(WEEK).unix)+'" "20"'
self.assertEqual(result, expect)
def today():
return Date.today().value
return queue
def _delete_old_indexes(self, candidates):
for c in candidates:
timestamp = unicode2Date(c.index[-15:], "%Y%m%d_%H%M%S")
if timestamp + self.rollover_interval < Date.today(
) - self.rollover_max:
# Log.warning("Will delete {{index}}", index=c.index)
try:
self.cluster.delete_index(c.index)
except Exception, e:
Log.warning("could not delete index {{index}}",
index=c.index,
cause=e)
for t, q in list(self.known_queues.items()):
if unix2Date(t) + self.rollover_interval < Date.today(
) - self.rollover_max:
del self.known_queues[t]
pass
# ADD keys() SO ETL LOOP CAN FIND WHAT'S GETTING REPLACED
def keys(self, prefix=None):
path = jx.reverse(etl2path(key2etl(prefix)))
result = self.es.search({
"fields": ["_id"],
"query": {
"filtered": {
"query": {
"match_all": {}
},
def test_two_simple(self):
today = Date.today()
result = replace_vars('"{{today|week}}" "{{today}}d"')
expect = '"'+unicode(today.floor(WEEK).unix)+'" "'+unicode(today.unix)+'d"'
self.assertEqual(result, expect)
# License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Author: Kyle Lahnakoski ([email protected]) # from __future__ import division from __future__ import unicode_literals from pyLibrary.dot import wrap from pyLibrary.times.dates import Date from pyLibrary.times.durations import WEEK, DAY from tests import NULL from tests.base_test_class import ActiveDataBaseTest, TEST_TABLE TODAY = Date.today() test_data_1 = [ {"a": "x", "t": Date("today").unix, "v": 2}, {"a": "x", "t": Date("today-day").unix, "v": 2}, {"a": "x", "t": Date("today-2day").unix, "v": 3}, {"a": "x", "t": Date("today-3day").unix, "v": 5}, {"a": "x", "t": Date("today-4day").unix, "v": 7}, {"a": "x", "t": Date("today-5day").unix, "v": 11}, {"a": "x", "t": NULL, "v": 27}, {"a": "y", "t": Date("today-day").unix, "v": 13}, {"a": "y", "t": Date("today-2day").unix, "v": 17}, {"a": "y", "t": Date("today-4day").unix, "v": 19}, {"a": "y", "t": Date("today-5day").unix, "v": 23} ]
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Author: Kyle Lahnakoski ([email protected]) # from __future__ import unicode_literals from __future__ import division import base_test_class from pyLibrary.times.dates import Date from pyLibrary.times.durations import DAY from tests.base_test_class import ActiveDataBaseTest, TEST_TABLE FROM_DATE = Date.today() - 7 * DAY TO_DATE = Date.today() simple_test_data = [{ "run": { "timestamp": Date("now-4day"), "value": 1 } }, { "run": { "timestamp": Date("now-4day"), "value": 2 } }, { "run": { "timestamp": Date("now-4day"),
def test_today(self):
self.assertAlmostEqual(parse("today").unix, Date.today().unix)
def today():
return Date.today().value
