def __init__(
self,
hg=None, # CONNECT TO hg
repo=None, # CONNECTION INFO FOR ES CACHE
branches=None, # CONNECTION INFO FOR ES CACHE
use_cache=False, # True IF WE WILL USE THE ES FOR DOWNLOADING BRANCHES
timeout=30 * SECOND,
settings=None
):
if not _hg_branches:
_late_imports()
self.settings = settings
self.timeout = Duration(timeout)
if branches == None:
self.branches = _hg_branches.get_branches(settings=settings)
self.es = None
return
self.es = elasticsearch.Cluster(settings=repo).get_or_create_index(settings=repo)
self.es.add_alias()
self.es.set_refresh_interval(seconds=1)
self.branches = _hg_branches.get_branches(use_cache=use_cache, settings=settings)
# TO ESTABLISH DATA
self.es.add({"id": "b3649fd5cd7a-mozilla-inbound-en-US", "value": {
"index": 247152,
"branch": {
"name": "mozilla-inbound",
"locale": DEFAULT_LOCALE
},
"changeset": {
"id": "b3649fd5cd7a76506d2cf04f45e39cbc972fb553",
"id12": "b3649fd5cd7a",
"author": "ryan",
"description": "Backed out changeset 7d0d8d304cd8 (bug 1171357) for bustage.",
"date": 1433429100,
"files": ["gfx/thebes/gfxTextRun.cpp"]
},
"push": {
"id": 60618,
"user": "******",
"date": 1433429138
},
"parents": ["7d0d8d304cd871f657effcc2d21d4eae5155fd1b"],
"children": ["411a9af141781c3c8fa883287966a4af348dbca8"]
}})
self.es.flush()
def _load_all_in_push(self, revision, locale=None):
# http://hg.mozilla.org/mozilla-central/json-pushes?full=1&changeset=57c461500a0c
found_revision = copy(revision)
if isinstance(found_revision.branch, basestring):
lower_name = found_revision.branch.lower()
else:
lower_name = found_revision.branch.name.lower()
if not lower_name:
Log.error("Defective revision? {{rev|json}}", rev=found_revision.branch)
b = found_revision.branch = self.branches[(lower_name, locale)]
if not b:
b = found_revision.branch = self.branches[(lower_name, DEFAULT_LOCALE)]
if not b:
Log.error("can not find branch ({{branch}}, {{locale}})", name=lower_name, locale=locale)
if Date.now() - Date(b.etl.timestamp) > _OLD_BRANCH:
self.branches = _hg_branches.get_branches(use_cache=True, settings=self.settings)
url = found_revision.branch.url.rstrip("/") + "/json-pushes?full=1&changeset=" + found_revision.changeset.id
Log.note(
"Reading pushlog for revision ({{branch}}, {{locale}}, {{changeset}}): {{url}}",
branch=found_revision.branch.name,
locale=locale,
changeset=found_revision.changeset.id,
url=url,
)
try:
data = self._get_and_retry(url, found_revision.branch)
revs = []
output = None
for index, _push in data.items():
push = Push(id=int(index), date=_push.date, user=_push.user)
for _, ids in jx.groupby(_push.changesets.node, size=200):
url_param = "&".join("node=" + c[0:12] for c in ids)
url = found_revision.branch.url.rstrip("/") + "/json-info?" + url_param
Log.note("Reading details from {{url}}", {"url": url})
raw_revs = self._get_and_retry(url, found_revision.branch)
for r in raw_revs.values():
rev = Revision(
branch=found_revision.branch,
index=r.rev,
changeset=Changeset(
id=r.node,
id12=r.node[0:12],
author=r.user,
description=r.description,
date=Date(r.date),
files=r.files,
),
parents=unwraplist(r.parents),
children=unwraplist(r.children),
push=push,
etl={"timestamp": Date.now().unix},
)
if r.node == found_revision.changeset.id:
output = rev
if r.node[0:12] == found_revision.changeset.id[0:12]:
output = rev
_id = (
coalesce(rev.changeset.id12, "")
+ "-"
+ rev.branch.name
+ "-"
+ coalesce(rev.branch.locale, DEFAULT_LOCALE)
)
revs.append({"id": _id, "value": rev})
self.es.extend(revs)
return output
except Exception, e:
Log.error("Problem pulling pushlog from {{url}}", url=url, cause=e)
