def find_build_info(self, date, fetch_txt_info=True, max_workers=2):
"""
Find build info for a nightly build, given a date.
Returns a :class:`NightlyBuildInfo` instance.
"""
# getting a valid build for a given date on nightly is tricky.
# there is multiple possible builds folders for one date,
# and some of them may be invalid (without binary for example)
# to save time, we will try multiple build folders at the same
# time in some threads. The first good one found is returned.
try:
build_urls = self._get_urls(date)
LOG.debug("got build_urls %s" % build_urls)
except HTTPError as exc:
raise BuildInfoNotFound(str(exc))
build_info = None
valid_builds = []
while build_urls:
some = build_urls[:max_workers]
threads = [
Thread(target=self._fetch_build_info_from_url,
args=(url, i, valid_builds))
for i, url in enumerate(some)
]
for thread in threads:
thread.daemon = True
thread.start()
for thread in threads:
while thread.is_alive():
thread.join(0.1)
LOG.debug("got valid_builds %s" % valid_builds)
if valid_builds:
infos = sorted(valid_builds, key=lambda b: b[0])[0][1]
if fetch_txt_info:
self._update_build_info_from_txt(infos)
build_info = NightlyBuildInfo(
self.fetch_config,
build_url=infos["build_url"],
build_date=date,
changeset=infos.get("changeset"),
repo_url=infos.get("repository"),
)
break
build_urls = build_urls[max_workers:]
if build_info is None:
raise BuildInfoNotFound("Unable to find build info for %s" % date)
return build_info
def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
data = {}
if not url.endswith("/"):
url += "/"
for link in url_links(url):
if "build_url" not in data and self.build_regex.match(link):
data["build_url"] = url + link
elif "build_txt_url" not in data and self.build_info_regex.match(
link):
data["build_txt_url"] = url + link
if data:
# Check that we found all required data. The URL in build_url is
# required. build_txt_url is optional.
if "build_url" not in data:
raise BuildInfoNotFound(
"Failed to find a build file in directory {} that "
"matches regex '{}'".format(url, self.build_regex.pattern))
with self._fetch_lock:
lst.append((index, data))
def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
LOG.debug("Fetching build info from {}".format(url))
data = {}
if not url.endswith("/"):
url += "/"
links = url_links(url)
if not self.fetch_config.has_build_info:
links += url_links(self.fetch_config.get_nightly_info_url(url))
for link in links:
name = os.path.basename(link)
if "build_url" not in data and self.build_regex.match(name):
data["build_url"] = link
elif "build_txt_url" not in data and self.build_info_regex.match(
name):
data["build_txt_url"] = link
if data:
# Check that we found all required data. The URL in build_url is
# required. build_txt_url is optional.
if "build_url" not in data:
raise BuildInfoNotFound(
"Failed to find a build file in directory {} that "
"matches regex '{}'".format(url, self.build_regex.pattern))
with self._fetch_lock:
lst.append((index, data))
def find_build_info(self, push):
"""
Find build info for an inbound build, given a Push, a changeset or a
date/datetime.
if `push` is not an instance of Push (e.g. it is a date, datetime, or
string representing the changeset), a query to json pushes will be
done.
Return a :class:`InboundBuildInfo` instance.
"""
if not isinstance(push, Push):
try:
push = self.jpushes.push(push)
except MozRegressionError, exc:
raise BuildInfoNotFound(str(exc))
def find_build_info(self,
changeset,
fetch_txt_info=True,
check_changeset=False):
"""
Find build info for an inbound build, given a changeset or a date.
if `check_changeset` is True, the given changeset might be partial
(< 40 chars) because it will be verified and updated using json pushes.
Return a :class:`InboundBuildInfo` instance.
"""
if is_date_or_datetime(changeset):
changeset = self.jpushes.revision_for_date(changeset)
check_changeset = False
# find a task id
if check_changeset:
try:
changeset = self._check_changeset(changeset)
except MozRegressionError, exc:
raise BuildInfoNotFound(str(exc))
def fetch(index):
if func(index):
raise BuildInfoNotFound("")
return index
def find_build_info(self, push):
"""
Find build info for an integration build, given a Push, a changeset or a
date/datetime.
if `push` is not an instance of Push (e.g. it is a date, datetime, or
string representing the changeset), a query to json pushes will be
done.
Return a :class:`IntegrationBuildInfo` instance.
"""
if not isinstance(push, Push):
try:
push = self.jpushes.push(push)
except MozRegressionError as exc:
raise BuildInfoNotFound(str(exc))
changeset = push.changeset
try:
# taskcluster builds have two possible root urls: we switched
# from taskcluster.net -> firefox-ci-tc.services.mozilla.com
# around November 9. to make things faster, we'll iterate through
# them based on the one that most likely applies to this push
possible_tc_root_urls = [TC_ROOT_URL, OLD_TC_ROOT_URL]
if push.utc_date < TC_ROOT_URL_MIGRATION_FLAG_DATE:
possible_tc_root_urls.reverse()
task_id = None
status = None
for tc_root_url in possible_tc_root_urls:
LOG.debug("using taskcluster root url %s" % tc_root_url)
options = self.fetch_config.tk_options(tc_root_url)
tc_index = taskcluster.Index(options)
tc_queue = taskcluster.Queue(options)
tk_routes = self.fetch_config.tk_routes(push)
stored_failure = None
for tk_route in tk_routes:
LOG.debug("using taskcluster route %r" % tk_route)
try:
task_id = tc_index.findTask(tk_route)["taskId"]
except TaskclusterFailure as ex:
LOG.debug("nothing found via route %r" % tk_route)
stored_failure = ex
continue
if task_id:
status = tc_queue.status(task_id)["status"]
break
if status:
break
if not task_id:
raise stored_failure
except TaskclusterFailure:
raise BuildInfoNotFound("Unable to find build info using the"
" taskcluster route %r" %
self.fetch_config.tk_route(push))
# find a completed run for that task
run_id, build_date = None, None
for run in reversed(status["runs"]):
if run["state"] == "completed":
run_id = run["runId"]
build_date = datetime.strptime(run["resolved"],
"%Y-%m-%dT%H:%M:%S.%fZ")
break
if run_id is None:
raise BuildInfoNotFound(
"Unable to find completed runs for task %s" % task_id)
artifacts = tc_queue.listArtifacts(task_id, run_id)["artifacts"]
# look over the artifacts of that run
build_url = None
for a in artifacts:
name = os.path.basename(a["name"])
if self.build_regex.search(name):
meth = tc_queue.buildUrl
if self.fetch_config.tk_needs_auth():
meth = tc_queue.buildSignedUrl
build_url = meth("getArtifact", task_id, run_id, a["name"])
break
if build_url is None:
raise BuildInfoNotFound("unable to find a build url for the"
" changeset %r" % changeset)
return IntegrationBuildInfo(
self.fetch_config,
build_url=build_url,
build_date=build_date,
changeset=changeset,
repo_url=self.jpushes.repo_url,
task_id=task_id,
)
def fetch(index):
# last build info can't be fetched
if index == 9:
raise BuildInfoNotFound("")
return index
class InboundInfoFetcher(InfoFetcher):
def __init__(self, fetch_config):
InfoFetcher.__init__(self, fetch_config)
options = fetch_config.tk_options()
self.index = taskcluster.client.Index(options)
self.queue = taskcluster.Queue(options)
self.jpushes = JsonPushes(branch=fetch_config.inbound_branch)
def _check_changeset(self, changeset):
# return the full changeset
return self.jpushes.pushlog_for_change(changeset)['changesets'][-1]
def find_build_info(self,
changeset,
fetch_txt_info=True,
check_changeset=False):
"""
Find build info for an inbound build, given a changeset or a date.
if `check_changeset` is True, the given changeset might be partial
(< 40 chars) because it will be verified and updated using json pushes.
Return a :class:`InboundBuildInfo` instance.
"""
if is_date_or_datetime(changeset):
changeset = self.jpushes.revision_for_date(changeset)
check_changeset = False
# find a task id
if check_changeset:
try:
changeset = self._check_changeset(changeset)
except MozRegressionError, exc:
raise BuildInfoNotFound(str(exc))
tk_route = self.fetch_config.tk_inbound_route(changeset)
LOG.debug('using taskcluster route %r' % tk_route)
try:
task_id = self.index.findTask(tk_route)['taskId']
except TaskclusterFailure:
# HACK because of
# https://bugzilla.mozilla.org/show_bug.cgi?id=1199618
# and https://bugzilla.mozilla.org/show_bug.cgi?id=1211251
# TODO remove the if statement once these tasks no longer exists
# (just raise BuildInfoNotFound)
err = True
if self.fetch_config.app_name in ('firefox', 'fennec',
'fennec-2.3'):
err = False
try:
tk_route = tk_route.replace(changeset, changeset[:12])
task_id = self.index.findTask(tk_route)['taskId']
except TaskclusterFailure:
err = True
if err:
raise BuildInfoNotFound("Unable to find build info using the"
" taskcluster route %r" % tk_route)
# find a completed run for that task
run_id, build_date = None, None
status = self.queue.status(task_id)['status']
for run in reversed(status['runs']):
if run['state'] == 'completed':
run_id = run['runId']
build_date = datetime.strptime(run["resolved"],
'%Y-%m-%dT%H:%M:%S.%fZ')
break
if run_id is None:
raise BuildInfoNotFound(
"Unable to find completed runs for task %s" % task_id)
artifacts = self.queue.listArtifacts(task_id, run_id)['artifacts']
# look over the artifacts of that run
build_url = None
for a in artifacts:
name = os.path.basename(a['name'])
if self.build_regex.search(name):
meth = self.queue.buildUrl
if self.fetch_config.tk_needs_auth():
meth = self.queue.buildSignedUrl
build_url = meth('getArtifact', task_id, run_id, a['name'])
break
if build_url is None:
raise BuildInfoNotFound("unable to find a build url for the"
" changeset %r" % changeset)
return InboundBuildInfo(
self.fetch_config,
build_url=build_url,
build_date=build_date,
changeset=changeset,
repo_url=self.jpushes.repo_url(),
task_id=task_id,
)
class InboundInfoFetcher(InfoFetcher):
def __init__(self, fetch_config):
InfoFetcher.__init__(self, fetch_config)
options = fetch_config.tk_options()
self.index = taskcluster.client.Index(options)
self.queue = taskcluster.Queue(options)
self.jpushes = JsonPushes(branch=fetch_config.inbound_branch)
def find_build_info(self, push):
"""
Find build info for an inbound build, given a Push, a changeset or a
date/datetime.
if `push` is not an instance of Push (e.g. it is a date, datetime, or
string representing the changeset), a query to json pushes will be
done.
Return a :class:`InboundBuildInfo` instance.
"""
if not isinstance(push, Push):
try:
push = self.jpushes.push(push)
except MozRegressionError, exc:
raise BuildInfoNotFound(str(exc))
changeset = push.changeset
tk_route = self.fetch_config.tk_inbound_route(push)
LOG.debug('using taskcluster route %r' % tk_route)
try:
task_id = self.index.findTask(tk_route)['taskId']
except TaskclusterFailure:
# HACK because of
# https://bugzilla.mozilla.org/show_bug.cgi?id=1199618
# and https://bugzilla.mozilla.org/show_bug.cgi?id=1211251
# TODO remove the if statement once these tasks no longer exists
# (just raise BuildInfoNotFound)
err = True
if self.fetch_config.app_name in ('firefox',
'fennec',
'fennec-2.3') \
and push.timestamp < TIMESTAMP_GECKO_V2:
err = False
try:
old_route = tk_route.replace(changeset, changeset[:12])
task_id = self.index.findTask(old_route)['taskId']
except TaskclusterFailure:
err = True
if err:
raise BuildInfoNotFound("Unable to find build info using the"
" taskcluster route %r" % tk_route)
# find a completed run for that task
run_id, build_date = None, None
status = self.queue.status(task_id)['status']
for run in reversed(status['runs']):
if run['state'] == 'completed':
run_id = run['runId']
build_date = datetime.strptime(run["resolved"],
'%Y-%m-%dT%H:%M:%S.%fZ')
break
if run_id is None:
raise BuildInfoNotFound(
"Unable to find completed runs for task %s" % task_id)
artifacts = self.queue.listArtifacts(task_id, run_id)['artifacts']
# look over the artifacts of that run
build_url = None
for a in artifacts:
name = os.path.basename(a['name'])
if self.build_regex.search(name):
meth = self.queue.buildUrl
if self.fetch_config.tk_needs_auth():
meth = self.queue.buildSignedUrl
build_url = meth('getArtifact', task_id, run_id, a['name'])
break
if build_url is None:
raise BuildInfoNotFound("unable to find a build url for the"
" changeset %r" % changeset)
return InboundBuildInfo(
self.fetch_config,
build_url=build_url,
build_date=build_date,
changeset=changeset,
repo_url=self.jpushes.repo_url,
task_id=task_id,
)
def find_build_info(self, push):
"""
Find build info for an inbound build, given a Push, a changeset or a
date/datetime.
if `push` is not an instance of Push (e.g. it is a date, datetime, or
string representing the changeset), a query to json pushes will be
done.
Return a :class:`InboundBuildInfo` instance.
"""
if not isinstance(push, Push):
try:
push = self.jpushes.push(push)
except MozRegressionError as exc:
raise BuildInfoNotFound(str(exc))
changeset = push.changeset
tk_routes = self.fetch_config.tk_inbound_routes(push)
try:
task_id = None
stored_failure = None
for tk_route in tk_routes:
LOG.debug('using taskcluster route %r' % tk_route)
try:
task_id = self.index.findTask(tk_route)['taskId']
except TaskclusterFailure as ex:
LOG.debug('nothing found via route %r' % tk_route)
stored_failure = ex
continue
if task_id:
status = self.queue.status(task_id)['status']
break
if not task_id:
raise stored_failure
except TaskclusterFailure:
raise BuildInfoNotFound("Unable to find build info using the"
" taskcluster route %r" %
self.fetch_config.tk_inbound_route(push))
# find a completed run for that task
run_id, build_date = None, None
for run in reversed(status['runs']):
if run['state'] == 'completed':
run_id = run['runId']
build_date = datetime.strptime(run["resolved"],
'%Y-%m-%dT%H:%M:%S.%fZ')
break
if run_id is None:
raise BuildInfoNotFound(
"Unable to find completed runs for task %s" % task_id)
artifacts = self.queue.listArtifacts(task_id, run_id)['artifacts']
# look over the artifacts of that run
build_url = None
for a in artifacts:
name = os.path.basename(a['name'])
if self.build_regex.search(name):
meth = self.queue.buildUrl
if self.fetch_config.tk_needs_auth():
meth = self.queue.buildSignedUrl
build_url = meth('getArtifact', task_id, run_id, a['name'])
break
if build_url is None:
raise BuildInfoNotFound("unable to find a build url for the"
" changeset %r" % changeset)
return InboundBuildInfo(
self.fetch_config,
build_url=build_url,
build_date=build_date,
changeset=changeset,
repo_url=self.jpushes.repo_url,
task_id=task_id,
)
