def get_latest_build_date(self):
"""Return date of latest available nightly build."""
if self.application not in ('fennec'):
url = urljoin(self.base_url, 'nightly', 'latest-%s/' % self.branch)
else:
url = urljoin(self.base_url, 'nightly', 'latest-%s-%s/' %
(self.branch, self.platform))
self.logger.info('Retrieving the build status file from %s' % url)
parser = self._create_directory_parser(url)
parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex)
if not parser.entries:
message = 'Status file for %s build cannot be found' % \
self.platform_regex
raise errors.NotFoundError(message, url)
# Read status file for the platform, retrieve build id,
# and convert to a date
headers = {'Cache-Control': 'max-age=0'}
r = self.session.get(url + parser.entries[-1], headers=headers)
try:
r.raise_for_status()
return datetime.strptime(r.text.split('\n')[0], '%Y%m%d%H%M%S')
finally:
r.close()
def get_build_info(self):
"""Define additional build information."""
ReleaseScraper.get_build_info(self)
# Internally we access builds via index
url = urljoin(self.base_url, self.candidate_build_list_regex)
self.logger.info('Retrieving list of candidate builds from %s' % url)
parser = self._create_directory_parser(url)
if not parser.entries:
message = 'Folder for specific candidate builds at %s has not' \
'been found' % url
raise errors.NotFoundError(message, url)
self.show_matching_builds(parser.entries)
self.builds = parser.entries
self.build_index = len(parser.entries) - 1
if self.build_number and \
('build%s' % self.build_number) in self.builds:
self.builds = ['build%s' % self.build_number]
self.build_index = 0
self.logger.info('Selected build: build%s' % self.build_number)
else:
self.logger.info('Selected build: %s' %
(parser.entries[self.build_index]))
def get_build_info(self):
"""Define additional build information."""
# Retrieve build by revision
if self.revision:
th = treeherder.Treeherder(
APPLICATIONS_TO_FTP_DIRECTORY.get(self.application,
self.application),
self.branch, self.platform)
builds = th.query_builds_by_revision(
self.revision,
job_type_name='L10n Nightly'
if self.locale_build else 'Nightly')
if not builds:
raise errors.NotFoundError(
'No builds have been found for revision', self.revision)
# Extract the build folders which are prefixed with the buildid
self.builds = [build.rsplit('/', 2)[1] for build in builds]
self.show_matching_builds(self.builds)
# There is only a single build per revision and platform
self.build_index = 0
self.logger.info('Selected build: %s' %
self.builds[self.build_index])
# Retrieve the date from the build folder which is always 19 chars long
self.date = datetime.strptime(self.builds[self.build_index][:19],
'%Y-%m-%d-%H-%M-%S')
return
# Internally we access builds via index
if self.build_number is not None:
self.build_index = int(self.build_number) - 1
else:
self.build_index = None
if self.build_id:
# A build id has been specified. Split up its components so the
# date and time can be extracted:
# '20111212042025' -> '2011-12-12 04:20:25'
self.date = datetime.strptime(self.build_id, '%Y%m%d%H%M%S')
elif self.date:
# A date (without time) has been specified. Use its value and the
# build index to find the requested build for that day.
try:
self.date = datetime.strptime(self.date, '%Y-%m-%d')
except Exception:
raise ValueError('%s is not a valid date' % self.date)
else:
# If no querying option has been specified the latest available
# build of the given branch has to be identified. We also have to
# retrieve the date of the build via its build id.
self.date = self.get_latest_build_date()
self.builds, self.build_index = self.get_build_info_for_date(
self.date, self.build_index)
def _get_binary():
# Retrieve all entries from the remote virtual folder
parser = self._create_directory_parser(self.path)
if not parser.entries:
raise errors.NotFoundError('No entries found', self.path)
# Download the first matched directory entry
pattern = re.compile(self.binary_regex, re.IGNORECASE)
for entry in parser.entries:
try:
self._binary = pattern.match(entry).group()
break
except Exception:
# No match, continue with next entry
continue
else:
raise errors.NotFoundError("Binary not found in folder",
self.path)
def get_build_info(self):
"""Define additional build information."""
# Retrieve build by revision
if self.revision:
th = treeherder.Treeherder(
APPLICATIONS_TO_FTP_DIRECTORY.get(self.application,
self.application),
self.branch, self.platform)
builds = th.query_builds_by_revision(self.revision,
job_type_name='Build',
debug_build=self.debug_build)
if not builds:
raise errors.NotFoundError(
'No builds have been found for revision', self.revision)
# Extract timestamp from each build folder
self.builds = [build.rsplit('/', 2)[1] for build in builds]
self.show_matching_builds(self.builds)
# There is only a single build
self.build_index = 0
self.logger.info('Selected build: %s' %
self.builds[self.build_index])
return
# Internally we access builds via index
if self.build_number is not None:
self.build_index = int(self.build_number) - 1
else:
self.build_index = None
if self.date is not None:
try:
# date is provided in the format 2013-07-23
self.date = datetime.strptime(self.date, '%Y-%m-%d')
except Exception:
try:
# date is provided as a unix timestamp
datetime.fromtimestamp(float(self.date))
self.timestamp = self.date
except Exception:
raise ValueError('%s is not a valid date' % self.date)
# For localized builds we do not have to retrieve the list of builds
# because only the last build is available
if not self.locale_build:
self.builds, self.build_index = self.get_build_info_for_index(
self.build_index)
# Always force a timestamp prefix in the filename
self.timestamp = self.builds[self.build_index]
def path_regex(self):
"""Return the regex for the path to the build folder."""
try:
path = '%s/' % urljoin(self.monthly_build_list_regex,
self.builds[self.build_index])
if self.application in APPLICATIONS_MULTI_LOCALE \
and self.locale != 'multi':
path = '%s/' % urljoin(path, self.locale)
return path
except Exception:
folder = urljoin(self.base_url, self.monthly_build_list_regex)
raise errors.NotFoundError("Specified sub folder cannot be found",
folder)
def _retry_check_404(self, func,
err_message="Specified build has not been found",
**retry_kwargs):
retry_kwargs.setdefault('retry_exceptions',
(errors.NotFoundError,
requests.exceptions.RequestException))
try:
self._retry(func, **retry_kwargs)
except requests.exceptions.HTTPError as exc:
if exc.response.status_code == 404:
raise errors.NotFoundError(err_message, exc.response.url)
else:
raise
def get_build_info_for_date(self, date, build_index=None):
"""Return the build information for a given date."""
url = urljoin(self.base_url, self.monthly_build_list_regex)
has_time = date and date.time(
) and date.strftime('%H-%M-%S') != '00-00-00'
self.logger.info('Retrieving list of builds from %s' % url)
parser = self._create_directory_parser(url)
regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' % {
'DATE':
date.strftime('%Y-%m-%d'),
'BRANCH':
self.branch,
# ensure to select the correct subfolder for localized builds
'L10N':
'(-l10n)?' if self.locale_build else '',
'PLATFORM':
'' if self.application not in ('fennec') else '-' + self.platform
}
parser.entries = parser.filter(regex)
parser.entries = parser.filter(self.is_build_dir)
if has_time:
# If a time is included in the date, use it to determine the
# build's index
regex = r'.*%s.*' % date.strftime('%H-%M-%S')
parser.entries = parser.filter(regex)
if not parser.entries:
date_format = '%Y-%m-%d-%H-%M-%S' if has_time else '%Y-%m-%d'
message = 'Folder for builds on %s has not been found' % \
self.date.strftime(date_format)
raise errors.NotFoundError(message, url)
# If no index has been given, set it to the last build of the day.
self.show_matching_builds(parser.entries)
# If no index has been given, set it to the last build of the day.
if build_index is None:
# Find the most recent non-empty entry.
build_index = len(parser.entries)
for build in reversed(parser.entries):
build_index -= 1
if not build_index or self.is_build_dir(build):
break
self.logger.info('Selected build: %s' % parser.entries[build_index])
return (parser.entries, build_index)
def _download():
try:
start_time = datetime.now()
# Enable streaming mode so we can download content in chunks
r = self.session.get(self.url, stream=True)
r.raise_for_status()
content_length = r.headers.get('Content-length')
# ValueError: Value out of range if only total_size given
if content_length:
total_size = int(content_length.strip())
max_value = ((total_size / CHUNK_SIZE) + 1) * CHUNK_SIZE
bytes_downloaded = 0
log_level = self.logger.getEffectiveLevel()
if log_level <= logging.INFO and content_length:
widgets = [pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA(),
' ', pb.FileTransferSpeed()]
pbar = pb.ProgressBar(widgets=widgets,
maxval=max_value).start()
with open(tmp_file, 'wb') as f:
for chunk in r.iter_content(CHUNK_SIZE):
f.write(chunk)
bytes_downloaded += CHUNK_SIZE
if log_level <= logging.INFO and content_length:
pbar.update(bytes_downloaded)
t1 = total_seconds(datetime.now() - start_time)
if self.timeout_download and \
t1 >= self.timeout_download:
raise errors.TimeoutError
if log_level <= logging.INFO and content_length:
pbar.finish()
except Exception as ex:
if os.path.isfile(tmp_file):
os.remove(tmp_file)
if type(ex) is requests.exceptions.HTTPError and \
ex.response.status_code == 404:
raise errors.NotFoundError("The requested url was not found", self.url)
else:
raise
def get_build_info(self):
"""Define additional build information."""
# Retrieve build by revision
th = treeherder.Treeherder(
APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application),
'try',
self.platform)
builds = th.query_builds_by_revision(
self.revision, job_type_name='Build', debug_build=self.debug_build)
if not builds:
raise errors.NotFoundError('No builds have been found for revision', self.revision)
# Extract username and revision from build folders
self.builds = [build.rsplit('/', 3)[1] for build in builds]
self.show_matching_builds(self.builds)
# There is only a single build per revision and platform
self.build_index = 0
self.logger.info('Selected build: %s' % self.builds[self.build_index])
def get_build_info_for_index(self, build_index=None):
"""Get additional information for the build at the given index."""
url = urljoin(self.base_url, self.build_list_regex)
self.logger.info('Retrieving list of builds from %s' % url)
parser = self._create_directory_parser(url)
parser.entries = parser.filter(r'^\d+$')
if self.timestamp:
# If a timestamp is given, retrieve the folder with the timestamp
# as name
parser.entries = self.timestamp in parser.entries and \
[self.timestamp]
elif self.date:
# If date is given, retrieve the subset of builds on that date
parser.entries = filter(self.date_matches, parser.entries)
if not parser.entries:
message = 'No builds have been found'
raise errors.NotFoundError(message, url)
self.show_matching_builds(parser.entries)
# If no index has been given, set it to the last build of the day.
if build_index is None:
# Find the most recent non-empty entry.
build_index = len(parser.entries)
for build in reversed(parser.entries):
build_index -= 1
if not build_index or self.is_build_dir(build):
break
self.logger.info('Selected build: %s' % parser.entries[build_index])
return (parser.entries, build_index)