def DownloadLatestFile(base_url, partial_name, dst):
"""Get the latest archived object with the given base url and partial name.
Args:
base_url: Base Google Storage archive URL (gs://...) containing the build.
partial_name: Partial name of the archive file to download.
dst: Destination file/directory where the file will be downloaded.
Raises:
Exception: If unable to find or download a file.
"""
base_url_glob = '%s/**' % base_url.rstrip('/')
result = slave_utils.GSUtilListBucket(base_url_glob, ['-l'])
if not result or result[0]:
raise Exception('Could not find any archived files.')
files = [b.split()[2] for b in result[1].split('\n') if partial_name in b]
if not files:
raise Exception('Could not find any matching files.')
files = [distutils.version.LooseVersion(x) for x in files]
newest_file = str(max(files))
slave_utils.GSUtilDownloadFile(newest_file, dst)
def DownloadFileWithPolling(url,
dst,
poll_interval=_POLL_INTERVAL_DEFAULT,
timeout=_TIMEOUT_DEFAULT):
"""Download the archived file with given |url|.
If not found, keep trying at |poll_interval| until we reach |timeout|.
Args:
url: Google Storage URL (gs://...).
dst: Destination file/directory where the file will be downloaded.
poll_interval: Polling interval in seconds.
timeout: Timeout in seconds.
Raises:
Exception: If there is a timeout.
"""
start_time = time.time()
time_passed = 0
while time_passed < timeout:
if not slave_utils.GSUtilDownloadFile(url, dst):
return
print 'Retrying in %d seconds...' % poll_interval
time.sleep(poll_interval)
time_passed = time.time() - start_time
raise Exception('Timed out trying to download %s' % url)
def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename):
"""Downloads the webpages archive and its required page set from GS."""
wpr_source = posixpath.join(
self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
wpr_data_file)
page_set_source = posixpath.join(
self._dest_gsbase, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive',
page_set_basename)
if (old_gs_utils.does_storage_object_exist(wpr_source) and
old_gs_utils.does_storage_object_exist(page_set_source)):
slave_utils.GSUtilDownloadFile(
src=wpr_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
slave_utils.GSUtilDownloadFile(
src=page_set_source, dst=LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR)
else:
raise Exception('%s and %s do not exist in Google Storage!' % (
wpr_source, page_set_source))
def _MaybeUploadReleaseNotes(version):
"""Upload release notes if conditions are right."""
# Check if the current version has already been released.
notes_name = 'notes.txt'
notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, version, notes_name)
prev_version = '.'.join(
[version.split('.')[0],
str(int(version.split('.')[1]) - 1)])
prev_notes_url = '%s/%s/%s' % (GS_CHROMEDRIVER_BUCKET, prev_version,
notes_name)
result, _ = slave_utils.GSUtilListBucket(notes_url, [])
if result == 0:
return
fixed_issues = []
query = ('https://code.google.com/p/chromedriver/issues/csv?'
'can=1&q=label%%3AChromeDriver-%s&colspec=ID%%20Summary' %
version)
issues = StringIO.StringIO(_GetWebPageContent(query).split('\n', 1)[1])
for issue in csv.reader(issues):
if not issue:
continue
issue_id = issue[0]
desc = issue[1]
labels = issue[2].split(', ')
labels.remove('ChromeDriver-%s' % version)
if 'Hotlist-GoodFirstBug' in labels:
labels.remove('Hotlist-GoodFirstBug')
fixed_issues += [
'Resolved issue %s: %s [%s]' % (issue_id, desc, labels)
]
old_notes = ''
temp_notes_fname = tempfile.mkstemp()[1]
if not slave_utils.GSUtilDownloadFile(prev_notes_url, temp_notes_fname):
with open(temp_notes_fname, 'rb') as f:
old_notes = f.read()
new_notes = '----------ChromeDriver v%s (%s)----------\n%s\n%s\n\n%s' % (
version, datetime.date.today().isoformat(), 'Supports Chrome v%s-%s' %
_GetSupportedChromeVersions(), '\n'.join(fixed_issues), old_notes)
with open(temp_notes_fname, 'w') as f:
f.write(new_notes)
if slave_utils.GSUtilCopy(temp_notes_fname,
notes_url,
mimetype='text/plain'):
util.MarkBuildStepError()
def read_timestamp_file(timestamp_file_name, gs_base, gs_relative_dir):
"""Reads the specified TIMESTAMP file from the specified GS dir.
Returns 0 if the file is empty or does not exist.
"""
src = posixpath.join(gs_base, gs_relative_dir, timestamp_file_name)
temp_file = tempfile.mkstemp()[1]
slave_utils.GSUtilDownloadFile(src=src, dst=temp_file)
storage_file_obj = open(temp_file, 'r')
try:
timestamp_value = storage_file_obj.read().strip()
return timestamp_value if timestamp_value else "0"
finally:
storage_file_obj.close()
def download_directory_contents_if_changed(gs_base, gs_relative_dir,
local_dir):
"""Compares the TIMESTAMP_LAST_UPLOAD_COMPLETED and downloads if different.
The goal of download_directory_contents_if_changed and
upload_directory_contents_if_changed is to attempt to replicate directory
level rsync functionality to the Google Storage directories we care about.
"""
if _are_timestamps_equal(gs_base, gs_relative_dir, local_dir):
print '\n\n=======Local directory is current=======\n\n'
else:
file_utils.create_clean_local_dir(local_dir)
gs_source = posixpath.join(gs_base, gs_relative_dir, '*')
slave_utils.GSUtilDownloadFile(src=gs_source, dst=local_dir)
if not _are_timestamps_equal(gs_base, gs_relative_dir, local_dir):
raise Exception('Failed to download from GS: %s' % gs_source)
def _GetTestResultsLog(platform):
"""Gets the test results log for the given platform.
Returns:
A dictionary where the keys are SVN revisions and the values are booleans
indicating whether the tests passed.
"""
temp_log = tempfile.mkstemp()[1]
log_name = TEST_LOG_FORMAT % platform
result = slave_utils.GSUtilDownloadFile(
'%s/%s' % (GS_CHROMEDRIVER_DATA_BUCKET, log_name), temp_log)
if result:
return {}
with open(temp_log, 'rb') as log_file:
json_dict = json.load(log_file)
# Workaround for json encoding dictionary keys as strings.
return dict([(int(v[0]), v[1]) for v in json_dict.items()])
def _are_timestamps_equal(gs_base, gs_relative_dir, local_dir):
"""Compares the local TIMESTAMP with the TIMESTAMP from Google Storage."""
local_timestamp_file = os.path.join(local_dir,
TIMESTAMP_COMPLETED_FILENAME)
# Make sure that the local TIMESTAMP file exists.
if not os.path.exists(local_timestamp_file):
return False
# Get the timestamp file from Google Storage.
src = posixpath.join(gs_base, gs_relative_dir,
TIMESTAMP_COMPLETED_FILENAME)
temp_file = tempfile.mkstemp()[1]
slave_utils.GSUtilDownloadFile(src=src, dst=temp_file)
local_file_obj = open(local_timestamp_file, 'r')
storage_file_obj = open(temp_file, 'r')
try:
local_timestamp = local_file_obj.read().strip()
storage_timestamp = storage_file_obj.read().strip()
return local_timestamp == storage_timestamp
finally:
local_file_obj.close()
storage_file_obj.close()
def Run(self):
"""Run the SkPicturePlayback BuildStep."""
# Ensure the right .boto file is used by gsutil.
if not self._skip_all_gs_access and old_gs_utils.read_timestamp_file(
timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
gs_base=self._dest_gsbase,
gs_relative_dir=posixpath.join(ROOT_PLAYBACK_DIR_NAME,
SKPICTURES_DIR_NAME)) == "0":
raise Exception(
'Missing .boto file or .boto does not have the right credentials.'
'Please see https://docs.google.com/a/google.com/document/d/1ZzHP6M5q'
'ACA9nJnLqOZr2Hl0rjYqE4yQsQWAfVjKCzs/edit '
'(may have to request access). The .boto file will need to be placed '
'under third_party/chromium_buildbot/site_config/')
# Download the credentials file if it was not previously downloaded.
if self._skip_all_gs_access:
print """\n\nPlease create a %s file that contains:
{
"google": {
"username": "******",
"password": "******"
},
"facebook": {
"username": "******",
"password": "******"
}
}\n\n""" % CREDENTIALS_FILE_PATH
raw_input("Please press a key when you are ready to proceed...")
elif not os.path.isfile(CREDENTIALS_FILE_PATH):
# Download the credentials.json file from Google Storage.
slave_utils.GSUtilDownloadFile(
src=CREDENTIALS_GS_LOCATION, dst=CREDENTIALS_FILE_PATH)
# Delete any left over data files in the data directory.
for archive_file in glob.glob(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')):
os.remove(archive_file)
# Delete the local root directory if it already exists.
if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR):
shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR)
# Create the required local storage directories.
self._CreateLocalStorageDirs()
# Start the timer.
start_time = time.time()
# Loop through all page_sets.
for page_set in self._page_sets:
page_set_basename = os.path.basename(page_set).split('.')[0] + '.json'
wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr'
if self._record:
# Create an archive of the specified webpages if '--record=True' is
# specified.
record_wpr_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
os.path.join(TELEMETRY_BINARIES_DIR, 'record_wpr'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
page_set
)
for _ in range(RETRY_RECORD_WPR_COUNT):
output = shell_utils.run(' '.join(record_wpr_cmd), shell=True)
if RECORD_FAILURE_MSG in output:
print output
else:
# Break out of the retry loop since there were no errors.
break
else:
# If we get here then record_wpr did not succeed and thus did not
# break out of the loop.
raise Exception('record_wpr failed for page_set: %s' % page_set)
else:
if not self._skip_all_gs_access:
# Get the webpages archive so that it can be replayed.
self._DownloadWebpagesArchive(wpr_data_file, page_set_basename)
run_measurement_cmd = (
'DISPLAY=%s' % X11_DISPLAY,
'timeout', '300',
os.path.join(TELEMETRY_BINARIES_DIR, 'run_measurement'),
'--extra-browser-args=--disable-setuid-sandbox',
'--browser=exact',
'--browser-executable=%s' % self._browser_executable,
SKP_BENCHMARK,
page_set,
'-o',
'/tmp/test.skp',
'--skp-outdir=%s' % TMP_SKP_DIR
)
for _ in range(RETRY_RUN_MEASUREMENT_COUNT):
try:
print '\n\n=======Capturing SKP of %s=======\n\n' % page_set
shell_utils.run(' '.join(run_measurement_cmd), shell=True)
except shell_utils.CommandFailedException:
# skpicture_printer sometimes fails with AssertionError but the
# captured SKP is still valid. This is a known issue.
pass
if self._record:
# Move over the created archive into the local webpages archive
# directory.
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file),
self._local_record_webpages_archive_dir)
shutil.move(
os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR,
page_set_basename),
self._local_record_webpages_archive_dir)
# Rename generated SKP files into more descriptive names.
try:
self._RenameSkpFiles(page_set)
# Break out of the retry loop since there were no errors.
break
except Exception:
# There was a failure continue with the loop.
traceback.print_exc()
print '\n\n=======Retrying %s=======\n\n' % page_set
time.sleep(10)
else:
# If we get here then run_measurement did not succeed and thus did not
# break out of the loop.
raise Exception('run_measurement failed for page_set: %s' % page_set)
print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % (
time.time() - start_time)
if self._skia_tools:
render_pictures_cmd = [
os.path.join(self._skia_tools, 'render_pictures'),
'-r', self._local_skp_dir
]
render_pdfs_cmd = [
os.path.join(self._skia_tools, 'render_pdfs'),
self._local_skp_dir
]
for tools_cmd in (render_pictures_cmd, render_pdfs_cmd):
print '\n\n=======Running %s=======' % ' '.join(tools_cmd)
proc = subprocess.Popen(tools_cmd)
(code, output) = shell_utils.log_process_after_completion(proc,
echo=False)
if code != 0:
raise Exception('%s failed!' % ' '.join(tools_cmd))
if not self._non_interactive:
print '\n\n=======Running debugger======='
os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'),
os.path.join(self._local_skp_dir, '*')))
print '\n\n'
if not self._skip_all_gs_access and self._upload_to_gs:
print '\n\n=======Uploading to Google Storage=======\n\n'
# Copy the directory structure in the root directory into Google Storage.
dest_dir_name = ROOT_PLAYBACK_DIR_NAME
if self._alternate_upload_dir:
dest_dir_name = self._alternate_upload_dir
gs_status = slave_utils.GSUtilCopyDir(
src_dir=LOCAL_PLAYBACK_ROOT_DIR,
gs_base=self._dest_gsbase,
dest_dir=dest_dir_name,
gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL)
if gs_status != 0:
raise Exception(
'ERROR: GSUtilCopyDir error %d. "%s" -> "%s/%s"' % (
gs_status, LOCAL_PLAYBACK_ROOT_DIR, self._dest_gsbase,
ROOT_PLAYBACK_DIR_NAME))
self._SetGoogleReadACLs(
posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
# Add a timestamp file to the SKP directory in Google Storage so we can
# use directory level rsync like functionality.
old_gs_utils.write_timestamp_file(
timestamp_file_name=old_gs_utils.TIMESTAMP_COMPLETED_FILENAME,
timestamp_value=time.time(),
gs_base=self._dest_gsbase,
gs_relative_dir=posixpath.join(dest_dir_name, SKPICTURES_DIR_NAME),
gs_acl=gs_utils.GSUtils.PLAYBACK_CANNED_ACL,
local_dir=LOCAL_PLAYBACK_ROOT_DIR)
print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % (
posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME))
else:
print '\n\n=======Not Uploading to Google Storage=======\n\n'
print 'Generated resources are available in %s\n\n' % (
LOCAL_PLAYBACK_ROOT_DIR)
return 0
def DownloadFromBucket(source_gsurl, dest):
status = slave_utils.GSUtilDownloadFile(source_gsurl, dest)
if status != 0:
raise Exception('ERROR: GSUtilDownloadFile error %d. "%s" -> "%s"' %
(status, source_gsurl, dest))
return 0
def upload_directory_contents_if_changed(gs_base,
gs_relative_dir,
gs_acl,
local_dir,
force_upload=False,
upload_chunks=False,
files_to_upload=None):
"""Compares the TIMESTAMP_LAST_UPLOAD_COMPLETED and uploads if different.
Args:
gs_base: str - The Google Storage base. Eg: gs://rmistry.
gs_relative_dir: str - Relative directory to the Google Storage base.
gs_acl: str - ACL to use when uploading to Google Storage.
local_dir: str - The local directory to upload.
force_upload: bool - Whether upload should be done regardless of timestamps
matching or not.
upload_chunks: bool - Whether upload should be done in chunks or in a single
command.
files_to_upload: str seq - Specific files that should be uploaded, if not
specified then all files in local_dir are uploaded. If upload_chunks is
True then files will be uploaded in chunks else they will be uploaded
one at a time. The Google Storage directory is not cleaned before upload
if files_to_upload is specified.
The goal of download_directory_contents_if_changed and
upload_directory_contents_if_changed is to attempt to replicate directory
level rsync functionality to the Google Storage directories we care about.
Returns True if contents were uploaded, else returns False.
"""
if not force_upload and _are_timestamps_equal(gs_base, gs_relative_dir,
local_dir):
print '\n\n=======Local directory is current=======\n\n'
return False
else:
local_src = os.path.join(local_dir, '*')
gs_dest = posixpath.join(gs_base, gs_relative_dir)
timestamp_value = time.time()
if not files_to_upload:
print '\n\n=======Delete Storage directory before uploading=======\n\n'
delete_storage_object(gs_dest)
print '\n\n=======Writing new TIMESTAMP_LAST_UPLOAD_STARTED=======\n\n'
write_timestamp_file(timestamp_file_name=TIMESTAMP_STARTED_FILENAME,
timestamp_value=timestamp_value,
gs_base=gs_base,
gs_relative_dir=gs_relative_dir,
local_dir=local_dir,
gs_acl=gs_acl)
if upload_chunks:
if files_to_upload:
local_files = [
os.path.join(local_dir, local_file)
for local_file in files_to_upload
]
else:
local_files = [
os.path.join(local_dir, local_file)
for local_file in os.listdir(local_dir)
]
for files_chunk in _get_chunks(local_files, FILES_CHUNK):
gsutil = slave_utils.GSUtilSetup()
command = [gsutil, 'cp'] + files_chunk + [gs_dest]
try:
shell_utils.run(command)
except shell_utils.CommandFailedException:
raise Exception(
'Could not upload the chunk to Google Storage! The chunk: %s'
% files_chunk)
else:
if files_to_upload:
for file_to_upload in files_to_upload:
if slave_utils.GSUtilDownloadFile(src=os.path.join(
local_dir, file_to_upload),
dst=gs_dest) != 0:
raise Exception(
'Could not upload %s to Google Storage!' %
file_to_upload)
else:
if slave_utils.GSUtilDownloadFile(src=local_src,
dst=gs_dest) != 0:
raise Exception('Could not upload %s to Google Storage!' %
local_src)
print '\n\n=======Writing new TIMESTAMP_LAST_UPLOAD_COMPLETED=======\n\n'
write_timestamp_file(timestamp_file_name=TIMESTAMP_COMPLETED_FILENAME,
timestamp_value=timestamp_value,
gs_base=gs_base,
gs_relative_dir=gs_relative_dir,
local_dir=local_dir,
gs_acl=gs_acl)
return True
