def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
"""Fetches file(s) from the Google Cloud Storage.
As a side-effect, this prints messages to stdout about what's happening.
Args:
bucket_name: Google Storage bucket name.
source_path: Source file path.
destination_dir: Destination file path.
Returns:
Local file path of downloaded file if it was downloaded. If the file does
not exist in the given bucket, or if there was an error while downloading,
None is returned.
"""
target_file = os.path.join(destination_dir, os.path.basename(source_path))
gs_url = 'gs://%s/%s' % (bucket_name, source_path)
try:
if cloud_storage.Exists(bucket_name, source_path):
logging.info('Fetching file from %s...', gs_url)
cloud_storage.Get(bucket_name, source_path, target_file)
if os.path.exists(target_file):
return target_file
else:
logging.info('File %s not found in cloud storage.', gs_url)
return None
except Exception as e:
logging.warn('Exception while fetching from cloud storage: %s', e)
if os.path.exists(target_file):
os.remove(target_file)
return None
def BuildIsAvailable(bucket_name, remote_path):
"""Checks whether a build is currently archived at some place."""
logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path))
try:
exists = cloud_storage.Exists(bucket_name, remote_path)
logging.info('Exists? %s' % exists)
return exists
except cloud_storage.CloudStorageError:
return False
def testExistsReturnsFalse(self):
stubs = system_stub.Override(cloud_storage, ['subprocess'])
orig_find_gs_util = cloud_storage.FindGsutil
try:
stubs.subprocess.Popen.communicate_result = (
'', 'CommandException: One or more URLs matched no objects.\n')
stubs.subprocess.Popen.returncode_result = 1
cloud_storage.FindGsutil = _FakeFindGsutil
self.assertFalse(
cloud_storage.Exists('fake bucket', 'fake remote path'))
finally:
stubs.Restore()
cloud_storage.FindGsutil = orig_find_gs_util
def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot):
"""Uploads the screenshot to cloud storage as the reference image
for this test, unless it already exists. Returns True if the
upload was actually performed."""
if not self.options.refimg_cloud_storage_bucket:
raise Exception('--refimg-cloud-storage-bucket argument is required')
cloud_name = self._FormatReferenceImageName(img_name, page, tab)
if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket,
cloud_name):
self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket,
cloud_name,
screenshot)
return True
return False
def _CheckWprShaFiles(input_api, output_api):
"""Check whether the wpr sha files have matching URLs."""
from telemetry.util import cloud_storage
results = []
for affected_file in input_api.AffectedFiles(include_deletes=False):
filename = affected_file.AbsoluteLocalPath()
if not filename.endswith('wpr.sha1'):
continue
expected_hash = cloud_storage.ReadHash(filename)
is_wpr_file_uploaded = any(
cloud_storage.Exists(bucket, expected_hash)
for bucket in cloud_storage.BUCKET_ALIASES.itervalues())
if not is_wpr_file_uploaded:
wpr_filename = filename[:-5]
results.append(
output_api.PresubmitError(
'There is no URLs matched for wpr sha file %s.\n'
'You can upload your new wpr archive file with the command:\n'
'depot_tools/upload_to_google_storage.py --bucket '
'<Your pageset\'s bucket> %s.\nFor more info: see '
'http://www.chromium.org/developers/telemetry/'
'record_a_page_set#TOC-Upload-the-recording-to-Cloud-Storage'
% (filename, wpr_filename)))
return results