def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
"""Fetches file(s) from the Google Cloud Storage.
As a side-effect, this prints messages to stdout about what's happening.
Args:
bucket_name: Google Storage bucket name.
source_path: Source file path.
destination_dir: Destination file path.
Returns:
Local file path of downloaded file if it was downloaded. If the file does
not exist in the given bucket, or if there was an error while downloading,
None is returned.
"""
target_file = os.path.join(destination_dir, os.path.basename(source_path))
gs_url = 'gs://%s/%s' % (bucket_name, source_path)
try:
if cloud_storage.Exists(bucket_name, source_path):
logging.info('Fetching file from %s...', gs_url)
cloud_storage.Get(bucket_name, source_path, target_file)
if os.path.exists(target_file):
return target_file
else:
logging.info('File %s not found in cloud storage.', gs_url)
return None
except Exception as e:
logging.warn('Exception while fetching from cloud storage: %s', e)
if os.path.exists(target_file):
os.remove(target_file)
return None
def _DownloadFromCloudStorage(self, img_name, page, tab):
"""Downloads the reference image for the given test from cloud
storage, returning it as a Telemetry Bitmap object."""
# TODO(kbr): there's a race condition between the deletion of the
# temporary file and gsutil's overwriting it.
if not self.options.refimg_cloud_storage_bucket:
raise Exception(
'--refimg-cloud-storage-bucket argument is required')
temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
cloud_storage.Get(self.options.refimg_cloud_storage_bucket,
self._FormatReferenceImageName(img_name, page, tab),
temp_file)
return image_util.FromPngFile(temp_file)
def testDisableCloudStorageIo(self, unused_lock_mock):
os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1'
dir_path = 'real_dir_path'
self.fs.CreateDirectory(dir_path)
file_path = os.path.join(dir_path, 'file1')
file_path_sha = file_path + '.sha1'
self.CreateFiles([file_path, file_path_sha])
with open(file_path_sha, 'w') as f:
f.write('hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1',
'remote_path2')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Get('bucket', 'foo', file_path)
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfChanged(file_path, 'foo')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetIfHashChanged('bar', file_path, 'bucket',
'hash1234')
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.Insert('bucket', 'foo', file_path)
with self.assertRaises(cloud_storage.CloudStorageIODisabled):
cloud_storage.GetFilesInDirectoryIfChanged(dir_path, 'bucket')
def _DownloadRemoteExtensions(self, remote_bucket, local_extensions_dir):
"""Downloads and unzips archive of common extensions to disk.
Args:
remote_bucket: bucket to download remote archive from.
local_extensions_dir: destination extensions directory.
Raises:
InvalidExtensionArchiveError if remote archive is not found.
"""
# Force Unix directory separator for remote path.
remote_zip_path = '%s/%s' % (REMOTE_DIR, ZIP_NAME)
local_zip_path = os.path.join(local_extensions_dir, ZIP_NAME)
try:
cloud_storage.Get(remote_bucket, remote_zip_path, local_zip_path)
except cloud_storage.ServerError:
raise InvalidExtensionArchiveError('Can\'t find archive at gs://%s/%s..'
% (remote_bucket, remote_zip_path))
try:
with zipfile.ZipFile(local_zip_path, 'r') as extensions_zip:
extensions_zip.extractall(local_extensions_dir)
finally:
os.remove(local_zip_path)