def FetchFromCloudStorage(bucket_name, source_path, destination_dir): """Fetches file(s) from the Google Cloud Storage. As a side-effect, this prints messages to stdout about what's happening. Args: bucket_name: Google Storage bucket name. source_path: Source file path. destination_dir: Destination file path. Returns: Local file path of downloaded file if it was downloaded. If the file does not exist in the given bucket, or if there was an error while downloading, None is returned. """ target_file = os.path.join(destination_dir, os.path.basename(source_path)) gs_url = 'gs://%s/%s' % (bucket_name, source_path) try: if cloud_storage.Exists(bucket_name, source_path): logging.info('Fetching file from %s...', gs_url) cloud_storage.Get(bucket_name, source_path, target_file) if os.path.exists(target_file): return target_file else: logging.info('File %s not found in cloud storage.', gs_url) return None except Exception as e: logging.warn('Exception while fetching from cloud storage: %s', e) if os.path.exists(target_file): os.remove(target_file) return None
def BuildIsAvailable(bucket_name, remote_path): """Checks whether a build is currently archived at some place.""" logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path)) try: exists = cloud_storage.Exists(bucket_name, remote_path) logging.info('Exists? %s' % exists) return exists except cloud_storage.CloudStorageError: return False
def testExistsReturnsFalse(self): stubs = system_stub.Override(cloud_storage, ['subprocess']) orig_find_gs_util = cloud_storage.FindGsutil try: stubs.subprocess.Popen.communicate_result = ( '', 'CommandException: One or more URLs matched no objects.\n') stubs.subprocess.Popen.returncode_result = 1 cloud_storage.FindGsutil = _FakeFindGsutil self.assertFalse( cloud_storage.Exists('fake bucket', 'fake remote path')) finally: stubs.Restore() cloud_storage.FindGsutil = orig_find_gs_util
def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): """Uploads the screenshot to cloud storage as the reference image for this test, unless it already exists. Returns True if the upload was actually performed.""" if not self.options.refimg_cloud_storage_bucket: raise Exception('--refimg-cloud-storage-bucket argument is required') cloud_name = self._FormatReferenceImageName(img_name, page, tab) if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, cloud_name): self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, cloud_name, screenshot) return True return False
def _CheckWprShaFiles(input_api, output_api): """Check whether the wpr sha files have matching URLs.""" from telemetry.util import cloud_storage results = [] for affected_file in input_api.AffectedFiles(include_deletes=False): filename = affected_file.AbsoluteLocalPath() if not filename.endswith('wpr.sha1'): continue expected_hash = cloud_storage.ReadHash(filename) is_wpr_file_uploaded = any( cloud_storage.Exists(bucket, expected_hash) for bucket in cloud_storage.BUCKET_ALIASES.itervalues()) if not is_wpr_file_uploaded: wpr_filename = filename[:-5] results.append( output_api.PresubmitError( 'There is no URLs matched for wpr sha file %s.\n' 'You can upload your new wpr archive file with the command:\n' 'depot_tools/upload_to_google_storage.py --bucket ' '<Your pageset\'s bucket> %s.\nFor more info: see ' 'http://www.chromium.org/developers/telemetry/' 'record_a_page_set#TOC-Upload-the-recording-to-Cloud-Storage' % (filename, wpr_filename))) return results