def CopyTarballToGCS(self, storage_client, gcs_object): """Copy a tarball of the snapshot to GCS. Args: storage_client: storage_api.StorageClient, The storage client to use for uploading. gcs_object: storage.objects Resource, The GCS object to write. Returns: storage_v1_messages.Object, The written GCS object. """ with files.ChDir(self.src_dir): with files.TemporaryDirectory() as tmp: archive_path = os.path.join(tmp, 'file.tgz') tf = self._MakeTarball(archive_path) tf.close() ignore_file_path = os.path.join(self.src_dir, gcloudignore.IGNORE_FILE_NAME) if self.any_files_ignored: if os.path.exists(ignore_file_path): log.info('Using gcloudignore file [{}]'.format( ignore_file_path)) else: log.status.Print( _IGNORED_FILE_MESSAGE.format( log_file=log.GetLogFilePath())) log.status.write( 'Uploading tarball of [{src_dir}] to ' '[gs://{bucket}/{object}]\n'.format( src_dir=self.src_dir, bucket=gcs_object.bucket, object=gcs_object.object, ), ) return storage_client.CopyFileToGCS(archive_path, gcs_object)
def _UploadToGcsGsutil(local_path, dest_path): """Uploads a local file to GCS using gsutil.""" retcode = storage_util.RunGsutilCommand('cp', [local_path, dest_path]) if retcode != 0: log.err.Print('Failed to upload file. See {} for details.'.format( log.GetLogFilePath())) raise exceptions.FailedSubCommand( ['gsutil', 'cp', local_path, dest_path], retcode) return dest_path
def CopyFilesToCodeBucket(modules, bucket, source_contexts): """Examines modules and copies files to a Google Cloud Storage bucket. Args: modules: [(str, ModuleYamlInfo)] List of pairs of module name, and parsed module information. bucket: str A URL to the Google Cloud Storage bucket where the files will be uploaded. source_contexts: [dict] List of json-serializable source contexts associated with the modules. Returns: A lookup from module name to a dictionary representing the manifest. See _BuildStagingDirectory. """ manifests = {} with file_utils.TemporaryDirectory() as staging_directory: for (module, info) in modules: source_directory = os.path.dirname(info.file) excluded_files_regex = info.parsed.skip_files.regex manifest = _BuildStagingDirectory(source_directory, staging_directory, bucket, excluded_files_regex, source_contexts) manifests[module] = manifest if any(manifest for manifest in manifests.itervalues()): log.status.Print('Copying files to Google Cloud Storage...') log.status.Print('Synchronizing files to [{b}].'.format(b=bucket)) try: log.SetUserOutputEnabled(False) def _StatusUpdate(result, unused_retry_state): log.info('Error synchronizing files. Return code: {0}. ' 'Retrying.'.format(result)) retryer = retry.Retryer(max_retrials=3, status_update_func=_StatusUpdate) def _ShouldRetry(return_code, unused_retry_state): return return_code != 0 try: retryer.RetryOnResult(cloud_storage.Rsync, (staging_directory, bucket), should_retry_if=_ShouldRetry) except retry.RetryException as e: raise exceptions.ToolException(( 'Could not synchronize files. The gsutil command exited with ' 'status [{s}]. Command output is available in [{l}].' ).format(s=e.last_result, l=log.GetLogFilePath())) finally: # Reset to the standard log level. log.SetUserOutputEnabled(None) return manifests
def FileIterator(base, skip_files): """Walks a directory tree, returning all the files. Follows symlinks. Args: base: The base path to search for files under. skip_files: A regular expression object for files/directories to skip. Yields: Paths of files found, relative to base. """ dirs = [''] contains_skipped_modules = False while dirs: current_dir = dirs.pop() entries = set(os.listdir(os.path.join(base, current_dir))) for entry in sorted(entries): true_name = os.path.join(current_dir, entry) fullname = os.path.join(base, true_name) # On Windows, os.path.join uses the path separator '\' instead of '/'. # However, the skip_files regular expression always uses '/'. # To handle this, we'll replace '\' characters with '/' characters. if os.path.sep == '\\': name = true_name.replace('\\', '/') else: name = true_name if os.path.isfile(fullname): if skip_files.match(name): log.info('Ignoring file [%s]: File matches ignore regex.', true_name) contains_skipped_modules = True else: yield true_name elif os.path.isdir(fullname): if skip_files.match(name): log.info( 'Ignoring directory [%s]: Directory matches ignore regex.', true_name) contains_skipped_modules = True else: dirs.append(true_name) if contains_skipped_modules: log.status.Print( 'Some files were skipped. Pass `--verbosity=info` to see which ones.' ) log_path = log.GetLogFilePath() if log_path: log.status.Print( ('You may also view the gcloud log file, found at\n' '[{0}].').format(log_path))
def CopyFilesToCodeBucket(service, source_dir, bucket_ref): """Examines services and copies files to a Google Cloud Storage bucket. Args: service: ServiceYamlInfo, The parsed service information. source_dir: str, path to the service's source directory bucket_ref: str A reference to a GCS bucket where the files will be uploaded. Returns: A dictionary representing the manifest. See _BuildStagingDirectory. """ with file_utils.TemporaryDirectory() as staging_directory: excluded_files_regex = service.parsed.skip_files.regex manifest = _BuildStagingDirectory(source_dir, staging_directory, bucket_ref, excluded_files_regex) if manifest: log.status.Print('Copying files to Google Cloud Storage...') log.status.Print('Synchronizing files to [{b}].' .format(b=bucket_ref.bucket)) try: log.SetUserOutputEnabled(False) def _StatusUpdate(result, unused_retry_state): log.info('Error synchronizing files. Return code: {0}. ' 'Retrying.'.format(result)) retryer = retry.Retryer(max_retrials=3, status_update_func=_StatusUpdate) def _ShouldRetry(return_code, unused_retry_state): return return_code != 0 # gsutil expects a trailing / dest_dir = bucket_ref.ToBucketUrl() try: retryer.RetryOnResult( storage_api.Rsync, (staging_directory, dest_dir), should_retry_if=_ShouldRetry) except retry.RetryException as e: raise exceptions.StorageError( ('Could not synchronize files. The gsutil command exited with ' 'status [{s}]. Command output is available in [{l}].').format( s=e.last_result, l=log.GetLogFilePath())) finally: # Reset to the standard log level. log.SetUserOutputEnabled(None) log.status.Print('File upload done.') return manifest
def _UploadToGcs(is_async, local_path, daisy_bucket, image_uuid): """Uploads a local file to GCS. Returns the gs:// URI to that file.""" file_name = os.path.basename(local_path).replace(' ', '-') dest_path = 'gs://{0}/tmpimage/{1}-{2}'.format(daisy_bucket, image_uuid, file_name) if is_async: log.status.Print('Async: Once upload is complete, your image will be ' 'imported from Cloud Storage asynchronously.') with progress_tracker.ProgressTracker('Copying [{0}] to [{1}]'.format( local_path, dest_path)): retcode = storage_util.RunGsutilCommand('cp', [local_path, dest_path]) if retcode != 0: log.err.Print('Failed to upload file. See {} for details.'.format( log.GetLogFilePath())) raise exceptions.FailedSubCommand( ['gsutil', 'cp', local_path, dest_path], retcode) return dest_path
def FileIterator(base, skip_files): """Walks a directory tree, returning all the files. Follows symlinks. Args: base: The base path to search for files under. skip_files: A regular expression object for files/directories to skip. Yields: Paths of files found, relative to base. """ dirs = [''] contains_skipped_modules = False while dirs: current_dir = dirs.pop() entries = set(os.listdir(os.path.join(base, current_dir))) for entry in sorted(entries): name = os.path.join(current_dir, entry) fullname = os.path.join(base, name) if os.path.isfile(fullname): if ShouldSkip(skip_files, name): log.info('Ignoring file [%s]: File matches ignore regex.', name) contains_skipped_modules = True else: yield name elif os.path.isdir(fullname): if ShouldSkip(skip_files, name): log.info( 'Ignoring directory [%s]: Directory matches ignore regex.', name) contains_skipped_modules = True else: dirs.append(name) if contains_skipped_modules: log.status.Print( 'Some files were skipped. Pass `--verbosity=info` to see which ones.' ) log_path = log.GetLogFilePath() if log_path: log.status.Print( ('You may also view the gcloud log file, found at\n' '[{0}].').format(log_path))
def CopyTarballToGCS(self, storage_client, gcs_object, ignore_file=None, hide_logs=False): """Copy a tarball of the snapshot to GCS. Args: storage_client: storage_api.StorageClient, The storage client to use for uploading. gcs_object: storage.objects Resource, The GCS object to write. ignore_file: Override .gcloudignore file to specify skip files. hide_logs: boolean, not print the status message if the flag is true. Returns: storage_v1_messages.Object, The written GCS object. """ with metrics.RecordDuration(metric_names.UPLOAD_SOURCE): with files.ChDir(self.src_dir): with files.TemporaryDirectory() as tmp: archive_path = os.path.join(tmp, 'file.tgz') tf = self._MakeTarball(archive_path) tf.close() ignore_file_path = os.path.join( self.src_dir, ignore_file or gcloudignore.IGNORE_FILE_NAME) if self.any_files_ignored: if os.path.exists(ignore_file_path): log.info('Using ignore file [{}]'.format( ignore_file_path)) elif not hide_logs: log.status.Print( _IGNORED_FILE_MESSAGE.format( log_file=log.GetLogFilePath())) if not hide_logs: log.status.write( 'Uploading tarball of [{src_dir}] to ' '[gs://{bucket}/{object}]\n'.format( src_dir=self.src_dir, bucket=gcs_object.bucket, object=gcs_object.object, ), ) return storage_client.CopyFileToGCS( archive_path, gcs_object)
def CopyFilesToCodeBucket(modules, bucket): """Examines modules and copies files to a Google Cloud Storage bucket. Args: modules: [(str, ModuleYamlInfo)] List of pairs of module name, and parsed module information. bucket: str A URL to the Google Cloud Storage bucket where the files will be uploaded. Returns: A lookup from module name to a dictionary representing the manifest. See _BuildStagingDirectory. """ manifests = {} with file_utils.TemporaryDirectory() as staging_directory: for (module, info) in modules: source_directory = os.path.dirname(info.file) excluded_files_regex = info.parsed.skip_files.regex manifest = _BuildStagingDirectory(source_directory, staging_directory, bucket, excluded_files_regex) manifests[module] = manifest if any(manifest for manifest in manifests.itervalues()): log.status.Print('Copying files to Google Cloud Storage...') log.status.Print('Synchronizing files to [{b}].'.format(b=bucket)) try: log.SetUserOutputEnabled(False) exit_code = cloud_storage.Rsync(staging_directory, bucket) if exit_code: raise exceptions.ToolException( ('Could not synchronize files. The gsutil command exited with ' 'status [{s}]. Command output is available in [{l}].').format( s=exit_code, l=log.GetLogFilePath())) finally: # Reset to the standard log level. log.SetUserOutputEnabled(None) return manifests
def _UploadToGcs(async, local_path, daisy_bucket, image_uuid): """Uploads a local file to GCS. Returns the gs:// URI to that file.""" file_name = os.path.basename(local_path).replace(' ', '-') dest_path = 'gs://{0}/tmpimage/{1}-{2}'.format(daisy_bucket, image_uuid, file_name) log.status.Print('\nCopying [{0}] to [{1}]'.format(local_path, dest_path)) if async: log.status.Print( 'Once completed, your image will be imported from Cloud' ' Storage asynchronously.') retcode = storage_util.RunGsutilCommand('cp', [local_path, dest_path]) if retcode != 0: log.err.Print('Failed to upload file. See {} for details.'.format( log.GetLogFilePath())) raise exceptions.FailedSubCommand( ['gsutil', 'cp', local_path, dest_path], retcode) return dest_path def _CopyToScratchBucket(source_uri, image_uuid, storage_client, daisy_bucket): """Copy image from source_uri to daisy scratch bucket.""" image_file = os.path.basename(source_uri) dest_uri = 'gs://{0}/tmpimage/{1}-{2}'.format(daisy_bucket, image_uuid, image_file) src_object = resources.REGISTRY.Parse(source_uri, collection='storage.objects') dest_object = resources.REGISTRY.Parse(dest_uri, collection='storage.objects') log.status.Print('\nCopying [{0}] to [{1}]'.format(source_uri, dest_uri))