def copy(self, src, dest): """Copies the given GCS object from src to dest. Args: src: GCS file path pattern in the form gs://<bucket>/<name>. dest: GCS file path pattern in the form gs://<bucket>/<name>. """ src_bucket, src_path = parse_gcs_path(src) dest_bucket, dest_path = parse_gcs_path(dest) request = storage.StorageObjectsCopyRequest( sourceBucket=src_bucket, sourceObject=src_path, destinationBucket=dest_bucket, destinationObject=dest_path) self.client.objects.Copy(request)
def copy_batch(self, src_dest_pairs): """Copies the given GCS object from src to dest. Args: src_dest_pairs: list of (src, dest) tuples of gs://<bucket>/<name> files paths to copy from src to dest, not to exceed MAX_BATCH_OPERATION_SIZE in length. Returns: List of tuples of (src, dest, exception) in the same order as the src_dest_pairs argument, where exception is None if the operation succeeded or the relevant exception if the operation failed. """ if not src_dest_pairs: return [] batch_request = BatchApiRequest( batch_url=GCS_BATCH_ENDPOINT, retryable_codes=retry.SERVER_ERROR_OR_TIMEOUT_CODES) for src, dest in src_dest_pairs: src_bucket, src_path = parse_gcs_path(src) dest_bucket, dest_path = parse_gcs_path(dest) request = storage.StorageObjectsCopyRequest( sourceBucket=src_bucket, sourceObject=src_path, destinationBucket=dest_bucket, destinationObject=dest_path) batch_request.Add(self.client.objects, 'Copy', request) api_calls = batch_request.Execute(self.client._http) # pylint: disable=protected-access result_statuses = [] for i, api_call in enumerate(api_calls): src, dest = src_dest_pairs[i] exception = None if api_call.is_error: exception = api_call.exception # Translate 404 to the appropriate not found exception. if isinstance(exception, HttpError) and exception.status_code == 404: exception = (GcsIOError(errno.ENOENT, 'Source file not found: %s' % src)) result_statuses.append((src, dest, exception)) return result_statuses
def copy(self, src, dest): """Copies the given GCS object from src to dest. Args: src: GCS file path pattern in the form gs://<bucket>/<name>. dest: GCS file path pattern in the form gs://<bucket>/<name>. """ src_bucket, src_path = parse_gcs_path(src) dest_bucket, dest_path = parse_gcs_path(dest) request = storage.StorageObjectsCopyRequest( sourceBucket=src_bucket, sourceObject=src_path, destinationBucket=dest_bucket, destinationObject=dest_path) try: self.client.objects.Copy(request) except HttpError as http_error: if http_error.status_code == 404: # This is a permanent error that should not be retried. Note that # FileBasedSink.finalize_write expects an IOError when the source # file does not exist. raise GcsIOError(errno.ENOENT, 'Source file not found: %s' % src) raise