def get_transfer_job_with_retries(
project_id: str, job_name: str, max_retry_duration: float):
"""
Check the latest transfer operation associated with a transfer job with
retries.
"""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# Storage Transfer Service job name
# job_name = 'transferJobs/1234567890'
# The maximum amount of time to delay in seconds
# max_retry_duration = 60
transfer_job = client.get_transfer_job({
'project_id': project_id,
'job_name': job_name,
},
retry=Retry(maximum=max_retry_duration)
)
print(f"Fetched transfer job: {transfer_job.name} "
f"with a max retry duration of {max_retry_duration}s")
def job_description_unique(project_id: str):
"""
Generate a unique job description. Attempts to find and delete a job with
this generated description after tests are ran.
"""
# Create description
client = storage_transfer.StorageTransferServiceClient()
description = f"Storage Transfer Service Samples Test - {uuid.uuid4().hex}"
yield description
# Remove job based on description as the job's name isn't predetermined
transfer_job_to_delete: TransferJob = None
transfer_jobs = client.list_transfer_jobs({
'filter': json.dumps({"projectId": project_id})
})
for transfer_job in transfer_jobs:
if transfer_job.description == description:
transfer_job_to_delete = transfer_job
break
if transfer_job_to_delete and \
transfer_job_to_delete.status != TransferJob.Status.DELETED:
client.update_transfer_job({
"job_name": transfer_job_to_delete.name,
"project_id": project_id,
"transfer_job": {
"status": storage_transfer.TransferJob.Status.DELETED
}
})
def check_latest_transfer_operation(project_id: str, job_name: str):
"""Checks the latest transfer operation for a given transfer job."""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# Storage Transfer Service job name
# job_name = 'transferJobs/1234567890'
transfer_job = client.get_transfer_job({
'project_id': project_id,
'job_name': job_name,
})
if transfer_job.latest_operation_name:
response = client._transport.operations_client.get_operation(
transfer_job.latest_operation_name)
operation = storage_transfer.TransferOperation.deserialize(
response.metadata.value)
print(f"Latest transfer operation for `{job_name}`: {operation}")
else:
print(f"Transfer job {job_name} has not ran yet.")
def transfer_check(project_id: str, job_name: str):
"""
Lists operations for a transfer job.
"""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# Storage Transfer Service job name
# job_name = 'transferJobs/1234567890'
job_filter = json.dumps({
'project_id': project_id,
'job_names': [job_name]
})
response = client._transport.operations_client.list_operations(
"transferOperations", job_filter)
operations = [
storage_transfer.TransferOperation.deserialize(item.metadata.value)
for item in response
]
print(f"Transfer operations for {job_name}`:", operations)
def create_one_time_aws_transfer(project_id: str, description: str,
source_bucket: str, aws_access_key_id: str,
aws_secret_access_key: str, sink_bucket: str):
"""Creates a one-time transfer job from Amazon S3 to Google Cloud
Storage."""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# A useful description for your transfer job
# description = 'My transfer job'
# AWS S3 source bucket name
# source_bucket = 'my-s3-source-bucket'
# AWS Access Key ID
# aws_access_key_id = 'AKIA...'
# AWS Secret Access Key
# aws_secret_access_key = 'HEAoMK2.../...ku8'
# Google Cloud Storage destination bucket name
# sink_bucket = 'my-gcs-destination-bucket'
now = datetime.utcnow()
# Setting the start date and the end date as
# the same time creates a one-time transfer
one_time_schedule = {'day': now.day, 'month': now.month, 'year': now.year}
transfer_job_request = storage_transfer.CreateTransferJobRequest({
'transfer_job': {
'project_id': project_id,
'description': description,
'status': storage_transfer.TransferJob.Status.ENABLED,
'schedule': {
'schedule_start_date': one_time_schedule,
'schedule_end_date': one_time_schedule
},
'transfer_spec': {
'aws_s3_data_source': {
'bucket_name': source_bucket,
'aws_access_key': {
'access_key_id': aws_access_key_id,
'secret_access_key': aws_secret_access_key,
}
},
'gcs_data_sink': {
'bucket_name': sink_bucket,
}
}
}
})
result = client.create_transfer_job(transfer_job_request)
print(f'Created transferJob: {result.name}')
def create_daily_nearline_30_day_migration(project_id: str, description: str,
source_bucket: str,
sink_bucket: str,
start_date: datetime):
"""Create a daily migration from a GCS bucket to a Nearline GCS bucket
for objects untouched for 30 days."""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# A useful description for your transfer job
# description = 'My transfer job'
# Google Cloud Storage source bucket name
# source_bucket = 'my-gcs-source-bucket'
# Google Cloud Storage destination bucket name
# sink_bucket = 'my-gcs-destination-bucket'
transfer_job_request = storage_transfer.CreateTransferJobRequest({
'transfer_job': {
'project_id': project_id,
'description': description,
'status': storage_transfer.TransferJob.Status.ENABLED,
'schedule': {
'schedule_start_date': {
'day': start_date.day,
'month': start_date.month,
'year': start_date.year
}
},
'transfer_spec': {
'gcs_data_source': {
'bucket_name': source_bucket,
},
'gcs_data_sink': {
'bucket_name': sink_bucket,
},
'object_conditions': {
'min_time_elapsed_since_last_modification':
Duration(seconds=2592000 # 30 days
)
},
'transfer_options': {
'delete_objects_from_source_after_transfer': True
}
}
}
})
result = client.create_transfer_job(transfer_job_request)
print(f'Created transferJob: {result.name}')
def transfer_job(project_id: str, source_bucket: Bucket,
destination_bucket: Bucket):
# Create job
client = storage_transfer.StorageTransferServiceClient()
transfer_job = {
"description": "Sample job",
"status": "ENABLED",
"project_id": project_id,
"schedule": {
"schedule_start_date": {
"day": 1,
"month": 1,
"year": 2000
},
"start_time_of_day": {
"hours": 0,
"minutes": 0,
"seconds": 0
},
},
"transfer_spec": {
"gcs_data_source": {
"bucket_name": source_bucket.name
},
"gcs_data_sink": {
"bucket_name": destination_bucket.name
},
"object_conditions": {
'min_time_elapsed_since_last_modification':
Duration(seconds=2592000 # 30 days
)
},
"transfer_options": {
"delete_objects_from_source_after_transfer": True
},
},
}
result = client.create_transfer_job({"transfer_job": transfer_job})
client.run_transfer_job({
'job_name': result.name,
'project_id': project_id
})
yield result.name
# Remove job
client.update_transfer_job({
"job_name": result.name,
"project_id": project_id,
"transfer_job": {
"status": storage_transfer.TransferJob.Status.DELETED
}
})
def create_one_time_transfer(project_id: str, source_bucket: str,
sink_bucket: str):
"""Creates a one-time transfer job."""
client = storage_transfer.StorageTransferServiceClient()
# The ID of the Google Cloud Platform Project that owns the job
# project_id = 'my-project-id'
# Google Cloud Storage source bucket name
# source_bucket = 'my-gcs-source-bucket'
# Google Cloud Storage destination bucket name
# sink_bucket = 'my-gcs-destination-bucket'
transfer_job_request = storage_transfer.CreateTransferJobRequest({
'transfer_job': {
'project_id': project_id,
'status': storage_transfer.TransferJob.Status.ENABLED,
'transfer_spec': {
'gcs_data_source': {
'bucket_name': source_bucket,
},
'gcs_data_sink': {
'bucket_name': sink_bucket,
}
}
}
})
transfer_job = client.create_transfer_job(transfer_job_request)
client.run_transfer_job({
'job_name': transfer_job.name,
'project_id': project_id
})
print(f'Created and ran transfer job: {transfer_job.name}')
def create_transfer_client():
return storage_transfer.StorageTransferServiceClient()
def sts_service_account(project_id):
client = storage_transfer.StorageTransferServiceClient()
account = client.get_google_service_account({'project_id': project_id})
yield account.account_email