def _UpdateLatestServerDebDirectory(gcs_bucket: storage.Bucket,
gcs_build_results_dir: str):
"""Updates the '_latest_server_deb' GCS directory with the latest results."""
logging.info("Updating latest server deb directory.")
old_build_results = list(
gcs_bucket.list_blobs(prefix=_LATEST_SERVER_DEB_GCS_DIR))
new_build_results = list(
gcs_bucket.list_blobs(prefix=gcs_build_results_dir))
if not new_build_results:
raise GCSUploadError(
"Failed to find build results for the server-deb Travis job.")
for gcs_blob in old_build_results:
logging.info("Deleting previous blob: %s", gcs_blob)
gcs_blob.delete()
for gcs_blob in new_build_results:
build_result_filename = gcs_blob.name.split("/")[-1]
latest_build_result_path = (
f"{_LATEST_SERVER_DEB_GCS_DIR}/{build_result_filename}")
logging.info("Copying blob %s (%s) -> %s", gcs_blob, gcs_bucket,
latest_build_result_path)
gcs_bucket.copy_blob(gcs_blob,
gcs_bucket,
new_name=latest_build_result_path)
def rename(bucket: str,
source: str,
destination: str,
credentials: Credentials = None):
"""Rename a file.
This is a copy/delete action as GCS has no actual rename option, however as
it is all within GCS it is BLAZING fast, to the extent that there is
essentially no limit on the maximum size of file we can rename.
Arguments:
bucket (str): destination bucket name
source (str): current name
destination (str): new name
credentials (Credentials): authentication, if needed
"""
client = storage.Client(
credentials=(credentials.credentials if credentials else None))
source_bucket = Bucket(client, name=bucket)
source_blob = source_bucket.blob(blob_name=source)
destination_bucket = client.get_bucket(bucket)
source_bucket.copy_blob(source_blob, destination_bucket, destination)
source_blob.delete()
logging.info(f'Renamed file %s as %s in %s.', bucket, source,
destination)
def copy_to_gcs(bucket_name: str,
report: Dict[str, Any],
credentials: Credentials = None):
"""copy from one bucket to another
This is a copy from the bucket defined in the report definition (as DV360
stores its reports in GCS) into the monitored bucket for upload. It's
BLAZING fast, to the extent that there is essentially no limit on the
maximum size of a DV360 report we can handle.
The destination file name is the report's id.
Arguments:
bucket_name (str): destination bucket name
report (Dict[str, Any]): report definition
"""
client = storage.Client(
credentials=(credentials.credentials if credentials else None))
path_segments = report['current_path'].split('/')
report_bucket = path_segments[-2]
report_blob_name = path_segments[-1].split('?')[0]
output_blob_name = report['id']
source_bucket = Bucket(client, report_bucket)
source_blob = source_bucket.blob(report_blob_name)
destination_bucket = client.get_bucket(bucket_name)
source_bucket.copy_blob(source_blob, destination_bucket,
'{id}.csv'.format(id=output_blob_name))
logging.info('File {report} copied from {source} to {bucket}.'.format(
report=report_blob_name, bucket=bucket_name, source=report_bucket))
def copy_index(index_folder_path: str, build_index_blob: Blob, build_index_generation: str, production_bucket: Bucket,
build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the build bucket index to the production bucket index path.
Args:
index_folder_path (str): index folder full path.
build_index_blob (Blob): google cloud storage object that represents build index.zip blob.
build_index_generation (str): downloaded build index generation.
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where index is copied from.
storage_base_path (str): the path to upload the index to.
build_bucket_base_path (str): the path in the build bucket of the index.
"""
try:
build_index_blob.reload()
build_current_index_generation = build_index_blob.generation
# disabling caching for prod index blob
prod_index_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.zip")
prod_index_blob = production_bucket.blob(prod_index_storage_path)
prod_index_blob.cache_control = "no-cache,max-age=0"
prod_index_json_storage_path = os.path.join(storage_base_path, f"{GCPConfig.INDEX_NAME}.json")
prod_index_json_blob = production_bucket.blob(prod_index_json_storage_path)
prod_index_json_blob.cache_control = "no-cache,max-age=0"
if build_current_index_generation == build_index_generation:
copied_index = build_bucket.copy_blob(
blob=build_index_blob, destination_bucket=production_bucket, new_name=prod_index_storage_path
)
if copied_index.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.zip to storage.")
else:
logging.error("Failed copying index.zip from build index - blob does not exist.")
sys.exit(1)
copied_index_json_blob = build_bucket.blob(
os.path.join(build_bucket_base_path, f"{GCPConfig.INDEX_NAME}.json")
)
copied_index_json = build_bucket.copy_blob(
blob=copied_index_json_blob, destination_bucket=production_bucket, new_name=prod_index_json_storage_path
)
if copied_index_json.exists():
logging.success(f"Finished uploading {GCPConfig.INDEX_NAME}.json to storage.")
else:
logging.error("Failed copying index.json from build index - blob does not exist.")
sys.exit(1)
else:
logging.error(f"Failed in uploading {GCPConfig.INDEX_NAME}, mismatch in index file generation")
logging.error(f"Downloaded build index generation: {build_index_generation}")
logging.error(f"Current build index generation: {build_current_index_generation}")
sys.exit(1)
except Exception as e:
logging.exception(f"Failed copying {GCPConfig.INDEX_NAME}. Additional Info: {str(e)}")
sys.exit(1)
finally:
shutil.rmtree(index_folder_path)
def copy_id_set(production_bucket: Bucket, build_bucket: Bucket):
""" Copies the id_set.json artifact from the build bucket to the production bucket.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied from.
"""
build_id_set_path = os.path.join(
os.path.dirname(GCPConfig.BUILD_BASE_PATH), 'id_set.json')
build_id_set_blob = build_bucket.blob(build_id_set_path)
if not build_id_set_blob.exists():
logging.error(
f"id_set.json file does not exists in build bucket in path: {build_id_set_path}"
)
prod_id_set_path = os.path.join(
os.path.dirname(GCPConfig.STORAGE_BASE_PATH), 'id_set.json')
copied_blob = build_bucket.copy_blob(blob=build_id_set_blob,
destination_bucket=production_bucket,
new_name=prod_id_set_path)
if not copied_blob.exists():
logging.error(f"Failed to upload id_set.json to {prod_id_set_path}")
else:
logging.success("Finished uploading id_set.json to storage.")
def copy_id_set(production_bucket: Bucket, build_bucket: Bucket, storage_base_path: str, build_bucket_base_path: str):
""" Copies the id_set.json artifact from the build bucket to the production bucket.
Args:
production_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied to.
build_bucket (google.cloud.storage.bucket.Bucket): gcs bucket where id_set is copied from.
storage_base_path (str): the path to upload the id_set.json to.
build_bucket_base_path (str): the path in the build bucket of the id_set.json.
"""
build_id_set_path = os.path.join(os.path.dirname(build_bucket_base_path), 'id_set.json')
build_id_set_blob = build_bucket.blob(build_id_set_path)
if not build_id_set_blob.exists():
logging.error(f"id_set.json file does not exists in build bucket in path: {build_id_set_path}")
sys.exit(1)
prod_id_set_path = os.path.join(os.path.dirname(storage_base_path), 'id_set.json')
try:
copied_blob = build_bucket.copy_blob(
blob=build_id_set_blob, destination_bucket=production_bucket, new_name=prod_id_set_path
)
if not copied_blob.exists():
logging.error(f"Failed to upload id_set.json to {prod_id_set_path}")
sys.exit(1)
else:
logging.success("Finished uploading id_set.json to storage.")
except Exception as e:
logging.exception(f"Failed copying ID Set. Additional Info: {str(e)}")
sys.exit(1)
