def upload_image(self, location: str, label: str, image_bytes, metadata: dict) -> typing.Optional[str]: blob = Blob(f"{location}/{label}", self.bucket) blob.metadata = metadata blob.upload_from_file(image_bytes, content_type="image/png") blob.make_public() return blob.public_url
def upload_blob_string(bucket_name, data_string, destination_blob_name, metadata, content_type): logging.debug( 'upload_blob_string(...) \n\tbucket_name={}\n\tlen data_string={}\n\t' 'destination_blob_name={}\n\tmetadata={}\n\tcontent_type={}'.format( bucket_name, len(data_string), destination_blob_name, metadata, content_type)) client = storage.Client() bucket = client.get_bucket(bucket_name) blob = Blob(destination_blob_name, bucket) if blob.metadata is None: blob.metadata = metadata else: blob.metadata.update(metadata) return blob.upload_from_string(data=data_string, content_type=content_type)
def write_blob_sha256(self, blob: gcs.Blob, force: bool = False) -> None: """ Calculates a blob's SHA256 and writes the value to the blob's custom metadata 'sha256' field. """ current_value = None if blob.metadata is None else blob.metadata.get( 'sha256') log.info('Current SHA256 value: %s', current_value) if current_value is None or force: file_sha256 = self.calculate_blob_sha256(blob) if current_value == file_sha256: log.info('Calculated SHA256 matches current value, no change.') else: log.info('Saving SHA256 value: %s', file_sha256) blob.metadata = {'sha256': file_sha256} blob.patch() else: log.info('Blob SHA256 not calculated or changed.')
def _set_format_metadata(formatted: OptimizedMetricRepresentation, blob: storage.Blob, should_compress: bool = False) -> None: """Sets metadata on the Cloud Storage blob that can be used to retrieve data points from the optimized representation. This includes the ordered dimension manifest, the ordered list of value keys, and the total number of data points to effectively "unflatten" the flattened matrix. Also sets the 'Content-Encoding: gzip' header if the content is going to be compressed. """ total_data_points = len( formatted.value_matrix[0]) if formatted.value_matrix else 0 metadata = { 'dimension_manifest': json.dumps(formatted.dimension_manifest), 'value_keys': json.dumps(formatted.value_keys), 'total_data_points': total_data_points, } blob.metadata = metadata if should_compress: blob.content_encoding = 'gzip'
def sync_box_to_gcs(box: BoxClient, bucket: Bucket, cache: dict) -> List[Future]: """Sync Box account files to a GCS bucket. For versioned Box files, the latest version will always be synced back to the GCS bucket. Non-current versions will not be deliberately preserved, though syncing to a versioned bucket will have this effect. Arguments: box {BoxClient} -- [description] bucket {Bucket} -- [description] cache {dict} -- A dictionary that will opportunistically be filled with Box item paths/objects. Returns: List[Future] -- [description] """ # constuct an executor for copy tasks executor = ThreadPoolExecutor(max_workers=cpu_count()) futures = [] # sync box files back to GCS for path, item in box_walk(box.root_folder()): LOG.debug("Box directory walk found: {}".format(path)) # cache the Box item in module scope cache[path] = item # get the blob to overwrite, or make a new one blob_exists = True blob = bucket.get_blob(path) if not blob: blob_exists = False blob = Blob(path, bucket) # branch depending on whether file or folder if item.type == "folder": if not blob_exists: blob.metadata = { # Not an important value. BOX_MTIME_KEY: datetime.now().isoformat() } # create directory marker, used by UI and FUSE LOG.info("Creating directory marker in GCS: {}".format( blob.name)) blob.upload_from_string(b'') elif item.type == "file": box_file = box.file(item.id) box_mtime = box_file.get().modified_at if should_copy_box_to_gcs(box_file, box_mtime, blob, blob_exists): LOG.info( "Box file {} is not found in GCS or updated since last sync. Copying to {}." .format(item.name, blob.name)) blob.metadata = { BOX_MTIME_KEY: box_mtime } # This change will "follow" the upload into GCS temp_file = BytesIO() reader = box_file.download_to writer = blob.upload_from_file future = executor.submit(concurrent_upload, reader, writer, temp_file) futures.append(future) else: LOG.info("Ignoring item of type {}".format(item.type)) return futures
def mark_complete(self, blob: storage.Blob): blob.metadata = {"export_completed": True} patch_retryer = retry.Retry( predicate=retry.if_exception_type(ServiceUnavailable), deadline=60) patch_retryer(lambda: blob.patch())()