def test_blob(): """Provides a pre-existing blob in the test bucket.""" bucket = storage.Client().bucket(BUCKET) blob = Blob('encryption_test_sigil', bucket, encryption_key=TEST_ENCRYPTION_KEY_DECODED) content = 'Hello, is it me you\'re looking for?' blob.upload_from_string(content) return blob.name, content
def download_to_file(client, to_delete): # [START download_to_file] from google.cloud.storage import Blob client = storage.Client(project="my-project") bucket = client.get_bucket("my-bucket") encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6" blob = Blob("secure-data", bucket, encryption_key=encryption_key) blob.upload_from_string("my secret message.") with open("/tmp/my-secure-file", "wb") as file_obj: blob.download_to_file(file_obj) # [END download_to_file] to_delete.append(blob)
def test_load_stage(self): # TODO check that extra files are skipped # the expected calls to load_table_from_uri # are made when all vocabulary files are present all_blobs = [ Blob(f'{table}.csv', self.bucket_name) for table in common.VOCABULARY_TABLES ] self.gcs_client.list_blobs.return_value = all_blobs load_vocab.load_stage(self.dst_dataset, self.bq_client, self.bucket_name, self.gcs_client) mock_ltfu = self.bq_client.load_table_from_uri expected_calls = [(f'gs://{self.bucket_name}/{table}.csv', self.dst_dataset.table(table)) for table in common.VOCABULARY_TABLES] actual_calls = [(source_uri, destination) for (source_uri, destination), _ in mock_ltfu.call_args_list] self.assertListEqual(expected_calls, actual_calls) # error is thrown when vocabulary files are missing expected_missing = [common.DOMAIN, common.CONCEPT_SYNONYM] incomplete_blobs = [ Blob(f'{table}.csv', self.bucket_name) for table in common.VOCABULARY_TABLES if table not in expected_missing ] self.gcs_client.list_blobs.return_value = incomplete_blobs expected_msg = f'Bucket {self.bucket_name} is missing files for tables {expected_missing}' with self.assertRaises(RuntimeError) as c: load_vocab.load_stage(self.dst_dataset, self.bq_client, self.bucket_name, self.gcs_client) self.assertIsInstance(c.exception, RuntimeError) self.assertEqual(str(c.exception), expected_msg)
def write_gcloud_blob(bucket_id: str, gpath: str, fpath: str): """ Write blob from Google Cloud Storage. References: https://pypi.org/project/google-cloud-storage/ :param bucket_id: id for google cloud bucket :param gpath: file path of item within bucket :param fpath: file path of item from disk :return: upload file blob from disk :rtype: None """ logger.info("Writing '{}' to '{}' at '{}'".\ format(fpath, bucket_id, gpath)) try: client = storage.Client() bucket = client.get_bucket(bucket_id) blob = Blob(gpath, bucket) with open(fpath, 'rb') as infile: blob.upload_from_file(infile) logger.info("SUCCESS -- uploaded '{}' to '{}' using '{}'".\ format(fpath, gpath, bucket_id)) except Exception as exc: logger.error("Unable to upload '{}' to '{}' using '{}'".\ format(fpath, gpath, bucket_id)) logger.exception(exc)
def get_gcp_service_account_credentials(gcp_project_id): # Retrieve service account information corresponding to the GCP Project ID provided # bucket, blob_name = get_gcp_service_account_infos(gcp_project_id) if (bucket is None) or (blob_name is None): return None try: # Read the credentials from GCS # gcs_client = Client() bucket = gcs_client.get_bucket(bucket) blob = Blob(blob_name, bucket) json_credentials = json.loads(blob.download_as_string()) # Build and return GCP Credentials # return service_account.Credentials.from_service_account_info( json_credentials) except Exception as ex: print("Cannot retrieve Service Account credentials.") print(ex) return None
def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key, base64_new_encryption_key): """Performs a key rotation by re-writing an encrypted blob with a new encryption key.""" storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) current_encryption_key = base64.b64decode(base64_encryption_key) new_encryption_key = base64.b64decode(base64_new_encryption_key) # Both source_blob and destination_blob refer to the same storage object, # but destination_blob has the new encryption key. source_blob = Blob( blob_name, bucket, encryption_key=current_encryption_key) destination_blob = Blob( blob_name, bucket, encryption_key=new_encryption_key) token = None while True: token, bytes_rewritten, total_bytes = destination_blob.rewrite( source_blob, token=token) if token is None: break print('Key rotation complete for Blob {}'.format(blob_name))
def test_get_latest_version_from_bucket(mocker): """ Given: - An id of a pack and the bucket. When: - Getting the latest version of the pack in the bucket. - Having a with_dependency.zip file in the bucket. Then: - Validate that the version is the one we expect for. - Skip over with_dependencies.zip file. """ dummy_prod_bucket = mocker.MagicMock() first_blob = Blob( f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/1.0.0/TestPack.zip', dummy_prod_bucket) second_blob = Blob( f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/1.0.1/TestPack.zip', dummy_prod_bucket) third_blob = Blob( f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/TestPack_with_dependencies.zip', dummy_prod_bucket) dummy_prod_bucket.list_blobs.return_value = [ first_blob, second_blob, third_blob ] assert script.get_latest_version_from_bucket('TestPack', dummy_prod_bucket) == '1.0.1'
def _download_from_blobstore(self, blob_to_download_name, blob_download_target_path, chunk_size=None): """Download file from blobstore. :type chunk_size: int :param chunk_size: If file size if greater than 5MB, it is recommended that, chunked downloads should be used. To do so, pass chunk_size param to this function. This must be a multiple of 256 KB per the API specification. """ log_prefix = '[Google Cloud Storage] [DOWNLOAD]' if self.container: self.logger.info('{} Started to download the tarball to target.'.format( log_prefix, blob_download_target_path)) try: blob = Blob(blob_to_download_name, self.container, chunk_size=chunk_size) blob.download_to_filename(blob_download_target_path) self.logger.info('{} SUCCESS: blob_to_download={}, blob_target_name={}, container={}' .format(log_prefix, blob_to_download_name, self.CONTAINER, blob_download_target_path)) return True except Exception as error: message = '{} ERROR: blob_to_download={}, blob_target_name={}, container={}\n{}'.format( log_prefix, blob_to_download_name, blob_download_target_path, self.CONTAINER, error) self.logger.error(message) raise Exception(message) else: message = '{} ERROR: blob_to_download={}, blob_target_name={}, container={}\n{}'.format( log_prefix, blob_to_download_name, blob_download_target_path, self.CONTAINER, "Container not found or accessible") self.logger.error(message) raise Exception(message)
def upload_image_to_gcs(self, image_file_path, bucket_name): """Uploads images to Google Cloud Storage. Arguments: file_name : Name of image to upload on local machine image_file_path: Path to image to upload on local machine. bucket_name: Name of the GCS bucket. """ project = "<GCP_project_id>" credentials = service_account.Credentials.from_service_account_file( '~/gcp-service-account.json') client = storage.Client(project, credentials) try: bucket = client.get_bucket(bucket_name) except Exception as e: # Error out if we're unable to locate the S3 bucket. raise MotionAlertError("Error connecting to GCS bucket: " "{0}".format(e)) try: temp_image = image_file_path.split('/') image_file = temp_image[3] + "/" + temp_image[4] blob = Blob(image_file, bucket) with open(image_file_path, 'rb') as my_file: blob.upload_from_file(my_file) except Exception as e: # Error out if we're unable to upload the image. raise MotionAlertError( "Error uploading file to GCS: {0}".format(e))
def blob_exists(bucket_name, blob_name): """ check if blob/key exists or not! """ tries = 0 while tries < NUM_TRIES: try: if bucket_exists(bucket_name): client = storage.Client() bucket = client.bucket(bucket_name) blob = Blob(blob_name, bucket) return blob.exists() else: return False except BadRequest: return False except Exception: time.sleep(300) tries += 1 logger.error( "Can not check the status of the blob {} after multiple attempts".format( blob_name ) ) return False
class Uploder(): def __init__(self): self.IMAGE_DIR = "/home/dumingzhex/Projects/WintersWrath/webspider/Image/" self.storage_client = storage.Client() try: self.bucket = self.storage_client.get_bucket('argus_space') print("bucket") except Exception as e: print(e) print('Sorry, that bucket does not exist!') def generator(self, file_name): #encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6' self.blob = Blob(file_name, self.bucket, encryption_key=None) self.blob.upload_from_filename(self.IMAGE_DIR + file_name) self.blob.make_public() def get_media_link(self): return self.blob.media_link def get_public_link(self): return self.blob.public_url def get_dir(self, dir_name): return os.listdir(dir_name)
def write_data(self, data: str, bucket_name: str, name: str, content_type: str = None): blob = Blob(name, self.get_bucket(bucket_name)) blob.upload_from_string(data, content_type=content_type)
def handle_duplicate_notification( gcs_client: storage.Client, blob_to_claim: storage.Blob, ): """ Need to handle potential duplicate Pub/Sub notifications. To achieve this we will drop an empty "claimed" file that indicates an invocation of this cloud function has picked up the success file with a certain creation timestamp. This will support republishing the success file as a mechanism of re-running the ingestion while avoiding duplicate ingestion due to multiple Pub/Sub messages for a success file with the same creation time. """ blob_to_claim.reload(client=gcs_client) created_unix_timestamp = blob_to_claim.time_created.timestamp() basename = os.path.basename(blob_to_claim.name) claim_blob: storage.Blob = blob_to_claim.bucket.blob( blob_to_claim.name.replace( basename, f"_claimed_{basename}_created_at_" f"{created_unix_timestamp}")) try: claim_blob.upload_from_string("", if_generation_match=0, client=gcs_client) except google.api_core.exceptions.PreconditionFailed as err: blob_to_claim.reload(client=gcs_client) raise exceptions.DuplicateNotificationException( f"gs://{blob_to_claim.bucket.name}/{blob_to_claim.name} appears " "to already have been claimed for created timestamp: " f"{created_unix_timestamp}." "This means that another invocation of this cloud function has " "claimed the work to be one for this file. " "This may be due to a rare duplicate delivery of the Pub/Sub " "storage notification.") from err
def handle_duplicate_notification(bkt: storage.Bucket, success_blob: storage.Blob, gsurl: str): """ Need to handle potential duplicate Pub/Sub notifications. To achieve this we will drop an empty "claimed" file that indicates an invocation of this cloud function has picked up the success file with a certain creation timestamp. This will support republishing the success file as a mechanism of re-running the ingestion while avoiding duplicate ingestion due to multiple Pub/Sub messages for a success file with the same creation time. """ success_blob.reload() success_created_unix_timestamp = success_blob.time_created.timestamp() claim_blob: storage.Blob = bkt.blob( success_blob.name.replace( SUCCESS_FILENAME, f"_claimed_{success_created_unix_timestamp}")) try: claim_blob.upload_from_string("", if_generation_match=0) except google.api_core.exceptions.PreconditionFailed as err: raise RuntimeError( f"The prefix {gsurl} appears to already have been claimed for " f"{gsurl}{SUCCESS_FILENAME} with created timestamp" f"{success_created_unix_timestamp}." "This means that another invocation of this cloud function has" "claimed the ingestion of this batch." "This may be due to a rare duplicate delivery of the Pub/Sub " "storage notification.") from err
def upload_item_as_raw_file(path, client=None): """Set things up, convert the file, and upload it.""" if client is None: client = get_storage_client() # Check that the bucket exists, make it if not. try: b = client.get_bucket(TRANSCRIPTS_BUCKET_NAME) except Forbidden as e: print("Received Forbidden (403) error while getting bucket. This could " "mean that you do not have billing set up for this " "account/project, or that somebody else has taken this bucket " "from the global namespace.") raise e except NotFound: b = client.bucket(TRANSCRIPTS_BUCKET_NAME) b.lifecycle_rules = [{ 'action': {'type': 'Delete'}, 'condition': {'age': 7}, }] b.create() b.make_public(future=True) # Re-encode the file as a temp file and upload it. When we leave the context # manager, the temp file gets automatically deleted. with NamedTemporaryFile(prefix='transcode_', suffix='.raw') as tmp: encode_as_linear16(path, tmp) # Name it after a SHA2 hash of the item, to avoid collisions. file_name = 'transcripts-%s' % hashlib.sha256(tmp.read()).hexdigest() blob = Blob(file_name, b) blob.upload_from_file(tmp, rewind=True) return {'blob_name': blob.name, 'bucket_name': blob.bucket.name}
def upload_blob_string(bucket_name, csvString, destination_blob_name): client = storage.Client() bucket = client.get_bucket(bucket_name) blob = Blob(destination_blob_name, bucket) return blob.upload_from_string( data=csvString, content_type='text/csv')
def delete_gcp_file(url): print('###### start removing %s' % url) client = storage.Client.from_service_account_json(settings.GCP_KEY_PATH) bucket = client.get_bucket(settings.BUCKET_NAME) blob = Blob(gcp_path(url), bucket) blob.delete() print('###### removing success %s' % url)
def google_stt(self): # Instantiates a client client = speech.SpeechClient() sound = AudioSegment.from_file(self.inputFilePath, format="webm") # if(sound.channels != 1):#If it's not mono sound = sound.set_channels(1) #Change it to mono sound.export(self.inputFilePath, format="wav") #Export them as wav files print('Conversion complete') # Instantiates a client and uploads file storage_client = storage.Client() # Parameter is the name of the Google Cloud bucket bucket = storage_client.lookup_bucket('celerfama2') folder = bucket.list_blobs() with open(self.inputFilePath, 'rb') as file: blob = Blob(os.path.basename(file.name), bucket) print("Google: Uploading: " + os.path.basename(file.name)) blob.upload_from_filename(self.inputFilePath) # Transcribes the file in the cloud for element in folder: print("Google: Transcribing " + element.name) audio = types.RecognitionAudio(uri="gs://celerfama2/" + element.name) config = types.RecognitionConfig( # Option to get word specific info like time stamps enable_word_time_offsets=True, # Language of the audio language_code='en-US') # Detects speech in the audio file operation = client.long_running_recognize(config, audio) print('Google: Waiting for operation to complete...') response = operation.result() file_name = element.name[:-4] output_file = open(file_name + "Google" + ".txt", "w") for result in response.results: for alternative in result.alternatives: output_file.write('Transcript: {}'.format( alternative.transcript.encode("utf-8")) + '\n') output_file.write("Confidence: " + str(alternative.confidence) + '\n') # # Below can be commented to get the detailed info for each word. # for word_info in alternative.words: # word = word_info.word # start_time = word_info.start_time # end_time = word_info.end_time # output_file.write('Word: {}, start_time: {}, end_time: {}'.format( # word, # start_time.seconds + start_time.nanos * 1e-9, # end_time.seconds + end_time.nanos * 1e-9)) # output_file.write("\n") output_file.close() print("Google: Operation Complete") element.delete() return
def upload_arr(arr: np.ndarray, blob: storage.Blob): logging.info(f'uploading blob {blob.name}') with io.BytesIO() as stream: # noinspection PyTypeChecker np.save(stream, arr) stream.seek(0) blob.upload_from_file(stream)
def process_zip(blob: storage.Blob, patient_id: str) -> np.ndarray: """ Downloads the blob and returns a 3D standardized numpy array. :param blob: :param patient_id: :return: """ old_wd = os.getcwd() dirname = f'/tmp/dicom_to_npy-{int(time.time())}' os.makedirs(dirname, exist_ok=True) os.chdir(dirname) blob.download_to_filename(patient_id + '.zip') logging.info('extracting zip file') shutil.unpack_archive(patient_id + '.zip', format='zip') dirpath = list(os.walk('.'))[3][0] logging.info(f'loading scans from {dirpath}') scan = parsers.load_scan(dirpath) processed_scan = preprocess_scan(scan) logging.info(f'processing dicom data') os.chdir(old_wd) shutil.rmtree(dirname) return processed_scan
def write(self, file_path, data, num_retries=10, content_type=None, bucket=None): bucket = self.client.get_bucket(self.bucket or bucket) try: blob = Blob(file_path, bucket) except: blob = bucket.get_blob(file_path) try: data = json.loads(data) except: pass if isinstance(data,(dict,list)): data = json.dumps(data) else: data = data data = _to_bytes(data, encoding="utf-8") string_buffer = BytesIO(data) blob.upload_from_file( file_obj = string_buffer, size = len(data), client = self.client, num_retries = num_retries or self.num_retries, content_type = _MEME_TYPES[self.content_type or content_type] ) return
def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key, base64_new_encryption_key): """Performs a key rotation by re-writing an encrypted blob with a new encryption key.""" storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) current_encryption_key = base64.b64decode(base64_encryption_key) new_encryption_key = base64.b64decode(base64_new_encryption_key) # Both source_blob and destination_blob refer to the same storage object, # but destination_blob has the new encryption key. source_blob = Blob(blob_name, bucket, encryption_key=current_encryption_key) destination_blob = Blob(blob_name, bucket, encryption_key=new_encryption_key) token = None while True: token, bytes_rewritten, total_bytes = destination_blob.rewrite( source_blob, token=token) if token is None: break print('Key rotation complete for Blob {}'.format(blob_name))
def _upload(self, payload: bytes, filename: str, bucket: str) -> None: """ Upload a payload to GCS """ client = Client(project=self.project_id) count = 0 while count < self.max_retries: try: bucket_obj = client.get_bucket(bucket) if self.use_encryption: payload = self._encrypt(payload) content = BytesIO(payload) blob = Blob(filename, bucket_obj) blob.upload_from_file(content) break except ( InvalidResponse, GoogleAPICallError, InternalServerError, SSLError, ) as e: if count >= self.max_retries: raise StoqPluginException( f'Failed to upload {bucket}/{filename} to GCS: {str(e)}' ) count += 1 sleep(randrange(0, 4))
def test_blob(): """Provides a pre-existing blob in the test bucket.""" bucket = storage.Client().bucket(BUCKET) blob_name = "test_blob_{}".format(uuid.uuid4().hex) blob = Blob( blob_name, bucket, encryption_key=TEST_ENCRYPTION_KEY_DECODED, ) content = "Hello, is it me you're looking for?" blob.upload_from_string(content) yield blob.name, content # To delete an encrypted blob, you have to provide the same key # used for the blob. When you provide a wrong key, you'll get # NotFound. try: # Clean up for the case that the rotation didn't occur. blob.delete() except NotFound as e: # For the case that the rotation succeeded. print("Ignoring 404, detail: {}".format(e)) blob = Blob(blob_name, bucket, encryption_key=TEST_ENCRYPTION_KEY_2_DECODED) blob.delete()
def download_bhavcopy(event, context): holiday_dict = create_holiday_dict() logging.info('Dictionary created for ' + str(len(holiday_dict)) + ' holidays') base_url = 'https://www.bseindia.com/download/BhavCopy/Equity/' pubsub_message = base64.b64decode(event['data']).decode('utf-8') print(pubsub_message) print(event['attributes']) fname = event['attributes']['objectId'] extracted_date = re.search(r'([eE][qQ])(\d\d\d\d\d\d)', fname).group(2) new_date = datetime.strptime(extracted_date, '%d%m%y') + timedelta(days=1) file_downloaded_locally, new_fname = check_and_download( new_date, holiday_dict, base_url) try: if file_downloaded_locally and ( not check_if_already_stored(new_fname)): client = storage.Client(project='bhavcopy') bucket = client.get_bucket('bhavcopy-store') blob = Blob(new_fname, bucket) with open('/tmp/' + new_fname, 'rb') as my_file: blob.upload_from_file(my_file) except Exception as e: logging.info( 'Not Downloaded: Cloud function exiting without storing file for date: ' + str(new_date) + '.Received error: ' + str(e))
class GoogleCloudFile(File): def __init__(self, name, mode, storage): self.name = name self.mime_type = mimetypes.guess_type(name)[0] self._mode = mode self._storage = storage self.blob = storage.bucket.get_blob(name) if not self.blob and 'w' in mode: self.blob = Blob(self.name, storage.bucket, chunk_size=storage.blob_chunk_size) self._file = None self._is_dirty = False @property def size(self): return self.blob.size def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, suffix=".GSStorageFile", dir=setting("FILE_UPLOAD_TEMP_DIR")) if 'r' in self._mode: self._is_dirty = False self.blob.download_to_file(self._file) self._file.seek(0) return self._file def _set_file(self, value): self._file = value file = property(_get_file, _set_file) def read(self, num_bytes=None): if 'r' not in self._mode: raise AttributeError("File was not opened in read mode.") if num_bytes is None: num_bytes = -1 return super().read(num_bytes) def write(self, content): if 'w' not in self._mode: raise AttributeError("File was not opened in write mode.") self._is_dirty = True return super().write(force_bytes(content)) def close(self): if self._file is not None: if self._is_dirty: self.blob.upload_from_file( self.file, rewind=True, content_type=self.mime_type, predefined_acl=self._storage.default_acl) self._file.close() self._file = None
def put(self, source, name=None): """ Parameters ---------- source name Returns ------- """ if isinstance(source, io.BytesIO): filebuff = io.BufferedReader(source) elif isinstance(source, (str, bytes)): filebuff = io.BufferedReader(io.BytesIO(source)) else: log.error( 'Source should be either a string, or bytes or io.BytesIO, got {}' .format(type(source))) return False key = self.base_address if name is None else os.path.join( self.base_address, name) key = key.strip('/') try: blob = Blob(key, self.bucket) blob.upload_from_file(filebuff, rewind=True) log.info("Uploaded {} bytes to \tbucket={}\tkey={}".format( len(source), self.bucket_name, key)) return True except Exception as e: log.error("{}\tbucket={}\tkey={}".format(e, self.bucket, key)) return False
def delete_file(module, client, name): try: bucket = client.get_bucket(module.params['bucket']) blob = Blob(name, bucket) blob.delete() return {} except google.cloud.exceptions.NotFound as e: module.fail_json(msg=str(e))
def exists(self, uri_path, bucket_name=None, exact=False, **kwargs): gcp_bucket = self._gcp_bucket(bucket_name) is_found = Blob(uri_path, gcp_bucket).exists() if exact is False and is_found is False: folder_name = '{0}_$folder$'.format(path.basename(uri_path)) folder_key = path.join(uri_path, folder_name) is_found = Blob(folder_key, gcp_bucket).exists() return is_found
def upload(csvfile, bucketname, blobname): client = storage.Client() bucket = client.get_bucket(bucketname) blob = Blob(blobname, bucket) blob.upload_from_filename(csvfile) gcslocation = 'gs://{}/{}'.format(bucketname, blobname) logging.info('Uploaded {} ...'.format(gcslocation)) return gcslocation
def upload_static_files(bucket, file_name): # Uploader Form blob = Blob(name=file_name, bucket=bucket) with open(os.path.abspath('../static-files/%s' % file_name), 'rb') as tmp_file: blob.upload_from_file(tmp_file) # Resource print(u'Info: File %s uploaded' % file_name) print(blob)
def download(): file_name = request.form["downloadingfilename"] client = storage.Client() bucket = client.get_bucket(request.form['downbucketname']) blob = Blob(file_name, bucket, encryption_key=key) stringfile = blob.download_as_string() return stringfile
def _set_data(self, **kwargs) -> dict: df = kwargs.get('data_frame') ts = time.time_ns() blob_name_parts = os.path.splitext(self.__blob_name) blob_name = blob_name_parts[0] + '_' + str(ts) + blob_name_parts[1] blob = Blob(blob_name, self.__bucket) blob.upload_from_string(df.to_csv(), self.__file_format) return dict(record_count=df.shape[0])
def upload_from_file(client, to_delete): # [START upload_from_file] from google.cloud.storage import Blob client = storage.Client(project="my-project") bucket = client.get_bucket("my-bucket") encryption_key = "aa426195405adee2c8081bb9e7e74b19" blob = Blob("secure-data", bucket, encryption_key=encryption_key) with open("my-file", "rb") as my_file: blob.upload_from_file(my_file) # [END upload_from_file] to_delete.append(blob)
def upload_from_file(client, to_delete): # [START upload_from_file] from google.cloud.storage import Blob client = storage.Client(project='my-project') bucket = client.get_bucket('my-bucket') encryption_key = 'aa426195405adee2c8081bb9e7e74b19' blob = Blob('secure-data', bucket, encryption_key=encryption_key) with open('my-file', 'rb') as my_file: blob.upload_from_file(my_file) # [END upload_from_file] to_delete.append(blob)
def download_to_file(client, to_delete): # [START download_to_file] from google.cloud.storage import Blob client = storage.Client(project='my-project') bucket = client.get_bucket('my-bucket') encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6' blob = Blob('secure-data', bucket, encryption_key=encryption_key) with open('/tmp/my-secure-file', 'wb') as file_obj: blob.download_to_file(file_obj) # [END download_to_file] to_delete.append(blob)
def download_encrypted_blob(bucket_name, source_blob_name, destination_file_name, base64_encryption_key): """Downloads a previously-encrypted blob from Google Cloud Storage. The encryption key provided must be the same key provided when uploading the blob. """ storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) # Encryption key must be an AES256 key represented as a bytestring with # 32 bytes. Since it's passed in as a base64 encoded string, it needs # to be decoded. encryption_key = base64.b64decode(base64_encryption_key) blob = Blob(source_blob_name, bucket, encryption_key=encryption_key) blob.download_to_filename(destination_file_name) print('Blob {} downloaded to {}.'.format( source_blob_name, destination_file_name))
def upload_encrypted_blob(bucket_name, source_file_name, destination_blob_name, base64_encryption_key): """Uploads a file to a Google Cloud Storage bucket using a custom encryption key. The file will be encrypted by Google Cloud Storage and only retrievable using the provided encryption key. """ storage_client = storage.Client() bucket = storage_client.get_bucket(bucket_name) # Encryption key must be an AES256 key represented as a bytestring with # 32 bytes. Since it's passed in as a base64 encoded string, it needs # to be decoded. encryption_key = base64.b64decode(base64_encryption_key) blob = Blob(destination_blob_name, bucket, encryption_key=encryption_key) blob.upload_from_filename(source_file_name) print('File {} uploaded to {}.'.format( source_file_name, destination_blob_name))