def test_blob():
    """Provides a pre-existing blob in the test bucket."""
    bucket = storage.Client().bucket(BUCKET)
    blob = Blob('encryption_test_sigil',
                bucket, encryption_key=TEST_ENCRYPTION_KEY_DECODED)
    content = 'Hello, is it me you\'re looking for?'
    blob.upload_from_string(content)
    return blob.name, content
Exemplo n.º 2
1
def download_to_file(client, to_delete):
    # [START download_to_file]
    from google.cloud.storage import Blob

    client = storage.Client(project="my-project")
    bucket = client.get_bucket("my-bucket")
    encryption_key = "c7f32af42e45e85b9848a6a14dd2a8f6"
    blob = Blob("secure-data", bucket, encryption_key=encryption_key)
    blob.upload_from_string("my secret message.")
    with open("/tmp/my-secure-file", "wb") as file_obj:
        blob.download_to_file(file_obj)
    # [END download_to_file]

    to_delete.append(blob)
Exemplo n.º 3
0
    def test_load_stage(self):
        # TODO check that extra files are skipped

        # the expected calls to load_table_from_uri
        # are made when all vocabulary files are present
        all_blobs = [
            Blob(f'{table}.csv', self.bucket_name)
            for table in common.VOCABULARY_TABLES
        ]
        self.gcs_client.list_blobs.return_value = all_blobs
        load_vocab.load_stage(self.dst_dataset, self.bq_client,
                              self.bucket_name, self.gcs_client)
        mock_ltfu = self.bq_client.load_table_from_uri
        expected_calls = [(f'gs://{self.bucket_name}/{table}.csv',
                           self.dst_dataset.table(table))
                          for table in common.VOCABULARY_TABLES]
        actual_calls = [(source_uri, destination)
                        for (source_uri,
                             destination), _ in mock_ltfu.call_args_list]
        self.assertListEqual(expected_calls, actual_calls)

        # error is thrown when vocabulary files are missing
        expected_missing = [common.DOMAIN, common.CONCEPT_SYNONYM]
        incomplete_blobs = [
            Blob(f'{table}.csv', self.bucket_name)
            for table in common.VOCABULARY_TABLES
            if table not in expected_missing
        ]
        self.gcs_client.list_blobs.return_value = incomplete_blobs
        expected_msg = f'Bucket {self.bucket_name} is missing files for tables {expected_missing}'
        with self.assertRaises(RuntimeError) as c:
            load_vocab.load_stage(self.dst_dataset, self.bq_client,
                                  self.bucket_name, self.gcs_client)
            self.assertIsInstance(c.exception, RuntimeError)
            self.assertEqual(str(c.exception), expected_msg)
Exemplo n.º 4
0
def write_gcloud_blob(bucket_id: str, gpath: str, fpath: str):
    """ Write blob from Google Cloud Storage.

    References:
      https://pypi.org/project/google-cloud-storage/

    :param bucket_id: id for google cloud bucket
    :param gpath: file path of item within bucket
    :param fpath: file path of item from disk
    :return: upload file blob from disk
    :rtype: None
    """
    logger.info("Writing '{}' to '{}' at '{}'".\
                format(fpath, bucket_id, gpath))
    try:
        client = storage.Client()
        bucket = client.get_bucket(bucket_id)
        blob = Blob(gpath, bucket)
        with open(fpath, 'rb') as infile:
            blob.upload_from_file(infile)

        logger.info("SUCCESS -- uploaded '{}' to '{}' using '{}'".\
                    format(fpath, gpath, bucket_id))

    except Exception as exc:
        logger.error("Unable to upload '{}' to '{}' using '{}'".\
                     format(fpath, gpath, bucket_id))
        logger.exception(exc)
Exemplo n.º 5
0
def get_gcp_service_account_credentials(gcp_project_id):

    # Retrieve service account information corresponding to the GCP Project ID provided
    #
    bucket, blob_name = get_gcp_service_account_infos(gcp_project_id)

    if (bucket is None) or (blob_name is None):
        return None

    try:
        # Read the credentials from GCS
        #
        gcs_client = Client()
        bucket = gcs_client.get_bucket(bucket)
        blob = Blob(blob_name, bucket)
        json_credentials = json.loads(blob.download_as_string())

        # Build and return GCP Credentials
        #
        return service_account.Credentials.from_service_account_info(
            json_credentials)

    except Exception as ex:
        print("Cannot retrieve Service Account credentials.")
        print(ex)
        return None
Exemplo n.º 6
0
def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key,
                          base64_new_encryption_key):
    """Performs a key rotation by re-writing an encrypted blob with a new
    encryption key."""
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    current_encryption_key = base64.b64decode(base64_encryption_key)
    new_encryption_key = base64.b64decode(base64_new_encryption_key)

    # Both source_blob and destination_blob refer to the same storage object,
    # but destination_blob has the new encryption key.
    source_blob = Blob(
        blob_name, bucket, encryption_key=current_encryption_key)
    destination_blob = Blob(
        blob_name, bucket, encryption_key=new_encryption_key)

    token = None

    while True:
        token, bytes_rewritten, total_bytes = destination_blob.rewrite(
            source_blob, token=token)
        if token is None:
            break

    print('Key rotation complete for Blob {}'.format(blob_name))
Exemplo n.º 7
0
def test_get_latest_version_from_bucket(mocker):
    """
       Given:
           - An id of a pack and the bucket.
       When:
           - Getting the latest version of the pack in the bucket.
           - Having a with_dependency.zip file in the bucket.
       Then:
           - Validate that the version is the one we expect for.
           - Skip over with_dependencies.zip file.
   """
    dummy_prod_bucket = mocker.MagicMock()
    first_blob = Blob(
        f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/1.0.0/TestPack.zip',
        dummy_prod_bucket)
    second_blob = Blob(
        f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/1.0.1/TestPack.zip',
        dummy_prod_bucket)
    third_blob = Blob(
        f'{GCPConfig.CONTENT_PACKS_PATH}/TestPack/TestPack_with_dependencies.zip',
        dummy_prod_bucket)
    dummy_prod_bucket.list_blobs.return_value = [
        first_blob, second_blob, third_blob
    ]
    assert script.get_latest_version_from_bucket('TestPack',
                                                 dummy_prod_bucket) == '1.0.1'
Exemplo n.º 8
0
 def _download_from_blobstore(self, blob_to_download_name, blob_download_target_path, chunk_size=None):
     """Download file from blobstore.
     :type chunk_size: int
     :param chunk_size: If file size if greater than 5MB, it is recommended that, 
                        chunked downloads should be used.
                        To do so, pass chunk_size param to this function.
                        This must be a multiple of 256 KB per the API specification.
     """
     log_prefix = '[Google Cloud Storage] [DOWNLOAD]'
     if self.container:
         self.logger.info('{} Started to download the tarball to target.'.format(
             log_prefix, blob_download_target_path))
         try:
             blob = Blob(blob_to_download_name,
                         self.container, chunk_size=chunk_size)
             blob.download_to_filename(blob_download_target_path)
             self.logger.info('{} SUCCESS: blob_to_download={}, blob_target_name={}, container={}'
                              .format(log_prefix, blob_to_download_name, self.CONTAINER,
                                      blob_download_target_path))
             return True
         except Exception as error:
             message = '{} ERROR: blob_to_download={}, blob_target_name={}, container={}\n{}'.format(
                 log_prefix, blob_to_download_name, blob_download_target_path, self.CONTAINER, error)
             self.logger.error(message)
             raise Exception(message)
     else:
         message = '{} ERROR: blob_to_download={}, blob_target_name={}, container={}\n{}'.format(
             log_prefix, blob_to_download_name, blob_download_target_path, self.CONTAINER, "Container not found or accessible")
         self.logger.error(message)
         raise Exception(message)
Exemplo n.º 9
0
    def upload_image_to_gcs(self, image_file_path, bucket_name):
        """Uploads images to Google Cloud Storage.

        Arguments:
            file_name : Name of image to upload on local machine
            image_file_path: Path to image to upload on local machine.
            bucket_name: Name of the GCS bucket.
        """
        project = "<GCP_project_id>"
        credentials = service_account.Credentials.from_service_account_file(
            '~/gcp-service-account.json')
        client = storage.Client(project, credentials)

        try:
            bucket = client.get_bucket(bucket_name)
        except Exception as e:
            # Error out if we're unable to locate the S3 bucket.
            raise MotionAlertError("Error connecting to GCS bucket: "
                                   "{0}".format(e))

        try:
            temp_image = image_file_path.split('/')
            image_file = temp_image[3] + "/" + temp_image[4]

            blob = Blob(image_file, bucket)

            with open(image_file_path, 'rb') as my_file:
                blob.upload_from_file(my_file)
        except Exception as e:
            # Error out if we're unable to upload the image.
            raise MotionAlertError(
                "Error uploading file to GCS: {0}".format(e))
Exemplo n.º 10
0
def blob_exists(bucket_name, blob_name):
    """
    check if blob/key exists or not!
    """
    tries = 0

    while tries < NUM_TRIES:
        try:
            if bucket_exists(bucket_name):
                client = storage.Client()
                bucket = client.bucket(bucket_name)
                blob = Blob(blob_name, bucket)
                return blob.exists()
            else:
                return False
        except BadRequest:
            return False
        except Exception:
            time.sleep(300)
            tries += 1

    logger.error(
        "Can not check the status of the blob {} after multiple attempts".format(
            blob_name
        )
    )
    return False
Exemplo n.º 11
0
class Uploder():
    def __init__(self):
        self.IMAGE_DIR = "/home/dumingzhex/Projects/WintersWrath/webspider/Image/"
        self.storage_client = storage.Client()
        try:
            self.bucket = self.storage_client.get_bucket('argus_space')
            print("bucket")
        except Exception as e:
            print(e)
            print('Sorry, that bucket does not exist!')

    def generator(self, file_name):
        #encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6'
        self.blob = Blob(file_name, self.bucket, encryption_key=None)
        self.blob.upload_from_filename(self.IMAGE_DIR + file_name)
        self.blob.make_public()

    def get_media_link(self):
        return self.blob.media_link

    def get_public_link(self):
        return self.blob.public_url

    def get_dir(self, dir_name):
        return os.listdir(dir_name)
Exemplo n.º 12
0
 def write_data(self,
                data: str,
                bucket_name: str,
                name: str,
                content_type: str = None):
     blob = Blob(name, self.get_bucket(bucket_name))
     blob.upload_from_string(data, content_type=content_type)
Exemplo n.º 13
0
def handle_duplicate_notification(
    gcs_client: storage.Client,
    blob_to_claim: storage.Blob,
):
    """
    Need to handle potential duplicate Pub/Sub notifications.
    To achieve this we will drop an empty "claimed" file that indicates
    an invocation of this cloud function has picked up the success file
    with a certain creation timestamp. This will support republishing the
    success file as a mechanism of re-running the ingestion while avoiding
    duplicate ingestion due to multiple Pub/Sub messages for a success file
    with the same creation time.
    """
    blob_to_claim.reload(client=gcs_client)
    created_unix_timestamp = blob_to_claim.time_created.timestamp()

    basename = os.path.basename(blob_to_claim.name)
    claim_blob: storage.Blob = blob_to_claim.bucket.blob(
        blob_to_claim.name.replace(
            basename, f"_claimed_{basename}_created_at_"
            f"{created_unix_timestamp}"))
    try:
        claim_blob.upload_from_string("",
                                      if_generation_match=0,
                                      client=gcs_client)
    except google.api_core.exceptions.PreconditionFailed as err:
        blob_to_claim.reload(client=gcs_client)
        raise exceptions.DuplicateNotificationException(
            f"gs://{blob_to_claim.bucket.name}/{blob_to_claim.name} appears "
            "to already have been claimed for created timestamp: "
            f"{created_unix_timestamp}."
            "This means that another invocation of this cloud function has "
            "claimed the work to be one for this file. "
            "This may be due to a rare duplicate delivery of the Pub/Sub "
            "storage notification.") from err
Exemplo n.º 14
0
def handle_duplicate_notification(bkt: storage.Bucket,
                                  success_blob: storage.Blob, gsurl: str):
    """
    Need to handle potential duplicate Pub/Sub notifications.
    To achieve this we will drop an empty "claimed" file that indicates
    an invocation of this cloud function has picked up the success file
    with a certain creation timestamp. This will support republishing the
    success file as a mechanism of re-running the ingestion while avoiding
    duplicate ingestion due to multiple Pub/Sub messages for a success file
    with the same creation time.
    """
    success_blob.reload()
    success_created_unix_timestamp = success_blob.time_created.timestamp()

    claim_blob: storage.Blob = bkt.blob(
        success_blob.name.replace(
            SUCCESS_FILENAME, f"_claimed_{success_created_unix_timestamp}"))
    try:
        claim_blob.upload_from_string("", if_generation_match=0)
    except google.api_core.exceptions.PreconditionFailed as err:
        raise RuntimeError(
            f"The prefix {gsurl} appears to already have been claimed for "
            f"{gsurl}{SUCCESS_FILENAME} with created timestamp"
            f"{success_created_unix_timestamp}."
            "This means that another invocation of this cloud function has"
            "claimed the ingestion of this batch."
            "This may be due to a rare duplicate delivery of the Pub/Sub "
            "storage notification.") from err
Exemplo n.º 15
0
def upload_item_as_raw_file(path, client=None):
    """Set things up, convert the file, and upload it."""
    if client is None:
        client = get_storage_client()

    # Check that the bucket exists, make it if not.
    try:
        b = client.get_bucket(TRANSCRIPTS_BUCKET_NAME)
    except Forbidden as e:
        print("Received Forbidden (403) error while getting bucket. This could "
              "mean that you do not have billing set up for this "
              "account/project, or that somebody else has taken this bucket "
              "from the global namespace.")
        raise e
    except NotFound:
        b = client.bucket(TRANSCRIPTS_BUCKET_NAME)
        b.lifecycle_rules = [{
            'action': {'type': 'Delete'},
            'condition': {'age': 7},
        }]
        b.create()
        b.make_public(future=True)

    # Re-encode the file as a temp file and upload it. When we leave the context
    # manager, the temp file gets automatically deleted.
    with NamedTemporaryFile(prefix='transcode_', suffix='.raw') as tmp:
        encode_as_linear16(path, tmp)

        # Name it after a SHA2 hash of the item, to avoid collisions.
        file_name = 'transcripts-%s' % hashlib.sha256(tmp.read()).hexdigest()
        blob = Blob(file_name, b)
        blob.upload_from_file(tmp, rewind=True)

    return {'blob_name': blob.name, 'bucket_name': blob.bucket.name}
Exemplo n.º 16
0
def upload_blob_string(bucket_name, csvString, destination_blob_name):
    client = storage.Client()
    bucket = client.get_bucket(bucket_name)
    blob = Blob(destination_blob_name, bucket)
    return blob.upload_from_string(
        data=csvString,
        content_type='text/csv')
Exemplo n.º 17
0
def delete_gcp_file(url):
    print('###### start removing %s' % url)
    client = storage.Client.from_service_account_json(settings.GCP_KEY_PATH)
    bucket = client.get_bucket(settings.BUCKET_NAME)
    blob = Blob(gcp_path(url), bucket)
    blob.delete()
    print('###### removing success %s' % url)
    def google_stt(self):
        # Instantiates a client
        client = speech.SpeechClient()

        sound = AudioSegment.from_file(self.inputFilePath, format="webm")
        # if(sound.channels != 1):#If it's not mono
        sound = sound.set_channels(1)  #Change it to mono
        sound.export(self.inputFilePath,
                     format="wav")  #Export them as wav files
        print('Conversion complete')
        # Instantiates a client and uploads file
        storage_client = storage.Client()
        # Parameter is the name of the Google Cloud bucket
        bucket = storage_client.lookup_bucket('celerfama2')
        folder = bucket.list_blobs()
        with open(self.inputFilePath, 'rb') as file:
            blob = Blob(os.path.basename(file.name), bucket)
            print("Google: Uploading: " + os.path.basename(file.name))
            blob.upload_from_filename(self.inputFilePath)

        # Transcribes the file in the cloud
        for element in folder:
            print("Google: Transcribing " + element.name)
            audio = types.RecognitionAudio(uri="gs://celerfama2/" +
                                           element.name)
            config = types.RecognitionConfig(
                # Option to get word specific info like time stamps
                enable_word_time_offsets=True,
                # Language of the audio
                language_code='en-US')

            # Detects speech in the audio file
            operation = client.long_running_recognize(config, audio)

            print('Google: Waiting for operation to complete...')
            response = operation.result()

            file_name = element.name[:-4]
            output_file = open(file_name + "Google" + ".txt", "w")

            for result in response.results:
                for alternative in result.alternatives:
                    output_file.write('Transcript: {}'.format(
                        alternative.transcript.encode("utf-8")) + '\n')
                    output_file.write("Confidence: " +
                                      str(alternative.confidence) + '\n')
                # # Below can be commented to get the detailed info for each word.
                # for word_info in alternative.words:
                #     word = word_info.word
                #     start_time = word_info.start_time
                #     end_time = word_info.end_time
                #     output_file.write('Word: {}, start_time: {}, end_time: {}'.format(
                #         word,
                #         start_time.seconds + start_time.nanos * 1e-9,
                #         end_time.seconds + end_time.nanos * 1e-9))
                #     output_file.write("\n")
            output_file.close()
            print("Google: Operation Complete")
            element.delete()
            return
Exemplo n.º 19
0
def upload_arr(arr: np.ndarray, blob: storage.Blob):
    logging.info(f'uploading blob {blob.name}')
    with io.BytesIO() as stream:
        # noinspection PyTypeChecker
        np.save(stream, arr)
        stream.seek(0)
        blob.upload_from_file(stream)
Exemplo n.º 20
0
def process_zip(blob: storage.Blob, patient_id: str) -> np.ndarray:
    """
    Downloads the blob and returns a 3D standardized numpy array.
    :param blob:
    :param patient_id:
    :return:
    """
    old_wd = os.getcwd()
    dirname = f'/tmp/dicom_to_npy-{int(time.time())}'
    os.makedirs(dirname, exist_ok=True)
    os.chdir(dirname)

    blob.download_to_filename(patient_id + '.zip')
    logging.info('extracting zip file')
    shutil.unpack_archive(patient_id + '.zip', format='zip')

    dirpath = list(os.walk('.'))[3][0]
    logging.info(f'loading scans from {dirpath}')
    scan = parsers.load_scan(dirpath)
    processed_scan = preprocess_scan(scan)
    logging.info(f'processing dicom data')

    os.chdir(old_wd)
    shutil.rmtree(dirname)
    return processed_scan
Exemplo n.º 21
0
    def write(self,
               file_path,               
               data,
               num_retries=10,
               content_type=None,               
               bucket=None):

        bucket = self.client.get_bucket(self.bucket or bucket)

        try:
            blob = Blob(file_path, bucket)
        except:
            blob = bucket.get_blob(file_path)

        try:
            data = json.loads(data)
        except:
            pass

        if isinstance(data,(dict,list)):
            data = json.dumps(data)
        else:
            data =  data

        data = _to_bytes(data, encoding="utf-8")
        string_buffer = BytesIO(data)    

        blob.upload_from_file(
            file_obj = string_buffer,
            size = len(data),
            client = self.client,
            num_retries = num_retries or self.num_retries,
            content_type = _MEME_TYPES[self.content_type or content_type]
        )
        return 
Exemplo n.º 22
0
def rotate_encryption_key(bucket_name, blob_name, base64_encryption_key,
                          base64_new_encryption_key):
    """Performs a key rotation by re-writing an encrypted blob with a new
    encryption key."""
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    current_encryption_key = base64.b64decode(base64_encryption_key)
    new_encryption_key = base64.b64decode(base64_new_encryption_key)

    # Both source_blob and destination_blob refer to the same storage object,
    # but destination_blob has the new encryption key.
    source_blob = Blob(blob_name,
                       bucket,
                       encryption_key=current_encryption_key)
    destination_blob = Blob(blob_name,
                            bucket,
                            encryption_key=new_encryption_key)

    token = None

    while True:
        token, bytes_rewritten, total_bytes = destination_blob.rewrite(
            source_blob, token=token)
        if token is None:
            break

    print('Key rotation complete for Blob {}'.format(blob_name))
Exemplo n.º 23
0
    def _upload(self, payload: bytes, filename: str, bucket: str) -> None:
        """
        Upload a payload to GCS

        """

        client = Client(project=self.project_id)
        count = 0
        while count < self.max_retries:
            try:
                bucket_obj = client.get_bucket(bucket)
                if self.use_encryption:
                    payload = self._encrypt(payload)
                content = BytesIO(payload)
                blob = Blob(filename, bucket_obj)
                blob.upload_from_file(content)
                break
            except (
                InvalidResponse,
                GoogleAPICallError,
                InternalServerError,
                SSLError,
            ) as e:
                if count >= self.max_retries:
                    raise StoqPluginException(
                        f'Failed to upload {bucket}/{filename} to GCS: {str(e)}'
                    )
                count += 1
                sleep(randrange(0, 4))
Exemplo n.º 24
0
def test_blob():
    """Provides a pre-existing blob in the test bucket."""
    bucket = storage.Client().bucket(BUCKET)
    blob_name = "test_blob_{}".format(uuid.uuid4().hex)
    blob = Blob(
        blob_name,
        bucket,
        encryption_key=TEST_ENCRYPTION_KEY_DECODED,
    )
    content = "Hello, is it me you're looking for?"
    blob.upload_from_string(content)

    yield blob.name, content

    # To delete an encrypted blob, you have to provide the same key
    # used for the blob. When you provide a wrong key, you'll get
    # NotFound.
    try:
        # Clean up for the case that the rotation didn't occur.
        blob.delete()
    except NotFound as e:
        # For the case that the rotation succeeded.
        print("Ignoring 404, detail: {}".format(e))
        blob = Blob(blob_name,
                    bucket,
                    encryption_key=TEST_ENCRYPTION_KEY_2_DECODED)
        blob.delete()
Exemplo n.º 25
0
def download_bhavcopy(event, context):

    holiday_dict = create_holiday_dict()
    logging.info('Dictionary created for ' + str(len(holiday_dict)) +
                 ' holidays')

    base_url = 'https://www.bseindia.com/download/BhavCopy/Equity/'

    pubsub_message = base64.b64decode(event['data']).decode('utf-8')
    print(pubsub_message)
    print(event['attributes'])
    fname = event['attributes']['objectId']

    extracted_date = re.search(r'([eE][qQ])(\d\d\d\d\d\d)', fname).group(2)
    new_date = datetime.strptime(extracted_date, '%d%m%y') + timedelta(days=1)

    file_downloaded_locally, new_fname = check_and_download(
        new_date, holiday_dict, base_url)
    try:
        if file_downloaded_locally and (
                not check_if_already_stored(new_fname)):
            client = storage.Client(project='bhavcopy')
            bucket = client.get_bucket('bhavcopy-store')
            blob = Blob(new_fname, bucket)
            with open('/tmp/' + new_fname, 'rb') as my_file:
                blob.upload_from_file(my_file)
    except Exception as e:
        logging.info(
            'Not Downloaded: Cloud function exiting without storing file for date: '
            + str(new_date) + '.Received error: ' + str(e))
Exemplo n.º 26
0
class GoogleCloudFile(File):
    def __init__(self, name, mode, storage):
        self.name = name
        self.mime_type = mimetypes.guess_type(name)[0]
        self._mode = mode
        self._storage = storage
        self.blob = storage.bucket.get_blob(name)
        if not self.blob and 'w' in mode:
            self.blob = Blob(self.name,
                             storage.bucket,
                             chunk_size=storage.blob_chunk_size)
        self._file = None
        self._is_dirty = False

    @property
    def size(self):
        return self.blob.size

    def _get_file(self):
        if self._file is None:
            self._file = SpooledTemporaryFile(
                max_size=self._storage.max_memory_size,
                suffix=".GSStorageFile",
                dir=setting("FILE_UPLOAD_TEMP_DIR"))
            if 'r' in self._mode:
                self._is_dirty = False
                self.blob.download_to_file(self._file)
                self._file.seek(0)
        return self._file

    def _set_file(self, value):
        self._file = value

    file = property(_get_file, _set_file)

    def read(self, num_bytes=None):
        if 'r' not in self._mode:
            raise AttributeError("File was not opened in read mode.")

        if num_bytes is None:
            num_bytes = -1

        return super().read(num_bytes)

    def write(self, content):
        if 'w' not in self._mode:
            raise AttributeError("File was not opened in write mode.")
        self._is_dirty = True
        return super().write(force_bytes(content))

    def close(self):
        if self._file is not None:
            if self._is_dirty:
                self.blob.upload_from_file(
                    self.file,
                    rewind=True,
                    content_type=self.mime_type,
                    predefined_acl=self._storage.default_acl)
            self._file.close()
            self._file = None
Exemplo n.º 27
0
    def put(self, source, name=None):
        """

        Parameters
        ----------
        source
        name

        Returns
        -------

        """

        if isinstance(source, io.BytesIO):
            filebuff = io.BufferedReader(source)
        elif isinstance(source, (str, bytes)):
            filebuff = io.BufferedReader(io.BytesIO(source))
        else:
            log.error(
                'Source should be either a string, or bytes or io.BytesIO, got {}'
                .format(type(source)))
            return False

        key = self.base_address if name is None else os.path.join(
            self.base_address, name)
        key = key.strip('/')
        try:
            blob = Blob(key, self.bucket)
            blob.upload_from_file(filebuff, rewind=True)
            log.info("Uploaded {} bytes to \tbucket={}\tkey={}".format(
                len(source), self.bucket_name, key))
            return True
        except Exception as e:
            log.error("{}\tbucket={}\tkey={}".format(e, self.bucket, key))
            return False
def delete_file(module, client, name):
    try:
        bucket = client.get_bucket(module.params['bucket'])
        blob = Blob(name, bucket)
        blob.delete()
        return {}
    except google.cloud.exceptions.NotFound as e:
        module.fail_json(msg=str(e))
Exemplo n.º 29
0
 def exists(self, uri_path, bucket_name=None, exact=False, **kwargs):
     gcp_bucket = self._gcp_bucket(bucket_name)
     is_found = Blob(uri_path, gcp_bucket).exists()
     if exact is False and is_found is False:
         folder_name = '{0}_$folder$'.format(path.basename(uri_path))
         folder_key = path.join(uri_path, folder_name)
         is_found = Blob(folder_key, gcp_bucket).exists()
     return is_found
Exemplo n.º 30
0
def upload(csvfile, bucketname, blobname):
   client = storage.Client()
   bucket = client.get_bucket(bucketname)
   blob = Blob(blobname, bucket)
   blob.upload_from_filename(csvfile)
   gcslocation = 'gs://{}/{}'.format(bucketname, blobname)
   logging.info('Uploaded {} ...'.format(gcslocation))
   return gcslocation
def upload_static_files(bucket, file_name):
	# Uploader Form
	blob = Blob(name=file_name, bucket=bucket)
	with open(os.path.abspath('../static-files/%s' % file_name), 'rb') as tmp_file:
		blob.upload_from_file(tmp_file)
	# Resource
	print(u'Info: File %s uploaded' % file_name)
	print(blob)
Exemplo n.º 32
0
def download():
	file_name = request.form["downloadingfilename"]
	client = storage.Client()
	bucket =  client.get_bucket(request.form['downbucketname'])
	blob = Blob(file_name, bucket, encryption_key=key)
	
	stringfile = blob.download_as_string()
	return stringfile
Exemplo n.º 33
0
 def _set_data(self, **kwargs) -> dict:
     df = kwargs.get('data_frame')
     ts = time.time_ns()
     blob_name_parts = os.path.splitext(self.__blob_name)
     blob_name = blob_name_parts[0] + '_' + str(ts) + blob_name_parts[1]
     blob = Blob(blob_name, self.__bucket)
     blob.upload_from_string(df.to_csv(), self.__file_format)
     return dict(record_count=df.shape[0])
Exemplo n.º 34
0
def upload(csvfile, bucketname, blobname):
    client = storage.Client()
    bucket = client.get_bucket(bucketname)
    blob = Blob(blobname, bucket)
    blob.upload_from_filename(csvfile)
    gcslocation = 'gs://{}/{}'.format(bucketname, blobname)
    logging.info('Uploaded {} ...'.format(gcslocation))
    return gcslocation
Exemplo n.º 35
0
def upload_from_file(client, to_delete):
    # [START upload_from_file]
    from google.cloud.storage import Blob

    client = storage.Client(project="my-project")
    bucket = client.get_bucket("my-bucket")
    encryption_key = "aa426195405adee2c8081bb9e7e74b19"
    blob = Blob("secure-data", bucket, encryption_key=encryption_key)
    with open("my-file", "rb") as my_file:
        blob.upload_from_file(my_file)
    # [END upload_from_file]

    to_delete.append(blob)
Exemplo n.º 36
0
def upload_from_file(client, to_delete):
    # [START upload_from_file]
    from google.cloud.storage import Blob

    client = storage.Client(project='my-project')
    bucket = client.get_bucket('my-bucket')
    encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
    blob = Blob('secure-data', bucket, encryption_key=encryption_key)
    with open('my-file', 'rb') as my_file:
        blob.upload_from_file(my_file)
    # [END upload_from_file]

    to_delete.append(blob)
Exemplo n.º 37
0
def download_to_file(client, to_delete):
    # [START download_to_file]
    from google.cloud.storage import Blob

    client = storage.Client(project='my-project')
    bucket = client.get_bucket('my-bucket')
    encryption_key = 'c7f32af42e45e85b9848a6a14dd2a8f6'
    blob = Blob('secure-data', bucket, encryption_key=encryption_key)
    with open('/tmp/my-secure-file', 'wb') as file_obj:
        blob.download_to_file(file_obj)
    # [END download_to_file]

    to_delete.append(blob)
Exemplo n.º 38
0
def download_encrypted_blob(bucket_name, source_blob_name,
                            destination_file_name, base64_encryption_key):
    """Downloads a previously-encrypted blob from Google Cloud Storage.

    The encryption key provided must be the same key provided when uploading
    the blob.
    """
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    # Encryption key must be an AES256 key represented as a bytestring with
    # 32 bytes. Since it's passed in as a base64 encoded string, it needs
    # to be decoded.
    encryption_key = base64.b64decode(base64_encryption_key)
    blob = Blob(source_blob_name, bucket, encryption_key=encryption_key)

    blob.download_to_filename(destination_file_name)

    print('Blob {} downloaded to {}.'.format(
        source_blob_name,
        destination_file_name))
Exemplo n.º 39
0
def upload_encrypted_blob(bucket_name, source_file_name,
                          destination_blob_name, base64_encryption_key):
    """Uploads a file to a Google Cloud Storage bucket using a custom
    encryption key.

    The file will be encrypted by Google Cloud Storage and only
    retrievable using the provided encryption key.
    """
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucket_name)
    # Encryption key must be an AES256 key represented as a bytestring with
    # 32 bytes. Since it's passed in as a base64 encoded string, it needs
    # to be decoded.
    encryption_key = base64.b64decode(base64_encryption_key)
    blob = Blob(destination_blob_name, bucket, encryption_key=encryption_key)

    blob.upload_from_filename(source_file_name)

    print('File {} uploaded to {}.'.format(
        source_file_name,
        destination_blob_name))