def _set_result_from_operation(self): """Set the result or exception from the operation if it is complete.""" # This must be done in a lock to prevent the polling thread # and main thread from both executing the completion logic # at the same time. with self._completion_lock: # If the operation isn't complete or if the result has already been # set, do not call set_result/set_exception again. # Note: self._result_set is set to True in set_result and # set_exception, in case those methods are invoked directly. if not self._operation.done or self._result_set: return if self._operation.HasField('response'): response = _helpers._from_any_pb( self._result_type, self._operation.response) self.set_result(response) elif self._operation.HasField('error'): exception = exceptions.GoogleCloudError( self._operation.error.message, errors=(self._operation.error)) self.set_exception(exception) else: exception = exceptions.GoogleCloudError( 'Unexpected state: Long-running operation had neither ' 'response nor error set.') self.set_exception(exception)
def wait_for_operation(self, operation: Operation, project_id: str = None) -> Operation: """ Given an operation, continuously fetches the status from Google Cloud until either completion or an error occurring :param operation: The Operation to wait for :type operation: google.cloud.container_V1.gapic.enums.Operation :param project_id: Google Cloud Platform project ID :type project_id: str :return: A new, updated operation fetched from Google Cloud """ self.log.info("Waiting for OPERATION_NAME %s", operation.name) time.sleep(OPERATIONAL_POLL_INTERVAL) while operation.status != Operation.Status.DONE: if operation.status == Operation.Status.RUNNING or operation.status == \ Operation.Status.PENDING: time.sleep(OPERATIONAL_POLL_INTERVAL) else: raise exceptions.GoogleCloudError( "Operation has failed with status: %s" % operation.status) # To update status of operation operation = self.get_operation(operation.name, project_id=project_id or self.project_id) return operation
def authenticate(self, options): """[authenticate function to authenticate the service (aws s3/gcp bucket)]. Args: options ([dict]): [options dict contains all the configuration settings] Returns: [object]: [a client object when authentication is successful else exception is raised] """ try: credentials = None if 'CREDENTIAL_JSON' in options and options['CREDENTIAL_JSON']: credentials = service_account.Credentials.from_service_account_info( json.loads(options['CREDENTIAL_JSON'])) else: credentials = service_account.Credentials.from_service_account_file( options['CREDENTIAL_FILE']) if credentials: client = storage.Client( credentials=credentials, project=options['PROJECT_NAME']) else: raise exceptions.Forbidden('Authentication Failed') if client: return client else: raise exceptions.GoogleCloudError('Authentication Failed') except Exception as E: logging.exception("Exception {err}".format()) raise Exception('Authentication Failed')
def test_retrying_on_google_cloud_error(self, mocked_function): mocked_function.side_effect = [ google_cloud_exceptions.GoogleCloudError('e'), 1 ] decorated = pubsub.retry(mocked_function) result = decorated() assert result == 1
def _get_bucket_connection(self, bucket_name, settings): try: storage_client = storage.Client( credentials=self._get_gcp_credentials(settings) ) return self._get_bucket(storage_client, bucket_name) except gcloud_exceptions.GoogleCloudError as error: self.log.error(f"Unable to get bucket {bucket_name}: {error}") raise gcloud_exceptions.GoogleCloudError(error)
def test_failing_check_of_gcp_bucket_is_handled(): # When with patch('app.file_sender.storage.Client') as bucket_client: bucket_client.side_effect = exceptions.GoogleCloudError( "bucket doesn't exist") try: check_gcp_bucket_ready() except Exception: assert False, "Exception msgs when checking GCP bucket should be handled"
def test_eof_deleted_when_task_creation_fails(self): self.tasks_client.return_value.push_tasks.side_effect = exceptions.GoogleCloudError( 'Dummy message') request_body = _build_request_body_for_start( delete_count=_DUMMY_DELETE_COUNT, expiring_count=_DUMMY_EXPIRING_COUNT, upsert_count=_DUMMY_UPSERT_COUNT) response = self.test_app_client.post(_START_PATH, json=request_body) self.storage_client.return_value.delete_eof.assert_called_once() self.assertEqual(http.HTTPStatus.INTERNAL_SERVER_ERROR, response.status_code)
def test_items_table_deleted_when_creating_processing_table_fails(self): self.bigquery_client.return_value.initialize_dataset_and_table.side_effect = exceptions.GoogleCloudError( 'Dummy message') request_body = _build_request_body_for_start( delete_count=_DUMMY_DELETE_COUNT, expiring_count=_DUMMY_EXPIRING_COUNT, upsert_count=_DUMMY_UPSERT_COUNT) response = self.test_app_client.post(_START_PATH, json=request_body) self.bigquery_client.return_value.delete_table.assert_called_once() self.assertEqual(http.HTTPStatus.INTERNAL_SERVER_ERROR, response.status_code)
def read(self, blob_name: str, **kwargs): """Downloads a blob from the bucket.""" blob = self.bucket.blob(blob_name) string_buffer = BytesIO() try: blob.download_to_file(string_buffer) except gcloud_exceptions.GoogleCloudError as error: self.log.error(f"Unable to read blob {blob_name}: {error}") raise gcloud_exceptions.GoogleCloudError(error) else: self.log.info(f"Blob {blob_name} read to string to string") return string_buffer.getvalue().decode()
def test_failing_write_to_gcp_bucket_is_handled(): # When with patch('app.file_sender.storage.Client') as bucket_client: # Simulate an error from the GCS storage client bucket_client.side_effect = exceptions.GoogleCloudError( "bucket doesn't exist") try: write_file_to_bucket( RESOURCE_FILE_PATH.joinpath('dummy_print_file.txt')) except Exception: assert False, "Exception msgs from writing to GCP bucket should be handled"
def test_create_google_cloud_error_with_args(): error = { 'domain': 'global', 'location': 'test', 'locationType': 'testing', 'message': 'Testing', 'reason': 'test', } exception = exceptions.GoogleCloudError('Testing', [error]) exception.code = 600 assert str(exception) == '600 Testing' assert exception.message == 'Testing' assert exception.errors == [error]
def test_create_google_cloud_error_with_args(): error = { "domain": "global", "location": "test", "locationType": "testing", "message": "Testing", "reason": "test", } exception = exceptions.GoogleCloudError("Testing", [error]) exception.code = 600 assert str(exception) == "600 Testing" assert exception.message == "Testing" assert exception.errors == [error]
def write( self, data: str, destination_blob_name: str, fmt: str, metadata: dict = None ): """Write string to a bucket.""" try: name = f"{destination_blob_name}.{fmt}" blob = self.bucket.blob(unquote(name)) blob.upload_from_string(data) except gcloud_exceptions.GoogleCloudError as error: self.log.error(f"Error writing file {name} to google storage: {error}") raise gcloud_exceptions.GoogleCloudError(error) else: self.log.info( f"{destination_blob_name} successfully written to {blob.public_url}" ) return f"{blob.public_url}"
def test_create_google_cloud_error(): exception = exceptions.GoogleCloudError('Testing') exception.code = 600 assert str(exception) == '600 Testing' assert exception.message == 'Testing' assert exception.errors == []