def list_table_change_logs(project_id: str, dataset_id: str) -> Generator[str, None, None]: '''Helper function that returns all relevant logs (directly from the Stackdriver API, not PubSub) :param project_id: The project_id that is associated with your GCP account and BigQuery Table :type project_id: str :param dataset_id: The name of the dataset that you want to track/replicate via CDC :type dataset_id: str :return: A Generator object representing an iterable of relevant logs :rtype: Generator ''' client = logging_v2.LoggingServiceV2Client() resource_names = ["projects/{}".format(project_id)] #filter logs that reflect a bigquery table insert/update/delete only (for the specified project and dataset) query = ''' resource.labels.dataset_id="{dataset_id}" resource.labels.project_id="{project_id}" protoPayload.metadata.@type="type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata" (protoPayload.metadata.tableDataChange.deletedRowsCount > "0" OR protoPayload.metadata.tableDataChange.insertedRowsCount > "0") '''.format(project_id=project_id, dataset_id=dataset_id) for log in client.list_log_entries(resource_names, filter_=query): yield log
def test_end_to_end(pubsub_topic): # Post the message "Runner" to the topic subprocess.run([ "gcloud", "pubsub", "topics", "publish", f"{pubsub_topic}", "--project", PROJECT, "--message", "Runner", "--quiet" ], check=True) # Check the logs for "Hello Runner" time.sleep(20) # Slight delay writing to stackdriver client = logging_v2.LoggingServiceV2Client() resource_names = [f"projects/{PROJECT}"] # We add timestamp for making the query faster. now = datetime.datetime.now(datetime.timezone.utc) filter_date = now - datetime.timedelta(minutes=1) filters = (f"timestamp>=\"{filter_date.isoformat('T')}\" " "resource.type=cloud_run_revision " f"AND resource.labels.service_name={CLOUD_RUN_SERVICE} ") # Retry a maximum number of 10 times to find results in stackdriver found = False for x in range(10): iterator = client.list_log_entries(resource_names, filter_=filters) for entry in iterator: if entry.text_payload == "Hello Runner!": found = True break # When message found, exit loop if found is True: break time.sleep(5) # Slight delay before retry assert found
def test_list_monitored_resource_descriptors(self): # Setup Expected Response next_page_token = "" resource_descriptors_element = {} resource_descriptors = [resource_descriptors_element] expected_response = { "next_page_token": next_page_token, "resource_descriptors": resource_descriptors, } expected_response = logging_pb2.ListMonitoredResourceDescriptorsResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() paged_list_response = client.list_monitored_resource_descriptors() resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.resource_descriptors[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListMonitoredResourceDescriptorsRequest( ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_logs(self): # Setup Expected Response next_page_token = '' log_names_element = 'logNamesElement-1079688374' log_names = [log_names_element] expected_response = { 'next_page_token': next_page_token, 'log_names': log_names } expected_response = logging_pb2.ListLogsResponse(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup Request parent = client.project_path('[PROJECT]') paged_list_response = client.list_logs(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.log_names[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListLogsRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_log_entries(self): # Setup Expected Response next_page_token = "" entries_element = {} entries = [entries_element] expected_response = { "next_page_token": next_page_token, "entries": entries } expected_response = logging_pb2.ListLogEntriesResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup Request resource_names = [] paged_list_response = client.list_log_entries(resource_names) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.entries[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListLogEntriesRequest( resource_names=resource_names) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_monitored_resource_descriptors(self): # Setup Expected Response next_page_token = '' resource_descriptors_element = {} resource_descriptors = [resource_descriptors_element] expected_response = { 'next_page_token': next_page_token, 'resource_descriptors': resource_descriptors } expected_response = logging_pb2.ListMonitoredResourceDescriptorsResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = logging_v2.LoggingServiceV2Client(channel=channel) paged_list_response = client.list_monitored_resource_descriptors() resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.resource_descriptors[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListMonitoredResourceDescriptorsRequest( ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_monitored_resource_descriptors_exception(self): channel = ChannelStub(responses=[CustomException()]) client = logging_v2.LoggingServiceV2Client(channel=channel) paged_list_response = client.list_monitored_resource_descriptors() with pytest.raises(CustomException): list(paged_list_response)
def test_list_log_entries(self): # Setup Expected Response next_page_token = '' entries_element = {} entries = [entries_element] expected_response = { 'next_page_token': next_page_token, 'entries': entries } expected_response = logging_pb2.ListLogEntriesResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup Request resource_names = [] paged_list_response = client.list_log_entries(resource_names) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.entries[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListLogEntriesRequest( resource_names=resource_names) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_logs(self): # Setup Expected Response next_page_token = "" log_names_element = "logNamesElement-1079688374" log_names = [log_names_element] expected_response = { "next_page_token": next_page_token, "log_names": log_names } expected_response = logging_pb2.ListLogsResponse(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup Request parent = client.project_path("[PROJECT]") paged_list_response = client.list_logs(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.log_names[0] == resources[0] assert len(channel.requests) == 1 expected_request = logging_pb2.ListLogsRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def sample_delete_log(): # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.DeleteLogRequest(log_name="log_name_value", ) # Make the request client.delete_log(request=request)
async def test_progress(self, monkeypatch): fake_creds = AnonymousCredentials() mock_client = mock.create_autospec( ClusterControllerClient(credentials=fake_creds)) mock_logging_client = mock.create_autospec( logging_v2.LoggingServiceV2Client(credentials=fake_creds)) spawner = DataprocSpawner(hub=Hub(), dataproc=mock_client, user=MockUser(), _mock=True, logging=mock_logging_client, gcs_notebooks=self.gcs_notebooks) spawner.project = "test-progress" async def collect(ait): items = [] async for value in ait: items.append(value) return items def create_logs(): entries = [] for i in range(5): e = LogEntry(insert_id=f'entry_{i}', json_payload=ParseDict( { 'method': 'method', 'message': f'message_{i}' }, Struct())) entries.append(e) return entries def create_expected(): progress = 5 expected = [] i = 0 for e in create_logs(): progress += math.ceil((90 - progress) / 4) expected.append({ 'progress': progress, 'message': f'method: message_{i}' }) i += 1 expected.append({'message': 'operation.done()', 'progress': 71}) return expected def test_list_log_entries(*args, **kwargs): return create_logs() op = MockOperation('op1', 'cluster1-op1') monkeypatch.setattr(mock_logging_client, 'list_log_entries', test_list_log_entries) monkeypatch.setattr(spawner, 'operation', op) _, _ = await spawner.start() assert await collect(spawner.progress()) == create_expected()
def test_list_log_entries_exception(self): channel = ChannelStub(responses=[CustomException()]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup request resource_names = [] paged_list_response = client.list_log_entries(resource_names) with pytest.raises(CustomException): list(paged_list_response)
def test_write_log_entries_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup request entries = [] with pytest.raises(CustomException): client.write_log_entries(entries)
def test_delete_log_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup request log_name = client.log_path('[PROJECT]', '[LOG]') with pytest.raises(CustomException): client.delete_log(log_name)
def test_list_logs_exception(self): channel = ChannelStub(responses=[CustomException()]) client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup request parent = client.project_path('[PROJECT]') paged_list_response = client.list_logs(parent) with pytest.raises(CustomException): list(paged_list_response)
def test_list_monitored_resource_descriptors_exception(self): channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() paged_list_response = client.list_monitored_resource_descriptors() with pytest.raises(CustomException): list(paged_list_response)
def sample_delete_log(): """Snippet for delete_log""" # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.DeleteLogRequest( ) # Make the request response = client.delete_log(request=request)
def __init__(self, buffer_size=_LOG_BUFFER_SIZE): self._buffer_size = buffer_size self._buffer = collections.deque() self._reset() self._logging_client = gcp_logging_v2.LoggingServiceV2Client() self._operation_pb2 = None # Used to determine how long a request took. self._first_log_ts = None
def test_write_log_entries(self): _, project_id = google.auth.default() client = logging_v2.LoggingServiceV2Client() log_name = client.log_path(project_id, "test-{0}".format(time.time())) resource = {} labels = {} entries = [] response = client.write_log_entries(entries, log_name=log_name, resource=resource, labels=labels)
def test_write_log_entries_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup request entries = [] with pytest.raises(CustomException): client.write_log_entries(entries)
def sample_list_monitored_resource_descriptors(): # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.ListMonitoredResourceDescriptorsRequest() # Make the request page_result = client.list_monitored_resource_descriptors(request=request) # Handle the response for response in page_result: print(response)
def sample_list_logs(): """Snippet for list_logs""" # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.ListLogsRequest() # Make the request page_result = client.list_logs(request=request) for response in page_result: print("{}".format(response))
def test_list_log_entries_exception(self): channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup request resource_names = [] paged_list_response = client.list_log_entries(resource_names) with pytest.raises(CustomException): list(paged_list_response)
def test_delete_log(self): channel = ChannelStub() client = logging_v2.LoggingServiceV2Client(channel=channel) # Setup Request log_name = client.log_path('[PROJECT]', '[LOG]') client.delete_log(log_name) assert len(channel.requests) == 1 expected_request = logging_pb2.DeleteLogRequest(log_name=log_name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def sample_list_logs(): # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.ListLogsRequest(parent="parent_value", ) # Make the request page_result = client.list_logs(request=request) # Handle the response for response in page_result: print(response)
def test_delete_log_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup request log_name = client.log_path("[PROJECT]", "[LOG]") with pytest.raises(CustomException): client.delete_log(log_name)
def test_list_logs_exception(self): channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = logging_v2.LoggingServiceV2Client() # Setup request parent = client.project_path("[PROJECT]") paged_list_response = client.list_logs(parent) with pytest.raises(CustomException): list(paged_list_response)
def sample_write_log_entries(): """Snippet for write_log_entries""" # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.WriteLogEntriesRequest() # Make the request response = client.write_log_entries(request=request) # Handle response print("{}".format(response))
def sample_list_log_entries(): # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.ListLogEntriesRequest( resource_names=['resource_names_value_1', 'resource_names_value_2'], ) # Make the request page_result = client.list_log_entries(request=request) # Handle the response for response in page_result: print(response)
def sample_tail_log_entries(): """Snippet for tail_log_entries""" # Create a client client = logging_v2.LoggingServiceV2Client() # Initialize request argument(s) request = logging_v2.TailLogEntriesRequest( ) # Make the request stream = client.tail_log_entries([]) for response in stream: print("{}".format(response))