def list_table_change_logs(project_id: str,
dataset_id: str) -> Generator[str, None, None]:
'''Helper function that returns all relevant logs (directly from the Stackdriver API, not PubSub)
:param project_id: The project_id that is associated with your GCP account and BigQuery Table
:type project_id: str
:param dataset_id: The name of the dataset that you want to track/replicate via CDC
:type dataset_id: str
:return: A Generator object representing an iterable of relevant logs
:rtype: Generator
'''
client = logging_v2.LoggingServiceV2Client()
resource_names = ["projects/{}".format(project_id)]
#filter logs that reflect a bigquery table insert/update/delete only (for the specified project and dataset)
query = '''
resource.labels.dataset_id="{dataset_id}"
resource.labels.project_id="{project_id}"
protoPayload.metadata.@type="type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata"
(protoPayload.metadata.tableDataChange.deletedRowsCount > "0" OR protoPayload.metadata.tableDataChange.insertedRowsCount > "0")
'''.format(project_id=project_id, dataset_id=dataset_id)
for log in client.list_log_entries(resource_names, filter_=query):
yield log
def test_end_to_end(pubsub_topic):
# Post the message "Runner" to the topic
subprocess.run([
"gcloud", "pubsub", "topics", "publish", f"{pubsub_topic}",
"--project", PROJECT, "--message", "Runner", "--quiet"
],
check=True)
# Check the logs for "Hello Runner"
time.sleep(20) # Slight delay writing to stackdriver
client = logging_v2.LoggingServiceV2Client()
resource_names = [f"projects/{PROJECT}"]
# We add timestamp for making the query faster.
now = datetime.datetime.now(datetime.timezone.utc)
filter_date = now - datetime.timedelta(minutes=1)
filters = (f"timestamp>=\"{filter_date.isoformat('T')}\" "
"resource.type=cloud_run_revision "
f"AND resource.labels.service_name={CLOUD_RUN_SERVICE} ")
# Retry a maximum number of 10 times to find results in stackdriver
found = False
for x in range(10):
iterator = client.list_log_entries(resource_names, filter_=filters)
for entry in iterator:
if entry.text_payload == "Hello Runner!":
found = True
break
# When message found, exit loop
if found is True:
break
time.sleep(5) # Slight delay before retry
assert found
def test_list_monitored_resource_descriptors(self):
# Setup Expected Response
next_page_token = ""
resource_descriptors_element = {}
resource_descriptors = [resource_descriptors_element]
expected_response = {
"next_page_token": next_page_token,
"resource_descriptors": resource_descriptors,
}
expected_response = logging_pb2.ListMonitoredResourceDescriptorsResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
paged_list_response = client.list_monitored_resource_descriptors()
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.resource_descriptors[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListMonitoredResourceDescriptorsRequest(
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_logs(self):
# Setup Expected Response
next_page_token = ''
log_names_element = 'logNamesElement-1079688374'
log_names = [log_names_element]
expected_response = {
'next_page_token': next_page_token,
'log_names': log_names
}
expected_response = logging_pb2.ListLogsResponse(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup Request
parent = client.project_path('[PROJECT]')
paged_list_response = client.list_logs(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.log_names[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListLogsRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_log_entries(self):
# Setup Expected Response
next_page_token = ""
entries_element = {}
entries = [entries_element]
expected_response = {
"next_page_token": next_page_token,
"entries": entries
}
expected_response = logging_pb2.ListLogEntriesResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup Request
resource_names = []
paged_list_response = client.list_log_entries(resource_names)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.entries[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListLogEntriesRequest(
resource_names=resource_names)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_monitored_resource_descriptors(self):
# Setup Expected Response
next_page_token = ''
resource_descriptors_element = {}
resource_descriptors = [resource_descriptors_element]
expected_response = {
'next_page_token': next_page_token,
'resource_descriptors': resource_descriptors
}
expected_response = logging_pb2.ListMonitoredResourceDescriptorsResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = logging_v2.LoggingServiceV2Client(channel=channel)
paged_list_response = client.list_monitored_resource_descriptors()
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.resource_descriptors[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListMonitoredResourceDescriptorsRequest(
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_monitored_resource_descriptors_exception(self):
channel = ChannelStub(responses=[CustomException()])
client = logging_v2.LoggingServiceV2Client(channel=channel)
paged_list_response = client.list_monitored_resource_descriptors()
with pytest.raises(CustomException):
list(paged_list_response)
def test_list_log_entries(self):
# Setup Expected Response
next_page_token = ''
entries_element = {}
entries = [entries_element]
expected_response = {
'next_page_token': next_page_token,
'entries': entries
}
expected_response = logging_pb2.ListLogEntriesResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup Request
resource_names = []
paged_list_response = client.list_log_entries(resource_names)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.entries[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListLogEntriesRequest(
resource_names=resource_names)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_logs(self):
# Setup Expected Response
next_page_token = ""
log_names_element = "logNamesElement-1079688374"
log_names = [log_names_element]
expected_response = {
"next_page_token": next_page_token,
"log_names": log_names
}
expected_response = logging_pb2.ListLogsResponse(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup Request
parent = client.project_path("[PROJECT]")
paged_list_response = client.list_logs(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.log_names[0] == resources[0]
assert len(channel.requests) == 1
expected_request = logging_pb2.ListLogsRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def sample_delete_log():
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.DeleteLogRequest(log_name="log_name_value", )
# Make the request
client.delete_log(request=request)
async def test_progress(self, monkeypatch):
fake_creds = AnonymousCredentials()
mock_client = mock.create_autospec(
ClusterControllerClient(credentials=fake_creds))
mock_logging_client = mock.create_autospec(
logging_v2.LoggingServiceV2Client(credentials=fake_creds))
spawner = DataprocSpawner(hub=Hub(),
dataproc=mock_client,
user=MockUser(),
_mock=True,
logging=mock_logging_client,
gcs_notebooks=self.gcs_notebooks)
spawner.project = "test-progress"
async def collect(ait):
items = []
async for value in ait:
items.append(value)
return items
def create_logs():
entries = []
for i in range(5):
e = LogEntry(insert_id=f'entry_{i}',
json_payload=ParseDict(
{
'method': 'method',
'message': f'message_{i}'
}, Struct()))
entries.append(e)
return entries
def create_expected():
progress = 5
expected = []
i = 0
for e in create_logs():
progress += math.ceil((90 - progress) / 4)
expected.append({
'progress': progress,
'message': f'method: message_{i}'
})
i += 1
expected.append({'message': 'operation.done()', 'progress': 71})
return expected
def test_list_log_entries(*args, **kwargs):
return create_logs()
op = MockOperation('op1', 'cluster1-op1')
monkeypatch.setattr(mock_logging_client, 'list_log_entries',
test_list_log_entries)
monkeypatch.setattr(spawner, 'operation', op)
_, _ = await spawner.start()
assert await collect(spawner.progress()) == create_expected()
def test_list_log_entries_exception(self):
channel = ChannelStub(responses=[CustomException()])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup request
resource_names = []
paged_list_response = client.list_log_entries(resource_names)
with pytest.raises(CustomException):
list(paged_list_response)
def test_write_log_entries_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup request
entries = []
with pytest.raises(CustomException):
client.write_log_entries(entries)
def test_delete_log_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup request
log_name = client.log_path('[PROJECT]', '[LOG]')
with pytest.raises(CustomException):
client.delete_log(log_name)
def test_list_logs_exception(self):
channel = ChannelStub(responses=[CustomException()])
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup request
parent = client.project_path('[PROJECT]')
paged_list_response = client.list_logs(parent)
with pytest.raises(CustomException):
list(paged_list_response)
def test_list_monitored_resource_descriptors_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
paged_list_response = client.list_monitored_resource_descriptors()
with pytest.raises(CustomException):
list(paged_list_response)
def sample_delete_log():
"""Snippet for delete_log"""
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.DeleteLogRequest(
)
# Make the request
response = client.delete_log(request=request)
def __init__(self, buffer_size=_LOG_BUFFER_SIZE):
self._buffer_size = buffer_size
self._buffer = collections.deque()
self._reset()
self._logging_client = gcp_logging_v2.LoggingServiceV2Client()
self._operation_pb2 = None
# Used to determine how long a request took.
self._first_log_ts = None
def test_write_log_entries(self):
_, project_id = google.auth.default()
client = logging_v2.LoggingServiceV2Client()
log_name = client.log_path(project_id, "test-{0}".format(time.time()))
resource = {}
labels = {}
entries = []
response = client.write_log_entries(entries,
log_name=log_name,
resource=resource,
labels=labels)
def test_write_log_entries_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup request
entries = []
with pytest.raises(CustomException):
client.write_log_entries(entries)
def sample_list_monitored_resource_descriptors():
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.ListMonitoredResourceDescriptorsRequest()
# Make the request
page_result = client.list_monitored_resource_descriptors(request=request)
# Handle the response
for response in page_result:
print(response)
def sample_list_logs():
"""Snippet for list_logs"""
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.ListLogsRequest()
# Make the request
page_result = client.list_logs(request=request)
for response in page_result:
print("{}".format(response))
def test_list_log_entries_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup request
resource_names = []
paged_list_response = client.list_log_entries(resource_names)
with pytest.raises(CustomException):
list(paged_list_response)
def test_delete_log(self):
channel = ChannelStub()
client = logging_v2.LoggingServiceV2Client(channel=channel)
# Setup Request
log_name = client.log_path('[PROJECT]', '[LOG]')
client.delete_log(log_name)
assert len(channel.requests) == 1
expected_request = logging_pb2.DeleteLogRequest(log_name=log_name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def sample_list_logs():
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.ListLogsRequest(parent="parent_value", )
# Make the request
page_result = client.list_logs(request=request)
# Handle the response
for response in page_result:
print(response)
def test_delete_log_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup request
log_name = client.log_path("[PROJECT]", "[LOG]")
with pytest.raises(CustomException):
client.delete_log(log_name)
def test_list_logs_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = logging_v2.LoggingServiceV2Client()
# Setup request
parent = client.project_path("[PROJECT]")
paged_list_response = client.list_logs(parent)
with pytest.raises(CustomException):
list(paged_list_response)
def sample_write_log_entries():
"""Snippet for write_log_entries"""
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.WriteLogEntriesRequest()
# Make the request
response = client.write_log_entries(request=request)
# Handle response
print("{}".format(response))
def sample_list_log_entries():
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.ListLogEntriesRequest(
resource_names=['resource_names_value_1', 'resource_names_value_2'], )
# Make the request
page_result = client.list_log_entries(request=request)
# Handle the response
for response in page_result:
print(response)
def sample_tail_log_entries():
"""Snippet for tail_log_entries"""
# Create a client
client = logging_v2.LoggingServiceV2Client()
# Initialize request argument(s)
request = logging_v2.TailLogEntriesRequest(
)
# Make the request
stream = client.tail_log_entries([])
for response in stream:
print("{}".format(response))
