def test_log_empty_w_explicit(self):
import datetime
from google.cloud.logging.resource import Resource
ALT_LOG_NAME = "projects/foo/logs/alt.log.name"
DEFAULT_LABELS = {"foo": "spam"}
LABELS = {"foo": "bar", "baz": "qux"}
IID = "IID"
SEVERITY = "CRITICAL"
METHOD = "POST"
URI = "https://api.example.com/endpoint"
STATUS = "500"
TRACE = "12345678-1234-5678-1234-567812345678"
SPANID = "000000000000004a"
REQUEST = {
"requestMethod": METHOD,
"requestUrl": URI,
"status": STATUS
}
TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999)
RESOURCE = Resource(type="gae_app",
labels={
"module_id": "default",
"version_id": "test"
})
ENTRIES = [{
"logName": ALT_LOG_NAME,
"labels": LABELS,
"insertId": IID,
"severity": SEVERITY,
"httpRequest": REQUEST,
"timestamp": "2016-12-31T00:01:02.999999Z",
"resource": RESOURCE._to_dict(),
"trace": TRACE,
"spanId": SPANID,
"traceSampled": True,
}]
client1 = _Client(self.PROJECT)
client2 = _Client(self.PROJECT)
api = client2.logging_api = _DummyLoggingAPI()
logger = self._make_one(self.LOGGER_NAME,
client=client1,
labels=DEFAULT_LABELS)
logger.log_empty(
log_name=ALT_LOG_NAME,
client=client2,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
)
self.assertEqual(api._write_entries_called_with,
(ENTRIES, None, None, None))
def test_from_api_repr_w_loggers_no_logger_match(self):
from datetime import datetime
from google.cloud._helpers import UTC
from google.cloud.logging.resource import Resource
klass = self._get_target_class()
client = _Client(self.PROJECT)
PAYLOAD = 'PAYLOAD'
SEVERITY = 'CRITICAL'
IID = 'IID'
NOW = datetime.utcnow().replace(tzinfo=UTC)
TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
LABELS = {'foo': 'bar', 'baz': 'qux'}
METHOD = 'POST'
URI = 'https://api.example.com/endpoint'
RESOURCE = Resource(
type='gae_app',
labels={
'type': 'gae_app',
'labels': {
'module_id': 'default',
'version': 'test',
}
}
)
STATUS = '500'
TRACE = '12345678-1234-5678-1234-567812345678'
API_REPR = {
'dummyPayload': PAYLOAD,
'logName': LOG_NAME,
'insertId': IID,
'timestamp': TIMESTAMP,
'labels': LABELS,
'severity': SEVERITY,
'httpRequest': {
'requestMethod': METHOD,
'requestUrl': URI,
'status': STATUS,
},
'resource': RESOURCE._to_dict(),
'trace': TRACE
}
loggers = {}
entry = klass.from_api_repr(API_REPR, client, loggers=loggers)
self.assertEqual(entry.payload, PAYLOAD)
self.assertEqual(entry.insert_id, IID)
self.assertEqual(entry.timestamp, NOW)
self.assertEqual(entry.labels, LABELS)
self.assertEqual(entry.severity, SEVERITY)
self.assertEqual(entry.http_request['requestMethod'], METHOD)
self.assertEqual(entry.http_request['requestUrl'], URI)
self.assertEqual(entry.http_request['status'], STATUS)
logger = entry.logger
self.assertIsInstance(logger, _Logger)
self.assertIs(logger.client, client)
self.assertEqual(logger.name, self.LOGGER_NAME)
self.assertEqual(loggers, {LOG_NAME: logger})
self.assertEqual(entry.resource, RESOURCE)
self.assertEqual(entry.trace, TRACE)
def test_commit_w_resource_specified(self):
from google.cloud.logging.logger import _GLOBAL_RESOURCE
from google.cloud.logging.resource import Resource
logger = _Logger()
client = _Client(project=self.PROJECT, connection=_make_credentials())
api = client.logging_api = _DummyLoggingAPI()
RESOURCE = Resource(type='gae_app',
labels={
'module_id': 'default',
'version_id': 'test',
})
batch = self._make_one(logger, client, resource=RESOURCE)
MESSAGE = 'This is the entry text'
ENTRIES = [
{
'textPayload': MESSAGE
},
{
'textPayload': MESSAGE,
'resource': _GLOBAL_RESOURCE._to_dict()
},
]
batch.log_text(MESSAGE, resource=None)
batch.log_text(MESSAGE)
batch.commit()
self.assertEqual(
api._write_entries_called_with,
(ENTRIES, logger.full_name, RESOURCE._to_dict(), None))
def test_commit_w_resource_specified(self):
from google.cloud.logging.logger import _GLOBAL_RESOURCE
from google.cloud.logging.resource import Resource
logger = _Logger()
client = _Client(project=self.PROJECT, connection=_make_credentials())
api = client.logging_api = _DummyLoggingAPI()
RESOURCE = Resource(
type='gae_app',
labels={
'module_id': 'default',
'version_id': 'test',
}
)
batch = self._make_one(logger, client, resource=RESOURCE)
MESSAGE = 'This is the entry text'
ENTRIES = [
{'textPayload': MESSAGE},
{'textPayload': MESSAGE, 'resource': _GLOBAL_RESOURCE._to_dict()},
]
batch.log_text(MESSAGE, resource=None)
batch.log_text(MESSAGE)
batch.commit()
self.assertEqual(api._write_entries_called_with,
(ENTRIES, logger.full_name,
RESOURCE._to_dict(), None))
def test_to_api_repr_explicit(self):
import datetime
from google.cloud.logging.resource import Resource
from google.cloud._helpers import _datetime_to_rfc3339
LOG_NAME = "test.log"
JSON_PAYLOAD = {"key": "value"}
LABELS = {"foo": "bar", "baz": "qux"}
IID = "IID"
SEVERITY = "CRITICAL"
METHOD = "POST"
URI = "https://api.example.com/endpoint"
STATUS = "500"
REQUEST = {"requestMethod": METHOD, "requestUrl": URI, "status": STATUS}
TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999)
RESOURCE = Resource(
type="gae_app", labels={"module_id": "default", "version_id": "test"}
)
TRACE = "12345678-1234-5678-1234-567812345678"
SPANID = "000000000000004a"
FILE = "my_file.py"
LINE = 123
FUNCTION = "my_function"
SOURCE_LOCATION = {"file": FILE, "line": LINE, "function": FUNCTION}
OP_ID = "OP_ID"
PRODUCER = "PRODUCER"
OPERATION = {"id": OP_ID, "producer": PRODUCER, "first": True, "last": False}
expected = {
"logName": LOG_NAME,
"jsonPayload": JSON_PAYLOAD,
"labels": LABELS,
"insertId": IID,
"severity": SEVERITY,
"httpRequest": REQUEST,
"timestamp": _datetime_to_rfc3339(TIMESTAMP),
"resource": RESOURCE._to_dict(),
"trace": TRACE,
"spanId": SPANID,
"traceSampled": True,
"sourceLocation": {"file": FILE, "line": str(LINE), "function": FUNCTION},
"operation": OPERATION,
}
entry = self._make_one(
log_name=LOG_NAME,
payload=JSON_PAYLOAD,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
source_location=SOURCE_LOCATION,
operation=OPERATION,
)
self.assertEqual(entry.to_api_repr(), expected)
def test_from_api_repr_w_loggers_no_logger_match(self):
from datetime import datetime
from google.cloud._helpers import UTC
from google.cloud.logging.resource import Resource
klass = self._get_target_class()
client = _Client(self.PROJECT)
PAYLOAD = 'PAYLOAD'
SEVERITY = 'CRITICAL'
IID = 'IID'
NOW = datetime.utcnow().replace(tzinfo=UTC)
TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME)
LABELS = {'foo': 'bar', 'baz': 'qux'}
METHOD = 'POST'
URI = 'https://api.example.com/endpoint'
RESOURCE = Resource(type='gae_app',
labels={
'type': 'gae_app',
'labels': {
'module_id': 'default',
'version': 'test',
}
})
STATUS = '500'
TRACE = '12345678-1234-5678-1234-567812345678'
API_REPR = {
'dummyPayload': PAYLOAD,
'logName': LOG_NAME,
'insertId': IID,
'timestamp': TIMESTAMP,
'labels': LABELS,
'severity': SEVERITY,
'httpRequest': {
'requestMethod': METHOD,
'requestUrl': URI,
'status': STATUS,
},
'resource': RESOURCE._to_dict(),
'trace': TRACE
}
loggers = {}
entry = klass.from_api_repr(API_REPR, client, loggers=loggers)
self.assertEqual(entry.payload, PAYLOAD)
self.assertEqual(entry.insert_id, IID)
self.assertEqual(entry.timestamp, NOW)
self.assertEqual(entry.labels, LABELS)
self.assertEqual(entry.severity, SEVERITY)
self.assertEqual(entry.http_request['requestMethod'], METHOD)
self.assertEqual(entry.http_request['requestUrl'], URI)
self.assertEqual(entry.http_request['status'], STATUS)
logger = entry.logger
self.assertIsInstance(logger, _Logger)
self.assertIs(logger.client, client)
self.assertEqual(logger.name, self.LOGGER_NAME)
self.assertEqual(loggers, {LOG_NAME: logger})
self.assertEqual(entry.resource, RESOURCE)
self.assertEqual(entry.trace, TRACE)
def test_should_read_logs_with_custom_resources(
self, mock_client, mock_get_creds_and_project_id):
mock_get_creds_and_project_id.return_value = ('creds', 'project_id')
resource = Resource(
type="cloud_composer_environment",
labels={
"environment.name": 'test-instancce',
"location": 'europpe-west-3',
"project_id": "asf-project",
},
)
self.stackdriver_task_handler = StackdriverTaskHandler(
transport=self.transport_mock, resource=resource)
entry = mock.MagicMock(payload={"message": "TEXT"})
page = [entry, entry]
mock_client.return_value.list_entries.return_value.pages = (
n for n in [page])
mock_client.return_value.list_entries.return_value.next_page_token = None
logs, metadata = self.stackdriver_task_handler.read(self.ti)
mock_client.return_value.list_entries.assert_called_once_with(
filter_='resource.type="cloud_composer_environment"\n'
'logName="projects/asf-project/logs/airflow"\n'
'resource.labels."environment.name"="test-instancce"\n'
'resource.labels.location="europpe-west-3"\n'
'resource.labels.project_id="asf-project"\n'
'labels.task_id="task_for_testing_file_log_handler"\n'
'labels.dag_id="dag_for_testing_file_task_handler"\n'
'labels.execution_date="2016-01-01T00:00:00+00:00"',
page_token=None)
self.assertEqual(['TEXT\nTEXT'], logs)
self.assertEqual([{'end_of_log': True}], metadata)
def test_ctor_explicit(self):
import io
from google.cloud.logging.resource import Resource
resource = Resource("resource_type", {"resource_label": "value"})
labels = {"handler_lable": "value"}
name = "test-logger"
client = _Client(self.PROJECT)
stream = io.BytesIO()
handler = self._make_one(
client,
name=name,
transport=_Transport,
resource=resource,
labels=labels,
stream=stream,
)
self.assertEqual(handler.name, name)
self.assertIs(handler.client, client)
self.assertIsInstance(handler.transport, _Transport)
self.assertIs(handler.transport.client, client)
self.assertEqual(handler.transport.name, name)
self.assertIs(handler.resource, resource)
self.assertEqual(handler.labels, labels)
self.assertIs(handler.stream, stream)
def setup_logging(name, log_level=20):
task_id = str(uuid4())
logging.getLogger().setLevel(log_level)
if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') is None:
return task_id
res = Resource(
type="generic_task",
labels={
"location":
"vast.ai/{}".format(os.environ.get('VAST_CONTAINERLABEL', '')),
"task_id":
task_id,
"namespace":
"samplernn-pytorch",
"job":
"gen.py",
},
)
# Instantiates a client
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client, name, resource=res)
google.cloud.logging.handlers.setup_logging(handler, log_level=log_level)
return task_id
def add_mock_log_entry(self,
payload,
logger,
insert_id=None,
timestamp=None,
labels=None,
severity=None,
http_request=None,
resource=None):
if isinstance(resource, dict):
resource = Resource(**resource)
entry = StructEntry(
http_request=http_request,
insert_id=insert_id,
labels=labels,
logger=logger,
payload=payload,
resource=resource,
severity=severity,
timestamp=timestamp,
)
self.mock_log_entries.append(entry)
def test_should_pass_message_to_client(self, mock_client,
mock_get_creds_and_project_id):
self.addCleanup(_remove_stackdriver_handlers)
mock_get_creds_and_project_id.return_value = ('creds', 'project_id')
transport_type = mock.MagicMock()
stackdriver_task_handler = StackdriverTaskHandler(
transport=transport_type, labels={"key": 'value'})
logger = logging.getLogger("logger")
logger.addHandler(stackdriver_task_handler)
logger.info("test-message")
stackdriver_task_handler.flush()
transport_type.assert_called_once_with(mock_client.return_value,
'airflow')
transport_type.return_value.send.assert_called_once_with(
mock.ANY,
'test-message',
labels={"key": 'value'},
resource=Resource(type='global', labels={}))
mock_client.assert_called_once_with(credentials='creds',
client_info=mock.ANY,
project="project_id")
def log_struct(self, msg: str, struct: dict = {}, **kw):
"""Logs a structured message
Annotated with metadata about the event being handled.
Args:
msg: Text message to log via message key in log structure.
struct: Additional key/value attributes to log in the log structure.
**kw: (optional) additional keyword arguments for the entry. See
:class:`~google.cloud.logging.entries.LogEntry`.
"""
# Note: If the log name has a prefix of
# `cloudfunctions.googleapis.com/cloud-functions` then message will not
# be parsed from jsonPayload in the Console UI.
log_name = ('projects/{}/logs/reports%2F{}').format(
self.function_project, self.function_name)
jsonPayload = {'vm_uri': self.vm_uri}
jsonPayload.update(struct)
resource_labels = {
'function_name': self.function_name,
'project_id': self.function_project,
'region': self.function_region,
}
resource = Resource(labels=resource_labels, type='cloud_function')
log_entry = {
'log_name': log_name,
'labels': {
'event_id': self.event_id,
},
'severity': 'INFO',
'resource': resource,
}
log_entry.update(kw)
jsonPayload['message'] = msg
self.cloud_log.log_struct(info=jsonPayload, **log_entry)
def test_get_default_handler_general(self):
import io
from google.cloud.logging.handlers import CloudLoggingHandler
from google.cloud.logging.resource import Resource
name = "test-logger"
resource = Resource("resource_type", {"resource_label": "value"})
labels = {"handler_label": "value"}
stream = io.BytesIO()
credentials = _make_credentials()
client = self._make_one(
project=self.PROJECT, credentials=credentials, _use_grpc=False
)
handler = client.get_default_handler(
name=name, resource=resource, labels=labels, stream=stream
)
handler.transport.worker.stop()
self.assertIsInstance(handler, CloudLoggingHandler)
self.assertEqual(handler.name, name)
self.assertEqual(handler.resource, resource)
self.assertEqual(handler.labels, labels)
def get_handler(self):
# gcp has three independent implementation of api bindings for python.
# The one used by logging is not yet supported by our test recording.
# TODO drop these grpc variants for the REST versions, and we can drop
# protobuf/grpc deps, and also so we can record tests..
# gcp has three different python sdks all independently maintained .. hmmm...
# and random monkey shims on top of those :-(
from google.cloud.logging import Client as LogClient
from google.cloud.logging.handlers import CloudLoggingHandler
from google.cloud.logging.resource import Resource
log_group = self.ctx.options.log_group
if log_group.endswith('*'):
log_group = "%s%s" % (log_group[:-1], self.ctx.policy.name)
project_id = local_session(
self.ctx.session_factory).get_default_project()
client = LogClient(project_id)
return CloudLoggingHandler(client,
log_group,
resource=Resource(
type='project',
labels={'project_id': project_id}))
def get_instance_resource():
def _get_instance_id():
resp = requests.get("http://metadata/computeMetadata/v1/instance/id",
headers={"Metadata-Flavor": "Google"})
return resp.content.decode()
instance_id = os.environ.get('GCP_INSTANCE_ID') or _get_instance_id()
return Resource(type="gce_instance", labels={"instance_id": instance_id})
def log_resource(self):
return Resource(type="cloud_function",
labels={
"function_name": self.function_name,
"region": self.region,
"correlation_id": self.correlation_id or "missing"
},
)
def test_log_proto_explicit(self):
import datetime
from google.cloud.logging.resource import Resource
from google.cloud.logging.entries import ProtobufEntry
from google.protobuf.struct_pb2 import Struct
from google.protobuf.struct_pb2 import Value
message = Struct(fields={'foo': Value(bool_value=True)})
LABELS = {'foo': 'bar', 'baz': 'qux'}
IID = 'IID'
SEVERITY = 'CRITICAL'
METHOD = 'POST'
URI = 'https://api.example.com/endpoint'
STATUS = '500'
TRACE = '12345678-1234-5678-1234-567812345678'
SPANID = '000000000000004a'
REQUEST = {
'requestMethod': METHOD,
'requestUrl': URI,
'status': STATUS,
}
TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999)
RESOURCE = Resource(
type='gae_app',
labels={
'module_id': 'default',
'version_id': 'test',
}
)
ENTRY = ProtobufEntry(
payload=message,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
)
client = _Client(project=self.PROJECT, connection=_make_credentials())
logger = _Logger()
batch = self._make_one(logger, client=client)
batch.log_proto(
message,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
)
self.assertEqual(batch.entries, [ENTRY])
def test_log_proto_explicit(self):
import datetime
from google.cloud.logging.resource import Resource
from google.cloud.logging.entries import ProtobufEntry
from google.protobuf.struct_pb2 import Struct
from google.protobuf.struct_pb2 import Value
message = Struct(fields={"foo": Value(bool_value=True)})
LABELS = {"foo": "bar", "baz": "qux"}
IID = "IID"
SEVERITY = "CRITICAL"
METHOD = "POST"
URI = "https://api.example.com/endpoint"
STATUS = "500"
TRACE = "12345678-1234-5678-1234-567812345678"
SPANID = "000000000000004a"
REQUEST = {
"requestMethod": METHOD,
"requestUrl": URI,
"status": STATUS
}
TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999)
RESOURCE = Resource(type="gae_app",
labels={
"module_id": "default",
"version_id": "test"
})
ENTRY = ProtobufEntry(
payload=message,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
)
client = _Client(project=self.PROJECT, connection=_make_credentials())
logger = _Logger()
batch = self._make_one(logger, client=client)
batch.log_proto(
message,
labels=LABELS,
insert_id=IID,
severity=SEVERITY,
http_request=REQUEST,
timestamp=TIMESTAMP,
resource=RESOURCE,
trace=TRACE,
span_id=SPANID,
trace_sampled=True,
)
self.assertEqual(batch.entries, [ENTRY])
def bigquery_job(job_name, bucket, file, dataset, table, truncate):
global logger, grupo_log, hora_ini, res
try:
#Inicio carga
hora_ini = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
#Configuracoes Estrutura Logging
log_name = 'log_data_ingestion'
grupo_log = 'DATA_INGESTION'
#Cria o objeto logger para o log de carga
res = Resource(type="cloud_function",
labels={
"function_name": job_name,
"region": "us-central1"
}
)
logging_client = logging.Client()
logger = logging_client.logger(log_name)
stg_client = storage.Client()
bkt = stg_client.get_bucket(bucket)
sql = bkt.get_blob(file).download_as_string().decode('utf-8')
bigquery_client = bigquery.Client()
dataset_ref = bigquery_client.get_dataset(dataset)
table_ref = dataset_ref.table(table)
table_id = bigquery_client.get_table(table_ref)
job_config = bigquery.QueryJobConfig()
if truncate == 'true':
sql_truncate = 'truncate table ' + dataset + '.' + table
query_job = bigquery_client.query(sql_truncate, location='US')
query_job.result()
else:
job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND
job_config.schema = table_id.schema
job_config.use_legacy_sql = False
job_config.destination = table_ref
job_config.create_disposition = 'CREATE_NEVER'
query_job = bigquery_client.query(sql, location='US', job_config=job_config)
query_job.result()
logger.log_struct({'Grupo_Log': grupo_log, 'Status_Execucao': 'Sucesso', 'Nome_Job': job_name,
'Dataset': dataset, 'Tabela': table, 'Inicio': hora_ini, 'Fim': datetime.now().strftime('%Y-%m-%d %H:%M:%S')},
severity='INFO', resource=res)
except Exception as ex:
logger.log_struct({'Grupo_Log': grupo_log, 'Status_Execucao': 'Erro', 'Nome_Job': job_name,
'Dataset': dataset, 'Tabela': table, 'Inicio': hora_ini,
'Fim': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'Msg_Erro': str(ex)},
severity='ERROR', resource=res)
def gen_google_cloud_logger():
log_client = google.cloud.logging.Client()
res = Resource(type="cloud_function",
labels={
"function_name": "refresh_classes",
"region": os.environ.get("FUNC_REGION")
})
return log_client.logger(
'cloudfunctions.googleapis.com%2Fcloud-functions'.format(
os.environ.get("DEFAULT_PROJECT_ID"))), res
def write(text,
severity='INFO',
show=None,
seq=None,
shot=None,
role=None,
**kwargs):
'''Wrapper method for assembling the payload to send to logger.log_text.'''
# Extract and build LOG_ID from environment.
# For example: 'myfilm.slb.0050.render'
if not show:
show = os.getenv('SHOW')
if not seq:
seq = os.getenv('SEQ')
if not shot:
shot = os.getenv('SHOT')
if not role:
role = os.getenv('ROLE')
if not show or not seq or not shot or not role:
raise Exception(
'One or more log name tokens are empty. Unable to log.')
# end if
# Assemble logger name.
logger_name = '.'.join([show, seq, shot, role])
print '# Logging to %s...' % logger_name
# Build logger object.
logging_client = logging.Client()
logger = logging_client.logger(logger_name)
# Assemble the required log metadata.
label_payload = {
"artist": getpass.getuser(),
"hostname": socket.gethostname(),
"show": show,
"seq": seq,
"shot": shot,
"role": role
}
# Add optional kwargs to payload.
label_payload.update(kwargs)
# Write log.
logger.log_text(text,
resource=Resource(type='project',
labels={'project_id': show}),
severity=severity,
labels=label_payload)
def logger_usage(client, to_delete):
"""Logger usage."""
LOG_NAME = "logger_usage_%d" % (_millis())
# [START logger_create]
logger = client.logger(LOG_NAME)
# [END logger_create]
to_delete.append(logger)
# [START logger_log_text]
logger.log_text("A simple entry") # API call
# [END logger_log_text]
# [START logger_log_struct]
logger.log_struct({
"message": "My second entry",
"weather": "partly cloudy"
}) # API call
# [END logger_log_struct]
# [START logger_log_resource_text]
from google.cloud.logging.resource import Resource
res = Resource(
type="generic_node",
labels={
"location": "us-central1-a",
"namespace": "default",
"node_id": "10.10.10.1",
},
)
logger.log_struct({
"message": "My first entry",
"weather": "partly cloudy"
},
resource=res)
# [END logger_log_resource_text]
# [START logger_list_entries]
from google.cloud.logging import DESCENDING
for entry in logger.list_entries(order_by=DESCENDING): # API call(s)
do_something_with(entry)
# [END logger_list_entries]
def _logger_delete():
# [START logger_delete]
logger.delete() # API call
# [END logger_delete]
_backoff_not_found(_logger_delete)
to_delete.remove(logger)
def log(self, level, message, *args, **kwargs):
"""
Override the log method because the log level for the current
message is not passed in process.
"""
if not self.isEnabledFor(level):
return
message = message.strip()
message, kwargs = self.process(message, kwargs)
app_id = self.extra['app_id']
version = self.extra['version']
if cloud_logger_bg is not None:
# The following usage is highly unconventional:
# There appears to be no way to log a structured log message
# asynchronously using the cloud logging client properly.
# However, internally, the library uses structured logging. As long
# as we queue the right payload into the worker's queue, we're OK.
# This works well with google-cloud-logging==1.9.0.
# It looks shady, but the only decent way to use the async logging
# support for Stackdriver.
cloud_logger_bg.worker._queue.put_nowait({
'info': {
'app_id': app_id,
'version': version,
'message': message,
'level': getLevelName(level)
},
'severity':
getLevelName(level),
'resource':
Resource(type='global', labels={})
})
if log_json:
json_log = {
'app_id': app_id,
'version': version,
'level': getLevelName(level),
'message': message
}
if kwargs.get('exc_info') is not None:
tb = traceback.format_exc()
json_log['message'] += '\n' + tb
self.logger.log(level, json_log, *args, **kwargs)
else:
message_pretty = f'{app_id}::{version} => {message}'
self.logger.log(level, message_pretty, *args, **kwargs)
def from_api_repr(cls, resource, client, loggers=None):
"""Factory: construct an entry given its API representation
:type resource: dict
:param resource: text entry resource representation returned from
the API
:type client: :class:`google.cloud.logging.client.Client`
:param client: Client which holds credentials and project
configuration.
:type loggers: dict
:param loggers:
(Optional) A mapping of logger fullnames -> loggers. If not
passed, the entry will have a newly-created logger.
:rtype: :class:`google.cloud.logging.entries._BaseEntry`
:returns: Text entry parsed from ``resource``.
"""
if loggers is None:
loggers = {}
logger_fullname = resource['logName']
logger = loggers.get(logger_fullname)
if logger is None:
logger_name = logger_name_from_path(logger_fullname)
logger = loggers[logger_fullname] = client.logger(logger_name)
payload = resource[cls._PAYLOAD_KEY]
insert_id = resource.get('insertId')
timestamp = resource.get('timestamp')
if timestamp is not None:
timestamp = _rfc3339_nanos_to_datetime(timestamp)
labels = resource.get('labels')
severity = resource.get('severity')
http_request = resource.get('httpRequest')
trace = resource.get('trace')
span_id = resource.get('spanId')
monitored_resource_dict = resource.get('resource')
monitored_resource = None
if monitored_resource_dict is not None:
monitored_resource = Resource._from_dict(monitored_resource_dict)
return cls(payload,
logger,
insert_id=insert_id,
timestamp=timestamp,
labels=labels,
severity=severity,
http_request=http_request,
resource=monitored_resource,
trace=trace,
span_id=span_id)
def get_handler(self):
# TODO drop these grpc variants for the REST versions, and we can drop
# protobuf/grpc deps, and also so we can record tests.
log_group = self.get_log_group()
project_id = local_session(self.ctx.session_factory).get_default_project()
client = LogClient(project_id)
return CloudLoggingHandler(
client,
log_group,
labels={
'policy': self.ctx.policy.name,
'resource': self.ctx.policy.resource_type},
resource=Resource(type='project', labels={'project_id': project_id}))
def _process_log(log, context):
"""Process logs by writing them out to CloudWatch and Stackdriver."""
res = Resource(
type='generic_task',
labels={
'location': REGION,
'namespace': 'default',
'job': 'router',
'task_id': context.aws_request_id,
},
)
stackdriver.log_struct(log, resource=res)
print(log)
def get_gae_resource(self):
"""Return the GAE resource using the environment variables.
:rtype: :class:`~google.cloud.logging.resource.Resource`
:returns: Monitored resource for GAE.
"""
gae_resource = Resource(
type="gae_app",
labels={
"project_id": self.project_id,
"module_id": self.module_id,
"version_id": self.version_id,
},
)
return gae_resource
def get_gae_resource(self):
"""Return the GAE resource using the environment variables.
:rtype: :class:`~google.cloud.logging.resource.Resource`
:returns: Monitored resource for GAE.
"""
gae_resource = Resource(
type='gae_app',
labels={
'project_id': os.environ.get(_GAE_PROJECT_ENV),
'module_id': os.environ.get(_GAE_SERVICE_ENV),
'version_id': os.environ.get(_GAE_VERSION_ENV),
},
)
return gae_resource
def test_should_set_labels(self, mock_client):
self.stackdriver_task_handler.set_context(self.ti)
self.logger.addHandler(self.stackdriver_task_handler)
self.logger.info("test-message")
self.stackdriver_task_handler.flush()
labels = {
'task_id': 'task_for_testing_file_log_handler',
'dag_id': 'dag_for_testing_file_task_handler',
'execution_date': '2016-01-01T00:00:00+00:00',
'try_number': '1'
}
resource = Resource(type='global', labels={})
self.transport_mock.return_value.send.assert_called_once_with(
mock.ANY, 'test-message', labels=labels, resource=resource)
def init_logging(client,
level=logging.INFO,
resource_type=None,
resource_labels=None,
labels={}):
"""
Call this once at the top of your main program to log via Stackdriver.
"""
if isinstance(level, str):
level = getattr(logging, level)
resource = (Resource(type=resource_type, labels=resource_labels)
if resource_type is not None else None)
handler = client.get_default_handler(resource=resource, labels=labels)
handler.setFormatter(LogFormatter(*FORMATTER_ARGS))
root_logger = logging.getLogger()
root_logger.addHandler(handler)
root_logger.setLevel(level)
def write_to_stackdriver(self, subs):
"""Writes an entry to Stackdriver log.
Args:
subs: number of subscriptions
Returns: None
"""
log_client = logging.Client()
logger = log_client.logger('quota-watcher')
res = Resource(
type='pubsub_subscription',
labels={
'usage': 'quota_watcher',
},
)
logger.log_struct({'total_subscriptions': subs}, resource=res)
def get_handler(self):
# gcp has three independent implementation of api bindings for python.
# The one used by logging is not yet supported by our test recording.
log_group = self.ctx.options.log_group
if log_group.endswith('*'):
log_group = "%s%s" % (log_group[:-1], self.ctx.policy.name)
project_id = local_session(
self.ctx.session_factory).get_default_project()
client = LogClient(project_id)
return CloudLoggingHandler(client,
log_group,
resource=Resource(
type='project',
labels={'project_id': project_id}))
def test_should_pass_message_to_client(self, mock_client):
transport_type = mock.MagicMock()
stackdriver_task_handler = StackdriverTaskHandler(
gcp_conn_id=None,
transport=transport_type,
labels={"key": 'value'}
)
logger = logging.getLogger("logger")
logger.addHandler(stackdriver_task_handler)
logger.info("test-message")
stackdriver_task_handler.flush()
transport_type.assert_called_once_with(mock_client.return_value, 'airflow')
transport_type.return_value.send.assert_called_once_with(
mock.ANY, 'test-message', labels={"key": 'value'}, resource=Resource(type='global', labels={})
)
mock_client.assert_called_once()
def from_api_repr(cls, resource, client, loggers=None):
"""Factory: construct an entry given its API representation
:type resource: dict
:param resource: text entry resource representation returned from
the API
:type client: :class:`google.cloud.logging.client.Client`
:param client: Client which holds credentials and project
configuration.
:type loggers: dict
:param loggers:
(Optional) A mapping of logger fullnames -> loggers. If not
passed, the entry will have a newly-created logger.
:rtype: :class:`google.cloud.logging.entries._BaseEntry`
:returns: Text entry parsed from ``resource``.
"""
if loggers is None:
loggers = {}
logger_fullname = resource['logName']
logger = loggers.get(logger_fullname)
if logger is None:
logger_name = logger_name_from_path(logger_fullname)
logger = loggers[logger_fullname] = client.logger(logger_name)
payload = resource[cls._PAYLOAD_KEY]
insert_id = resource.get('insertId')
timestamp = resource.get('timestamp')
if timestamp is not None:
timestamp = _rfc3339_nanos_to_datetime(timestamp)
labels = resource.get('labels')
severity = resource.get('severity')
http_request = resource.get('httpRequest')
trace = resource.get('trace')
monitored_resource_dict = resource.get('resource')
monitored_resource = None
if monitored_resource_dict is not None:
monitored_resource = Resource._from_dict(monitored_resource_dict)
return cls(payload, logger, insert_id=insert_id, timestamp=timestamp,
labels=labels, severity=severity, http_request=http_request,
resource=monitored_resource, trace=trace)
def test_from_api_repr_w_loggers_no_logger_match(self):
from datetime import datetime
from google.cloud._helpers import UTC
from google.cloud.logging.resource import Resource
klass = self._get_target_class()
client = _Client(self.PROJECT)
SEVERITY = "CRITICAL"
IID = "IID"
NOW = datetime.utcnow().replace(tzinfo=UTC)
TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW)
LOG_NAME = "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME)
LABELS = {"foo": "bar", "baz": "qux"}
METHOD = "POST"
URI = "https://api.example.com/endpoint"
RESOURCE = Resource(
type="gae_app",
labels={
"type": "gae_app",
"labels": {"module_id": "default", "version": "test"},
},
)
STATUS = "500"
TRACE = "12345678-1234-5678-1234-567812345678"
SPANID = "000000000000004a"
FILE = "my_file.py"
LINE_NO = 123
FUNCTION = "my_function"
SOURCE_LOCATION = {"file": FILE, "line": str(LINE_NO), "function": FUNCTION}
OP_ID = "OP_ID"
PRODUCER = "PRODUCER"
OPERATION = {"id": OP_ID, "producer": PRODUCER, "first": True, "last": False}
API_REPR = {
"logName": LOG_NAME,
"insertId": IID,
"timestamp": TIMESTAMP,
"labels": LABELS,
"severity": SEVERITY,
"httpRequest": {
"requestMethod": METHOD,
"requestUrl": URI,
"status": STATUS,
},
"resource": RESOURCE._to_dict(),
"trace": TRACE,
"spanId": SPANID,
"traceSampled": True,
"sourceLocation": SOURCE_LOCATION,
"operation": OPERATION,
}
loggers = {}
entry = klass.from_api_repr(API_REPR, client, loggers=loggers)
self.assertEqual(entry.log_name, LOG_NAME)
logger = entry.logger
self.assertIsInstance(logger, _Logger)
self.assertEqual(logger.name, self.LOGGER_NAME)
self.assertEqual(entry.insert_id, IID)
self.assertEqual(entry.timestamp, NOW)
self.assertIsNone(entry.received_timestamp)
self.assertEqual(entry.labels, LABELS)
self.assertEqual(entry.severity, SEVERITY)
self.assertEqual(entry.http_request["requestMethod"], METHOD)
self.assertEqual(entry.http_request["requestUrl"], URI)
self.assertEqual(entry.http_request["status"], STATUS)
self.assertIs(logger.client, client)
self.assertEqual(logger.name, self.LOGGER_NAME)
self.assertEqual(loggers, {LOG_NAME: logger})
self.assertEqual(entry.resource, RESOURCE)
self.assertEqual(entry.trace, TRACE)
self.assertEqual(entry.span_id, SPANID)
self.assertTrue(entry.trace_sampled)
source_location = entry.source_location
self.assertEqual(source_location["file"], FILE)
self.assertEqual(source_location["line"], LINE_NO)
self.assertEqual(source_location["function"], FUNCTION)
self.assertEqual(entry.operation, OPERATION)
self.assertIsNone(entry.payload)
def from_api_repr(cls, resource, client, loggers=None):
"""Factory: construct an entry given its API representation
:type resource: dict
:param resource: text entry resource representation returned from
the API
:type client: :class:`google.cloud.logging.client.Client`
:param client: Client which holds credentials and project
configuration.
:type loggers: dict
:param loggers:
(Optional) A mapping of logger fullnames -> loggers. If not
passed, the entry will have a newly-created logger.
:rtype: :class:`google.cloud.logging.entries.LogEntry`
:returns: Log entry parsed from ``resource``.
"""
if loggers is None:
loggers = {}
logger_fullname = resource["logName"]
logger = loggers.get(logger_fullname)
if logger is None:
logger_name = logger_name_from_path(logger_fullname)
logger = loggers[logger_fullname] = client.logger(logger_name)
payload = cls._extract_payload(resource)
insert_id = resource.get("insertId")
timestamp = resource.get("timestamp")
if timestamp is not None:
timestamp = _rfc3339_nanos_to_datetime(timestamp)
labels = resource.get("labels")
severity = resource.get("severity")
http_request = resource.get("httpRequest")
trace = resource.get("trace")
span_id = resource.get("spanId")
trace_sampled = resource.get("traceSampled")
source_location = resource.get("sourceLocation")
if source_location is not None:
line = source_location.pop("line", None)
source_location["line"] = _int_or_none(line)
operation = resource.get("operation")
monitored_resource_dict = resource.get("resource")
monitored_resource = None
if monitored_resource_dict is not None:
monitored_resource = Resource._from_dict(monitored_resource_dict)
inst = cls(
log_name=logger_fullname,
insert_id=insert_id,
timestamp=timestamp,
labels=labels,
severity=severity,
http_request=http_request,
resource=monitored_resource,
trace=trace,
span_id=span_id,
trace_sampled=trace_sampled,
source_location=source_location,
operation=operation,
logger=logger,
payload=payload,
)
received = resource.get("receiveTimestamp")
if received is not None:
inst.received_timestamp = _rfc3339_nanos_to_datetime(received)
return inst
