def test_log_empty_w_explicit(self): import datetime from google.cloud.logging.resource import Resource ALT_LOG_NAME = "projects/foo/logs/alt.log.name" DEFAULT_LABELS = {"foo": "spam"} LABELS = {"foo": "bar", "baz": "qux"} IID = "IID" SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" TRACE = "12345678-1234-5678-1234-567812345678" SPANID = "000000000000004a" REQUEST = { "requestMethod": METHOD, "requestUrl": URI, "status": STATUS } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource(type="gae_app", labels={ "module_id": "default", "version_id": "test" }) ENTRIES = [{ "logName": ALT_LOG_NAME, "labels": LABELS, "insertId": IID, "severity": SEVERITY, "httpRequest": REQUEST, "timestamp": "2016-12-31T00:01:02.999999Z", "resource": RESOURCE._to_dict(), "trace": TRACE, "spanId": SPANID, "traceSampled": True, }] client1 = _Client(self.PROJECT) client2 = _Client(self.PROJECT) api = client2.logging_api = _DummyLoggingAPI() logger = self._make_one(self.LOGGER_NAME, client=client1, labels=DEFAULT_LABELS) logger.log_empty( log_name=ALT_LOG_NAME, client=client2, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None))
def test_from_api_repr_w_loggers_no_logger_match(self): from datetime import datetime from google.cloud._helpers import UTC from google.cloud.logging.resource import Resource klass = self._get_target_class() client = _Client(self.PROJECT) PAYLOAD = 'PAYLOAD' SEVERITY = 'CRITICAL' IID = 'IID' NOW = datetime.utcnow().replace(tzinfo=UTC) TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW) LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME) LABELS = {'foo': 'bar', 'baz': 'qux'} METHOD = 'POST' URI = 'https://api.example.com/endpoint' RESOURCE = Resource( type='gae_app', labels={ 'type': 'gae_app', 'labels': { 'module_id': 'default', 'version': 'test', } } ) STATUS = '500' TRACE = '12345678-1234-5678-1234-567812345678' API_REPR = { 'dummyPayload': PAYLOAD, 'logName': LOG_NAME, 'insertId': IID, 'timestamp': TIMESTAMP, 'labels': LABELS, 'severity': SEVERITY, 'httpRequest': { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, }, 'resource': RESOURCE._to_dict(), 'trace': TRACE } loggers = {} entry = klass.from_api_repr(API_REPR, client, loggers=loggers) self.assertEqual(entry.payload, PAYLOAD) self.assertEqual(entry.insert_id, IID) self.assertEqual(entry.timestamp, NOW) self.assertEqual(entry.labels, LABELS) self.assertEqual(entry.severity, SEVERITY) self.assertEqual(entry.http_request['requestMethod'], METHOD) self.assertEqual(entry.http_request['requestUrl'], URI) self.assertEqual(entry.http_request['status'], STATUS) logger = entry.logger self.assertIsInstance(logger, _Logger) self.assertIs(logger.client, client) self.assertEqual(logger.name, self.LOGGER_NAME) self.assertEqual(loggers, {LOG_NAME: logger}) self.assertEqual(entry.resource, RESOURCE) self.assertEqual(entry.trace, TRACE)
def test_commit_w_resource_specified(self): from google.cloud.logging.logger import _GLOBAL_RESOURCE from google.cloud.logging.resource import Resource logger = _Logger() client = _Client(project=self.PROJECT, connection=_make_credentials()) api = client.logging_api = _DummyLoggingAPI() RESOURCE = Resource(type='gae_app', labels={ 'module_id': 'default', 'version_id': 'test', }) batch = self._make_one(logger, client, resource=RESOURCE) MESSAGE = 'This is the entry text' ENTRIES = [ { 'textPayload': MESSAGE }, { 'textPayload': MESSAGE, 'resource': _GLOBAL_RESOURCE._to_dict() }, ] batch.log_text(MESSAGE, resource=None) batch.log_text(MESSAGE) batch.commit() self.assertEqual( api._write_entries_called_with, (ENTRIES, logger.full_name, RESOURCE._to_dict(), None))
def test_commit_w_resource_specified(self): from google.cloud.logging.logger import _GLOBAL_RESOURCE from google.cloud.logging.resource import Resource logger = _Logger() client = _Client(project=self.PROJECT, connection=_make_credentials()) api = client.logging_api = _DummyLoggingAPI() RESOURCE = Resource( type='gae_app', labels={ 'module_id': 'default', 'version_id': 'test', } ) batch = self._make_one(logger, client, resource=RESOURCE) MESSAGE = 'This is the entry text' ENTRIES = [ {'textPayload': MESSAGE}, {'textPayload': MESSAGE, 'resource': _GLOBAL_RESOURCE._to_dict()}, ] batch.log_text(MESSAGE, resource=None) batch.log_text(MESSAGE) batch.commit() self.assertEqual(api._write_entries_called_with, (ENTRIES, logger.full_name, RESOURCE._to_dict(), None))
def test_to_api_repr_explicit(self): import datetime from google.cloud.logging.resource import Resource from google.cloud._helpers import _datetime_to_rfc3339 LOG_NAME = "test.log" JSON_PAYLOAD = {"key": "value"} LABELS = {"foo": "bar", "baz": "qux"} IID = "IID" SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" REQUEST = {"requestMethod": METHOD, "requestUrl": URI, "status": STATUS} TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource( type="gae_app", labels={"module_id": "default", "version_id": "test"} ) TRACE = "12345678-1234-5678-1234-567812345678" SPANID = "000000000000004a" FILE = "my_file.py" LINE = 123 FUNCTION = "my_function" SOURCE_LOCATION = {"file": FILE, "line": LINE, "function": FUNCTION} OP_ID = "OP_ID" PRODUCER = "PRODUCER" OPERATION = {"id": OP_ID, "producer": PRODUCER, "first": True, "last": False} expected = { "logName": LOG_NAME, "jsonPayload": JSON_PAYLOAD, "labels": LABELS, "insertId": IID, "severity": SEVERITY, "httpRequest": REQUEST, "timestamp": _datetime_to_rfc3339(TIMESTAMP), "resource": RESOURCE._to_dict(), "trace": TRACE, "spanId": SPANID, "traceSampled": True, "sourceLocation": {"file": FILE, "line": str(LINE), "function": FUNCTION}, "operation": OPERATION, } entry = self._make_one( log_name=LOG_NAME, payload=JSON_PAYLOAD, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, source_location=SOURCE_LOCATION, operation=OPERATION, ) self.assertEqual(entry.to_api_repr(), expected)
def test_from_api_repr_w_loggers_no_logger_match(self): from datetime import datetime from google.cloud._helpers import UTC from google.cloud.logging.resource import Resource klass = self._get_target_class() client = _Client(self.PROJECT) PAYLOAD = 'PAYLOAD' SEVERITY = 'CRITICAL' IID = 'IID' NOW = datetime.utcnow().replace(tzinfo=UTC) TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW) LOG_NAME = 'projects/%s/logs/%s' % (self.PROJECT, self.LOGGER_NAME) LABELS = {'foo': 'bar', 'baz': 'qux'} METHOD = 'POST' URI = 'https://api.example.com/endpoint' RESOURCE = Resource(type='gae_app', labels={ 'type': 'gae_app', 'labels': { 'module_id': 'default', 'version': 'test', } }) STATUS = '500' TRACE = '12345678-1234-5678-1234-567812345678' API_REPR = { 'dummyPayload': PAYLOAD, 'logName': LOG_NAME, 'insertId': IID, 'timestamp': TIMESTAMP, 'labels': LABELS, 'severity': SEVERITY, 'httpRequest': { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, }, 'resource': RESOURCE._to_dict(), 'trace': TRACE } loggers = {} entry = klass.from_api_repr(API_REPR, client, loggers=loggers) self.assertEqual(entry.payload, PAYLOAD) self.assertEqual(entry.insert_id, IID) self.assertEqual(entry.timestamp, NOW) self.assertEqual(entry.labels, LABELS) self.assertEqual(entry.severity, SEVERITY) self.assertEqual(entry.http_request['requestMethod'], METHOD) self.assertEqual(entry.http_request['requestUrl'], URI) self.assertEqual(entry.http_request['status'], STATUS) logger = entry.logger self.assertIsInstance(logger, _Logger) self.assertIs(logger.client, client) self.assertEqual(logger.name, self.LOGGER_NAME) self.assertEqual(loggers, {LOG_NAME: logger}) self.assertEqual(entry.resource, RESOURCE) self.assertEqual(entry.trace, TRACE)
def test_should_read_logs_with_custom_resources( self, mock_client, mock_get_creds_and_project_id): mock_get_creds_and_project_id.return_value = ('creds', 'project_id') resource = Resource( type="cloud_composer_environment", labels={ "environment.name": 'test-instancce', "location": 'europpe-west-3', "project_id": "asf-project", }, ) self.stackdriver_task_handler = StackdriverTaskHandler( transport=self.transport_mock, resource=resource) entry = mock.MagicMock(payload={"message": "TEXT"}) page = [entry, entry] mock_client.return_value.list_entries.return_value.pages = ( n for n in [page]) mock_client.return_value.list_entries.return_value.next_page_token = None logs, metadata = self.stackdriver_task_handler.read(self.ti) mock_client.return_value.list_entries.assert_called_once_with( filter_='resource.type="cloud_composer_environment"\n' 'logName="projects/asf-project/logs/airflow"\n' 'resource.labels."environment.name"="test-instancce"\n' 'resource.labels.location="europpe-west-3"\n' 'resource.labels.project_id="asf-project"\n' 'labels.task_id="task_for_testing_file_log_handler"\n' 'labels.dag_id="dag_for_testing_file_task_handler"\n' 'labels.execution_date="2016-01-01T00:00:00+00:00"', page_token=None) self.assertEqual(['TEXT\nTEXT'], logs) self.assertEqual([{'end_of_log': True}], metadata)
def test_ctor_explicit(self): import io from google.cloud.logging.resource import Resource resource = Resource("resource_type", {"resource_label": "value"}) labels = {"handler_lable": "value"} name = "test-logger" client = _Client(self.PROJECT) stream = io.BytesIO() handler = self._make_one( client, name=name, transport=_Transport, resource=resource, labels=labels, stream=stream, ) self.assertEqual(handler.name, name) self.assertIs(handler.client, client) self.assertIsInstance(handler.transport, _Transport) self.assertIs(handler.transport.client, client) self.assertEqual(handler.transport.name, name) self.assertIs(handler.resource, resource) self.assertEqual(handler.labels, labels) self.assertIs(handler.stream, stream)
def setup_logging(name, log_level=20): task_id = str(uuid4()) logging.getLogger().setLevel(log_level) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') is None: return task_id res = Resource( type="generic_task", labels={ "location": "vast.ai/{}".format(os.environ.get('VAST_CONTAINERLABEL', '')), "task_id": task_id, "namespace": "samplernn-pytorch", "job": "gen.py", }, ) # Instantiates a client client = google.cloud.logging.Client() handler = CloudLoggingHandler(client, name, resource=res) google.cloud.logging.handlers.setup_logging(handler, log_level=log_level) return task_id
def add_mock_log_entry(self, payload, logger, insert_id=None, timestamp=None, labels=None, severity=None, http_request=None, resource=None): if isinstance(resource, dict): resource = Resource(**resource) entry = StructEntry( http_request=http_request, insert_id=insert_id, labels=labels, logger=logger, payload=payload, resource=resource, severity=severity, timestamp=timestamp, ) self.mock_log_entries.append(entry)
def test_should_pass_message_to_client(self, mock_client, mock_get_creds_and_project_id): self.addCleanup(_remove_stackdriver_handlers) mock_get_creds_and_project_id.return_value = ('creds', 'project_id') transport_type = mock.MagicMock() stackdriver_task_handler = StackdriverTaskHandler( transport=transport_type, labels={"key": 'value'}) logger = logging.getLogger("logger") logger.addHandler(stackdriver_task_handler) logger.info("test-message") stackdriver_task_handler.flush() transport_type.assert_called_once_with(mock_client.return_value, 'airflow') transport_type.return_value.send.assert_called_once_with( mock.ANY, 'test-message', labels={"key": 'value'}, resource=Resource(type='global', labels={})) mock_client.assert_called_once_with(credentials='creds', client_info=mock.ANY, project="project_id")
def log_struct(self, msg: str, struct: dict = {}, **kw): """Logs a structured message Annotated with metadata about the event being handled. Args: msg: Text message to log via message key in log structure. struct: Additional key/value attributes to log in the log structure. **kw: (optional) additional keyword arguments for the entry. See :class:`~google.cloud.logging.entries.LogEntry`. """ # Note: If the log name has a prefix of # `cloudfunctions.googleapis.com/cloud-functions` then message will not # be parsed from jsonPayload in the Console UI. log_name = ('projects/{}/logs/reports%2F{}').format( self.function_project, self.function_name) jsonPayload = {'vm_uri': self.vm_uri} jsonPayload.update(struct) resource_labels = { 'function_name': self.function_name, 'project_id': self.function_project, 'region': self.function_region, } resource = Resource(labels=resource_labels, type='cloud_function') log_entry = { 'log_name': log_name, 'labels': { 'event_id': self.event_id, }, 'severity': 'INFO', 'resource': resource, } log_entry.update(kw) jsonPayload['message'] = msg self.cloud_log.log_struct(info=jsonPayload, **log_entry)
def test_get_default_handler_general(self): import io from google.cloud.logging.handlers import CloudLoggingHandler from google.cloud.logging.resource import Resource name = "test-logger" resource = Resource("resource_type", {"resource_label": "value"}) labels = {"handler_label": "value"} stream = io.BytesIO() credentials = _make_credentials() client = self._make_one( project=self.PROJECT, credentials=credentials, _use_grpc=False ) handler = client.get_default_handler( name=name, resource=resource, labels=labels, stream=stream ) handler.transport.worker.stop() self.assertIsInstance(handler, CloudLoggingHandler) self.assertEqual(handler.name, name) self.assertEqual(handler.resource, resource) self.assertEqual(handler.labels, labels)
def get_handler(self): # gcp has three independent implementation of api bindings for python. # The one used by logging is not yet supported by our test recording. # TODO drop these grpc variants for the REST versions, and we can drop # protobuf/grpc deps, and also so we can record tests.. # gcp has three different python sdks all independently maintained .. hmmm... # and random monkey shims on top of those :-( from google.cloud.logging import Client as LogClient from google.cloud.logging.handlers import CloudLoggingHandler from google.cloud.logging.resource import Resource log_group = self.ctx.options.log_group if log_group.endswith('*'): log_group = "%s%s" % (log_group[:-1], self.ctx.policy.name) project_id = local_session( self.ctx.session_factory).get_default_project() client = LogClient(project_id) return CloudLoggingHandler(client, log_group, resource=Resource( type='project', labels={'project_id': project_id}))
def get_instance_resource(): def _get_instance_id(): resp = requests.get("http://metadata/computeMetadata/v1/instance/id", headers={"Metadata-Flavor": "Google"}) return resp.content.decode() instance_id = os.environ.get('GCP_INSTANCE_ID') or _get_instance_id() return Resource(type="gce_instance", labels={"instance_id": instance_id})
def log_resource(self): return Resource(type="cloud_function", labels={ "function_name": self.function_name, "region": self.region, "correlation_id": self.correlation_id or "missing" }, )
def test_log_proto_explicit(self): import datetime from google.cloud.logging.resource import Resource from google.cloud.logging.entries import ProtobufEntry from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={'foo': Value(bool_value=True)}) LABELS = {'foo': 'bar', 'baz': 'qux'} IID = 'IID' SEVERITY = 'CRITICAL' METHOD = 'POST' URI = 'https://api.example.com/endpoint' STATUS = '500' TRACE = '12345678-1234-5678-1234-567812345678' SPANID = '000000000000004a' REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, 'status': STATUS, } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource( type='gae_app', labels={ 'module_id': 'default', 'version_id': 'test', } ) ENTRY = ProtobufEntry( payload=message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) client = _Client(project=self.PROJECT, connection=_make_credentials()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto( message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) self.assertEqual(batch.entries, [ENTRY])
def test_log_proto_explicit(self): import datetime from google.cloud.logging.resource import Resource from google.cloud.logging.entries import ProtobufEntry from google.protobuf.struct_pb2 import Struct from google.protobuf.struct_pb2 import Value message = Struct(fields={"foo": Value(bool_value=True)}) LABELS = {"foo": "bar", "baz": "qux"} IID = "IID" SEVERITY = "CRITICAL" METHOD = "POST" URI = "https://api.example.com/endpoint" STATUS = "500" TRACE = "12345678-1234-5678-1234-567812345678" SPANID = "000000000000004a" REQUEST = { "requestMethod": METHOD, "requestUrl": URI, "status": STATUS } TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) RESOURCE = Resource(type="gae_app", labels={ "module_id": "default", "version_id": "test" }) ENTRY = ProtobufEntry( payload=message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) client = _Client(project=self.PROJECT, connection=_make_credentials()) logger = _Logger() batch = self._make_one(logger, client=client) batch.log_proto( message, labels=LABELS, insert_id=IID, severity=SEVERITY, http_request=REQUEST, timestamp=TIMESTAMP, resource=RESOURCE, trace=TRACE, span_id=SPANID, trace_sampled=True, ) self.assertEqual(batch.entries, [ENTRY])
def bigquery_job(job_name, bucket, file, dataset, table, truncate): global logger, grupo_log, hora_ini, res try: #Inicio carga hora_ini = datetime.now().strftime('%Y-%m-%d %H:%M:%S') #Configuracoes Estrutura Logging log_name = 'log_data_ingestion' grupo_log = 'DATA_INGESTION' #Cria o objeto logger para o log de carga res = Resource(type="cloud_function", labels={ "function_name": job_name, "region": "us-central1" } ) logging_client = logging.Client() logger = logging_client.logger(log_name) stg_client = storage.Client() bkt = stg_client.get_bucket(bucket) sql = bkt.get_blob(file).download_as_string().decode('utf-8') bigquery_client = bigquery.Client() dataset_ref = bigquery_client.get_dataset(dataset) table_ref = dataset_ref.table(table) table_id = bigquery_client.get_table(table_ref) job_config = bigquery.QueryJobConfig() if truncate == 'true': sql_truncate = 'truncate table ' + dataset + '.' + table query_job = bigquery_client.query(sql_truncate, location='US') query_job.result() else: job_config.write_disposition = bigquery.WriteDisposition.WRITE_APPEND job_config.schema = table_id.schema job_config.use_legacy_sql = False job_config.destination = table_ref job_config.create_disposition = 'CREATE_NEVER' query_job = bigquery_client.query(sql, location='US', job_config=job_config) query_job.result() logger.log_struct({'Grupo_Log': grupo_log, 'Status_Execucao': 'Sucesso', 'Nome_Job': job_name, 'Dataset': dataset, 'Tabela': table, 'Inicio': hora_ini, 'Fim': datetime.now().strftime('%Y-%m-%d %H:%M:%S')}, severity='INFO', resource=res) except Exception as ex: logger.log_struct({'Grupo_Log': grupo_log, 'Status_Execucao': 'Erro', 'Nome_Job': job_name, 'Dataset': dataset, 'Tabela': table, 'Inicio': hora_ini, 'Fim': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'Msg_Erro': str(ex)}, severity='ERROR', resource=res)
def gen_google_cloud_logger(): log_client = google.cloud.logging.Client() res = Resource(type="cloud_function", labels={ "function_name": "refresh_classes", "region": os.environ.get("FUNC_REGION") }) return log_client.logger( 'cloudfunctions.googleapis.com%2Fcloud-functions'.format( os.environ.get("DEFAULT_PROJECT_ID"))), res
def write(text, severity='INFO', show=None, seq=None, shot=None, role=None, **kwargs): '''Wrapper method for assembling the payload to send to logger.log_text.''' # Extract and build LOG_ID from environment. # For example: 'myfilm.slb.0050.render' if not show: show = os.getenv('SHOW') if not seq: seq = os.getenv('SEQ') if not shot: shot = os.getenv('SHOT') if not role: role = os.getenv('ROLE') if not show or not seq or not shot or not role: raise Exception( 'One or more log name tokens are empty. Unable to log.') # end if # Assemble logger name. logger_name = '.'.join([show, seq, shot, role]) print '# Logging to %s...' % logger_name # Build logger object. logging_client = logging.Client() logger = logging_client.logger(logger_name) # Assemble the required log metadata. label_payload = { "artist": getpass.getuser(), "hostname": socket.gethostname(), "show": show, "seq": seq, "shot": shot, "role": role } # Add optional kwargs to payload. label_payload.update(kwargs) # Write log. logger.log_text(text, resource=Resource(type='project', labels={'project_id': show}), severity=severity, labels=label_payload)
def logger_usage(client, to_delete): """Logger usage.""" LOG_NAME = "logger_usage_%d" % (_millis()) # [START logger_create] logger = client.logger(LOG_NAME) # [END logger_create] to_delete.append(logger) # [START logger_log_text] logger.log_text("A simple entry") # API call # [END logger_log_text] # [START logger_log_struct] logger.log_struct({ "message": "My second entry", "weather": "partly cloudy" }) # API call # [END logger_log_struct] # [START logger_log_resource_text] from google.cloud.logging.resource import Resource res = Resource( type="generic_node", labels={ "location": "us-central1-a", "namespace": "default", "node_id": "10.10.10.1", }, ) logger.log_struct({ "message": "My first entry", "weather": "partly cloudy" }, resource=res) # [END logger_log_resource_text] # [START logger_list_entries] from google.cloud.logging import DESCENDING for entry in logger.list_entries(order_by=DESCENDING): # API call(s) do_something_with(entry) # [END logger_list_entries] def _logger_delete(): # [START logger_delete] logger.delete() # API call # [END logger_delete] _backoff_not_found(_logger_delete) to_delete.remove(logger)
def log(self, level, message, *args, **kwargs): """ Override the log method because the log level for the current message is not passed in process. """ if not self.isEnabledFor(level): return message = message.strip() message, kwargs = self.process(message, kwargs) app_id = self.extra['app_id'] version = self.extra['version'] if cloud_logger_bg is not None: # The following usage is highly unconventional: # There appears to be no way to log a structured log message # asynchronously using the cloud logging client properly. # However, internally, the library uses structured logging. As long # as we queue the right payload into the worker's queue, we're OK. # This works well with google-cloud-logging==1.9.0. # It looks shady, but the only decent way to use the async logging # support for Stackdriver. cloud_logger_bg.worker._queue.put_nowait({ 'info': { 'app_id': app_id, 'version': version, 'message': message, 'level': getLevelName(level) }, 'severity': getLevelName(level), 'resource': Resource(type='global', labels={}) }) if log_json: json_log = { 'app_id': app_id, 'version': version, 'level': getLevelName(level), 'message': message } if kwargs.get('exc_info') is not None: tb = traceback.format_exc() json_log['message'] += '\n' + tb self.logger.log(level, json_log, *args, **kwargs) else: message_pretty = f'{app_id}::{version} => {message}' self.logger.log(level, message_pretty, *args, **kwargs)
def from_api_repr(cls, resource, client, loggers=None): """Factory: construct an entry given its API representation :type resource: dict :param resource: text entry resource representation returned from the API :type client: :class:`google.cloud.logging.client.Client` :param client: Client which holds credentials and project configuration. :type loggers: dict :param loggers: (Optional) A mapping of logger fullnames -> loggers. If not passed, the entry will have a newly-created logger. :rtype: :class:`google.cloud.logging.entries._BaseEntry` :returns: Text entry parsed from ``resource``. """ if loggers is None: loggers = {} logger_fullname = resource['logName'] logger = loggers.get(logger_fullname) if logger is None: logger_name = logger_name_from_path(logger_fullname) logger = loggers[logger_fullname] = client.logger(logger_name) payload = resource[cls._PAYLOAD_KEY] insert_id = resource.get('insertId') timestamp = resource.get('timestamp') if timestamp is not None: timestamp = _rfc3339_nanos_to_datetime(timestamp) labels = resource.get('labels') severity = resource.get('severity') http_request = resource.get('httpRequest') trace = resource.get('trace') span_id = resource.get('spanId') monitored_resource_dict = resource.get('resource') monitored_resource = None if monitored_resource_dict is not None: monitored_resource = Resource._from_dict(monitored_resource_dict) return cls(payload, logger, insert_id=insert_id, timestamp=timestamp, labels=labels, severity=severity, http_request=http_request, resource=monitored_resource, trace=trace, span_id=span_id)
def get_handler(self): # TODO drop these grpc variants for the REST versions, and we can drop # protobuf/grpc deps, and also so we can record tests. log_group = self.get_log_group() project_id = local_session(self.ctx.session_factory).get_default_project() client = LogClient(project_id) return CloudLoggingHandler( client, log_group, labels={ 'policy': self.ctx.policy.name, 'resource': self.ctx.policy.resource_type}, resource=Resource(type='project', labels={'project_id': project_id}))
def _process_log(log, context): """Process logs by writing them out to CloudWatch and Stackdriver.""" res = Resource( type='generic_task', labels={ 'location': REGION, 'namespace': 'default', 'job': 'router', 'task_id': context.aws_request_id, }, ) stackdriver.log_struct(log, resource=res) print(log)
def get_gae_resource(self): """Return the GAE resource using the environment variables. :rtype: :class:`~google.cloud.logging.resource.Resource` :returns: Monitored resource for GAE. """ gae_resource = Resource( type="gae_app", labels={ "project_id": self.project_id, "module_id": self.module_id, "version_id": self.version_id, }, ) return gae_resource
def get_gae_resource(self): """Return the GAE resource using the environment variables. :rtype: :class:`~google.cloud.logging.resource.Resource` :returns: Monitored resource for GAE. """ gae_resource = Resource( type='gae_app', labels={ 'project_id': os.environ.get(_GAE_PROJECT_ENV), 'module_id': os.environ.get(_GAE_SERVICE_ENV), 'version_id': os.environ.get(_GAE_VERSION_ENV), }, ) return gae_resource
def test_should_set_labels(self, mock_client): self.stackdriver_task_handler.set_context(self.ti) self.logger.addHandler(self.stackdriver_task_handler) self.logger.info("test-message") self.stackdriver_task_handler.flush() labels = { 'task_id': 'task_for_testing_file_log_handler', 'dag_id': 'dag_for_testing_file_task_handler', 'execution_date': '2016-01-01T00:00:00+00:00', 'try_number': '1' } resource = Resource(type='global', labels={}) self.transport_mock.return_value.send.assert_called_once_with( mock.ANY, 'test-message', labels=labels, resource=resource)
def init_logging(client, level=logging.INFO, resource_type=None, resource_labels=None, labels={}): """ Call this once at the top of your main program to log via Stackdriver. """ if isinstance(level, str): level = getattr(logging, level) resource = (Resource(type=resource_type, labels=resource_labels) if resource_type is not None else None) handler = client.get_default_handler(resource=resource, labels=labels) handler.setFormatter(LogFormatter(*FORMATTER_ARGS)) root_logger = logging.getLogger() root_logger.addHandler(handler) root_logger.setLevel(level)
def write_to_stackdriver(self, subs): """Writes an entry to Stackdriver log. Args: subs: number of subscriptions Returns: None """ log_client = logging.Client() logger = log_client.logger('quota-watcher') res = Resource( type='pubsub_subscription', labels={ 'usage': 'quota_watcher', }, ) logger.log_struct({'total_subscriptions': subs}, resource=res)
def get_handler(self): # gcp has three independent implementation of api bindings for python. # The one used by logging is not yet supported by our test recording. log_group = self.ctx.options.log_group if log_group.endswith('*'): log_group = "%s%s" % (log_group[:-1], self.ctx.policy.name) project_id = local_session( self.ctx.session_factory).get_default_project() client = LogClient(project_id) return CloudLoggingHandler(client, log_group, resource=Resource( type='project', labels={'project_id': project_id}))
def test_should_pass_message_to_client(self, mock_client): transport_type = mock.MagicMock() stackdriver_task_handler = StackdriverTaskHandler( gcp_conn_id=None, transport=transport_type, labels={"key": 'value'} ) logger = logging.getLogger("logger") logger.addHandler(stackdriver_task_handler) logger.info("test-message") stackdriver_task_handler.flush() transport_type.assert_called_once_with(mock_client.return_value, 'airflow') transport_type.return_value.send.assert_called_once_with( mock.ANY, 'test-message', labels={"key": 'value'}, resource=Resource(type='global', labels={}) ) mock_client.assert_called_once()
def from_api_repr(cls, resource, client, loggers=None): """Factory: construct an entry given its API representation :type resource: dict :param resource: text entry resource representation returned from the API :type client: :class:`google.cloud.logging.client.Client` :param client: Client which holds credentials and project configuration. :type loggers: dict :param loggers: (Optional) A mapping of logger fullnames -> loggers. If not passed, the entry will have a newly-created logger. :rtype: :class:`google.cloud.logging.entries._BaseEntry` :returns: Text entry parsed from ``resource``. """ if loggers is None: loggers = {} logger_fullname = resource['logName'] logger = loggers.get(logger_fullname) if logger is None: logger_name = logger_name_from_path(logger_fullname) logger = loggers[logger_fullname] = client.logger(logger_name) payload = resource[cls._PAYLOAD_KEY] insert_id = resource.get('insertId') timestamp = resource.get('timestamp') if timestamp is not None: timestamp = _rfc3339_nanos_to_datetime(timestamp) labels = resource.get('labels') severity = resource.get('severity') http_request = resource.get('httpRequest') trace = resource.get('trace') monitored_resource_dict = resource.get('resource') monitored_resource = None if monitored_resource_dict is not None: monitored_resource = Resource._from_dict(monitored_resource_dict) return cls(payload, logger, insert_id=insert_id, timestamp=timestamp, labels=labels, severity=severity, http_request=http_request, resource=monitored_resource, trace=trace)
def test_from_api_repr_w_loggers_no_logger_match(self): from datetime import datetime from google.cloud._helpers import UTC from google.cloud.logging.resource import Resource klass = self._get_target_class() client = _Client(self.PROJECT) SEVERITY = "CRITICAL" IID = "IID" NOW = datetime.utcnow().replace(tzinfo=UTC) TIMESTAMP = _datetime_to_rfc3339_w_nanos(NOW) LOG_NAME = "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME) LABELS = {"foo": "bar", "baz": "qux"} METHOD = "POST" URI = "https://api.example.com/endpoint" RESOURCE = Resource( type="gae_app", labels={ "type": "gae_app", "labels": {"module_id": "default", "version": "test"}, }, ) STATUS = "500" TRACE = "12345678-1234-5678-1234-567812345678" SPANID = "000000000000004a" FILE = "my_file.py" LINE_NO = 123 FUNCTION = "my_function" SOURCE_LOCATION = {"file": FILE, "line": str(LINE_NO), "function": FUNCTION} OP_ID = "OP_ID" PRODUCER = "PRODUCER" OPERATION = {"id": OP_ID, "producer": PRODUCER, "first": True, "last": False} API_REPR = { "logName": LOG_NAME, "insertId": IID, "timestamp": TIMESTAMP, "labels": LABELS, "severity": SEVERITY, "httpRequest": { "requestMethod": METHOD, "requestUrl": URI, "status": STATUS, }, "resource": RESOURCE._to_dict(), "trace": TRACE, "spanId": SPANID, "traceSampled": True, "sourceLocation": SOURCE_LOCATION, "operation": OPERATION, } loggers = {} entry = klass.from_api_repr(API_REPR, client, loggers=loggers) self.assertEqual(entry.log_name, LOG_NAME) logger = entry.logger self.assertIsInstance(logger, _Logger) self.assertEqual(logger.name, self.LOGGER_NAME) self.assertEqual(entry.insert_id, IID) self.assertEqual(entry.timestamp, NOW) self.assertIsNone(entry.received_timestamp) self.assertEqual(entry.labels, LABELS) self.assertEqual(entry.severity, SEVERITY) self.assertEqual(entry.http_request["requestMethod"], METHOD) self.assertEqual(entry.http_request["requestUrl"], URI) self.assertEqual(entry.http_request["status"], STATUS) self.assertIs(logger.client, client) self.assertEqual(logger.name, self.LOGGER_NAME) self.assertEqual(loggers, {LOG_NAME: logger}) self.assertEqual(entry.resource, RESOURCE) self.assertEqual(entry.trace, TRACE) self.assertEqual(entry.span_id, SPANID) self.assertTrue(entry.trace_sampled) source_location = entry.source_location self.assertEqual(source_location["file"], FILE) self.assertEqual(source_location["line"], LINE_NO) self.assertEqual(source_location["function"], FUNCTION) self.assertEqual(entry.operation, OPERATION) self.assertIsNone(entry.payload)
def from_api_repr(cls, resource, client, loggers=None): """Factory: construct an entry given its API representation :type resource: dict :param resource: text entry resource representation returned from the API :type client: :class:`google.cloud.logging.client.Client` :param client: Client which holds credentials and project configuration. :type loggers: dict :param loggers: (Optional) A mapping of logger fullnames -> loggers. If not passed, the entry will have a newly-created logger. :rtype: :class:`google.cloud.logging.entries.LogEntry` :returns: Log entry parsed from ``resource``. """ if loggers is None: loggers = {} logger_fullname = resource["logName"] logger = loggers.get(logger_fullname) if logger is None: logger_name = logger_name_from_path(logger_fullname) logger = loggers[logger_fullname] = client.logger(logger_name) payload = cls._extract_payload(resource) insert_id = resource.get("insertId") timestamp = resource.get("timestamp") if timestamp is not None: timestamp = _rfc3339_nanos_to_datetime(timestamp) labels = resource.get("labels") severity = resource.get("severity") http_request = resource.get("httpRequest") trace = resource.get("trace") span_id = resource.get("spanId") trace_sampled = resource.get("traceSampled") source_location = resource.get("sourceLocation") if source_location is not None: line = source_location.pop("line", None) source_location["line"] = _int_or_none(line) operation = resource.get("operation") monitored_resource_dict = resource.get("resource") monitored_resource = None if monitored_resource_dict is not None: monitored_resource = Resource._from_dict(monitored_resource_dict) inst = cls( log_name=logger_fullname, insert_id=insert_id, timestamp=timestamp, labels=labels, severity=severity, http_request=http_request, resource=monitored_resource, trace=trace, span_id=span_id, trace_sampled=trace_sampled, source_location=source_location, operation=operation, logger=logger, payload=payload, ) received = resource.get("receiveTimestamp") if received is not None: inst.received_timestamp = _rfc3339_nanos_to_datetime(received) return inst