def __connect_gcl(logger_name: str) -> Logger: """ Sets the ``GOOGLE_APPLICATION_CREDENTIALS`` environmental variable for connecting to Stackdriver-Logging and initiates a new logger with name **logger_name**. :param logger_name: The name of the logger. :return: A Stackdriver Logger. .. note:: A google credentials file should exist in the current directory with the name ``meetup-analysis.json``. """ pprint(f"Initiating {logger_name}...") try: logging_client = logging.Client() except DefaultCredentialsError: google_env_var = "GOOGLE_APPLICATION_CREDENTIALS" credentials_path = os.path.join(os.getcwd(), '../meetup-analysis.json') if os.environ.get(google_env_var) is None: if os.path.exists(credentials_path): os.environ[google_env_var] = credentials_path pprint("Found google credentials.", pformat=BColors.OKGREEN) else: raise FileNotFoundError( f'Could not find environmental variable [{google_env_var}]' f' or the credentials file [{credentials_path}]') logging_client = logging.Client() logger = logging_client.logger(logger_name) # Initiate a new logger. return logger
def instantiate_client(_unused_client, _unused_to_delete): """Instantiate client.""" # [START client_create_default] from google.cloud import logging client = logging.Client() # [END client_create_default] credentials = object() # [START client_create_explicit] from google.cloud import logging client = logging.Client(project='my-project', credentials=credentials)
def test_create_sink(capfd): try: # run the function logs_operator = export_logs_utility( test_variables["sink_name"], test_variables["project_id"], test_variables["dataset_name"], test_variables["dataset_location"], "create", test_variables["filter_"], ) logs_operator.create_sink() out, err = capfd.readouterr() logging_client = logging.Client() sink = logging_client.sink(logs_operator.sink_name) # assert that the sink exists assert sink.exists() # assert that the print message exists in the terminal output assert out == "Created sink {}\n".format(test_variables["sink_name"]) except AssertionError: # assert that the sink exists assert sink.exists() assert out == "Sink {} already exists.\n".format( test_variables["sink_name"])
def test_delete_sink(capfd): # run the function logs_operator = export_logs_utility( test_variables["sink_name"], test_variables["project_id"], test_variables["dataset_name"], test_variables["dataset_location"], "delete", test_variables["filter_"], ) # create sink logs_operator.create_sink() logging_client = logging.Client() sink = logging_client.sink(logs_operator.sink_name) # assert that the sink exists assert sink.exists() # delete sink logs_operator.delete_sink() out, err = capfd.readouterr() # assert that the print message exists in the terminal output assert "Deleted sink {}\n".format(test_variables["sink_name"]) in out # assert that it does not exist assert not sink.exists()
def create_sink(self): """Creates a sink to export logs to the given Cloud Storage bucket. The filter determines which logs this sink matches and will be exported to the destination. For example a filter of 'severity>=INFO' will send all logs that have a severity of INFO or greater to the destination. See https://cloud.google.com/logging/docs/view/advanced_filters for more filter information. """ logging_client = logging.Client() # The destination can be a Cloud Storage bucket, a Cloud Pub/Sub topic, # or a BigQuery dataset. In this case, it is a Cloud Storage Bucket. # See https://cloud.google.com/logging/docs/api/tasks/exporting-logs for # information on the destination format. destination = f"bigquery.googleapis.com/projects/{self.project_id}/datasets/{self.dataset_name}" sink = logging_client.sink(self.sink_name, self.filter_, destination) if sink.exists(): print("Sink {} already exists.".format(sink.name)) return sink.create(unique_writer_identity=True) print("Created sink {}".format(sink.name))
def process(self, KVelement, offsetData): try: key, element = KVelement imgMetadata = element['imgMetadata'] fileName = imgMetadata['fileName'] if not 'Nt' in imgMetadata: assert imgMetadata['Nz'] == 1 assert offsetData[key].size > 0 imgMetadata['Nt'] = offsetData[key].size else: assert 'Nz' in imgMetadata # Nframes = imgMetadata['Nt']*imgMetadata['Nz'] # assert Nframes == offsetData[key].size splitter = self._getSplitter(imgMetadata) IFD = element['IFD'] sortedIFDs = offsetData[key] assert IFD in sortedIFDs n = searchsorted(sortedIFDs, IFD) + 1 for chunk in splitter.iterChunks(n, element['frame']): yield beam.pvalue.TaggedOutput('chunks', chunk) except: client = error_reporting.Client() client.report('File Not Processed: ' + fileName) client.report_exception() logging_client = logging.Client() log_name = 'TIF-reader' logger = logging_client.logger(log_name) logmessage = {'Error': 'File cannot be read', 'Filename': fileName} logger.log_struct(logmessage) yield beam.pvalue.TaggedOutput('readError', ('File Not Processed', fileName))
def post(instring): errorLog = error_reporting.Client(project='gcpoc-173120', service="GCPOCGetService", version="1.0.0") logClient = logging.Client() logger = logClient.logger('GCPOCPostLog') connection = pymysql.connect( host=os.environ['GCPOC_DB_HOST'], user=os.environ['GCPOC_DB_USER'], password=os.environ['GCPOC_DB_PASSWORD'], db=os.environ['GCPOC_DB_DATABASE'], charset='utf8mb4', cursorclass=pymysql.cursors.Cursor) try: with connection.cursor() as cursor: sql = "INSERT INTO gcpoc (instring) VALUES (%s)" cursor.execute(sql, (instring,)) logger.log_text("Added %s to database" % instring) connection.commit() except: errorLog.report_exception() return instring,500 finally: connection.close() return instring,201
def __init__(self, **kwargs): StreamHandler.__init__(self) if 'log_name' not in kwargs.keys(): raise ValueError("Missing log_name value'") log_name = kwargs['log_name'] if not isinstance(log_name, str): raise ValueError("Invalid value: log_name must be of type string") self.parse_json = False if 'parse_json' in kwargs.keys(): parse_json = kwargs['parse_json'] if not isinstance(parse_json, bool): raise ValueError( "Invalid value: parse_json must be of type bool") self.parse_json = parse_json self.add_hostname = False if 'add_hostname' in kwargs.keys(): add_hostname = kwargs['add_hostname'] if not isinstance(add_hostname, bool): raise ValueError( "Invalid value: parse_json must be of type bool") self.add_hostname = add_hostname self.hostname = socket.gethostname() self.logger = logging.Client().logger(log_name)
def get_log_entries(unique_id): cloud_client = cloud_logging.Client() log_name = 'cloudfunctions.googleapis.com%2Fcloud-functions' cloud_logger = cloud_client.logger(log_name) start_time = time.time() is_existing = {'pdf': False, 'xml': False} while True: if time.time() - start_time > 60: raise TimeoutError('No matching logs found') else: logging.info('Refreshing logs...') entries = request_log(cloud_logger=cloud_logger, unique_id=unique_id) for entry in entries: logging.info('Found logging: {}'.format(entry.payload)) if 'PDF invoice sent' in entry.payload: is_existing['pdf'] = True elif 'XML invoice sent' in entry.payload: is_existing['xml'] = True if is_existing['pdf'] and is_existing['xml']: logging.info('Both files succesful posted') break else: time.sleep(15.0 - ((time.time() - start_time) % 15.0))
def create_sink(sink_name, destination_bucket, filter_): """Creates a sink to export logs to the given Cloud Storage bucket. The filter determines which logs this sink matches and will be exported to the destination. For example a filter of 'severity>=INFO' will send all logs that have a severity of INFO or greater to the destination. See https://cloud.google.com/logging/docs/view/advanced_filters for more filter information. """ logging_client = logging.Client() # The destination can be a Cloud Storage bucket, a Cloud Pub/Sub topic, # or a BigQuery dataset. In this case, it is a Cloud Storage Bucket. # See https://cloud.google.com/logging/docs/api/tasks/exporting-logs for # information on the destination format. destination = 'storage.googleapis.com/{bucket}'.format( bucket=destination_bucket) sink = logging_client.sink( sink_name, filter_, destination) if sink.exists(): print('Sink {} already exists.'.format(sink.name)) return sink.create() print('Created sink {}'.format(sink.name))
def __init__(self, config, callback_handler): self.config = config self._callback_handler = callback_handler if 'gcp_cred' not in config: LOGGER.info('No gcp_cred credential specified in config') self._pubber = None self._storage = None self._firestore = None self._client_name = None return cred_file = self.config['gcp_cred'] LOGGER.info('Loading gcp credentials from %s', cred_file) # Normal execution assumes default credentials. # pylint: disable=protected-access (self._credentials, self._project) = google_auth._load_credentials_from_file(cred_file) self._client_name = self._parse_creds(cred_file) self._site_name = self._get_site_name() self._pubber = pubsub_v1.PublisherClient(credentials=self._credentials) LOGGER.info('Initialized gcp pub/sub %s:%s:%s', self._project, self._client_name, self._site_name) self._firestore = self._initialize_firestore(cred_file) self._report_bucket_name = self.REPORT_BUCKET_FORMAT % self._project self._storage = storage.Client(project=self._project, credentials=self._credentials) self._ensure_report_bucket() self._config_callbacks = {} self._logging = logging.Client(credentials=self._credentials, project=self._project) LOGGER.info('Connection initialized at %s', get_timestamp())
def process(self, element): decoded_data = element.data.decode("utf-8") data = json.loads(decoded_data) api_url = f"{aq_baseurl}?city=Richmond&state=Virginia&country=USA&key={AQ_API_KEY}" r = requests.get(api_url) aq_dict = r.json() air_q = data['air_quality'] air_q.append(aq_dict) data['air_quality'] = air_q currentTime = time.time() data['ma_arrival'] = currentTime uuid = data['uuid'] enriched_bird_str = json.dumps(data).encode('utf-8') updated_element = element updated_element.data = enriched_bird_str client = logging.Client() log_name = data['uuid'] logger = client.logger(log_name) logger.log_text( f"MID ATLANTIC, v.{version['__version__']}, uuid: {uuid}, Elapsed time since last step: {currentTime - data['ne_arrival']}" ) yield updated_element
def get_handler(self): """Create a fully configured CloudLoggingHandler. Returns: (obj): Instance of `google.cloud.logging.handlers. CloudLoggingHandler` """ gcl_client = gcl_logging.Client(project=self.project_id, credentials=self.credentials) handler = gcl_handlers.CloudLoggingHandler(gcl_client, resource=self.resource, labels={ 'resource_id': self.instance_id, 'resource_project': self.project_id, 'resource_zone': self.zone, 'resource_host': self.hostname }) handler.setFormatter(self.get_formatter()) self._set_worker_thread_level() return handler
def _ParseLoggingArguments(self, options): """Parses the --logging flag. Args: options (argparse.Namespace): the parsed command-line arguments. Raises: errors.BadConfigOption: if the options are invalid. """ logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s') self._logger = logging.getLogger(self.__class__.__name__) if 'stackdriver' in options.logging: if not self._gcs_settings: raise errors.BadConfigOption( 'Please provide a valid --gs_keyfile to enable StackDriver ' 'logging') gcp_credentials = service_account.Credentials.from_service_account_file( options.gs_keyfile) project_id = self._gcs_settings.get('project_id', None) gcp_logging_client = google_logging.Client( project=project_id, credentials=gcp_credentials) self._stackdriver_handler = CloudLoggingHandler(gcp_logging_client, name='GiftStick') self._logger.addHandler(self._stackdriver_handler) if options.log_progress: if 'stackdriver' not in options.logging: raise errors.BadConfigOption( 'Progress logging requires Stackdriver logging to be enabled' ) self._progress_logger = google_logging.logger.Logger( 'GiftStick', gcp_logging_client)
def __init__(self, config, callback_handler): self.config = config self._callback_handler = callback_handler cred_file = self.config.get('gcp_cred') if not cred_file: LOGGER.info( 'No gcp_cred file specified in config, disabling gcp use.') self._pubber = None self._storage = None self._firestore = None self._client_name = None return assert SUCCESSFUL_IMPORTS, "Missing google cloud python dependencies." LOGGER.info('Loading gcp credentials from %s', cred_file) # Normal execution assumes default credentials. (self._credentials, self._project) = google_auth.load_credentials_from_file(cred_file) self._client_name = self._parse_creds(cred_file) self._site_name = self._get_site_name() self._pubber = pubsub_v1.PublisherClient(credentials=self._credentials) LOGGER.info('Initialized gcp pub/sub %s:%s:%s', self._project, self._client_name, self._site_name) self._firestore = self._initialize_firestore(cred_file) self._report_bucket_name = self.REPORT_BUCKET_FORMAT % self._project self._storage = storage.Client(project=self._project, credentials=self._credentials) self._bucket = self._ensure_report_bucket() self._config_callbacks = {} self._logging = logging.Client(credentials=self._credentials, project=self._project) LOGGER.info('Connection initialized at %s', get_timestamp())
def configure_logging(): """Configure DNA to use cloud logging. Invoke once at application startup, before any log calls. """ client = cloud_logging.Client() client.setup_logging(log_level=logging.DEBUG)
def configure_logging(): """ Configure the loggers for Talos. Sets up the Talos loggers and discord.py loggers separately, so they can be easily configured independently. """ fh = logging.FileHandler(utils.log_folder / "dtalos.log") dfh = logging.FileHandler(utils.log_folder / "dpy.log") sh = logging.StreamHandler(sys.stderr) gh = None try: import google.cloud.logging as glog client = glog.Client() gh = client.get_default_handler() gh.name = "dtalos" gh.setLevel(logging.WARNING) except (ImportError, OSError): print("Could not setup GCloud logging, setup google dependencies") ff = logging.Formatter("%(levelname)s:%(name)s:%(message)s") dlog = logging.getLogger("discord") utils.configure_logger(log, handlers=[fh, sh, gh], formatter=ff, level=logging.INFO, propagate=False) utils.configure_logger(dlog, handlers=[dfh, sh], formatter=ff, level=logging.INFO, propagate=False)
def main(argv): del argv # Unused creds, project_id = google.auth.default(scopes=SCOPE) logging_client = gcp_logging.Client(project=project_id, credentials=creds) logging_client.setup_logging(log_level=logging.INFO) movie_service = imdb.Service(FLAGS.min_votes, FLAGS.movie_file, FLAGS.rating_file) shutdown_handler = util.GracefulShutdown() signal.signal(signal.SIGTERM, shutdown_handler.exit) while not shutdown_handler.is_exit(): logging.info("Start iteration.") sheets_service = sheets.BuildService(creds) titles = sheets.ReadMovieNames(sheets_service, FLAGS.sheet_id) ratings = movie_service.GetRating(titles) logging.info( str(sheets.WriteMovieInfo(ratings, sheets_service, FLAGS.sheet_id))) logging.info("End iteration.") time.sleep(FLAGS.polling_interval) logging.info("Exit")
def search() -> list: errorLog = error_reporting.Client(project='gcpoc-173120', service="GCPOCGetService", version="1.0.0") logClient = logging.Client() logger = logClient.logger('GCPOCGetLog') connection = pymysql.connect(host=os.environ['GCPOC_DB_HOST'], user=os.environ['GCPOC_DB_USER'], password=os.environ['GCPOC_DB_PASSWORD'], db=os.environ['GCPOC_DB_DATABASE'], charset='utf8mb4', cursorclass=pymysql.cursors.Cursor) try: with connection.cursor() as cursor: sql = "SELECT instring from gcpoc" cursor.execute(sql) result = [item[0] for item in cursor.fetchall()] logger.log_text("Found %d items in database" % len(result)) except: errorLog.report_exception() return None, 500 finally: connection.close return result, 200
def setup_logging(): """Connects the default logger to Google Cloud Logging. Only logs at INFO level or higher will be captured. """ client = logging.Client() client.get_default_handler() client.setup_logging()
def log(text, severity=LOG_SEVERITY_DEFAULT, log_name=app_name): logging_client = logging.Client() logger = logging_client.logger(log_name) return logger.log_text(text, severity=severity, resource=google.cloud.logging.Resource(type="cloud_function", labels={}))
def add_stackdriver_sink(loguru_logger, log_name): """Google cloud log sink in "Global" i.e. https://console.cloud.google.com/logs/viewer?project=silken-impulse-217423&minLogLevel=0&expandAll=false&resource=global """ global stackdriver_client if not in_test() and stackdriver_client is None and \ not blconfig.disable_cloud_log_sinks: stackdriver_client = gcloud_logging.Client() stackdriver_logger = stackdriver_client.logger(log_name) def sink(message): record = message.record level = str(record['level']) if level == 'SUCCESS': severity = 'NOTICE' elif level == 'TRACE': # Nothing lower than DEBUG in stackdriver severity = 'DEBUG' elif level == 'EXCEPTION': severity = 'ERROR' elif level in VALID_STACK_DRIVER_LEVELS: severity = level else: severity = 'INFO' if not in_test(): stackdriver_logger.log_text(message, severity=severity) loguru_logger.add(sink)
async def get_sinks(self, project_id: str): try: client = stackdriverlogging.Client(project=project_id) return await run_concurrently( lambda: [sink for sink in client.list_sinks()]) except Exception as e: print_exception('Failed to retrieve sinks: {}'.format(e)) return []
def log_event(self): """ Saves the event to the specified Log Sink """ record = self.format_event() self.logging_client = logging.Client() self.logger = self.logging_client.logger(self.log) self.logger.log_struct(record)
def delete_sink(sink_name): """Deletes a sink.""" logging_client = logging.Client() sink = logging_client.sink(sink_name) sink.delete() print('Deleted sink {}'.format(sink.name))
def gcp_connect_service(service, credentials=None, region_name=None): logging.getLogger('googleapiclient.discovery_cache').setLevel( logging.ERROR) # Set logging level to error for GCP services as otherwise generates a lot of warnings logging.getLogger().setLevel(logging.ERROR) try: if service == 'cloudresourcemanager': return discovery.build('cloudresourcemanager', 'v1', cache_discovery=False, cache=MemoryCache()) elif service == 'cloudresourcemanager-v2': return discovery.build('cloudresourcemanager', 'v2', cache_discovery=False, cache=MemoryCache()) elif service == 'cloudstorage': return storage.Client() elif service == 'cloudsql': return discovery.build('sqladmin', 'v1beta4', cache_discovery=False, cache=MemoryCache()) elif service == 'iam': return discovery.build('iam', 'v1', cache_discovery=False, cache=MemoryCache()) if service == 'stackdriverlogging': return stackdriver_logging.Client() if service == 'stackdrivermonitoring': return monitoring_v3.MetricServiceClient() elif service == 'computeengine': return discovery.build('compute', 'v1', cache_discovery=False, cache=MemoryCache()) elif service == 'kubernetesengine': return container_v1.ClusterManagerClient() else: printException('Service %s not supported' % service) return None except Exception as e: printException(e) return None
def _client(self) -> gcp_logging.Client: """The Cloud Library API client""" credentials, project = self._credentials_and_project client = gcp_logging.Client( credentials=credentials, project=project, client_info=ClientInfo(client_library_version='airflow_v' + version.version), ) return client
def __init__(self, log_name): log_client = logging.Client() # we can't set the resource type, this will default to 'global' # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2673 self.logger = log_client.logger(log_name) self.logger.log_text('Logging initialized.') rospy.Subscriber('/rosout_agg', rosgraph_msgs.msg.Log, self.callback) rospy.loginfo('forward "rosout_agg" to "%s"' % self.logger.path)
def _client(self) -> gcp_logging.Client: """The Cloud Library API client""" credentials, project = self._credentials_and_project client = gcp_logging.Client( credentials=credentials, project=project, client_info=CLIENT_INFO, ) return client
def create_logger(): """ Creates a log in Stackdriver """ instance_name = get_hostname() logname = '%s.log' % instance_name logging_client = logging.Client() logger = logging_client.logger(logname) return logger