def __init__(self, app, config): super(FileEventWriter, self).__init__(app, config) self._logger_name = config.get('logger', 'default') + '_metric_events' self._logger = log.Logs().get_logger(self._logger_name) self._log_level = config.get('loglevel', 'INFO') log.Logs().set_level(self._log_level, self._logger_name) # reset the formater of log handler for handler in self._logger.handlers: handler.setFormatter(logging.Formatter('%(message)s'))
def test_set_level(self, monkeypatch): logger = log.Logs().get_logger("set_level") logger.debug("this is a test log") log.Logs().set_level(log.logging.DEBUG) logger.warning("this is a test log that can show") log.Logs().set_level(log.logging.ERROR, name="set_level") logger.warning("this is a test log that can not show")
def update_config(self, config): super(FileEventWriter, self).update_config(config) l_name = config.get('logger', 'default') + '_metric_events' if l_name != self._logger_name: self._logger_name = l_name self._logger = log.Logs().get_logger(self._logger_name) l_level = config.get('loglevel', 'INFO') if self._log_level != l_level: self._log_level = l_level log.Logs().set_level(self._log_level, self._logger_name)
def test_log_enter_exit(monkeypatch): logger1 = log.Logs().get_logger("enter_exit1") logger2 = log.Logs().get_logger("enter_exit2") @log.log_enter_exit(logger1) def test1(): pass @log.log_enter_exit(logger2) def test2(): pass test1() test2()
def worker(logger_ref): native_logger = log.Logs().get_logger("test_multi_process") for i in range(100): logger_ref.debug("Log info from child process") native_logger.debug( "Log info from child process on native logger")
def get_logger(loglevel=None): if loglevel: set_log_level(loglevel) global global_logger if global_logger is None: global_logger = logs.Logs().get_logger(LOGGER_NAME) global_logger.setLevel(global_loglevel) return global_logger
def __init__(self, ta_name, alert_name): self._alert_name = alert_name self._logger_name = alert_name + "_modalert" self._logger = log.Logs().get_logger(self._logger_name) super().__init__(sys.stdin.read(), self._logger, alert_name) self.setup_util_module = None self.setup_util = None self.result_handle = None self.ta_name = ta_name self.splunk_uri = self.settings.get("server_uri") self.setup_util = Setup_Util(self.splunk_uri, self.session_key, self._logger) self.rest_helper = TARestHelper(self._logger)
def update_log_level(level): log_level = logging.INFO level = level.lower() if level == "debug": log_level = logging.DEBUG elif level == "info": log_level = logging.INFO elif level == "error": log_level = logging.ERROR else: log_level = logging.INFO _LOGGER.info("set the loglevel to %s", level) log.Logs().set_level(log_level, "validation_mi") return log_level
def _get_logger(name, level=logging.INFO): with g_logger_lock: l = None if name in g_loggers: l = g_loggers[name] else: l = log.Logs().get_logger(name) g_loggers[name] = l lv = g_log_levels.get(name, 10000) if level < lv: # logging.DEBUG < INFO < WARN < ERROR l.setLevel(level) g_log_levels[name] = level return l
def test_multi_thread(self, monkeypatch): log.Logs.set_context(directory="/tmp/", namespace="unittest") logger = log.Logs().get_logger("test_multi_thread") logger.debug("Log info from main thread") def worker(logger_ref): native_logger = log.Logs().get_logger("test_multi_thread") for i in range(100): logger_ref.debug("Log info from child thread") native_logger.debug( "Log info from child thread on native logger") for i in range(20): t = threading.Thread(target=worker, args=(logger, )) t.start() time.sleep(1)
def test_multi_process(self, monkeypatch): log.Logs.set_context(directory="/tmp/", namespace="unittest") logger = log.Logs().get_logger("test_multi_process") logger.debug("Log info from main process") def worker(logger_ref): native_logger = log.Logs().get_logger("test_multi_process") for i in range(100): logger_ref.debug("Log info from child process") native_logger.debug( "Log info from child process on native logger") for _ in range(20): p = multiprocessing.Process(target=worker, args=(logger, )) p.start() time.sleep(1)
def set_log_level(loglevel): if loglevel: global global_loglevel global_loglevel = loglevel if global_logger: logs.Logs().set_level(global_loglevel, LOGGER_NAME)
""" import urllib try: from urllib import urlencode except: from urllib.parse import urlencode from httplib2 import Http, ProxyInfo, socks import splunk.admin as admin from solnlib import log from solnlib import conf_manager from solnlib.utils import is_true import json log.Logs.set_context() logger = log.Logs().get_logger('splunk_ta_uccexample_rh_oauth2_token') # Map for available proxy type _PROXY_TYPE_MAP = { 'http': socks.PROXY_TYPE_HTTP, 'http_no_tunnel': socks.PROXY_TYPE_HTTP_NO_TUNNEL, 'socks4': socks.PROXY_TYPE_SOCKS4, 'socks5': socks.PROXY_TYPE_SOCKS5, } """ REST Endpoint of getting token by OAuth2 in Splunk Add-on UI Framework. T """ class splunk_ta_uccexample_rh_oauth2_token(admin.MConfigHandler): """
handleEdit method: controls the parameters and saves the values corresponds to handleractions = edit in restmap.conf """ import ta_sigsci_blacklist_alert_declare import json import splunk.clilib.cli_common as scc import splunk.admin as admin import solnlib.utils as utils import solnlib.log as log import solnlib.conf_manager as conf import ta_sigsci_blacklist_alert_consts as setup_const log.Logs.set_context(namespace="ta_sigsci_blacklist_alert") logger = log.Logs().get_logger("setup") def get_or_create_conf_file(conf_mgr, file_name): try: conf_file = conf_mgr.get_conf(file_name) return conf_file except conf.ConfManagerException as cme: conf_mgr._confs.create(file_name) return conf_mgr.get_conf(file_name, refresh=True) def filter_eai_property(stanza): if isinstance(stanza, dict): for k in list(stanza.keys()): if k.startswith('eai:'):
# encoding = utf-8 from builtins import object from aob.aob_common.metric_collector import event_writer from aob.aob_common.metric_collector import memory_event_writer from aob.aob_common.metric_collector.metric_exception import MetricException from aob.aob_common.metric_collector.number_metric_collector import NumberMetricCollector from solnlib import pattern from solnlib import log import threading from future.utils import with_metaclass __all__ = ['Monitor'] logger = log.Logs().get_logger('metric_collector') class Monitor(with_metaclass(pattern.Singleton, object)): EVENT_WRITERS = { 'memory': memory_event_writer.MemoryEventWriter, 'file': event_writer.FileEventWriter, 'hec': event_writer.FileEventWriter # TODO; implement a HEC writer } def __init__(self): self._app = None self._ewriter = None self._ewriter_type = None self.number_metric_collector = None self.worker_start_lock = threading.Lock()
def stream_events(self, inputs, ew): input_items = {} input_name = list(inputs.inputs.keys())[0] input_items = inputs.inputs[input_name] # Create UTC timezone for conversion utc = pytz.utc params = {} start = 0 zenoss_server = input_items.get("zenoss_server") username = input_items.get("zenoss_username") zenoss_realm = input_items.get("zenoss_realm") no_ssl_cert_check = int(input_items.get("no_ssl_cert_check")) # Since Disable=1 and Enable=0, negate bool() to keep alignment ssl_cert_check = not bool(no_ssl_cert_check) cafile = input_items.get("cafile") interval = int(input_items.get("interval", HOUR)) start_date = input_items.get("start_date") index_closed = int(input_items.get("index_closed")) index_cleared = int(input_items.get("index_cleared")) index_archived = int(input_items.get("index_archived")) index_suppressed = int(input_items.get("index_suppressed")) index_repeats = int(input_items.get("index_repeats")) archive_threshold = int(input_items.get("archive_threshold")) checkpoint_delete_threshold = int( input_items.get("checkpoint_delete_threshold")) tzone = input_items.get("tzone") proxy_uri = input_items.get("proxy_uri") proxy_username = input_items.get("proxy_username") proxy_realm = input_items.get("proxy_realm") proxy_password = None meta_configs = self._input_definition.metadata # Generate logger with input name _, input_name = (input_name.split('//', 2)) self.logger = log.Logs().get_logger('{}_input'.format(APP_NAME)) # Log level configuration self.logger.setLevel('INFO') if index_closed: params = dict(index_closed=True) if index_cleared: params = dict(index_closed=True) if index_suppressed: params = dict(index_suppressed=True) if index_repeats: params = dict(index_repeats=True) try: if tzone: zenoss_tz = pytz.timezone(tzone) else: zenoss_tz = pytz.timezone(str(get_localzone())) except pytz.UnknownTimeZoneError as e: self.logger.warn( "Unknown Timezone {} - Using default UTC".format(e)) zenoss_tz = pytz.timezone("utc") # Get UTC timestamp utc_now = datetime.utcnow().replace(tzinfo=utc) # Convert to Zenoss server timezone now_local = utc_now.astimezone(zenoss_tz) # Create local time string now_str = now_local.strftime(DATE_FORMAT) # Load checkpoint file self.chk = checkpointer.FileCheckpointer( meta_configs['checkpoint_dir']) if self.chk.get("run_from") is None: # Initializing keys in checkpoint self.chk.update("run_from", start_date) self.chk.update("last_run", None) self.chk.update("last_cleaned", now_str) try: device = input_items.get("device") except Exception: device = None # Get password from storage password try: password = self.get_password(zenoss_realm, username, meta_configs['session_key']) except Exception as e: self.logger.error( "Failed to get password for user %s, realm %s. Verify credential account exists. User who scheduled alert must have Admin privileges. - %s" % (username, zenoss_realm, e)) sys.exit(1) if proxy_username is not None: try: proxy_password = self.get_password(proxy_realm, proxy_username, meta_configs['session_key']) except Exception as e: self.logger.error( "Failed to get password for user %s, realm %s. Verify credential account exists. User who scheduled alert must have Admin privileges. - %s" % (proxy_username, proxy_realm, e)) sys.exit(1) while True: run_from = self.chk.get("run_from") # When none --> get ALL events, otherwise from users' specified date # Work with datetimes in UTC and then convert to timezone of Zenoss server utc_dt = utc.localize(datetime.utcnow()) now_local = zenoss_tz.normalize(utc_dt.astimezone(zenoss_tz)) now_epoch = calendar.timegm(now_local.utctimetuple()) cur_time = now_local.strftime(DATE_FORMAT) # Connect to Zenoss web interface and get events try: z = ZenossAPI(zenoss_server, username, password, proxy_uri, proxy_username, proxy_password, ssl_cert_check, cafile) except Exception as e: log_message = "Zenoss Events: Failed to connect to server %s as user %s - Error: %s" % ( zenoss_server, username, e) self.logger.error("{}. Exiting.".format(log_message)) sys.exit(1) # Initializing data events_dict = { "run_from": self.chk.get("run_from"), "last_run": self.chk.get("last_run"), "last_cleaned": self.chk.get("last_cleaned") } # Get Events events_dict = self.get_and_process_events(z, events_dict, ew, params, device, start, run_from, index_closed, index_cleared, index_suppressed) # Update last run timestamp events_dict['last_run'] = cur_time # Processed archived events if index_archived: # Get last archive read, convert and create epoch timestamp try: last_archive_read = self.chk.get('last_archive_read') if last_archive_read is None: # Key does not exist in checkpoint raise Exception archive_delta = self.calc_epoch_delta( last_archive_read, DATE_FORMAT, now_epoch, zenoss_tz, HOUR) except Exception: last_archive_read = None archive_delta = 0 # Read the archived events table if it hasn't been read or # last read exceeds archive threshold if archive_delta >= archive_threshold or \ not last_archive_read: log_message = "Zenoss Events: Processing Archived Events\n" % params self.logger.info(log_message) self.get_and_process_events(z, events_dict, ew, params, device, start, run_from, index_closed, index_cleared, index_suppressed, archive=True) events_dict['last_archive_read'] = cur_time # Clean checkpoint file try: last_cleaned = events_dict['last_cleaned'] if last_cleaned is None: # Key does not exist in checkpoint raise Exception except Exception: last_cleaned = cur_time # Check to see if we need to clean the checkpoint file based on the # checkpoint delta threshold last_cleaned_delta = self.calc_epoch_delta(last_cleaned, DATE_FORMAT, now_epoch, zenoss_tz, DAY) keys_toclean = [] # Clean checkpoint file of old archive records if last_cleaned_delta >= CHECKPOINT_CLEAN_FREQUENCY: for k in events_dict.keys(): if isinstance(events_dict[k], dict) and 'last_time' in events_dict[k]: last_time = events_dict[k]['last_time'] epoch_delta = self.calc_epoch_delta( last_time, "%Y-%m-%d %H:%M:%S", now_epoch, zenoss_tz, DAY) if epoch_delta >= int(checkpoint_delete_threshold): keys_toclean.append(k) self.chk.delete(k) # Update checkpoint file for key in events_dict.keys(): if key in keys_toclean: continue # dict2str to save among checkpoints value = events_dict[key] if isinstance(value, dict): value = json.dumps(value) self.chk.update(key, value) time.sleep(float(interval))
def stream_events(self, inputs, ew): meta_configs = self._input_definition.metadata session_key = meta_configs['session_key'] input_items = {} input_name = list(inputs.inputs.keys())[0] input_items = inputs.inputs[input_name] # Generate logger with input name _, input_name = (input_name.split('//', 2)) logger = log.Logs().get_logger('{}_input'.format(APP_NAME)) # Log level configuration log_level = get_log_level(session_key, logger) logger.setLevel(log_level) logger.debug("Modular input invoked.") try: account_name = input_items.get('global_account') account_details = get_account_details(session_key, account_name, logger) username = account_details.get('username') password = account_details.get('password') if not username: logger.error( "Username is required {} to resume updates".format( account_name)) sys.exit(1) if not password: logger.error( "password is required {} to resume updates".format( account_name)) sys.exit(1) logger.debug(f"username is {username}") # Proxy configuration proxy_settings = get_proxy_settings(session_key, logger) edition_ids = input_items.get('edition_ids') with tempfile.NamedTemporaryFile(mode='w', suffix=".conf", prefix="GeoIP") as file: file.write("\nAccountID " + username) file.write("\nLicenseKey " + password) file.write("\nEditionIDs " + edition_ids + "\n") if proxy_settings == {}: logger.debug("no proxy") else: file.write("\nProxy " + proxy_settings["proxy_url"] + ":" + proxy_settings["proxy_port"]) if not proxy_settings["proxy_username"] is None: file.write("\nProxyUserPassword " + proxy_settings["proxy_username"] + ":" + proxy_settings["proxy_password"]) file.flush() guargs = str( os.path.expandvars( "-v -d $SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/data/ -f " + file.name)) try: subprocess.check_output([ "$SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/bin/geoipupdate/linux_amd64/geoipupdate " + guargs ], shell=True, stderr=subprocess.STDOUT) except CalledProcessError as e: logger.exception(e) logger.error("command args:\n") logger.error(e.cmd) logger.error("output:\n") logger.error(e.output.decode("utf-8")) sys.exit(1) mmdb_dir = os.path.expandvars( '$SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/data/') files = os.listdir(mmdb_dir) for name in files: if name.endswith('.mmdb'): inode = os.stat(os.path.join(mmdb_dir, name)) logger.info('mmdb=' + name + ' size=' + str(inode.st_size) + ' mtime=' + str(inode.st_mtime)) except Exception as e: logger.exception(e) sys.exit(1) logger.debug("Modular input completed")
import urllib try: from urllib import urlencode except: from urllib.parse import urlencode import requests from httplib2 import Http, ProxyInfo, socks import splunk.admin as admin from solnlib import log from solnlib import conf_manager from solnlib.utils import is_true import json log.Logs.set_context() logger = log.Logs().get_logger('ta_lansweeper_rh_oauth') # Map for available proxy type _PROXY_TYPE_MAP = { 'http': socks.PROXY_TYPE_HTTP, 'http_no_tunnel': socks.PROXY_TYPE_HTTP_NO_TUNNEL, 'socks4': socks.PROXY_TYPE_SOCKS4, 'socks5': socks.PROXY_TYPE_SOCKS5, } """ REST Endpoint of getting token by OAuth2 in Splunk Add-on UI Framework. """ class ta_lansweeper_add_on_for_splunk_rh_oauth2_token(admin.MConfigHandler): """ This method checks which action is getting called and what parameters are required for the request.
def test_get_logger(self, monkeypatch): logger = log.Logs().get_logger("logging") logger.debug("this is a test log") logger.warning("this is a test log that can show")
def stream_events(self, inputs, ew): meta_configs = self._input_definition.metadata session_key = meta_configs['session_key'] input_items = {} input_name = list(inputs.inputs.keys())[0] input_items = inputs.inputs[input_name] # Generate logger with input name _, input_name = (input_name.split('//', 2)) logger = log.Logs().get_logger( 'ta_lansweeper_input_{}'.format(input_name)) log_level = get_log_level(session_key, logger) logger.setLevel(log_level) logger.info("Modular input invoked.") account_name = input_items.get('account_name') account_details = get_account_details(session_key, account_name, logger) proxy_settings = get_proxy_settings(session_key, logger) client_id = account_details.get('client_id') client_secret = account_details.get('client_secret') access_token = account_details.get('access_token') refresh_token = account_details.get('refresh_token') site_name = input_items.get('site').split(',') logger.info("Site names: " + str(site_name)) index = input_items.get('index') sites = [] # Note - Do not uncomment below line in the production # logger.debug('Access token={}, ||| Refresh token={}'.format(access_token, refresh_token)) lansweeper = Lansweeper(client_id=client_id, client_secret=client_secret, access_token=access_token, refresh_token=refresh_token, proxy_settings=proxy_settings, logger=logger) try: # Get site id status_code, response = lansweeper.get_site_id(site_name) if status_code != 200: is_expired_response = lansweeper.is_token_expired( status_code, response.text) if is_expired_response: lansweeper.access_token = is_expired_response[ 'access_token'] # Updating the access token and refresh token in the conf files try: update_access_token( access_token=is_expired_response['access_token'], refresh_token=refresh_token, client_secret=client_secret, session_key=session_key, stanza_name=account_name) logger.info( 'Successfully updated the new access token and refresh token in the conf file' ) except Exception as exception: logger.warning( 'Error while updating the access token and refresh token in the conf file, error={}' .format(exception)) logger.debug("Retrying for site-id with new tokens.") status_code, response = lansweeper.get_site_id(site_name) if status_code != 200: logger.error( 'Error while fetching the site id for site={}, status code={} response={}' .format(site_name, status_code, response.text)) sys.exit(1) else: sites = response else: logger.error( 'Error while fetching the site id for site={}, status code={} response={}' .format(site_name, status_code, response.text)) sys.exit(1) else: logger.info( 'Successfully fetch the site code for site={}'.format( site_name)) sites = response except Exception as exception: logger.exception( 'Error while fetching the site id for site={}, exception={}'. format(site_name, exception)) sys.exit(1) logger.info("Site id: {}".format(sites)) #Get asset information for all the sites logger.info('Starting to fetch assets data') for site in sites: try: cursor = '' page = 'FIRST' is_data = True site_name = list(site.keys())[0] site_id = site[site_name] while is_data: status, response_code, response = lansweeper.get_asset_info( site_id, cursor, page) if not status: is_expired_response = lansweeper.is_token_expired( response_code, response.text) if is_expired_response: lansweeper.access_token = is_expired_response[ 'access_token'] try: # Updating the access token and refresh token in the conf files update_access_token( access_token=is_expired_response[ 'access_token'], refresh_token=refresh_token, client_secret=client_secret, session_key=session_key, stanza_name=account_name) logger.info( 'Successfully updated the new access token and refresh token in the conf file' ) except Exception as exception: logger.warning( 'Error while updating the access token and refresh token in the conf file, error={}' .format(exception)) continue else: logger.error( 'Error while fetching the assets for site={}, status code={} response={}' .format(site_name, response_code, response.text)) break else: logger.info( 'Successfully fetch the assets for site={} page={}' .format(site_name, page)) cursor, asset_data = response_code, response write_event(asset_data, site_name, ew, index, logger) is_data = cursor page = 'NEXT' logger.info( 'Fetching the next group of assets for site={} cursor={}' .format(site_name, cursor)) except Exception as exception: logger.exception( 'Error while fetching the assets for site={}'.format( site_name)) logger.info('Completed the data collection for the input')