def shutdown(self): self.__init__() try: logging.shutdown() logging.captureWarnings(None) except: pass
def __remove_temp_logging_handler(): ''' This function will run once logging has been configured. It just removes the temporary stream Handler from the logging handlers. ''' if is_logging_configured(): # In this case, the temporary logging handler has been removed, return! return # This should already be done, but... __remove_null_logging_handler() root_logger = logging.getLogger() global LOGGING_TEMP_HANDLER for handler in root_logger.handlers: if handler is LOGGING_TEMP_HANDLER: root_logger.removeHandler(LOGGING_TEMP_HANDLER) # Redefine the null handler to None so it can be garbage collected LOGGING_TEMP_HANDLER = None break if sys.version_info >= (2, 7): # Python versions >= 2.7 allow warnings to be redirected to the logging # system now that it's configured. Let's enable it. logging.captureWarnings(True)
def initialize_logging(self): log_file = self.config['main']['log_file'] log_level = self.config['main']['log_level'] level_map = {'CRITICAL': logging.CRITICAL, 'ERROR': logging.ERROR, 'WARNING': logging.WARNING, 'INFO': logging.INFO, 'DEBUG': logging.DEBUG } handler = logging.FileHandler(os.path.expanduser(log_file)) formatter = logging.Formatter( '%(asctime)s (%(process)d/%(threadName)s) ' '%(name)s %(levelname)s - %(message)s') handler.setFormatter(formatter) root_logger = logging.getLogger('mycli') root_logger.addHandler(handler) root_logger.setLevel(level_map[log_level.upper()]) logging.captureWarnings(True) root_logger.debug('Initializing mycli logging.') root_logger.debug('Log file %r.', log_file)
def make_logger(self, f=None): self.fhandler = None """self.make_logger_240(file) -> logger. Creates a logging object to be used within the DebugPrint class that sends the debugging output to file. We will configure log so it outputs to both stderr and a file. """ # Setup root logger self.logger = logging.getLogger('VisLog') self.logger.setLevel(logging.DEBUG) self.format = logging.Formatter("%(asctime)s %(levelname)s:\n%(message)s") # Setup warnings logger if hasattr(logging, 'captureWarnings'): wlogger = logging.getLogger('py.warnings') wlogger.propagate = False wlogger.addHandler(EmitWarnings(self.logger)) logging.captureWarnings(True) # first we define a handler for logging to a file if f: self.set_logfile(f) #then we define a handler to log to the console self.console = logging.StreamHandler() self.console.setFormatter(self.format) self.console.setLevel(logging.WARNING) self.logger.addHandler(self.console)
def setup(self, verbose_level, error_level, logdir): self.presetup() logger_dnf = logging.getLogger("dnf") # setup file logger logfile = os.path.join(logdir, dnf.const.LOG) handler = _create_filehandler(logfile) logger_dnf.addHandler(handler) # temporarily turn off stdout/stderr handlers: self.stdout_handler.setLevel(SUPERCRITICAL) self.stderr_handler.setLevel(SUPERCRITICAL) # put the marker in the file now: _paint_mark(logger_dnf) # bring std handlers to the preferred level self.stdout_handler.setLevel(verbose_level) self.stderr_handler.setLevel(error_level) # setup Python warnings logging.captureWarnings(True) logger_warnings = logging.getLogger("py.warnings") logger_warnings.addHandler(self.stderr_handler) logger_warnings.addHandler(handler) # setup RPM callbacks logger logger_rpm = logging.getLogger("dnf.rpm") logger_rpm.propagate = False logger_rpm.setLevel(SUBDEBUG) logfile = os.path.join(logdir, dnf.const.LOG_RPM) handler = _create_filehandler(logfile) logger_rpm.addHandler(handler) _paint_mark(logger_rpm)
def init_log(args): """Init loggers based on the argparse namespace passed.""" level = args.loglevel.upper() try: numeric_level = getattr(logging, level) except AttributeError: raise ValueError("Invalid log level: {}".format(args.loglevel)) if numeric_level > logging.DEBUG and args.debug: numeric_level = logging.DEBUG console, ram = _init_handlers(numeric_level, args.color, args.force_color, args.json_logging, args.loglines) root = logging.getLogger() if console is not None: if args.logfilter is not None: console.addFilter(LogFilter(args.logfilter.split(','))) root.addHandler(console) if ram is not None: root.addHandler(ram) root.setLevel(logging.NOTSET) logging.captureWarnings(True) _init_py_warnings() QtCore.qInstallMessageHandler(qt_message_handler) global _log_inited _log_inited = True
def start(self, level): self._level = level logging.captureWarnings(True) # Set new log file path. Ensure that the directory exists # Ensure that the log directory is not too full. We'll determine this by size and not date archiving # This is suboptimal compared native Windows or UNIX size listing, but python does not offer anything faster # without system scripting. However, based on size limit (10MB), this method will not take more than 5ms. if platform.system() is "Linux" or platform.system() is "Darwin": self.__config_unix__() # window case removed. TODO else: raise lexcep("Error: Incompatible OS detected. Only Linux, Mac, or Windows is supported").with_traceback(sys.exc_info()[2]) # Verify that the file can be created and does not exist try: with open(self._fn) as f: pass except IOError as e: raise lexcep("log file\n\n" + self._fn + "\n\nexists. Uniquely named log file should not exist prior to initialization.\nPlease contact your System Administrator\n\n" + str(e)).with_traceback(sys.exc_info()[2]) try: logging.basicConfig(filename=self._fn, level=self._level) except FileNotFoundError as e: raise lexcep("log file\n\n" + self._fn + "\n\nwas not created for logging.\nCheck log directory permissions or contact your System Administrator\n\n" + str(e)).with_traceback(sys.exc_info()[2]) except ValueError as e: raise lexcep("ValueError was encountered while initializing logging:\n\n" + str(e)).with_traceback(sys.exc_info()[2]) except Exception as e: raise lexcep("Internal Error: unhandled exception occured:\n\n" + str(e)).with_traceback(sys.exc_info()[2])
def config_logging(self): import logging.config logging_config = {'version': 1, 'incremental': False, 'formatters': {'standard': {'format': '-- %(levelname)-8s [%(name)s] -- %(message)s'}, 'debug': {'format': '''\ -- %(levelname)-8s %(asctime)24s PID %(process)-12d TID %(thread)-20d from logger "%(name)s" at location %(pathname)s:%(lineno)d [%(funcName)s()] :: %(message)s '''}}, 'handlers': {'console': {'class': 'logging.StreamHandler', 'stream': 'ext://sys.stdout', 'formatter': 'standard'}}, 'loggers': {'mdtools': {'handlers': ['console'], 'propagate': False}, 'work_managers': {'handlers': ['console'], 'propagate': False}, 'py.warnings': {'handlers': ['console'], 'propagate': False}}, 'root': {'handlers': ['console']}} logging_config['loggers'][self.process_name] = {'handlers': ['console'], 'propagate': False} if self.verbosity == 'debug': logging_config['root']['level'] = 5 #'DEBUG' logging_config['handlers']['console']['formatter'] = 'debug' elif self.verbosity == 'verbose': logging_config['root']['level'] = 'INFO' else: logging_config['root']['level'] = 'WARNING' logging.config.dictConfig(logging_config) logging_config['incremental'] = True logging.captureWarnings(True)
def configure_stream_logger(level, logger): """ Configure the default stream handler for logging messages to the console. This also configures the basic logging environment for the application. :param level: The level to set the logger to. :type level: int, str :param str logger: The logger to add the stream handler for. :return: The new configured stream handler. :rtype: :py:class:`logging.StreamHandler` """ if isinstance(level, str): level = getattr(logging, level) root_logger = logging.getLogger('') for handler in root_logger.handlers: root_logger.removeHandler(handler) logging.getLogger(logger).setLevel(logging.DEBUG) console_log_handler = logging.StreamHandler() console_log_handler.setLevel(level) if its.on_linux: console_log_handler.setFormatter(color.ColoredLogFormatter("%(levelname)s %(message)s")) else: console_log_handler.setFormatter(logging.Formatter("%(levelname)-8s %(message)s")) logging.getLogger(logger).addHandler(console_log_handler) logging.captureWarnings(True) return console_log_handler
def _configure_logging(self): if self.options.debug: warnings.simplefilter('default') try: logging.captureWarnings(True) except AttributeError: pass root_logger = logging.getLogger('') root_logger.setLevel(logging.INFO) # Set up logging to a file if self.options.log_file: file_handler = logging.FileHandler(filename=self.options.log_file) formatter = logging.Formatter(self.LOG_GEARBOX_FORMAT, datefmt=self.LOG_DATE_FORMAT) file_handler.setFormatter(formatter) root_logger.addHandler(file_handler) # Always send higher-level messages to the console via stderr console = logging.StreamHandler(sys.stderr) console_level = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG, }.get(self.options.verbose_level, logging.DEBUG) console.setLevel(console_level) formatter = logging.Formatter(self.LOG_GEARBOX_FORMAT, datefmt=self.LOG_DATE_FORMAT) console.setFormatter(formatter) root_logger.addHandler(console)
def config_production(self): logger = logging.getLogger(self.name) if not logger.root.handlers: self._read_config_file() logging.captureWarnings(True) logger = logging.getLogger(self.name) return logger
def main(earliest_night, latest_night, data_dir, jar, xml, db, out, queue, walltime, engine, num_runs, vmem, log_level, port, source, conditions, max_delta_t, local, password): level=logging.INFO if log_level is 'DEBUG': level = logging.DEBUG elif log_level is 'WARN': level = logging.WARN elif log_level is 'INFO': level = logging.INFO logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s'), level=level) jarpath = os.path.abspath(jar) xmlpath =os. path.abspath(xml) outpath = os.path.abspath(out) erna.ensure_output(out) db_path = os.path.abspath(db) output_directory = os.path.dirname(outpath) #create dir if it doesnt exist os.makedirs(output_directory, exist_ok=True) logger.info("Writing output data to {}".format(out)) factdb = sqlalchemy.create_engine("mysql+pymysql://factread:{}@129.194.168.95/factdata".format(password)) data_conditions=dcc.conditions[conditions] df_runs = erna.load(earliest_night, latest_night, data_dir, source_name=source, timedelta_in_minutes=max_delta_t, factdb=factdb, data_conditions=data_conditions) logger.info("Would process {} jobs with {} runs per job".format(len(df_runs)//num_runs, num_runs)) click.confirm('Do you want to continue processing and start jobs?', abort=True) job_list = make_jobs(jarpath, xmlpath, db_path, output_directory, df_runs, engine, queue, vmem, num_runs, walltime) job_outputs = gridmap.process_jobs(job_list, max_processes=len(job_list), local=local) erna.collect_output(job_outputs, out, df_runs)
def _configure_logging(self): """ Setup logging from LOGGING_CONFIG and LOGGING settings. """ try: # Route warnings through python logging logging.captureWarnings(True) # Allow DeprecationWarnings through the warnings filters warnings.simplefilter("default", DeprecationWarning) except AttributeError: # No captureWarnings on Python 2.6, DeprecationWarnings are on anyway pass if self.LOGGING_CONFIG: from django.utils.log import DEFAULT_LOGGING # First find the logging configuration function ... logging_config_path, logging_config_func_name = self.LOGGING_CONFIG.rsplit('.', 1) logging_config_module = importlib.import_module(logging_config_path) logging_config_func = getattr(logging_config_module, logging_config_func_name) logging_config_func(DEFAULT_LOGGING) if self.LOGGING: # Backwards-compatibility shim for #16288 fix compat_patch_logging_config(self.LOGGING) # ... then invoke it with the logging settings logging_config_func(self.LOGGING)
def initialize_logger(): """sets up the logger including a console, file and qt handler """ # initialize logger logging.basicConfig(format="%(message)s", level=logging.INFO) logging.addLevelName(logging.CRITICAL, 'critical') logging.addLevelName(logging.ERROR, 'error') logging.addLevelName(logging.WARNING, 'warning') logging.addLevelName(logging.INFO, 'info') logging.addLevelName(logging.DEBUG, 'debug') logging.addLevelName(logging.NOTSET, 'not set') logging.captureWarnings(True) logger = logging.getLogger() logger.setLevel(logging.INFO) # set level of stream handler which logs to stderr logger.handlers[0].setLevel(logging.WARNING) # add file logger rotating_file_handler = logging.handlers.RotatingFileHandler( 'qudi.log', maxBytes=10*1024*1024, backupCount=5) rotating_file_handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s %(name)s %(message)s', datefmt="%Y-%m-%d %H:%M:%S")) rotating_file_handler.doRollover() rotating_file_handler.setLevel(logging.DEBUG) logger.addHandler(rotating_file_handler) # add Qt log handler qt_log_handler = QtLogHandler() qt_log_handler.setLevel(logging.DEBUG) logging.getLogger().addHandler(qt_log_handler) for logger_name in ['core', 'gui', 'logic', 'hardware']: logging.getLogger(logger_name).setLevel(logging.DEBUG)
def main(): parser = argparse.ArgumentParser(description='King Phisher SMTP Debug Server', conflict_handler='resolve') parser.add_argument('-v', '--version', action='version', version=parser.prog + ' Version: ' + version.version) parser.add_argument('-L', '--log', dest='loglvl', action='store', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='CRITICAL', help='set the logging level') parser.add_argument('-f', '--foreground', dest='foreground', action='store_true', default=False, help='run in forground (do not fork)') parser.add_argument('-a', '--address', dest='address', default='127.0.0.1', help='address to listen on') parser.add_argument('-p', '--port', dest='port', type=int, default=2525, help='port to listen on') arguments = parser.parse_args() logging.getLogger('').setLevel(logging.DEBUG) console_log_handler = logging.StreamHandler() console_log_handler.setLevel(getattr(logging, arguments.loglvl)) console_log_handler.setFormatter(logging.Formatter("%(levelname)-8s %(message)s")) logging.getLogger('').addHandler(console_log_handler) logging.captureWarnings(True) del parser if (not arguments.foreground) and os.fork(): return bind_address = (arguments.address, arguments.port) server = smtp_server.KingPhisherSMTPServer(bind_address, None, debugging=True) color.print_status("smtp server listening on {0}:{1}".format(bind_address[0], bind_address[1])) try: server.serve_forever() except KeyboardInterrupt: color.print_status('keyboard interrupt caught, now exiting')
def setuplogger(consolelevel, filename=None, filelevel=None): """ setup the python root logger to log to the console with defined log level. Optionally also log to file with the provided level """ if filelevel == None: filelevel = consolelevel if sys.version.startswith("2.7"): logging.captureWarnings(True) rootlogger = logging.getLogger() rootlogger.setLevel(min(consolelevel, filelevel)) formatter = logging.Formatter('%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S') if filename != None: filehandler = logging.FileHandler(filename) filehandler.setLevel(filelevel) filehandler.setFormatter(formatter) rootlogger.addHandler(filehandler) consolehandler = logging.StreamHandler() consolehandler.setLevel(consolelevel) consolehandler.setFormatter(formatter) rootlogger.addHandler(consolehandler)
def set_up_log(filename): # Add file extension. filename += '.log' print 'Preparing log file:', filename # Capture warnings. logging.captureWarnings(True) # Set output format. formatter = logging.Formatter(fmt='%(asctime)s %(message)s', datefmt='%d/%m/%Y %H:%M:%S') # Create file handler. fh = logging.FileHandler(filename=filename, mode='w') fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) # Create log. log = logging.getLogger('log') log.setLevel(logging.DEBUG) log.addHandler(fh) # Send test message. log.info('The log file has been set-up.') return log
def load_log(level): """ Load a log that controls the amount of information that is shown to the user. Parameter --------- level : logging.Level Return ------ log : logging.log """ logging.captureWarnings(False) formatter = logging.Formatter() handler = logging.StreamHandler() handler.setFormatter(formatter) log = logging.getLogger(__name__) log.addHandler(handler) log.setLevel(level) return log
def main(): logging.captureWarnings(True) # Request the list of contributors print('GET {}'.format(URL)) resp = requests.get(URL) contributors = resp.json() lines = [] for contributor in contributors: time.sleep(1.0) # Request each contributor individually to get the full name print('GET {}'.format(contributor['url'])) resp = requests.get(contributor['url']) user = resp.json() name = user.get('name') or user['login'] url = user['html_url'] lines.append('* `{} <{}>`_'.format(name, url)) print('Writing to {}'.format(FILENAME)) text = HEADER + '\n'.join(lines) text = text.encode('utf-8') with open(FILENAME, 'wb') as fp: fp.write(text)
def log(debug=False, path=None): """Log messages. If path is None, logging messages will be printed to the console (stdout). If it not None, logging messages will be appended to the file at that path. Typically someone using Nengo as a library will set up their own logging things, and Nengo will just populate their log. However, if the user is using Nengo directly, they can use this function to get log output. """ level = logging.DEBUG if debug else logging.WARNING logging.root.setLevel(level) if path is None: handler = console_handler else: for handler in logging.root.handlers: if (isinstance(handler, logging.FileHandler) and handler.baseFilename == path and handler.formatter == file_formatter): break else: handler = logging.FileHandler(path, encoding='utf-8') handler.setFormatter(file_formatter) if handler not in logging.root.handlers: logging.root.addHandler(handler) handler.setLevel(level) try: logging.captureWarnings(True) except AttributeError: # logging.captureWarnings doesn't exist in Python 2.6; ignore it pass
def test_warnings(self): with warnings.catch_warnings(): logging.captureWarnings(True) try: warnings.filterwarnings("always", category=UserWarning) file = io.StringIO() h = logging.StreamHandler(file) logger = logging.getLogger("py.warnings") logger.addHandler(h) warnings.warn("I'm warning you...") logger.removeHandler(h) s = file.getvalue() h.close() self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) #See if an explicit file uses the original implementation file = io.StringIO() warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, file, "Dummy line") s = file.getvalue() file.close() self.assertEqual(s, "dummy.py:42: UserWarning: Explicit\n Dummy line\n") finally: logging.captureWarnings(False)
def configure_logger(level): """ Configure a root logger to print records in pretty format. The format is more readable for end users, since it's not necessary at all to know a record's dateime and a source of the record. Examples:: [INFO] message [WARN] message [ERRO] message :param level: a minimum logging level to be printed """ class _Formatter(logging.Formatter): def format(self, record): record.levelname = record.levelname[:4] return super(_Formatter, self).format(record) # create stream handler with custom formatter stream_handler = logging.StreamHandler() stream_handler.setFormatter(_Formatter('[%(levelname)s] %(message)s')) # configure root logger logger = logging.getLogger() logger.addHandler(stream_handler) logger.setLevel(level) # capture warnings issued by 'warnings' module logging.captureWarnings(True)
def _configure_logging(self): """Configure logging for Insights. It will load configuration from logging.conf if present in root directory, otherwise custom logging format is used by default """ if self.configured: LOGGER.info("Already configured") return # All output should be made by the logging module, including warnings logging.captureWarnings(True) # Allow overriding logging config based on the presence of logging.conf # file on Insights's project root logging_conf_path = os.path.join(get_project_root(), 'logging.conf') if os.path.isfile(logging_conf_path): config.fileConfig(logging_conf_path) else: logging.basicConfig( format='%(levelname)s %(module)s:%(lineno)d: %(message)s' )
def set_logging_config(config, debug, verbosity, uncaught_logger, uncaught_handler): # configure logging globally import logging.config as logconfig logconfig.dictConfig(config) # make sure we log any warnings log.captureWarnings(True) import warnings categories = (DeprecationWarning, PendingDeprecationWarning) if verbosity > 2: warnings.simplefilter("always") elif debug or verbosity > 0: for category in categories: warnings.simplefilter("always", category=category) # make sure we also log any uncaught exceptions if uncaught_logger is None: logger = log.getLogger(__name__) else: logger = log.getLogger(uncaught_logger) if uncaught_handler is None: def exception_logger(exc_type, exc_value, exc_tb): logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_tb)) uncaught_handler = exception_logger sys.excepthook = uncaught_handler return logger
def sync_netscaler_config(collector, stack, **kwargs): """Sync netscaler configuration with the specified netscaler""" logging.captureWarnings(True) logging.getLogger("py.warnings").setLevel(logging.ERROR) configuration = collector.configuration if stack.netscaler is NotSpecified or stack.netscaler.configuration is NotSpecified: raise BespinError("Please configure {netscaler.configuration}") if stack.netscaler.syncable_environments is not NotSpecified: if configuration["environment"] not in stack.netscaler.syncable_environments: raise BespinError("Sorry, can only sync netscaler config for particular environments", wanted=configuration["environment"], available=list(stack.netscaler.syncable_environments)) for_layers = [] all_configuration = {} for vkey, value in stack.netscaler.configuration.items(): for key, thing in value.items(): if thing.environments is NotSpecified or configuration["environment"] in thing.environments: for_layers.append(thing.long_name) all_configuration[thing.long_name] = thing if vkey not in all_configuration: all_configuration[vkey] = {} all_configuration[vkey][key] = thing layers = Layers(for_layers, all_stacks=all_configuration) layers.add_all_to_layers() stack.netscaler.syncing_configuration = True with stack.netscaler as netscaler: for layer in layers.layered: for _, thing in layer: netscaler.sync(all_configuration, configuration["environment"], thing)
def cli(debug, config_dir, output, query): """CloudMunch CLI.\n This is a CLI tool for executing commands against CloudMunch API. """ log = logging.getLogger('cloudmunch') stdoutHanlder = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') stdoutHanlder.setFormatter(formatter) log.addHandler(stdoutHanlder) if debug: log.setLevel(logging.DEBUG) import httplib httplib.HTTPConnection.debuglevel = 1 requests_log = logging.getLogger("requests.packages.urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True if sys.version_info >= (2,7): logging.captureWarnings(True) else: log.setLevel(logging.NOTSET) urllib3.disable_warnings() if sys.version_info >= (2,7): logging.captureWarnings(False) log.info('Setting config dir to %s' % config_dir) log.info('Python version is %s - %s' % (sys.version, sys.hexversion)) cloudmunch.config.config_dir = config_dir cloudmunch.config.credentials = cloudmunch.config.Credentials(cloudmunch.config.config_dir) cloudmunch.config.config = cloudmunch.config.Config(cloudmunch.config.config_dir)
def main(): """ main entry point for script """ opts = getoptions() logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%dT%H:%M:%S', level=opts['log']) logging.captureWarnings(True) config = Config() threads = opts['threads'] if threads <= 1: processjobs(config, opts, None) return else: proclist = [] for procid in xrange(threads): p = Process(target=processjobs, args=(config, opts, procid)) p.start() proclist.append(p) for proc in proclist: p.join()
def __init__(self, default_logging_levels={}): logging.basicConfig(level=self.logging_levels.get('logging_level', logging.INFO)) self.system_logger = logging.getLogger("system") self.world_logger = logging.getLogger("world") self.nodenet_logger = logging.getLogger("nodenet") self.system_logger.setLevel(self.logging_levels.get(default_logging_levels.get('system', {}), logging.WARNING)) self.world_logger.setLevel(self.logging_levels.get(default_logging_levels.get('world', {}), logging.WARNING)) self.nodenet_logger.setLevel(self.logging_levels.get(default_logging_levels.get('nodenet', {}), logging.WARNING)) logging.captureWarnings(True) self.handlers = { 'system': RecordWebStorageHandler(self.system_record_storage), 'world': RecordWebStorageHandler(self.world_record_storage), 'nodenet': RecordWebStorageHandler(self.nodenet_record_storage) } logging.getLogger("py.warnings").addHandler(self.handlers['system']) logging.getLogger("system").addHandler(self.handlers['system']) logging.getLogger("world").addHandler(self.handlers['world']) logging.getLogger("nodenet").addHandler(self.handlers['nodenet']) logging.getLogger("system").debug("System logger ready.") logging.getLogger("world").debug("World logger ready.") logging.getLogger("nodenet").debug("Nodenet logger ready.")
def configLogging(build): global log logging.captureWarnings(True) term = os.environ.get('TERM') if term and term.startswith('screen.'): term = term[7:] record_format = '%(name)s: %(message)s' try: clifmt = utils.ColoredFormatter( blessings.Terminal(term, force_styling=build.force_color), record_format) except curses.error: try: # try falling back to basic term type clifmt = utils.ColoredFormatter( blessings.Terminal('linux', force_styling=build.force_color), record_format) except curses.error: # fall back to uncolored formatter clifmt = logging.Formatter(record_format) root_logger = logging.getLogger() clihandler = logging.StreamHandler(sys.stdout) clihandler.setFormatter(clifmt) if isinstance(build.log_level, str): build.log_level = build.log_level.upper() root_logger.setLevel(build.log_level) log.setLevel(build.log_level) root_logger.addHandler(clihandler) requests_logger = logging.getLogger("requests") requests_logger.setLevel(logging.WARN) urllib_logger = logging.getLogger("urllib3") urllib_logger.setLevel(logging.CRITICAL)
def init(self, parser, opts, args): if len(args) < 1: parser.error("No application module specified.") self.cfg.set("default_proc_name", args[0]) self.app_uri = args[0] logging.captureWarnings(True) if opts.auto_config: # Default config_dir config_dir = os.path.join(self.cfg.chdir, 'etc', self.app_uri.split(":", 1)[0]) if os.path.exists(config_dir): self.cfg.set('config_dir', config_dir) # Define dev etc folder as default config directory # generate config_dir directly: egg or chicken problem if opts.config_dir: self.cfg.set('config_dir', opts.config_dir) if not opts.config: opts.config = os.path.join(self.cfg.config_dir, 'api_hour/gunicorn_conf.py') if not self.cfg.logconfig: self.cfg.set('logconfig', os.path.join(self.cfg.config_dir, 'api_hour/logging.ini')) else: # To avoid with Gunicorn 19 that the console it's empty when you test if not opts.errorlog: opts.errorlog = '-' if not opts.accesslog: opts.accesslog = '-'
pyFAI-average is a small utility that averages out a serie of files, for example for dark, or flat or calibration images """ __author__ = "Jerome Kieffer, Picca Frédéric-Emmanuel" __contact__ = "*****@*****.**" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" __date__ = "05/03/2018" __status__ = "production" import os import logging logger = logging.getLogger("average") logging.basicConfig(level=logging.INFO) logging.captureWarnings(True) import pyFAI.utils.shell import pyFAI.utils.stringutil from pyFAI import average from pyFAI.third_party.argparse import ArgumentParser class PreEmitStreamHandler(logging.Handler): """Handler allowing to hook a function before the emit function. The main logging feature is delegated to a sub handler. """ def __init__(self, handler): self._handler = handler
def main(argv=None): """ Handles command line arguments and gets things started. Parameters ---------- argv : list of str List of arguments, as if specified on the command-line. If None, ``sys.argv[1:]`` is used instead. """ parser = argparse.ArgumentParser(description="Prints out the weights of a \ given model.", conflict_handler='resolve', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('model_file', help='model file to load') group = parser.add_mutually_exclusive_group() group.add_argument('--k', help='number of top features to print (0 for all)', type=int, default=50) group.add_argument("--sort_by_labels", '-s', action='store_true', default=False, help="order the features by classes") parser.add_argument('--sign', choices=['positive', 'negative', 'all'], default='all', help='show only positive, only negative, ' + 'or all weights') parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__)) args = parser.parse_args(argv) # Make warnings from built-in warnings module get formatted more nicely logging.captureWarnings(True) logging.basicConfig(format=('%(asctime)s - %(name)s - %(levelname)s - ' + '%(message)s')) k = args.k if args.k > 0 else None learner = Learner.from_file(args.model_file) (weights, intercept) = learner.model_params multiclass = False model = learner._model if (isinstance(model, LinearSVC) or (isinstance(model, LogisticRegression) and len(learner.label_list) > 2) or (isinstance(model, SVC) and model.kernel == 'linear')): multiclass = True weight_items = iteritems(weights) if args.sign == 'positive': weight_items = (x for x in weight_items if x[1] > 0) elif args.sign == 'negative': weight_items = (x for x in weight_items if x[1] < 0) if intercept is not None: # subclass of LinearModel if '_intercept_' in intercept: # Some learners (e.g. LinearSVR) may return an array of intercepts but # sometimes that array is of length 1 so we don't need to print that # as an array/list. First, let's normalize these cases. model_intercepts = intercept['_intercept_'] intercept_is_array = isinstance(model_intercepts, np.ndarray) num_intercepts = len(model_intercepts) if intercept_is_array else 1 if intercept_is_array and num_intercepts == 1: model_intercepts = model_intercepts[0] intercept_is_array = False # now print out the intercepts print("intercept = {:.12f}".format(model_intercepts)) else: print("== intercept values ==") for (label, val) in intercept.items(): print("{: .12f}\t{}".format(val, label)) print() print("Number of nonzero features:", len(weights), file=sys.stderr) weight_by_class = defaultdict(dict) if multiclass and args.sort_by_labels: for label_feature, weight in weight_items: label, feature = label_feature.split() weight_by_class[label][feature] = weight for label in sorted(weight_by_class): for feat, val in sorted(weight_by_class[label].items(), key=lambda x: -abs(x[1])): print("{: .12f}\t{}\t{}".format(val, label, feat)) else: for feat, val in sorted(weight_items, key=lambda x: -abs(x[1]))[:k]: print("{: .12f}\t{}".format(val, feat))
def vls_init_logger(): import logging logging.getLogger("pytorch_lightning").setLevel(logging.ERROR) logging.captureWarnings(True)
def main(): logging.captureWarnings(True) parser = argparse.ArgumentParser( description="Used to register new users with a given homeserver when" " registration has been disabled. The homeserver must be" " configured with the 'registration_shared_secret' option" " set.") parser.add_argument( "-u", "--user", default=None, help="Local part of the new user. Will prompt if omitted.", ) parser.add_argument( "-p", "--password", default=None, help="New password for user. Will prompt if omitted.", ) parser.add_argument( "-t", "--user_type", default=None, help="User type as specified in synapse.api.constants.UserTypes", ) admin_group = parser.add_mutually_exclusive_group() admin_group.add_argument( "-a", "--admin", action="store_true", help=("Register new user as an admin. " "Will prompt if --no-admin is not set either."), ) admin_group.add_argument( "--no-admin", action="store_true", help=("Register new user as a regular user. " "Will prompt if --admin is not set either."), ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( "-c", "--config", type=argparse.FileType("r"), help="Path to server config file. Used to read in shared secret.", ) group.add_argument("-k", "--shared-secret", help="Shared secret as defined in server config file.") parser.add_argument( "server_url", default="https://localhost:8448", nargs="?", help="URL to use to talk to the homeserver. Defaults to " " 'https://localhost:8448'.", ) args = parser.parse_args() if "config" in args and args.config: config = yaml.safe_load(args.config) secret = config.get("registration_shared_secret", None) if not secret: print("No 'registration_shared_secret' defined in config.") sys.exit(1) else: secret = args.shared_secret admin = None if args.admin or args.no_admin: admin = args.admin register_new_user(args.user, args.password, args.server_url, secret, admin, args.user_type)
def config_logging(self): import logging.config logging_config = { 'version': 1, 'incremental': False, 'formatters': { 'standard': { 'format': '-- %(levelname)-8s [%(name)s] -- %(message)s' }, 'debug': { 'format': '''\ -- %(levelname)-8s %(asctime)24s PID %(process)-12d TID %(thread)-20d from logger "%(name)s" at location %(pathname)s:%(lineno)d [%(funcName)s()] :: %(message)s ''' } }, 'handlers': { 'console': { 'class': 'logging.StreamHandler', 'stream': 'ext://sys.stdout', 'formatter': 'standard' } }, 'loggers': { 'sim': { 'handlers': ['console'], 'propagate': False }, 'ssa': { 'handlers': ['console'], 'propagate': False }, 'work_managers': { 'handlers': ['console'], 'propagate': False }, 'py.warnings': { 'handlers': ['console'], 'propagate': False } }, 'root': { 'handlers': ['console'] } } logging_config['loggers'][self.process_name] = { 'handlers': ['console'], 'propagate': False } if self.verbosity == 'debug': logging_config['root']['level'] = 5 #'DEBUG' logging_config['handlers']['console']['formatter'] = 'debug' elif self.verbosity == 'verbose': logging_config['root']['level'] = 'INFO' else: logging_config['root']['level'] = 'WARNING' logging.config.dictConfig(logging_config) logging_config['incremental'] = True logging.captureWarnings(True)
def suppress_urllib_warnings(): """Capture urllib warnings if possible, else disable them (python 2.6).""" try: lg.captureWarnings(True) except AttributeError: disable_warnings()
# treat humanfriendly as optional dependency humanfriendly_available = False try: from humanfriendly import AutomaticSpinner from humanfriendly.text import pluralize humanfriendly_available = True except ImportError: # pragma: no cover def pluralize(_1, _2, plural): return plural logging.basicConfig() log = logging.getLogger(sys.argv[0] if __name__ == "__main__" else __name__) logging.captureWarnings( True ) # see https://urllib3.readthedocs.org/en/latest/security.html#disabling-warnings class DownloadError(Exception): """content could not be downloaded as requested.""" pass class Browser(object): """download relative or absolute url and return soup.""" def __init__(self, args, root_url): """Construct a browser object with options.""" self.save = args.save if hasattr(args, 'save') else False self.load = args.load if hasattr(args, 'load') else False self.load_dir = args.load_dir if hasattr(args, 'load_dir') else '.'
def setup_logger(logfile="/tmp/dynafed_storagestats.log", logid=False, loglevel="WARNING", verbose=False): """Setup the logger format to be used throughout the script. Arguments: logfile -- string defining path to write logs to. logid -- string defining an ID to be logged. loglevel -- string defining level to log: "DEBUG, INFO, WARNING, ERROR" verbose -- boolean. 'True' prints log messages to stderr. """ # To capture warnings emitted by modules. logging.captureWarnings(True) # Create file logger. _logger = logging.getLogger("dynafed_storagestats") # Set log level to use. _num_loglevel = getattr(logging, loglevel.upper()) _logger.setLevel(_num_loglevel) # Set file where to log and the mode to use and set the format to use. _log_handler_file = logging.handlers.TimedRotatingFileHandler( logfile, when="midnight", backupCount=15, ) # Set the format depending whether a log id is requested. if logid: # Add ContextFilter _logid_context = ContextFilter(logid) # Add logid filter. _log_handler_file.addFilter(_logid_context) # Set logger format _log_format_file = logging.Formatter( '%(asctime)s - [%(logid)s] - [%(levelname)s]%(message)s') else: # Set logger format _log_format_file = logging.Formatter( '%(asctime)s - [%(levelname)s]%(message)s') # Set the format to the file handler. _log_handler_file.setFormatter(_log_format_file) # Add the file handler. _logger.addHandler(_log_handler_file) # Create STDERR handler if verbose is requested and add it to logger. if verbose: log_handler_stderr = logging.StreamHandler() if logid: log_format_stderr = logging.Formatter( '%(asctime)s - [%(logid)s] - [%(levelname)s]%(message)s') log_handler_stderr.addFilter(_logid_context) else: log_format_stderr = logging.Formatter( '%(asctime)s - [%(levelname)s]%(message)s') log_handler_stderr.setLevel(_num_loglevel) log_handler_stderr.setFormatter(log_format_stderr) # Add handler _logger.addHandler(log_handler_stderr)
def tearDown(self): # Be sure to reset the warning capture logging.captureWarnings(False) super(TestDaiquiri, self).tearDown()
def init_logger(): global _logger_init if _logger_init: return _logger_init = True old_factory = logging.getLogRecordFactory() def record_factory(*args, **kwargs): record = old_factory(*args, **kwargs) record.perf_info = "" return record logging.setLogRecordFactory(record_factory) logging.addLevelName(25, "INFO") logging.captureWarnings(True) from .tools.translate import resetlocale resetlocale() # create a format for log messages and dates format = '%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s %(perf_info)s' # Normal Handler on stderr handler = logging.StreamHandler() if tools.config['syslog']: # SysLog Handler if os.name == 'nt': handler = logging.handlers.NTEventLogHandler( "%s %s" % (release.description, release.version)) elif platform.system() == 'Darwin': handler = logging.handlers.SysLogHandler('/var/run/log') else: handler = logging.handlers.SysLogHandler('/dev/log') format = '%s %s' % (release.description, release.version) \ + ':%(dbname)s:%(levelname)s:%(name)s:%(message)s' elif tools.config['logfile']: # LogFile Handler logf = tools.config['logfile'] try: # We check we have the right location for the log files dirname = os.path.dirname(logf) if dirname and not os.path.isdir(dirname): os.makedirs(dirname) if os.name == 'posix': handler = logging.handlers.WatchedFileHandler(logf) else: handler = logging.FileHandler(logf) except Exception: sys.stderr.write( "ERROR: couldn't create the logfile directory. Logging to the standard output.\n" ) # Check that handler.stream has a fileno() method: when running OpenERP # behind Apache with mod_wsgi, handler.stream will have type mod_wsgi.Log, # which has no fileno() method. (mod_wsgi.Log is what is being bound to # sys.stderr when the logging.StreamHandler is being constructed above.) def is_a_tty(stream): return hasattr(stream, 'fileno') and os.isatty(stream.fileno()) if os.name == 'posix' and isinstance( handler, logging.StreamHandler) and is_a_tty(handler.stream): formatter = ColoredFormatter(format) perf_filter = ColoredPerfFilter() else: formatter = DBFormatter(format) perf_filter = PerfFilter() handler.setFormatter(formatter) logging.getLogger().addHandler(handler) logging.getLogger('werkzeug').addFilter(perf_filter) if tools.config['log_db']: db_levels = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL, } postgresqlHandler = PostgreSQLHandler() postgresqlHandler.setLevel( int( db_levels.get(tools.config['log_db_level'], tools.config['log_db_level']))) logging.getLogger().addHandler(postgresqlHandler) # Configure loggers levels pseudo_config = PSEUDOCONFIG_MAPPER.get(tools.config['log_level'], []) logconfig = tools.config['log_handler'] logging_configurations = DEFAULT_LOG_CONFIGURATION + pseudo_config + logconfig for logconfig_item in logging_configurations: loggername, level = logconfig_item.split(':') level = getattr(logging, level, logging.INFO) logger = logging.getLogger(loggername) logger.setLevel(level) for logconfig_item in logging_configurations: _logger.debug('logger level set: "%s"', logconfig_item)
def main(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument('-u', '--user', help='User to analyze') parser.add_argument('-o', '--organization', help='Organization to analyze') parser.add_argument('-r', '--repo', help='Repository to analyze') parser.add_argument('-t', '--token', help='Personal access token') parser.add_argument('--token-file', default='.token', help='File to load personal access token from, if ' '--token isn\'t specified (default: %(default)s)') parser.add_argument('--lfs', action='store_true', help='Clone repo to calculate LFS usage') parser.add_argument( '-s', '--sort', default='full_name', help='Repository metadata field to sort by. Prefix with ' '- to sort descending. (default: %(default)s)'), parser.add_argument('--fields', metavar='FIELD[,FIELD...]', help='Comma-separated list of fields to show') parser.add_argument('-h', '--humanize', action='store_true', help='Convert certain fields to human-readable format') parser.add_argument('--totals', action='store_true', help='Generate totals row') parser.add_argument( '-f', '--format', choices=('list', 'table', 'json', 'csv'), default='list', help='Format the output as a list, table, json, or csv ' '(default: %(default)s)') parser.add_argument('-c', '--csv', action='store_const', dest='format', const='csv', help='Output as CSV') parser.add_argument('-j', '--json', action='store_const', dest='format', const='json', help='Output as JSON') parser.add_argument('-l', '--log', action='store', metavar='FILE', help='Log output to a file') parser.add_argument('-v', '--verbose', action='count', default=1, help='Increase verbosity.') parser.add_argument('-q', '--quiet', action='count', default=0, help='Decrease verbosity.') parser.add_argument('--help', action='help') args = parser.parse_args() args.verbose -= args.quiet console = logging.StreamHandler() console.setFormatter(logging.Formatter('%(levelname)-8s %(message)s')) console.setLevel(logging.WARNING) logging.getLogger().addHandler(console) if args.verbose >= 2: console.setLevel(logging.DEBUG) elif args.verbose >= 1: console.setLevel(logging.INFO) else: console.setLevel(logging.WARNING) logging.getLogger().addHandler(console) if args.log: logfile = logging.FileHandler(args.log, 'w') logfile.setLevel(logging.DEBUG) logfile.setFormatter( logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')) logging.getLogger().addHandler(logfile) logging.getLogger().setLevel(logging.DEBUG) logging.captureWarnings(True) if not args.token and args.token_file and os.path.exists(args.token_file): logging.debug('Reading token from {}...'.format(args.token_file)) with open(args.token_file) as fp: args.token = fp.read().strip() repos = fetch_repos(user=args.user, organization=args.organization, repo=args.repo, access_token=args.token) if args.lfs: for repo in repos: try: repo['lfs'] = get_lfs_usage(repo) except: logging.exception('Error getting LFS usage for {}'.format( repo['full_name'])) repos.sort(key=sort_key(args.sort), reverse=args.sort.startswith('-')) multi_owners = len(set(repo['owner']['login'] for repo in repos)) > 1 fields = get_fields(args.fields, format=args.format, multi_owners=multi_owners, lfs=args.lfs) id_field = 'full_name' if multi_owners else 'name' if fields is not None: for field in ('full_name', 'name'): if field in fields: id_field = field break else: fields.insert(0, id_field) totals = {id_field: 'Totals'} if args.totals else None formatted = [ format_repo(repo, fields=fields, format=args.format, humanize=args.humanize, totals=totals) for repo in repos ] if args.totals and fields: totals_row = collections.OrderedDict() for field in fields: value = totals.get(field, None) if (args.humanize and value is not None and FIELD_ALIASES.get(field, field) in HUMANIZE): value = HUMANIZE[FIELD_ALIASES.get(field, field)](value) if args.format == 'csv': value = str(value) if value is not None else '' totals_row[field] = value formatted.append(totals_row) if args.format == 'json': print(json.dumps(formatted, indent=2)) elif args.format == 'csv': writer = csv.DictWriter(sys.stdout, fields) writer.writeheader() writer.writerows(formatted) elif args.format == 'table': print(tabulate(formatted, headers='keys')) else: non_id_fields = [field for field in fields if field != id_field] if len(non_id_fields) == 1: widest_id = max(len(repo[id_field]) for repo in formatted) for repo in formatted: print('{1:{0}} {2}'.format(widest_id, repo[id_field], repo[non_id_fields[0]])) elif len(non_id_fields) == 0: for repo in formatted: print(repo[id_field]) else: widest_field = max(len(field) for field in non_id_fields) for repo in formatted: print(repo[id_field]) for field in fields: if field == id_field: continue print(' - {1:>{0}}: {2}'.format(widest_field, field, repo.get(field))) print()
def main(): signal.signal(signal.SIGTERM, signal_handler) parser = argparse.ArgumentParser( description='Export ES query results to Prometheus.') parser.add_argument( '-e', '--es-cluster', default='localhost', help= 'addresses of nodes in a Elasticsearch cluster to run queries on. Nodes should be separated by commas e.g. es1,es2. Ports can be provided if non-standard (9200) e.g. es1:9999 (default: localhost)' ) parser.add_argument( '--ca-certs', help= 'path to a CA certificate bundle. Can be absolute, or relative to the current working directory. If not specified, SSL certificate verification is disabled.' ) parser.add_argument( '-p', '--port', type=int, default=9206, help='port to serve the metrics endpoint on. (default: 9206)') parser.add_argument('--basic-user', help='User for authentication. (default: no user)') parser.add_argument( '--basic-password', help='Password for authentication. (default: no password)') parser.add_argument( '--query-disable', action='store_true', help= 'disable query monitoring. Config file does not need to be present if query monitoring is disabled.' ) parser.add_argument( '-c', '--config-file', default='exporter.cfg', help= 'path to query config file. Can be absolute, or relative to the current working directory. (default: exporter.cfg)' ) parser.add_argument('--cluster-health-disable', action='store_true', help='disable cluster health monitoring.') parser.add_argument( '--cluster-health-timeout', type=float, default=10.0, help= 'request timeout for cluster health monitoring, in seconds. (default: 10)' ) parser.add_argument( '--cluster-health-level', default='indices', choices=['cluster', 'indices', 'shards'], help= 'level of detail for cluster health monitoring. (default: indices)') parser.add_argument('--nodes-stats-disable', action='store_true', help='disable nodes stats monitoring.') parser.add_argument( '--nodes-stats-timeout', type=float, default=10.0, help= 'request timeout for nodes stats monitoring, in seconds. (default: 10)' ) parser.add_argument( '--nodes-stats-metrics', type=nodes_stats_metrics_parser, help= 'limit nodes stats to specific metrics. Metrics should be separated by commas e.g. indices,fs.' ) parser.add_argument('--indices-stats-disable', action='store_true', help='disable indices stats monitoring.') parser.add_argument( '--indices-stats-timeout', type=float, default=10.0, help= 'request timeout for indices stats monitoring, in seconds. (default: 10)' ) parser.add_argument( '--indices-stats-mode', default='cluster', choices=['cluster', 'indices'], help='detail mode for indices stats monitoring. (default: cluster)') parser.add_argument( '--indices-stats-metrics', type=indices_stats_metrics_parser, help= 'limit indices stats to specific metrics. Metrics should be separated by commas e.g. indices,fs.' ) parser.add_argument( '--indices-stats-fields', type=indices_stats_fields_parser, help= 'include fielddata info for specific fields. Fields should be separated by commas e.g. indices,fs. Use \'*\' for all.' ) parser.add_argument( '--zero-query-gauges-on-exception', action='store_true', help= 'set query gauges to zero after got exception on Elasticsearch query. (default: false)' ) parser.add_argument('-j', '--json-logging', action='store_true', help='turn on json logging.') parser.add_argument( '--log-level', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='detail level to log. (default: INFO)') parser.add_argument( '-v', '--verbose', action='store_true', help='turn on verbose (DEBUG) logging. Overrides --log-level.') args = parser.parse_args() if args.basic_user and args.basic_password is None: parser.error('Username provided with no password.') elif args.basic_user is None and args.basic_password: parser.error('Password provided with no username.') elif args.basic_user: http_auth = (args.basic_user, args.basic_password) else: http_auth = None log_handler = logging.StreamHandler() log_format = '[%(asctime)s] %(name)s.%(levelname)s %(threadName)s %(message)s' formatter = JogFormatter( log_format) if args.json_logging else logging.Formatter(log_format) log_handler.setFormatter(formatter) log_level = getattr(logging, args.log_level) logging.basicConfig(handlers=[log_handler], level=logging.DEBUG if args.verbose else log_level) logging.captureWarnings(True) port = args.port es_cluster = args.es_cluster.split(',') if args.ca_certs: es_client = Elasticsearch(es_cluster, verify_certs=True, ca_certs=args.ca_certs, http_auth=http_auth) else: es_client = Elasticsearch(es_cluster, verify_certs=False, http_auth=http_auth) scheduler = None if not args.query_disable: scheduler = sched.scheduler() config = configparser.ConfigParser() config.read_file(open(args.config_file)) query_prefix = 'query_' queries = {} for section in config.sections(): if section.startswith(query_prefix): query_name = section[len(query_prefix):] query_interval = config.getfloat(section, 'QueryIntervalSecs', fallback=15) query_timeout = config.getfloat(section, 'QueryTimeoutSecs', fallback=10) query_indices = config.get(section, 'QueryIndices', fallback='_all') query = json.loads(config.get(section, 'QueryJson')) zero_query_gauges_on_exception = args.zero_query_gauges_on_exception queries[query_name] = (query_interval, query_timeout, query_indices, query, zero_query_gauges_on_exception) if queries: for name, (interval, timeout, indices, query, zero_query_gauges_on_exception) in queries.items(): func = partial(run_query, es_client, name, indices, query, timeout, zero_query_gauges_on_exception) run_scheduler(scheduler, interval, func) else: logging.warn('No queries found in config file %s', args.config_file) if not args.cluster_health_disable: REGISTRY.register( ClusterHealthCollector(es_client, args.cluster_health_timeout, args.cluster_health_level)) if not args.nodes_stats_disable: REGISTRY.register( NodesStatsCollector(es_client, args.nodes_stats_timeout, metrics=args.nodes_stats_metrics)) if not args.indices_stats_disable: parse_indices = args.indices_stats_mode == 'indices' REGISTRY.register( IndicesStatsCollector(es_client, args.indices_stats_timeout, parse_indices=parse_indices, metrics=args.indices_stats_metrics, fields=args.indices_stats_fields)) logging.info('Starting server...') start_http_server(port) logging.info('Server started on port %s', port) try: if scheduler: scheduler.run() else: while True: time.sleep(5) except KeyboardInterrupt: pass shutdown()
def set_cli_logger(**kwargs): logging.captureWarnings(True) set_logger(name='atmcorr', **kwargs)
"""FEniCS Form Compiler (FFC). FFC compiles finite element variational forms into C code. """ import logging import pkg_resources from FIAT import supported_elements __version__ = pkg_resources.get_distribution("fenics-ffc").version logging.basicConfig() logger = logging.getLogger("ffc") logging.captureWarnings(capture=True) # Import main function, entry point to script from ffc.main import main # noqa: F401 # Import default parameters from ffc.parameters import default_parameters # noqa: F401 # Duplicate list of supported elements from FIAT and emove elements from # list that we don't support or don't trust supported_elements = sorted(supported_elements.keys()) supported_elements.remove("Argyris") supported_elements.remove("Hermite") supported_elements.remove("Morley")
def master(taskAlias, **kwargs): ''' The main function ''' Tabc.pypmanConfigPath = os.path.join(os.getcwd(), 'pypman') logging.captureWarnings(True) log.info('Welcome to pypman!') log.info('Current dir: %s' % os.getcwd()) log.info('PYPMAN_CONFIG_PATH:%s' % Tabc.pypmanConfigPath) mapClasses = { 'gb': GlobalTask, 'python': PythonNodeTask, 'node': PythonNodeTask } file = os.path.join(Tabc.pypmanConfigPath, 'pypman.yml') log.info('Master config file: %s' % file) Tabc.masterConfig = util.loadYml(file) processInfo = collections.namedtuple('processInfo', 'process tname') processInfoList = [] targetHosts = Tabc.masterConfig['targetHosts'] for thost in targetHosts: Tabc.setHost(thost) log.debug(Tabc.host) Tabc.setProject(taskAlias) log.debug(Tabc.project) tList = Tabc.taskList(taskAlias) log.info('Task list %s' % tList) for tname in tList: Tabc.setTask(tname) log.info('Running task:%s' % tname) executer = {} if Tabc.task.fname == "node" or Tabc.task.fname == "python": executer = mapClasses[Tabc.task.fname](**kwargs) else: executer = mapClasses["gb"](**kwargs) # global task log.info('executing task %s on host %s ' % (tname, Tabc.host.cfg['name'])) if Tabc.task.synch: executer.execute_task( ) # tasks are executed in series (when user interaction is needed or when one task depends on the other) else: processInfoList.append( processInfo( executer.execute_task(), tname)) #asynch exec; tasks are executed in parallel if not Tabc.task.synch: i = 0 while processInfoList: i = i + 1 log.debug('i= %s' % i) j = 0 for pinf in processInfoList: j = j + 1 log.debug('j=%s' % j) rcd = pinf.process.poll() log.debug(rcd) if rcd is None: # No process is done, wait a bit and check again. log.info('running task %s and buffering output.' % pinf.tname) time.sleep( 0.5 ) # too small or too large is not good; optimize case by case else: # Process finished. log.info('%s %s' % (pinf.tname, pinf.process.stdout.read().decode('utf-8'))) log.info('Task %s is done.' % pinf.tname) processInfoList.remove(pinf) break return
def init( prog: Optional[str] = None, description: Optional[str] = None, add_help: bool = True, check_run: bool = False, argv: Optional[List[str]] = None, **load: bool, ) -> Tuple[argparse.ArgumentParser, List[str], Section]: argv = (argv or sys.argv) assert len(argv) > 0 args_parser = argparse.ArgumentParser(prog=(prog or argv[0]), description=description, add_help=add_help) args_parser.add_argument("-c", "--config", dest="config_path", default="/etc/kvmd/main.yaml", metavar="<file>", type=valid_abs_file, help="Set config file path") args_parser.add_argument( "-o", "--set-options", dest="set_options", default=[], nargs="+", help="Override config options list (like sec/sub/opt=value)") args_parser.add_argument( "-m", "--dump-config", dest="dump_config", action="store_true", help="View current configuration (include all overrides)") if check_run: args_parser.add_argument("--run", dest="run", action="store_true", help="Run the service") (options, remaining) = args_parser.parse_known_args(argv) if options.dump_config: _dump_config( _init_config( config_path=options.config_path, override_options=options.set_options, load_auth=True, load_hid=True, load_atx=True, load_msd=True, load_gpio=True, )) raise SystemExit() config = _init_config(options.config_path, options.set_options, **load) logging.captureWarnings(True) logging.config.dictConfig(config.logging) if check_run and not options.run: raise SystemExit( "To prevent accidental startup, you must specify the --run option to start.\n" "Try the --help option to find out what this service does.\n" "Make sure you understand exactly what you are doing!") return (args_parser, remaining, config)
def setup(self): self.queues = queues.declare_queues(self) ################# PROVIDERS self.media_provider = select_provider( self.config, self.pkg_resources_working_set, "r2.provider.media", self.media_provider, ) self.startup_timer.intermediate("providers") ################# CONFIGURATION # AMQP is required if not self.amqp_host: raise ValueError("amqp_host not set in the .ini") if not self.cassandra_seeds: raise ValueError("cassandra_seeds not set in the .ini") # heavy load mode is read only mode with a different infobar if self.heavy_load_mode: self.read_only_mode = True origin_prefix = self.domain_prefix + "." if self.domain_prefix else "" self.origin = "http://" + origin_prefix + self.domain self.trusted_domains = set([self.domain]) if self.https_endpoint: https_url = urlparse(self.https_endpoint) self.trusted_domains.add(https_url.hostname) # load the unique hashed names of files under static static_files = os.path.join(self.paths.get('static_files'), 'static') names_file_path = os.path.join(static_files, 'names.json') if os.path.exists(names_file_path): with open(names_file_path) as handle: self.static_names = json.load(handle) else: self.static_names = {} # make python warnings go through the logging system logging.captureWarnings(capture=True) log = logging.getLogger('reddit') # when we're a script (paster run) just set up super simple logging if self.running_as_script: log.setLevel(logging.INFO) log.addHandler(logging.StreamHandler()) # if in debug mode, override the logging level to DEBUG if self.debug: log.setLevel(logging.DEBUG) # attempt to figure out which pool we're in and add that to the # LogRecords. try: with open("/etc/ec2_asg", "r") as f: pool = f.read().strip() # clean up the pool name since we're putting stuff after "-" pool = pool.partition("-")[0] except IOError: pool = "reddit-app" self.log = logging.LoggerAdapter(log, {"pool": pool}) # set locations self.locations = {} if not self.media_domain: self.media_domain = self.domain if self.media_domain == self.domain: print >> sys.stderr, ("Warning: g.media_domain == g.domain. " + "This may give untrusted content access to user cookies") for arg in sys.argv: tokens = arg.split("=") if len(tokens) == 2: k, v = tokens self.log.debug("Overriding g.%s to %s" % (k, v)) setattr(self, k, v) self.reddit_host = socket.gethostname() self.reddit_pid = os.getpid() if hasattr(signal, 'SIGUSR1'): # not all platforms have user signals signal.signal(signal.SIGUSR1, thread_dump) locale.setlocale(locale.LC_ALL, self.locale) # Pre-calculate ratelimit values self.RL_RESET_SECONDS = self.config["RL_RESET_MINUTES"] * 60 self.RL_MAX_REQS = int(self.config["RL_AVG_REQ_PER_SEC"] * self.RL_RESET_SECONDS) self.RL_OAUTH_RESET_SECONDS = self.config["RL_OAUTH_RESET_MINUTES"] * 60 self.RL_OAUTH_MAX_REQS = int(self.config["RL_OAUTH_AVG_REQ_PER_SEC"] * self.RL_OAUTH_RESET_SECONDS) self.startup_timer.intermediate("configuration") ################# ZOOKEEPER # for now, zookeeper will be an optional part of the stack. # if it's not configured, we will grab the expected config from the # [live_config] section of the ini file zk_hosts = self.config.get("zookeeper_connection_string") if zk_hosts: from r2.lib.zookeeper import (connect_to_zookeeper, LiveConfig, LiveList) zk_username = self.config["zookeeper_username"] zk_password = self.config["zookeeper_password"] self.zookeeper = connect_to_zookeeper(zk_hosts, (zk_username, zk_password)) self.live_config = LiveConfig(self.zookeeper, LIVE_CONFIG_NODE) self.secrets = fetch_secrets(self.zookeeper) self.throttles = LiveList(self.zookeeper, "/throttles", map_fn=ipaddress.ip_network, reduce_fn=ipaddress.collapse_addresses) else: self.zookeeper = None parser = ConfigParser.RawConfigParser() parser.optionxform = str parser.read([self.config["__file__"]]) self.live_config = extract_live_config(parser, self.plugins) self.secrets = extract_secrets(parser) self.throttles = tuple() # immutable since it's not real self.startup_timer.intermediate("zookeeper") ################# PRIVILEGED USERS self.admins = PermissionFilteredEmployeeList( self.live_config, type="admin") self.sponsors = PermissionFilteredEmployeeList( self.live_config, type="sponsor") self.employees = PermissionFilteredEmployeeList( self.live_config, type="employee") ################# MEMCACHE num_mc_clients = self.num_mc_clients # the main memcache pool. used for most everything. self.memcache = CMemcache( self.memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) # a pool just used for @memoize results memoizecaches = CMemcache( self.memoizecaches, min_compress_len=50 * 1024, num_clients=num_mc_clients, ) # a pool just for srmember rels srmembercaches = CMemcache( self.srmembercaches, min_compress_len=96, num_clients=num_mc_clients, ) ratelimitcaches = CMemcache( self.ratelimitcaches, min_compress_len=96, num_clients=num_mc_clients, ) # a smaller pool of caches used only for distributed locks. # TODO: move this to ZooKeeper self.lock_cache = CMemcache(self.lockcaches, num_clients=num_mc_clients) self.make_lock = make_lock_factory(self.lock_cache, self.stats) # memcaches used in front of the permacache CF in cassandra. # XXX: this is a legacy thing; permacache was made when C* didn't have # a row cache. if self.permacache_memcaches: permacache_memcaches = CMemcache(self.permacache_memcaches, min_compress_len=50 * 1024, num_clients=num_mc_clients) else: permacache_memcaches = None # the stalecache is a memcached local to the current app server used # for data that's frequently fetched but doesn't need to be fresh. if self.stalecaches: stalecaches = CMemcache(self.stalecaches, num_clients=num_mc_clients) else: stalecaches = None # rendercache holds rendered partial templates. rendercaches = CMemcache( self.rendercaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=480, ) # pagecaches hold fully rendered pages pagecaches = CMemcache( self.pagecaches, noreply=True, no_block=True, num_clients=num_mc_clients, min_compress_len=1400, ) self.startup_timer.intermediate("memcache") ################# CASSANDRA keyspace = "reddit" self.cassandra_pools = { "main": StatsCollectingConnectionPool( keyspace, stats=self.stats, logging_name="main", server_list=self.cassandra_seeds, pool_size=self.cassandra_pool_size, timeout=4, max_retries=3, prefill=False ), } permacache_cf = CassandraCache( 'permacache', self.cassandra_pools[self.cassandra_default_pool], read_consistency_level=self.cassandra_rcl, write_consistency_level=self.cassandra_wcl ) self.startup_timer.intermediate("cassandra") ################# POSTGRES self.dbm = self.load_db_params() self.startup_timer.intermediate("postgres") ################# CHAINS # initialize caches. Any cache-chains built here must be added # to cache_chains (closed around by reset_caches) so that they # can properly reset their local components cache_chains = {} localcache_cls = (SelfEmptyingCache if self.running_as_script else LocalCache) if stalecaches: self.cache = StaleCacheChain( localcache_cls(), stalecaches, self.memcache, ) else: self.cache = MemcacheChain((localcache_cls(), self.memcache)) cache_chains.update(cache=self.cache) if stalecaches: self.memoizecache = StaleCacheChain( localcache_cls(), stalecaches, memoizecaches, ) else: self.memoizecache = MemcacheChain( (localcache_cls(), memoizecaches)) cache_chains.update(memoizecache=self.memoizecache) if stalecaches: self.srmembercache = StaleCacheChain( localcache_cls(), stalecaches, srmembercaches, ) else: self.srmembercache = MemcacheChain( (localcache_cls(), srmembercaches)) cache_chains.update(srmembercache=self.srmembercache) self.ratelimitcache = MemcacheChain( (localcache_cls(), ratelimitcaches)) cache_chains.update(ratelimitcache=self.ratelimitcache) self.rendercache = MemcacheChain(( localcache_cls(), rendercaches, )) cache_chains.update(rendercache=self.rendercache) self.pagecache = MemcacheChain(( localcache_cls(), pagecaches, )) cache_chains.update(pagecache=self.pagecache) # the thing_cache is used in tdb_cassandra. self.thing_cache = CacheChain((localcache_cls(),)) cache_chains.update(thing_cache=self.thing_cache) self.permacache = CassandraCacheChain( localcache_cls(), permacache_cf, memcache=permacache_memcaches, lock_factory=self.make_lock, ) cache_chains.update(permacache=self.permacache) # hardcache is used for various things that tend to expire # TODO: replace hardcache w/ cassandra stuff self.hardcache = HardcacheChain( (localcache_cls(), self.memcache, HardCache(self)), cache_negative_results=True, ) cache_chains.update(hardcache=self.hardcache) # I know this sucks, but we need non-request-threads to be # able to reset the caches, so we need them be able to close # around 'cache_chains' without being able to call getattr on # 'g' def reset_caches(): for name, chain in cache_chains.iteritems(): chain.reset() chain.stats = CacheStats(self.stats, name) self.cache_chains = cache_chains self.reset_caches = reset_caches self.reset_caches() self.startup_timer.intermediate("cache_chains") # try to set the source control revision numbers self.versions = {} r2_root = os.path.dirname(os.path.dirname(self.paths["root"])) r2_gitdir = os.path.join(r2_root, ".git") self.short_version = self.record_repo_version("r2", r2_gitdir) if I18N_PATH: i18n_git_path = os.path.join(os.path.dirname(I18N_PATH), ".git") self.record_repo_version("i18n", i18n_git_path) self.startup_timer.intermediate("revisions")
def __init__(self, config, name=None): self._logger = None self._is_master = is_master() self.timer = Timer() self.config = config self.save_dir = get_mmf_env(key="save_dir") self.log_format = config.training.log_format self.time_format = "%Y_%m_%dT%H_%M_%S" self.log_filename = "train_" self.log_filename += self.timer.get_time_hhmmss(None, format=self.time_format) self.log_filename += ".log" self.log_folder = os.path.join(self.save_dir, "logs") env_log_dir = get_mmf_env(key="log_dir") if env_log_dir: self.log_folder = env_log_dir if not PathManager.exists(self.log_folder): PathManager.mkdirs(self.log_folder) self.log_filename = os.path.join(self.log_folder, self.log_filename) if not self._is_master: return if self._is_master: print("Logging to:", self.log_filename) logging.captureWarnings(True) if not name: name = __name__ self._logger = logging.getLogger(name) self._file_only_logger = logging.getLogger(name) self._warnings_logger = logging.getLogger("py.warnings") # Set level level = config.training.logger_level self._logger.setLevel(getattr(logging, level.upper())) self._file_only_logger.setLevel(getattr(logging, level.upper())) # Capture stdout to logger self._stdout_logger = None if self.config.training.stdout_capture: self._stdout_logger = StreamToLogger( logging.getLogger("stdout"), getattr(logging, level.upper()) ) sys.stdout = self._stdout_logger formatter = logging.Formatter( "%(asctime)s | %(levelname)s | %(name)s : %(message)s", datefmt="%Y-%m-%dT%H:%M:%S", ) # Add handler to file channel = logging.StreamHandler(PathManager.open(self.log_filename, mode="a")) channel.setFormatter(formatter) self.add_handlers(channel) # Add handler to train.log. train.log is full log that is also used # by slurm/fbl output channel = logging.StreamHandler( PathManager.open(os.path.join(self.save_dir, "train.log"), mode="a") ) channel.setFormatter(formatter) self.add_handlers(channel) # Add handler to stdout. Only when we are not capturing stdout in # the logger if not self._stdout_logger: channel = logging.StreamHandler(sys.stdout) channel.setFormatter(formatter) self._logger.addHandler(channel) self._warnings_logger.addHandler(channel) should_not_log = self.config.training.should_not_log self.should_log = not should_not_log # Single log wrapper map self._single_log_map = set()