def configure_logger(logger, level='INFO', output="-", loggerconfig=None, name=None): if loggerconfig is None or loggerconfig.lower().strip() == "default": root_logger = logging.getLogger() loglevel = LOG_LEVELS.get(level.lower(), logging.INFO) root_logger.setLevel(loglevel) datefmt = LOG_DATE_FMT if output in ("-", "stdout"): handler = logging.StreamHandler() elif output.startswith('syslog://'): # URLs are syslog://host[:port]?facility or syslog:///path?facility info = urlparse(output) facility = 'user' if info.query in logging.handlers.SysLogHandler.facility_names: facility = info.query if info.netloc: address = (info.netloc, info.port or 514) else: address = info.path datefmt = LOG_DATE_SYSLOG_FMT handler = logging.handlers.SysLogHandler( address=address, facility=facility) else: if not IS_WINDOWS: handler = logging.handlers.WatchedFileHandler(output) close_on_exec(handler.stream.fileno()) else: # WatchedFileHandler is not supported on Windows, # but a FileHandler should be a good drop-in replacement # as log files are locked handler = logging.FileHandler(output) formatter = logging.Formatter(fmt=LOG_FMT, datefmt=datefmt) handler.setFormatter(formatter) root_logger.handlers = [handler] else: loggerconfig = os.path.abspath(loggerconfig) if loggerconfig.lower().endswith(".ini"): logging.config.fileConfig(loggerconfig, disable_existing_loggers=True) elif loggerconfig.lower().endswith(".json"): if not hasattr(logging.config, "dictConfig"): raise Exception("Logger configuration file %s appears to be " "a JSON file but this version of Python " "does not support the " "logging.config.dictConfig function. Try " "Python 2.7.") with open(loggerconfig, "r") as fh: logging.config.dictConfig(json.loads(fh.read())) elif loggerconfig.lower().endswith(".yaml"): if not hasattr(logging.config, "dictConfig"): raise Exception("Logger configuration file %s appears to be " "a YAML file but this version of Python " "does not support the " "logging.config.dictConfig function. Try " "Python 2.7.") if yaml is None: raise Exception("Logger configuration file %s appears to be " "a YAML file but PyYAML is not available. " "Try: pip install PyYAML" % (shell_escape_arg(loggerconfig),)) with open(loggerconfig, "r") as fh: logging.config.dictConfig(yaml.load(fh.read())) else: raise Exception("Logger configuration file %s is not in one " "of the recognized formats. The file name " "should be: *.ini, *.json or *.yaml." % (shell_escape_arg(loggerconfig),))
def run_circusd( options=(), config=(), log_capture_path="log.txt", additional_files=()): options = list(options) additional_files = dict(additional_files) config_ini_update = { "watcher:touch.cmd": ("bash -c 'touch workerstart.txt; " "while true; do sleep 0.1; done'"), } config_ini_update.update(dict(config)) config_ini = ConfigParser() config_ini.read(CONFIG_PATH) for dottedkey in config_ini_update: section, key = dottedkey.split(".", 1) if section not in config_ini.sections(): config_ini.add_section(section) config_ini.set(section, key, config_ini_update[dottedkey]) temp_dir = tempfile.mkdtemp() try: circus_ini_path = os.path.join(temp_dir, "circus.ini") with open(circus_ini_path, "w") as fh: config_ini.write(fh) for relpath in additional_files: path = os.path.join(temp_dir, relpath) with open(path, "w") as fh: fh.write(additional_files[relpath]) # argv2 = ["cat", "circus.ini"] # subprocess.check_call(argv2, cwd=temp_dir) argv = ["circus.circusd"] + options + [circus_ini_path] if sys.gettrace() is None: argv = ["python", "-m"] + argv else: argv = ["coverage", "run", "-p", "-m"] + argv # print "+", " ".join(shell_escape_arg(a) for a in argv) child = subprocess.Popen(argv, cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) try: touch_path = os.path.join(temp_dir, "workerstart.txt") # subprocess.call(["bash"], cwd=temp_dir) while True: child.poll() if os.path.exists(touch_path): break if child.returncode is not None: break time.sleep(0.01) finally: argv2 = ["python", "-m", "circus.circusctl", "quit"] subprocess.call(argv2, cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) child.wait() log_file_path = os.path.join(temp_dir, log_capture_path) # raise Exception(child.stdout.read()) if os.path.exists(log_file_path): with open(log_file_path, "r") as fh: return fh.read() else: if child.stdout is not None: raise Exception(child.stdout.read().decode("ascii")) assert child.returncode == 0, \ " ".join(shell_escape_arg(a) for a in argv) finally: for basename in sorted(os.listdir(temp_dir)): if basename.startswith(".coverage."): source = os.path.join(temp_dir, basename) target = os.path.abspath(basename) shutil.copy(source, target) shutil.rmtree(temp_dir)
def run_circusd(options=(), config=(), log_capture_path="log.txt", additional_files=()): options = list(options) additional_files = dict(additional_files) config_ini_update = { "watcher:touch.cmd": ("bash -c 'touch workerstart.txt; " "while true; do sleep 0.1; done'"), } config_ini_update.update(dict(config)) config_ini = ConfigParser() config_ini.read(CONFIG_PATH) for dottedkey in config_ini_update: section, key = dottedkey.split(".", 1) if section not in config_ini.sections(): config_ini.add_section(section) config_ini.set( section, key, config_ini_update[dottedkey]) temp_dir = tempfile.mkdtemp() try: circus_ini_path = os.path.join(temp_dir, "circus.ini") with open(circus_ini_path, "w") as fh: config_ini.write(fh) for relpath in additional_files: path = os.path.join(temp_dir, relpath) with open(path, "w") as fh: fh.write(additional_files[relpath]) # argv2 = ["cat", "circus.ini"] # subprocess.check_call(argv2, cwd=temp_dir) argv = ["circus.circusd"] + options + [circus_ini_path] if sys.gettrace() is None: argv = ["python", "-m"] + argv else: argv = ["coverage", "run", "-p", "-m"] + argv # print "+", " ".join(shell_escape_arg(a) for a in argv) child = subprocess.Popen(argv, cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) try: touch_path = os.path.join(temp_dir, "workerstart.txt") # subprocess.call(["bash"], cwd=temp_dir) while True: child.poll() if os.path.exists(touch_path): break if child.returncode is not None: break time.sleep(0.01) finally: argv2 = ["python", "-m", "circus.circusctl", "quit"] subprocess.call(argv2, cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) child.wait() log_file_path = os.path.join(temp_dir, log_capture_path) # raise Exception(child.stdout.read()) if os.path.exists(log_file_path): with open(log_file_path, "r") as fh: return fh.read() else: if child.stdout is not None: raise Exception(child.stdout.read().decode("ascii")) assert child.returncode == 0, \ " ".join(shell_escape_arg(a) for a in argv) finally: for basename in sorted(os.listdir(temp_dir)): if basename.startswith(".coverage."): source = os.path.join(temp_dir, basename) target = os.path.abspath(basename) shutil.copy(source, target) shutil.rmtree(temp_dir)
def run_circusd(options=(), config=(), log_capture_path="log.txt", additional_files=()): options = list(options) additional_files = dict(additional_files) config_ini_update = { "watcher:touch.cmd": PYTHON, "watcher:touch.args": "-c \"open('workerstart.txt', 'w+').close()\"", "watcher:touch.respawn": 'False' } config_ini_update.update(dict(config)) config_ini = ConfigParser() config_ini.read(CONFIG_PATH) for dottedkey in config_ini_update: section, key = dottedkey.split(".", 1) if section not in config_ini.sections(): config_ini.add_section(section) config_ini.set( section, key, config_ini_update[dottedkey]) temp_dir = tempfile.mkdtemp() try: circus_ini_path = os.path.join(temp_dir, "circus.ini") with open(circus_ini_path, "w") as fh: config_ini.write(fh) for relpath in additional_files: path = os.path.join(temp_dir, relpath) with open(path, "w") as fh: fh.write(additional_files[relpath]) env = os.environ.copy() sep = ';' if IS_WINDOWS else ':' # We're going to run circus from a process with a different # cwd, so we need to make sure that Python will import the # current version of circus pythonpath = env.get('PYTHONPATH', '') pythonpath += '%s%s' % (sep, os.path.abspath( os.path.join(HERE, os.pardir, os.pardir))) env['PYTHONPATH'] = pythonpath argv = ["circus.circusd"] + options + [circus_ini_path] if sys.gettrace() is None or IS_WINDOWS: # killing a coverage run process leaves a zombie on # Windows so we should skip coverage argv = [PYTHON, "-m"] + argv else: exe_dir = os.path.dirname(PYTHON) coverage = os.path.join(exe_dir, "coverage") if not os.path.isfile(coverage): coverage = "coverage" argv = [coverage, "run", "-p", "-m"] + argv child = subprocess.Popen(argv, cwd=temp_dir, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) try: touch_path = os.path.join(temp_dir, "workerstart.txt") while True: child.poll() if os.path.exists(touch_path): break if child.returncode is not None: break time.sleep(0.01) finally: child.terminate() child.wait() log_file_path = os.path.join(temp_dir, log_capture_path) try: if os.path.exists(log_file_path): with open(log_file_path, "r") as fh: return fh.read() else: if child.stdout is not None: raise Exception(child.stdout.read().decode("ascii")) finally: if child.stdout is not None: child.stdout.close() if child.stderr is not None: child.stderr.close() if child.stdin is not None: child.stdin.close() assert child.returncode == 0, \ " ".join(shell_escape_arg(a) for a in argv) finally: for basename in sorted(os.listdir(temp_dir)): if basename.startswith(".coverage."): source = os.path.join(temp_dir, basename) target = os.path.abspath(basename) shutil.copy(source, target) try: shutil.rmtree(temp_dir) except OSError: # Sometimes on Windows we can't delete the # logging file because it says it's still in # use (lock). pass
def configure_logger(level='INFO', output='-', loggerconfig=None): if loggerconfig is None or loggerconfig.lower().strip() == 'default': logger = logging.getLogger() loglevel = LOG_LEVELS.get(level.lower(), logging.INFO) logger.setLevel(loglevel) datefmt = LOG_DATE_FMT if output in ('-', 'stdout'): handler = logging.StreamHandler() elif output.startswith('syslog://'): # URLs are syslog://host[:port]?facility or syslog:///path?facility info = urlparse(output) facility = 'user' if info.query in logging.handlers.SysLogHandler.facility_names: facility = info.query if info.netloc: address = (info.netloc, info.port or 514) else: address = info.path datefmt = LOG_DATE_SYSLOG_FMT handler = logging.handlers.SysLogHandler( address=address, facility=facility) else: # if not IS_WINDOWS: # handler = logging.handlers.WatchedFileHandler(output) # close_on_exec(handler.stream.fileno()) # else: # WatchedFileHandler is not supported on Windows, # but a FileHandler should be a good drop-in replacement # as log files are locked handler = logging.handlers.RotatingFileHandler( output, maxBytes=10 * 1024 * 1024, backupCount=5) formatter = logging.Formatter(fmt=LOG_FMT, datefmt=datefmt) handler.setFormatter(formatter) logger.handlers = [handler] else: loggerconfig = os.path.realpath(loggerconfig) if loggerconfig.lower().endswith('.ini'): logging.config.fileConfig( loggerconfig, disable_existing_loggers=True) elif loggerconfig.lower().endswith('.json'): if not hasattr(logging.config, 'dictConfig'): raise Exception('Logger configuration file %s appears to be ' 'a JSON file but this version of Python ' 'does not support the ' 'logging.config.dictConfig function. Try ' 'Python 2.7.') with open(loggerconfig, 'r') as fh: logging.config.dictConfig(json.loads(fh.read())) elif loggerconfig.lower().endswith('.yaml'): if not hasattr(logging.config, 'dictConfig'): raise Exception('Logger configuration file %s appears to be ' 'a YAML file but this version of Python ' 'does not support the ' 'logging.config.dictConfig function. Try ' 'Python 2.7.') if yaml is None: raise Exception('Logger configuration file %s appears to be ' 'a YAML file but PyYAML is not available. ' 'Try: pip install PyYAML' % (shell_escape_arg(loggerconfig),)) with open(loggerconfig, 'r') as fh: logging.config.dictConfig(yaml.load(fh.read())) else: raise Exception('Logger configuration file %s is not in one ' 'of the recognized formats. The file name ' 'should be: *.ini, *.json or *.yaml.' % (shell_escape_arg(loggerconfig),))