def configure_logger(logger, level='INFO', output="-", loggerconfig=None,
name=None):
if loggerconfig is None or loggerconfig.lower().strip() == "default":
root_logger = logging.getLogger()
loglevel = LOG_LEVELS.get(level.lower(), logging.INFO)
root_logger.setLevel(loglevel)
datefmt = LOG_DATE_FMT
if output in ("-", "stdout"):
handler = logging.StreamHandler()
elif output.startswith('syslog://'):
# URLs are syslog://host[:port]?facility or syslog:///path?facility
info = urlparse(output)
facility = 'user'
if info.query in logging.handlers.SysLogHandler.facility_names:
facility = info.query
if info.netloc:
address = (info.netloc, info.port or 514)
else:
address = info.path
datefmt = LOG_DATE_SYSLOG_FMT
handler = logging.handlers.SysLogHandler(
address=address, facility=facility)
else:
if not IS_WINDOWS:
handler = logging.handlers.WatchedFileHandler(output)
close_on_exec(handler.stream.fileno())
else:
# WatchedFileHandler is not supported on Windows,
# but a FileHandler should be a good drop-in replacement
# as log files are locked
handler = logging.FileHandler(output)
formatter = logging.Formatter(fmt=LOG_FMT, datefmt=datefmt)
handler.setFormatter(formatter)
root_logger.handlers = [handler]
else:
loggerconfig = os.path.abspath(loggerconfig)
if loggerconfig.lower().endswith(".ini"):
logging.config.fileConfig(loggerconfig,
disable_existing_loggers=True)
elif loggerconfig.lower().endswith(".json"):
if not hasattr(logging.config, "dictConfig"):
raise Exception("Logger configuration file %s appears to be "
"a JSON file but this version of Python "
"does not support the "
"logging.config.dictConfig function. Try "
"Python 2.7.")
with open(loggerconfig, "r") as fh:
logging.config.dictConfig(json.loads(fh.read()))
elif loggerconfig.lower().endswith(".yaml"):
if not hasattr(logging.config, "dictConfig"):
raise Exception("Logger configuration file %s appears to be "
"a YAML file but this version of Python "
"does not support the "
"logging.config.dictConfig function. Try "
"Python 2.7.")
if yaml is None:
raise Exception("Logger configuration file %s appears to be "
"a YAML file but PyYAML is not available. "
"Try: pip install PyYAML"
% (shell_escape_arg(loggerconfig),))
with open(loggerconfig, "r") as fh:
logging.config.dictConfig(yaml.load(fh.read()))
else:
raise Exception("Logger configuration file %s is not in one "
"of the recognized formats. The file name "
"should be: *.ini, *.json or *.yaml."
% (shell_escape_arg(loggerconfig),))
def run_circusd(
options=(), config=(), log_capture_path="log.txt", additional_files=()):
options = list(options)
additional_files = dict(additional_files)
config_ini_update = {
"watcher:touch.cmd": ("bash -c 'touch workerstart.txt; "
"while true; do sleep 0.1; done'"),
}
config_ini_update.update(dict(config))
config_ini = ConfigParser()
config_ini.read(CONFIG_PATH)
for dottedkey in config_ini_update:
section, key = dottedkey.split(".", 1)
if section not in config_ini.sections():
config_ini.add_section(section)
config_ini.set(section, key, config_ini_update[dottedkey])
temp_dir = tempfile.mkdtemp()
try:
circus_ini_path = os.path.join(temp_dir, "circus.ini")
with open(circus_ini_path, "w") as fh:
config_ini.write(fh)
for relpath in additional_files:
path = os.path.join(temp_dir, relpath)
with open(path, "w") as fh:
fh.write(additional_files[relpath])
# argv2 = ["cat", "circus.ini"]
# subprocess.check_call(argv2, cwd=temp_dir)
argv = ["circus.circusd"] + options + [circus_ini_path]
if sys.gettrace() is None:
argv = ["python", "-m"] + argv
else:
argv = ["coverage", "run", "-p", "-m"] + argv
# print "+", " ".join(shell_escape_arg(a) for a in argv)
child = subprocess.Popen(argv,
cwd=temp_dir,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
try:
touch_path = os.path.join(temp_dir, "workerstart.txt")
# subprocess.call(["bash"], cwd=temp_dir)
while True:
child.poll()
if os.path.exists(touch_path):
break
if child.returncode is not None:
break
time.sleep(0.01)
finally:
argv2 = ["python", "-m", "circus.circusctl", "quit"]
subprocess.call(argv2,
cwd=temp_dir,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
child.wait()
log_file_path = os.path.join(temp_dir, log_capture_path)
# raise Exception(child.stdout.read())
if os.path.exists(log_file_path):
with open(log_file_path, "r") as fh:
return fh.read()
else:
if child.stdout is not None:
raise Exception(child.stdout.read().decode("ascii"))
assert child.returncode == 0, \
" ".join(shell_escape_arg(a) for a in argv)
finally:
for basename in sorted(os.listdir(temp_dir)):
if basename.startswith(".coverage."):
source = os.path.join(temp_dir, basename)
target = os.path.abspath(basename)
shutil.copy(source, target)
shutil.rmtree(temp_dir)
def configure_logger(logger, level='INFO', output="-", loggerconfig=None,
name=None):
if loggerconfig is None or loggerconfig.lower().strip() == "default":
root_logger = logging.getLogger()
loglevel = LOG_LEVELS.get(level.lower(), logging.INFO)
root_logger.setLevel(loglevel)
datefmt = LOG_DATE_FMT
if output in ("-", "stdout"):
handler = logging.StreamHandler()
elif output.startswith('syslog://'):
# URLs are syslog://host[:port]?facility or syslog:///path?facility
info = urlparse(output)
facility = 'user'
if info.query in logging.handlers.SysLogHandler.facility_names:
facility = info.query
if info.netloc:
address = (info.netloc, info.port or 514)
else:
address = info.path
datefmt = LOG_DATE_SYSLOG_FMT
handler = logging.handlers.SysLogHandler(
address=address, facility=facility)
else:
if not IS_WINDOWS:
handler = logging.handlers.WatchedFileHandler(output)
close_on_exec(handler.stream.fileno())
else:
# WatchedFileHandler is not supported on Windows,
# but a FileHandler should be a good drop-in replacement
# as log files are locked
handler = logging.FileHandler(output)
formatter = logging.Formatter(fmt=LOG_FMT, datefmt=datefmt)
handler.setFormatter(formatter)
root_logger.handlers = [handler]
else:
loggerconfig = os.path.abspath(loggerconfig)
if loggerconfig.lower().endswith(".ini"):
logging.config.fileConfig(loggerconfig,
disable_existing_loggers=True)
elif loggerconfig.lower().endswith(".json"):
if not hasattr(logging.config, "dictConfig"):
raise Exception("Logger configuration file %s appears to be "
"a JSON file but this version of Python "
"does not support the "
"logging.config.dictConfig function. Try "
"Python 2.7.")
with open(loggerconfig, "r") as fh:
logging.config.dictConfig(json.loads(fh.read()))
elif loggerconfig.lower().endswith(".yaml"):
if not hasattr(logging.config, "dictConfig"):
raise Exception("Logger configuration file %s appears to be "
"a YAML file but this version of Python "
"does not support the "
"logging.config.dictConfig function. Try "
"Python 2.7.")
if yaml is None:
raise Exception("Logger configuration file %s appears to be "
"a YAML file but PyYAML is not available. "
"Try: pip install PyYAML"
% (shell_escape_arg(loggerconfig),))
with open(loggerconfig, "r") as fh:
logging.config.dictConfig(yaml.load(fh.read()))
else:
raise Exception("Logger configuration file %s is not in one "
"of the recognized formats. The file name "
"should be: *.ini, *.json or *.yaml."
% (shell_escape_arg(loggerconfig),))
def run_circusd(options=(), config=(), log_capture_path="log.txt",
additional_files=()):
options = list(options)
additional_files = dict(additional_files)
config_ini_update = {
"watcher:touch.cmd": ("bash -c 'touch workerstart.txt; "
"while true; do sleep 0.1; done'"),
}
config_ini_update.update(dict(config))
config_ini = ConfigParser()
config_ini.read(CONFIG_PATH)
for dottedkey in config_ini_update:
section, key = dottedkey.split(".", 1)
if section not in config_ini.sections():
config_ini.add_section(section)
config_ini.set(
section, key, config_ini_update[dottedkey])
temp_dir = tempfile.mkdtemp()
try:
circus_ini_path = os.path.join(temp_dir, "circus.ini")
with open(circus_ini_path, "w") as fh:
config_ini.write(fh)
for relpath in additional_files:
path = os.path.join(temp_dir, relpath)
with open(path, "w") as fh:
fh.write(additional_files[relpath])
# argv2 = ["cat", "circus.ini"]
# subprocess.check_call(argv2, cwd=temp_dir)
argv = ["circus.circusd"] + options + [circus_ini_path]
if sys.gettrace() is None:
argv = ["python", "-m"] + argv
else:
argv = ["coverage", "run", "-p", "-m"] + argv
# print "+", " ".join(shell_escape_arg(a) for a in argv)
child = subprocess.Popen(argv, cwd=temp_dir, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
try:
touch_path = os.path.join(temp_dir, "workerstart.txt")
# subprocess.call(["bash"], cwd=temp_dir)
while True:
child.poll()
if os.path.exists(touch_path):
break
if child.returncode is not None:
break
time.sleep(0.01)
finally:
argv2 = ["python", "-m", "circus.circusctl", "quit"]
subprocess.call(argv2, cwd=temp_dir, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
child.wait()
log_file_path = os.path.join(temp_dir, log_capture_path)
# raise Exception(child.stdout.read())
if os.path.exists(log_file_path):
with open(log_file_path, "r") as fh:
return fh.read()
else:
if child.stdout is not None:
raise Exception(child.stdout.read().decode("ascii"))
assert child.returncode == 0, \
" ".join(shell_escape_arg(a) for a in argv)
finally:
for basename in sorted(os.listdir(temp_dir)):
if basename.startswith(".coverage."):
source = os.path.join(temp_dir, basename)
target = os.path.abspath(basename)
shutil.copy(source, target)
shutil.rmtree(temp_dir)
def run_circusd(options=(), config=(), log_capture_path="log.txt",
additional_files=()):
options = list(options)
additional_files = dict(additional_files)
config_ini_update = {
"watcher:touch.cmd": PYTHON,
"watcher:touch.args": "-c \"open('workerstart.txt', 'w+').close()\"",
"watcher:touch.respawn": 'False'
}
config_ini_update.update(dict(config))
config_ini = ConfigParser()
config_ini.read(CONFIG_PATH)
for dottedkey in config_ini_update:
section, key = dottedkey.split(".", 1)
if section not in config_ini.sections():
config_ini.add_section(section)
config_ini.set(
section, key, config_ini_update[dottedkey])
temp_dir = tempfile.mkdtemp()
try:
circus_ini_path = os.path.join(temp_dir, "circus.ini")
with open(circus_ini_path, "w") as fh:
config_ini.write(fh)
for relpath in additional_files:
path = os.path.join(temp_dir, relpath)
with open(path, "w") as fh:
fh.write(additional_files[relpath])
env = os.environ.copy()
sep = ';' if IS_WINDOWS else ':'
# We're going to run circus from a process with a different
# cwd, so we need to make sure that Python will import the
# current version of circus
pythonpath = env.get('PYTHONPATH', '')
pythonpath += '%s%s' % (sep, os.path.abspath(
os.path.join(HERE, os.pardir, os.pardir)))
env['PYTHONPATH'] = pythonpath
argv = ["circus.circusd"] + options + [circus_ini_path]
if sys.gettrace() is None or IS_WINDOWS:
# killing a coverage run process leaves a zombie on
# Windows so we should skip coverage
argv = [PYTHON, "-m"] + argv
else:
exe_dir = os.path.dirname(PYTHON)
coverage = os.path.join(exe_dir, "coverage")
if not os.path.isfile(coverage):
coverage = "coverage"
argv = [coverage, "run", "-p", "-m"] + argv
child = subprocess.Popen(argv, cwd=temp_dir, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
try:
touch_path = os.path.join(temp_dir, "workerstart.txt")
while True:
child.poll()
if os.path.exists(touch_path):
break
if child.returncode is not None:
break
time.sleep(0.01)
finally:
child.terminate()
child.wait()
log_file_path = os.path.join(temp_dir, log_capture_path)
try:
if os.path.exists(log_file_path):
with open(log_file_path, "r") as fh:
return fh.read()
else:
if child.stdout is not None:
raise Exception(child.stdout.read().decode("ascii"))
finally:
if child.stdout is not None:
child.stdout.close()
if child.stderr is not None:
child.stderr.close()
if child.stdin is not None:
child.stdin.close()
assert child.returncode == 0, \
" ".join(shell_escape_arg(a) for a in argv)
finally:
for basename in sorted(os.listdir(temp_dir)):
if basename.startswith(".coverage."):
source = os.path.join(temp_dir, basename)
target = os.path.abspath(basename)
shutil.copy(source, target)
try:
shutil.rmtree(temp_dir)
except OSError:
# Sometimes on Windows we can't delete the
# logging file because it says it's still in
# use (lock).
pass
def configure_logger(level='INFO', output='-', loggerconfig=None):
if loggerconfig is None or loggerconfig.lower().strip() == 'default':
logger = logging.getLogger()
loglevel = LOG_LEVELS.get(level.lower(), logging.INFO)
logger.setLevel(loglevel)
datefmt = LOG_DATE_FMT
if output in ('-', 'stdout'):
handler = logging.StreamHandler()
elif output.startswith('syslog://'):
# URLs are syslog://host[:port]?facility or syslog:///path?facility
info = urlparse(output)
facility = 'user'
if info.query in logging.handlers.SysLogHandler.facility_names:
facility = info.query
if info.netloc:
address = (info.netloc, info.port or 514)
else:
address = info.path
datefmt = LOG_DATE_SYSLOG_FMT
handler = logging.handlers.SysLogHandler(
address=address, facility=facility)
else:
# if not IS_WINDOWS:
# handler = logging.handlers.WatchedFileHandler(output)
# close_on_exec(handler.stream.fileno())
# else:
# WatchedFileHandler is not supported on Windows,
# but a FileHandler should be a good drop-in replacement
# as log files are locked
handler = logging.handlers.RotatingFileHandler(
output, maxBytes=10 * 1024 * 1024, backupCount=5)
formatter = logging.Formatter(fmt=LOG_FMT, datefmt=datefmt)
handler.setFormatter(formatter)
logger.handlers = [handler]
else:
loggerconfig = os.path.realpath(loggerconfig)
if loggerconfig.lower().endswith('.ini'):
logging.config.fileConfig(
loggerconfig, disable_existing_loggers=True)
elif loggerconfig.lower().endswith('.json'):
if not hasattr(logging.config, 'dictConfig'):
raise Exception('Logger configuration file %s appears to be '
'a JSON file but this version of Python '
'does not support the '
'logging.config.dictConfig function. Try '
'Python 2.7.')
with open(loggerconfig, 'r') as fh:
logging.config.dictConfig(json.loads(fh.read()))
elif loggerconfig.lower().endswith('.yaml'):
if not hasattr(logging.config, 'dictConfig'):
raise Exception('Logger configuration file %s appears to be '
'a YAML file but this version of Python '
'does not support the '
'logging.config.dictConfig function. Try '
'Python 2.7.')
if yaml is None:
raise Exception('Logger configuration file %s appears to be '
'a YAML file but PyYAML is not available. '
'Try: pip install PyYAML' %
(shell_escape_arg(loggerconfig),))
with open(loggerconfig, 'r') as fh:
logging.config.dictConfig(yaml.load(fh.read()))
else:
raise Exception('Logger configuration file %s is not in one '
'of the recognized formats. The file name '
'should be: *.ini, *.json or *.yaml.' %
(shell_escape_arg(loggerconfig),))
