def cli(ctx: click.Context, verbose: bool=False, no_version_check: bool=False,
change_dir: str=None, no_log_file: bool=False,
log_file: str="chaostoolkit.log"):
if verbose:
logzero.loglevel(logging.DEBUG, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
else:
logzero.loglevel(logging.INFO, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
if not no_log_file:
# let's ensure we log at DEBUG level
logger.setLevel(logging.DEBUG)
logzero.logfile(
click.format_filename(log_file), mode='a',
loglevel=logging.DEBUG)
logzero.formatter(
formatter=logzero.LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S"),
update_custom_handlers=False)
subcommand = ctx.invoked_subcommand
# make it nicer for going through the log file
logger.debug("#" * 79)
logger.debug("Running command '{}'".format(subcommand))
if not no_version_check:
check_newer_version(command=subcommand)
if change_dir:
logger.warning("Moving to {d}".format(d=change_dir))
os.chdir(change_dir)
def get_energy_levels(self, temp=23.27, verbose=0):
if verbose == 0: logger.setLevel(logging.ERROR)
unique_notations = self.get_unique_entries()
logger.info("Found unique notations = {0}".format(unique_notations))
# spec = unique_notations[1]
for spec in unique_notations:
direc = os.path.expanduser("~") + '/.nistasd/'
filename = 'nist_energylevels_' + spec + '.pkl'
logger.info(
"Searching for saved energy levels in {0}".format(direc))
if not os.path.isfile(direc + filename):
logger.info(
"Found no energy levels in {0} for {1}. Downloading energy levels ..."
.format(direc, self.spectrum))
self.energy_levels[spec] = self._parse_energy_levels(
spec, temp)
if not os.path.isdir(direc):
os.makedirs(direc)
pickle.dump(self.energy_levels[spec],
open(direc + filename, 'wb'),
protocol=2)
else:
logger.info("Found energy levels in {0}".format(direc))
with open(direc + filename, 'rb') as f:
self.energy_levels[spec] = pickle.load(f)
return self.energy_levels
def get_lines(self, verbose=0):
if verbose == 0: logger.setLevel(logging.ERROR)
# direc = str(pathlib.Path(__file__).resolve().parent) + '/NIST_data/'
direc = os.path.expanduser("~") + '/.nist-asd/'
filename = 'nist_lines_' + self.spectrum + '.pkl'
logger.info("Searching for saved spectrum in {0}".format(direc))
if not os.path.isfile(direc + filename):
logger.info(
"Found no spectrum in {0} for {1}. Downloading spectra ...".
format(direc, self.spectrum))
tmp_nistasd = NISTASD(self.spectrum, 0.01, 10000., self.order)
self.nistasd_obj = tmp_nistasd
if not os.path.isdir(direc):
os.makedirs(direc)
pickle.dump(self.nistasd_obj,
open(direc + filename, 'wb'),
protocol=2) # python 2 compat
else:
logger.info("Found spectrum in {0}".format(direc))
with open(direc + filename, 'rb') as f:
self.nistasd_obj = pickle.load(f)
self.lines = self.nistasd_obj.lines
return self.lines
def get_logger(name):
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(filename)s:%(lineno)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
return logger
def init_logging(app:Flask):
# logging.basicConfig(format=FORMAT)
logger = logging.getLogger("timeTravel")
logger.setLevel(Config.LOG_LEVEL)
if Config.LOG_PATH:
logfile(
Config.LOG_PATH,
maxBytes=1000000,
backupCount=3,
loglevel=Config.LOG_LEVEL,
)
_register_before_request(app)
_register_after_request(app)
def configure_logger(verbose: bool = False,
log_format: str = "string",
log_file: str = None,
logger_name: str = "chaostoolkit",
context_id: str = None):
"""
Configure the chaostoolkit logger.
By default logs as strings to stdout and the given file. When `log_format`
is `"json"`, records are set to the console as JSON strings but remain
as strings in the log file. The rationale is that the log file is mostly
for grepping purpose while records written to the console can be forwarded
out of band to anywhere else.
"""
log_level = logging.INFO
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
if verbose:
log_level = logging.DEBUG
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
formatter = LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S")
if log_format == 'json':
fmt = "(process) (asctime) (levelname) (module) (lineno) (message)"
if context_id:
fmt = "(context_id) {}".format(fmt)
formatter = jsonlogger.JsonFormatter(fmt,
json_default=encoder,
timestamp=True)
# sadly, no other way to specify the name of the default logger publicly
LOGZERO_DEFAULT_LOGGER = logger_name
logger = setup_default_logger(level=log_level, formatter=formatter)
if context_id:
logger.addFilter(ChaosToolkitContextFilter(logger_name, context_id))
if log_file:
# always everything as strings in the log file
logger.setLevel(logging.DEBUG)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
formatter = LogFormatter(fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S")
logzero.logfile(log_file,
formatter=formatter,
mode='a',
loglevel=logging.DEBUG)
def cli(ctx: click.Context,
verbose: bool = False,
no_version_check: bool = False,
change_dir: str = None,
no_log_file: bool = False,
log_file: str = "chaostoolkit.log",
settings: str = CHAOSTOOLKIT_CONFIG_PATH):
if verbose:
logzero.loglevel(logging.DEBUG, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s] "\
"[%(module)s:%(lineno)d]%(end_color)s %(message)s"
else:
logzero.loglevel(logging.INFO, update_custom_handlers=False)
fmt = "%(color)s[%(asctime)s %(levelname)s]%(end_color)s %(message)s"
if not no_log_file:
# let's ensure we log at DEBUG level
logger.setLevel(logging.DEBUG)
logzero.logfile(click.format_filename(log_file),
mode='a',
loglevel=logging.DEBUG)
colors = logzero.LogFormatter.DEFAULT_COLORS.copy()
colors[logging.CRITICAL] = logzero.ForegroundColors.RED
logzero.formatter(formatter=logzero.LogFormatter(
fmt=fmt, datefmt="%Y-%m-%d %H:%M:%S", colors=colors),
update_custom_handlers=False)
subcommand = ctx.invoked_subcommand
# make it nicer for going through the log file
logger.debug("#" * 79)
logger.debug("Running command '{}'".format(subcommand))
ctx.obj = {}
ctx.obj["settings_path"] = click.format_filename(settings)
logger.debug("Using settings file '{}'".format(ctx.obj["settings_path"]))
if not no_version_check:
check_newer_version(command=subcommand)
if change_dir:
logger.warning("Moving to {d}".format(d=change_dir))
os.chdir(change_dir)
def _get_logger(logdir,
logname,
loglevel=logging.INFO): # https://wikidocs.net/3736
fmt = "[%(asctime)s] %(levelname)s: %(message)s"
formatter = logging.Formatter(fmt)
# handler = logging.handlers.RotatingFileHandler(
handler = logging.handlers.TimedRotatingFileHandler(
filename=os.path.join(logdir, logname),
# maxBytes=2 * 1024 * 1024 * 1024,
# backupCount=10,
when='midnight',
interval=1,
encoding='utf-8')
handler.setFormatter(formatter)
logger = logging.getLogger("")
logger.addHandler(handler)
logger.setLevel(loglevel)
return logger
def setup_logger(name, save_dir, distributed_rank, level="INFO"):
logger = logging.getLogger(name)
logger.setLevel(10)
# don't log results for the non-master process
if distributed_rank > 0:
return logger
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(getattr(logging, level.upper()))
formatter = logging.Formatter(
"%(asctime)s %(name)s %(levelname)s: %(message)s")
ch.setFormatter(formatter)
logger.addHandler(ch)
if save_dir:
fh = logging.FileHandler(os.path.join(save_dir, "log.txt"), mode='w')
fh.setLevel(getattr(logging, level.upper()))
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
def __init__(self, dyda_config_path='', debug=False):
""" __init__ of ClassifierTensorRT
Trainer Variables:
input_data: a list of image array
results: defined by lab_tools.output_pred_classification (dict)
Arguments:
dyda_config_path -- Trainer config filepath
"""
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# Setup dyda config
super(ClassifierTensorRT,
self).__init__(dyda_config_path=dyda_config_path)
self.set_param(self.class_name)
self.check_param_keys()
# Create engine
self.engine = self._load_engine(self.param['model_file'])
# Get Dim:
self.c, self.h, self.w = self._get_input_shape()
# Create execution context
self.context = self.engine.create_execution_context()
(self.inputs, self.outputs, self.bindings,
self.stream) = self._allocate_buffers(self.engine)
# Setup DL model
with open(self.param['label_file'], 'r') as f:
self.labels_map = f.read().split('\n')
#logger = logging.getLogger()
"""
for handler in logger.handlers:
handler.setFormatter(logging.Formatter(fmt=__logFormat))
"""
# we set a new handler
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(logzero.LogFormatter(fmt=__logFormat, color=True))
logger.handlers = []
logger.addHandler(handler)
logzero.formatter(logging.Formatter(fmt=__logFormat))
logger.setLevel(conf.loglevel)
"""
Code shortcuts
"""
debug = logger.debug
info = logger.info
warn = logger.warn
error = logger.error
critical = logger.critical
setLevel = logger.setLevel
getLevel = logger.getEffectiveLevel
DEBUG = logging.DEBUG
INFO = logging.INFO
WARN = logging.WARN
def entrypoint():
"""
Entry-point to be used by CLI.
"""
logger.setLevel(DEFAULT_VERBOSITY)
main(sys.argv[1:])
import sys import logging import logzero from logzero import logger __logFormat = '[%(asctime)s] %(levelname)-6s| [%(module)s.%(funcName)s] %(message)-80s (%(lineno)d)' # we set a new handler handler = logging.StreamHandler(sys.stdout) # stream to stdout for pycharm handler.setFormatter(logzero.LogFormatter(fmt=__logFormat)) logger.handlers = [] logger.addHandler(handler) logger.setLevel(logging.INFO) # set default level """ Code shortcuts """ debug = logger.debug info = logger.info warn = logger.warn error = logger.error critical = logger.critical setLevel = logger.setLevel getLevel = logger.getEffectiveLevel DEBUG = logging.DEBUG INFO = logging.INFO WARN = logging.WARN
def _set_log_level(self):
logger.setLevel(eval(f"logging.{self._config['twitch']['log_level'].get().upper()}"))
import sys
import os
import argparse
from pathlib import Path
from datetime import datetime
from logzero import logger
logger.setLevel('DEBUG')
def main():
parser = argparse.ArgumentParser(
description='Create HTML from excel file.')
parser.add_argument(
'input',
help=
'Input filename. Latest file is selected when <input> is a directory.',
metavar='<input>')
parser.add_argument('output',
help='Output html filename.',
metavar='<output>')
parser.add_argument(
'--meta',
help='Translation file for metadata. default: %(default)s',
metavar='<name>',
default='items.csv')
args = parser.parse_args()
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import logging
from peewee import Proxy, SqliteDatabase
from logzero import logger
logger = logging.getLogger('peewee')
logger.setLevel(logging.ERROR)
class PWDatabase:
__proxy = None
@staticmethod
def DBProxy():
if not PWDatabase.__proxy:
PWDatabase.__proxy = Proxy()
return PWDatabase.__proxy
_db = None
def __init__(self, path):
try:
self._db = SqliteDatabase(path, check_same_thread=False)
PWDatabase.DBProxy().initialize(self._db)
self.startup()
except Exception as e:
logger.error("database file does not exist, or incorrect permissions")
def close(self):
self._db.close()
self._db = None
import logging
from peewee import Proxy, SqliteDatabase
from logzero import logger
logger = logging.getLogger('peewee')
logger.setLevel(logging.ERROR)
class PWDatabase(object):
__proxy = None
@staticmethod
def DBProxy():
if not PWDatabase.__proxy:
PWDatabase.__proxy = Proxy()
return PWDatabase.__proxy
_db = None
def __init__(self, path):
try:
self._db = SqliteDatabase(path, check_same_thread=False)
PWDatabase.DBProxy().initialize(self._db)
self.startup()
except Exception as e:
logger.error(
"database file does not exist, or incorrect permissions")
def close(self):
self._db.close()
import logging
from logzero import logger
import multiprocessing as mp
import numpy as np
import os
import PIL.Image as Image
import Queue
from rgb_histo import calc_1d_hist_flatten
import threading
import time
from s3dexp.utils import recursive_glob
if int(os.getenv('VERBOSE', 0)) >= 1:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
def minidiamond(base_dir, pattern="*.*", limit=None, fetcher_only=False, async_fetcher=True, use_mp=False):
"""Speed test for Scopelist + Fetcher + RGB
Arguments:
base_dir {string} -- Base directory to find files
Keyword Arguments:
pattern {str} -- File name pattern (default: {"*.jpg"})
limit {integer} -- Stop after (default: {None})
fetcher_only -- only run fetcher (default: {False})
async_fetcher {bool} -- run fetcher in a separate thread/process (default: {True})
import logzero
from logzero import logger
import os
from os import path
import sys
abspath = os.path.dirname(os.path.abspath(__file__))
sys.path.append(abspath)
import setting
log_save_path = setting.log_save_path
# ログの出力名を設定
# logger = logging.getLogger("stock_patterns")
# ログレベルの設定
logger.setLevel(10)
# ログのファイル出力先を設定
if not os.path.exists(log_save_path):
os.mkdir(log_save_path)
log_file = log_save_path + "/stock_strategy.log"
if not os.path.exists(log_file):
f = open(log_file, "a")
f.close()
logzero.logfile(log_file)
# fh = logging.FileHandler(log_file)
# logger.addHandler(fh)
def __exit__(self, exc_type, exc_value, traceback):
logger.setLevel(self.curent_level)
def __enter__(self):
logger.setLevel(self.tmp_level)
import logging
import os
from typing import List
import graphviz as gv
from logzero import logger as log
from mock import Mock
import util
from core import MessageUnderstanding, States, Context
from core.dialogstates import DialogStates
from logic.responsecomposer import ResponseComposer
from logic.rules.dialogcontroller import application_router
from model import User
log.setLevel(logging.INFO)
Digraph = functools.partial(gv.Digraph, format='svg')
def add_nodes(graph, nodes) -> gv.Digraph:
for n in nodes:
if isinstance(n, tuple):
graph.node(n[0], **n[1])
else:
graph.node(n)
return graph
def add_edges(graph, edges) -> gv.Digraph:
for e in edges:
def main(drive_ip, port=5567, verbose=False):
if verbose:
logger.setLevel(logging.DEBUG)
context = zmq.Context()
router = context.socket(zmq.ROUTER)
router.bind("tcp://*:{}".format(port))
logger.info("Listening on port {}".format(port))
poller = zmq.Poller()
poller.register(router, zmq.POLLIN)
kvclient = Client(drive_ip)
kvclient.connect()
assert kvclient.is_connected, "Failed to connect to drive"
logger.info("kv_client connected {}".format(drive_ip))
kvclient.queue_depth = 16
# use these to avoid redefining the callback function every time
# assumption: there are no requests for the same key in near future (no collision)
pending_requests = dict() # key -> [address, source proxy message, value]
ready_requests = dict() # same
def data_callbak(msg, cmd, value):
# fills the value in pending requests and move it to ready_requests
key = bytes(cmd.body.keyValue.key)
if cmd.status.code != kinetic_pb2.Command.Status.SUCCESS:
logger.error("\t Key: " + str(cmd.body.keyValue.key) + \
", BC: received ackSeq: "+str(cmd.header.ackSequence)+\
", msgType: "+str(MsgTypes.Name(cmd.header.messageType))+\
", statusCode: "+str(StatusCodes.Name(cmd.status.code)))
value = b''
else:
logger.debug("[get] Success: GET " + str(cmd.body.keyValue.key))
value = value
t = pending_requests.pop(key)
t[2] = bytes(value)
ready_requests[key] = t
kvclient.callback_delegate = data_callbak
while True:
# Wait for next request from client
events = poller.poll(0)
if events:
address, _, body = router.recv_multipart()
proxy_msg = Message()
proxy_msg.ParseFromString(body)
logger.debug("Recv request from {}, opcode {}".format(
address, str(proxy_msg.opcode)))
if proxy_msg.opcode == Message.Opcode.PING: # trivial
resp_msg = Message()
resp_msg.value = b'PONG'
router.send_multipart(
[address, b'', resp_msg.SerializeToString()])
elif proxy_msg.opcode in (Message.Opcode.GET,
Message.Opcode.GETSMART):
key = proxy_msg.key
pending_requests[key] = [address, proxy_msg, None]
kvclient.get(key)
else:
raise NotImplementedError
# send response
if ready_requests:
key, (address, req_msg, value) = ready_requests.popitem()
if req_msg.opcode == Message.Opcode.GET:
resp_msg = Message()
resp_msg.key = key
resp_msg.opcode = req_msg.opcode
resp_msg.value = value
router.send_multipart(
[address, b'', resp_msg.SerializeToString()])
elif req_msg.opcode == Message.Opcode.GETSMART:
# hack for speed: assume the client won't parse proto message
router.send_multipart([address, b'', b'\0' * req_msg.size])
else:
raise ValueError("Other opcode should not land here: " +
str(req_msg.opcode))
def with_log_level(logger, level):
"""Set log level to warning temporarily."""
old_level = logger.level
logger.setLevel(level)
yield
logger.setLevel(old_level)
from .commands.ide import ide
from .commands.projects import project
from .commands.runs import run
from .commands.tracing import tracing
config = ConfigParser()
if os.path.exists(str(Path.home() / ".dkube.ini")):
config.read(str(Path.home() / ".dkube.ini"))
elif os.path.exists(str(Path.home() / ".da.ini")):
config.read(str(Path.home() / ".da.ini"))
else:
print("you need to run dkube configure first")
sys.exit(-1)
logger.setLevel(logging.INFO)
@click.group()
@click_log.simple_verbosity_option(logger)
@click.pass_context
def main(ctx):
"""Console script for dkube_cli."""
ctx.obj = None
if "default" in config.sections():
name = config.get("default", "name")
if name in config.sections():
url = config.get(name, "url")
token = config.get(name, "token")
api = DkubeApi(URL=url, token=token)
username = config.get(name, "username")
def main():
global SSHCMD
parser = OptionParser()
parser.add_option('--iterations', type=int, default=10)
parser.add_option('--controlpersist', action='store_true')
parser.add_option('--selectors', action='store_true')
parser.add_option('--use_plugin', action='store_true')
parser.add_option('--vcount', type=int, default=None)
parser.add_option('--debug', action='store_true')
parser.add_option('--hostname', default=None)
parser.add_option('--username', default=None)
parser.add_option('--keyfile', default=None)
parser.add_option('--command', default=None)
(options, args) = parser.parse_args()
if not options.debug:
logger.setLevel('INFO')
# munge the example ssh command if not using the connection plugin
if not options.use_plugin:
validate_control_socket(SSHCMD)
if not options.controlpersist:
SSHCMD = remove_control_persist(SSHCMD)
if options.hostname:
SSHCMD = set_hostname(SSHCMD, options.hostname)
if options.username:
SSHCMD = set_username(SSHCMD, options.username)
if options.keyfile:
SSHCMD = set_keyfile(SSHCMD, options.keyfile)
if options.vcount is not None:
SSHCMD = set_vcount(SSHCMD, count=options.vcount)
if options.command is not None:
SSHCMD[-1] = '/bin/sh -c "%s"' % options.command
logger.info(SSHCMD)
# run the command X times and record the durations + speeds
durations = []
for x in range(0, options.iterations):
logger.info('iteration %s' % x)
start = datetime.now()
if options.use_plugin:
(rc, so, se) = run_ssh_exec(
command=options.command,
hostname=options.hostname,
username=options.username,
keyfile=options.keyfile,
)
else:
(rc, so, se) = run_ssh_cmd(
SSHCMD,
hostname=options.hostname,
username=options.username,
use_selectors=options.selectors
)
stop = datetime.now()
durations.append(stop - start)
stats = extract_speeed_from_stdtout(se)
logger.info('transfer stats ...')
for k, v in stats.items():
for k2, v2 in v.items():
logger.info('%s.%s = %s' % (k, k2, v2))
logger.info('rc: %s' % rc)
logger.info('so:%s' % so.strip())
if rc != 0:
logger.error(se)
logger.error('sshcmd: %s' % ' '.join(SSHCMD))
durations = [x.total_seconds() for x in durations]
logger.info('durations ...')
for idx, x in enumerate(durations):
logger.info('%s. %s' % (idx, x))
logger.info('duration min: %s' % min(durations))
logger.info('duration max: %s' % max(durations))
avg = sum(durations) / float(len(durations))
logger.info('duration avg: %s' % avg)