def test_logger_caching():
'''
Test that loggers are properly cached
'''
logwood.basic_config(handlers=[], level=logwood.DEBUG)
logger = logwood.get_logger('A')
logger2 = logwood.get_logger('A')
# Those two loggers should be completely the same
assert logger is logger2
# Store logger id so we can compare it later
logger_id = id(logger)
# Try to send something
logger.info('Test message')
logger2.info('Test message')
# Then delete both loggers
del logger
del logger2
# Collect all removed instances
gc.collect()
logger3 = logwood.get_logger('A')
assert logger_id != id(logger3)
def no_handler():
'''
Logwood with no handlers.
'''
logwood.basic_config(handlers=[])
logger = logwood.get_logger(__name__)
return logger
def __init__(self, safe=False):
self._logger = logwood.get_logger(f"{self.__class__.__name__}")
self.safe = safe
self.action_space = 140 # 140 possible actions in total
self.n_players = 2
self.reset()
def load_layout(self, path: str) -> KeyboardLayout:
logger = logwood.get_logger(self.__class__.__name__)
bindings_by_action = {}
with open(path) as f:
for raw_line in f:
line, *_ = raw_line.split('#')
if not line.strip():
continue
action, *bindings = line.split()
if action in bindings_by_action:
raise ConfigurationError(f'Duplicate action {action}')
bindings_by_action[action] = bindings
actions_by_key = {}
actions_by_scancode = {}
help_texts = []
for action in self._actions:
if action.name not in bindings_by_action:
logger.warning(f'Missing keybindings for action {action.name}')
help_texts.append((action.description or action.name, '<unassigned>'))
else:
bindings = bindings_by_action.pop(action.name)
help_texts.append((action.description or action.name, ', '.join(bindings) or '<unassigned>'))
self._parse_bindings_for_action(actions_by_key, actions_by_scancode, action.name, bindings)
for action_name in bindings_by_action:
logger.warning(f'Extra keybindings for unknown action {action_name}')
return KeyboardLayout(actions_by_key=actions_by_key, actions_by_scancode=actions_by_scancode, help=help_texts)
def file_handler():
'''
Logwood with FileHandler.
'''
logwood.testing.reset_state()
logwood.basic_config(handlers=[FileHandler(filename='example.log')])
logger = logwood.get_logger(__name__)
return logger
def syslog_handler():
'''
Logwood with syslog handler.
'''
logwood.testing.reset_state()
logwood.basic_config(handlers=[SysLogHandler()])
logger = logwood.get_logger(__name__)
return logger
def __init__(self, node: ComponentNode, main_queue: queue.Queue):
self._node = node
self._component = node.component # type: C
self._stopped = asyncio.Event()
self._queue = main_queue
self.logger = logwood.get_logger(self.__class__.__name__)
self.duration = AveragingOutput(DURATION_EVENT, interval=METRIC_AVERAGING_INTERVAL_SECONDS)
self.duration.set_queue(main_queue)
def std_err_handler():
'''
Logwood with stderr handler.
'''
logwood.testing.reset_state()
logwood.basic_config(handlers=[ColoredStderrHandler(logwood.DEBUG)])
logger = logwood.get_logger(__name__)
return logger
def __new__(cls, *args, **kwargs):
if cls.__name__ not in cls.__count:
cls.__count[cls.__name__] = 0
__instance = super().__new__(cls)
__instance._numeric_id = cls.__count[cls.__name__]
__instance.logger = logwood.get_logger(__instance.id)
__instance.__shutdown = Output(SHUTDOWN_EVENT)
cls.__count[cls.__name__] += 1
return __instance
def syslog_threaded_handler():
'''
Logwood with syslog handler run via ThreadedHandler.
'''
logwood.testing.reset_state()
logwood.basic_config(
handlers=[ThreadedHandler(underlying_handler=SysLogHandler())])
logger = logwood.get_logger(__name__)
return logger
def test_emit(syslog_mock):
'''
emit on libsyslog will call syslog.syslog
'''
logwood.basic_config(handlers=[logwood.handlers.syslog.SysLogLibHandler()])
logger = logwood.get_logger('Test')
logger.warning('Warning')
logwood.shutdown()
assert syslog_mock.called
def test_SysLogHandler_emit():
'''
Test that the refactored syslog handler's emit is working.
'''
with unittest.mock.patch('socket.socket'):
handler = logwood.handlers.logging.SysLogHandler()
logwood.basic_config(handlers=[handler])
logger = logwood.get_logger('Test')
logger.warning('Warning')
assert handler.socket.sendto.called
def test_chunk_long_message(socket):
message = '1234567890' * 100
handler = ChunkedSysLogHandler(address='/not/existing', chunk_size=10)
logwood.basic_config(format='%(message)s', handlers=[handler])
logger = logwood.get_logger('Test')
logger.error(message)
# We have chunk size 10 so emit should be called 100 times
assert handler.socket.send.call_count == 100
def emit(self, record):
''' Pass logs to logwood with logger name equal to original logger name. '''
name = record.name
level = record.levelno
# Format message using the original logging formatter so we keep the stack trace.
message = self._base_formatter.format(record)
try:
# Cache logger reference so we do not call get_logger for every message.
logger = self._loggers[name]
except KeyError:
self._loggers[name] = logger = logwood.get_logger(name)
logger.log(level, message)
def test_chunk_short_message(socket):
message = 'Short message'
handler = ChunkedSysLogHandler(address='/not/existing',
chunk_size=len(message))
logwood.basic_config(format='%(message)s', handlers=[handler])
logger = logwood.get_logger('Test')
logger.error(message)
# Do not call parent emit more then once for a short message
assert handler.socket.send.call_count == 1
# It was called just once (first zero) and we read the record (second zero)
assert message in str(handler.socket.send.call_args[0][0])
def __init__(self):
super().__init__()
self.logger = logwood.get_logger(self.__class__.__name__)
self._registry = Registry()
self._queue: queue.Queue = queue.Queue()
self._last_read_from_queue: float = 0
self._component_runners: Dict[Component, BaseComponentRunner] = {}
self._component_runner_tasks: Dict[Component, asyncio.Task] = {}
self._internal_tasks: List[asyncio.Task] = []
self._loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
self._internal_tasks_stopped = asyncio.Event(loop=self._loop)
self._main_loop_stopped = asyncio.Event(loop=self._loop)
self._main_loop_task: asyncio.Task = None
self._has_error_occurred = threading.Event()
self._shutting_down = threading.Event()
def __init__(self, model, **kwargs):
name = kwargs.get("name")
if name:
self.name = name
else:
self.name = self.__class__.__name__ + str(self.n_models)
self._logger = logwood.get_logger(self.name)
self.model = model
# Allows the model to be multithreaded
self.model._make_predict_function()
self._graph = tf.get_default_graph()
self.kwargs = kwargs
def test_underlying_handler_is_called():
'''
Assert that underlying handler got called in a thread.
'''
underlying_handler = unittest.mock.Mock()
handler = ThreadedHandler(underlying_handler = underlying_handler)
logwood.basic_config(level = logwood.DEBUG, handlers = [handler])
logger = logwood.get_logger('Test')
logger.error('Error message')
# After the message we can safely close all handlers
handler.close()
# Wait for a while for the scheduler to run our thread
time.sleep(0.1)
assert underlying_handler.emit.called
record = underlying_handler.emit.call_args[0][0]
assert record['message'] == 'Error message'
assert underlying_handler.close.called
def test_get_logger():
'''
get_logger returns a logger with configured handlers.
'''
handler1, handler2, handler3 = unittest.mock.Mock(), unittest.mock.Mock(
), unittest.mock.Mock()
logwood.basic_config(level=logwood.DEBUG, handlers=[handler1, handler2])
logger = logwood.get_logger('Test')
logger.add_handler(handler3)
assert handler1 not in logger.handlers
assert handler2 not in logger.handlers
assert handler3 in logger.handlers
logger.error('Error')
assert handler1.handle.call_count == 1
assert handler2.handle.call_count == 1
assert handler3.handle.call_count == 1
def __init__(
self,
env,
config,
replay,
evaluator,
callbacks=[],
model_dir=None,
replay_dir=None,
):
"""Initializes a StagedModelTrainer.
This class trains a neural-net guided by MCTS in three stages.
- Stage 1: Data is generated by playing games with the MCTS.
- Stage 2: The data that was generated is used to train a neural net.
- Stage 3: The trained neural net and original neural net compete.
The model that wins this stage takes over guiding the neural net.
Arguments:
env {Environment} -- The game environment.
config {dict} -- A configuration for the MCTS to run. Must contain a `model` key.
replay {mcts.nn.replay.Replay} -- The replay table used to store data points.
evaluator {mcts.evaluators.Evaluator} -- The evaluator to use during the evaluation stage.
Keyword Arguments:
callbacks {list} -- A list of Keras Callbacks to use in training (default: {[]})
model_dir {str} -- A filepath to save models (default: {None})
replay_dir {str} -- A filepath to save replay tables (default: {None})
"""
self.environment = env
self.config = config
self.training_model = config["model"]
self.generation_model = self.training_model.clone()
self.savepath = model_dir
self.replay_path = replay_dir
self.callbacks = callbacks
self.replay = replay
self.evaluator = evaluator
self._logger = logwood.get_logger(self.__class__.__name__)
def __init__(self,
environment,
calculation_time=5,
terminal_callback=None,
name=None):
self.tree = GameTree()
# Configure logger
if name == None:
name = self.__class__.__name__
self._logger = logwood.get_logger(name)
self.environment = environment
self.calculation_time = calculation_time
if terminal_callback:
self._handle_terminal = terminal_callback
self._handle_terminal.add_tree(self.tree)
self._builder = ConfigBuilder()
self.configured = False
self.reset()
def test_calling_basic_config_multiple_times():
'''
After first logger is created a call to basic_config should raise an exception.
'''
handler1, handler2 = unittest.mock.Mock(), unittest.mock.Mock()
logwood.basic_config(level=logwood.DEBUG, handlers=[handler1, handler2])
assert logwood.state.config_called
# But until the first logger is created it can be called multiple times
logwood.basic_config(handlers=[])
logwood.basic_config(handlers=[])
# now create some loggers
logwood.get_logger('A')
logwood.get_logger('B')
logwood.get_logger('C')
# Created loggers should be defined in state
assert len(logwood.state.defined_loggers) == 3
# After loggers are created it is impossible to change logging settings
with pytest.raises(AssertionError):
logwood.basic_config(handlers=[])
def __init__(self, environment, config, win_threshold=0.55):
self._logger = logwood.get_logger(self.__class__.__name__)
self.environment = environment.clone()
self.config = config
self.win_threshold = 0.55
def logger(handler):
logwood.basic_config(level=logwood.WARNING, handlers=[
handler,
])
return logwood.get_logger('TestLogger')
help='date since (e.g. 2019-04-29 or 2019-01-02T23:59.59Z)')
cli.add_argument('--until', type=str, default='2019-01-02T23:59.59Z',
help='date until (e.g. 2019-04-29 or 2019-01-02T23:59.59Z)')
cli.add_argument('--split', action='store_const', const=True,
default=False,
help='include when you want to split data info into multiples CSVs')
cli.add_argument('--debug', action='store_const', const=True,
default=False,
help='include when you want to see debug-level logs')
sysargv = cli.parse_args()
LOG_FORMAT = '{timestamp:.3f} [{level}] {message}'
logwood.basic_config(
level=logwood.DEBUG if sysargv.debug else logwood.INFO,
handlers=[StderrHandler(format=LOG_FORMAT)])
L = logwood.get_logger('GLOBAL')
# ------------------------------------------------------------------------------
import asyncio
import aiohttp
import certifi
import csv
import os
import ssl
import time
import ujson
from datetime import datetime
from dateutil import parser
from typing import List, Callable
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
def __init__(self):
self._logger = logwood.get_logger(self.__class__.__name__)
import pygame
import logwood
from logwood.handlers.stderr import ColoredStderrHandler
from environment.quoridor import Quoridor
from agents.base import BaseAgent
from agents.manual import ManualPygameAgent
logwood.basic_config(level=logwood.INFO, handlers=[ColoredStderrHandler()])
logger = logwood.get_logger('Visuals')
# Define Colors
BLACK = (0, 0, 0)
WHITE = (240, 255, 240)
LIGHTBROWN = (222, 184, 135)
BROWN = (128, 0, 0)
LIGHTRED = (240, 128, 128)
RED = (205, 92, 92)
LIGHTBLUE = (221, 160, 221)
BLUE = (186, 85, 211)
DARKBLUE = (0, 0, 128)
SCREEN_WIDTH = 600
SCREEN_HEIGHT = SCREEN_WIDTH - 200
TILE_WIDTH = SCREEN_HEIGHT / 10.6
TILE_HEIGHT = SCREEN_HEIGHT / 10.6
WALL_WIDTH = 0.2 * TILE_WIDTH
WALL_HEIGHT = TILE_WIDTH * 2 + WALL_WIDTH
def test_get_logger_before_basic_config():
'''
A Logger instance cannot be created before a call to basic_config.
'''
with pytest.raises(AssertionError):
logwood.get_logger('Test')
