def _run_saver(db, message_queue):
if not (dispatchers.repository.DispatcherRepository.get_repo().has_dispatcher(message_queue) and
dispatchers.repository.ConsumerRepository.get_repo().has_consumer(message_queue)):
click.echo(f"Error: no dispatcher/consumer pair for {message_queue.scheme}")
return -1
database = _get_db_or_die(db)
saver_repo = repository.Repository.get()
tees = []
for saver in saver_repo.handlers():
# TODO: consider creating a single consumer for all save topics that can route to correct saver.
with logging.log_exception(module_logger, to_suppress=(RuntimeError, Exception),
format=lambda x: f"Error running saver service {db}: {x}"):
consumer = dispatchers.get_topic_consumer(
configuration.get_parsed_data_topic_name(saver.target),
message_queue
)
tee = dispatchers.tee.Tee(consumer, db_dispatcher.DBDispatcher(database))
runner = PluginRunner(repository.Repository.get(), json.loads, json.dumps)
runner.run_with_tee(saver.target, tee)
tees.append(tee)
with suppress(KeyboardInterrupt):
while True:
time.sleep(1)
module_logger.info("Stopping all tees...")
for tee in tees:
tee.stop()
def test_log_exception_does_log_and_suppresses(capsys):
logger = logging.get_logger("name1")
exception_message = "This is a message"
with logging.log_exception(logger, to_suppress=(Exception, )):
raise Exception(exception_message)
output = capsys.readouterr().out
assert exception_message in output
assert 'ERROR' in output
def parse(name, path):
with logging.log_exception(
module_logger,
format=lambda x: f"Error running parser {name}: {x}"):
runner = PluginRunner(repository.Repository.get(),
snapshot_xcoder.snapshot_decoder)
out = runner.run(name, pathlib.Path(path).read_bytes())
click.echo(json.dumps(str(out)))
def test_log_exception_does_log(capsys):
logger = logging.get_logger("name")
exception_message = "This is a message"
with pytest.raises(Exception):
with logging.log_exception(logger):
raise Exception(exception_message)
output = capsys.readouterr().out
assert exception_message in output
assert 'ERROR' in output
def test_log_exception_does_not_log_on_ignore(capsys):
logger = logging.get_logger("name2")
typeerror_message = "this is a TypeError"
with pytest.raises(TypeError):
with logging.log_exception(logger, to_ignore=(TypeError, )):
raise TypeError(typeerror_message)
output = capsys.readouterr().out
assert typeerror_message not in output
assert 'ERROR' not in output
def run_server_with_url(host,
port,
publish_url,
run_threaded=False,
encoder=None):
publisher = None
with logging.log_exception(logger=module_logger,
format="Could not find publisher"):
publisher = dispatchers.repository.DispatcherRepository.get_repo(
).get_dispatcher(
publish_url,
configuration.get_config()[
configuration.CONFIG_SERVER_PUBLISH_TOPICS])
module_logger.info(f"got publisher {publisher}")
with logging.log_exception(logger=module_logger,
to_suppress=(Exception, )):
_run_server(host, port, publisher)
def get_handler(self, target):
"""
returns a single parser whose target matches the request
"""
with log_exception(self._logger,
to_suppress=(IndexError, ),
format=f"No hadnler for target {target}"):
target_parsers = ([
i for i in self.handlers() if i.target == target
])
out = target_parsers.pop(0)
self._logger.debug(f"got handler {out} for {target}")
return out
def snapshot_decoder(message_string):
d = loads(message_string)
with log_exception(
logger,
to_suppress=(
FileNotFoundError,
KeyError,
),
format=lambda x:
f"Could not read MessageRecord: {repr(x)}, parsed_message: {d}"):
with MessageRecord.open(d['snapshot']) as mr:
d['snapshot'] = mr.read()
return d
return None
def _run_all_parsers(url):
repo = repository.Repository.get()
waiting = []
for parser in repo.handlers():
t = threading.Thread(target=run_parser,
args=(parser.target, url),
kwargs=dict(blocking=False))
t.start()
waiting.append(t)
with logging.log_exception(module_logger,
to_suppress=(KeyboardInterrupt, ),
format=lambda x: f"stopping all parsers..."):
while True:
time.sleep(1)
def run_parser(name, url, blocking=True):
if not (dispatchers.repository.DispatcherRepository.get_repo(
).has_dispatcher(url) and dispatchers.repository.ConsumerRepository.
get_repo().has_consumer(url)):
module_logger.error(
f"Error: no dispatcher/consumer pair for {url.scheme}")
return -1
with logging.log_exception(
module_logger,
to_suppress=(RuntimeError, Exception),
format=lambda x: f"Error running parser {name}: {x}"):
runner = PluginRunner(repository.Repository.get(),
snapshot_xcoder.snapshot_decoder, json.dumps)
runner.run_with_uri(name, uri=url)
def _consume_indefinitely(self, topic, record):
# assumes that the channel is connected and ready at this point
while True:
with log_exception(
self._logger,
to_suppress=(pika.exceptions.StreamLostError, ),
format=lambda e:
f"thread {threading.current_thread()} lost stream, reconnecting"
):
self._logger.info(f"start consuming {topic}")
record.channel.start_consuming()
record.channel = self._make_channel(
topic,
handler=record.callback,
message_decoder=record.message_decoder)
def _run_consumer(self, topic):
with self._io_list_lock:
record = self.handlers.get(topic, None)
if record is None:
self._logger.error(f"handler for {topic} is none")
else:
with log_exception(
self._logger,
to_suppress=(Exception, RuntimeError),
format=lambda e:
f"thread {threading.current_thread()} exception while consuming: {e}"
):
self._consume_indefinitely(topic, record)
with self._io_list_lock:
self.handlers.pop(topic)
if not len(self.handlers):
self._running = False
def run_server(host, port, database):
with log_exception(logger, to_suppress=(ValueError, Exception)):
run_api_server(host=host, port=port, database_url=database)
def saver(name, path, database):
db = _get_db_or_die(database)
saver = functools.partial(repository.get_saver(name).handler, db)
with logging.log_exception(module_logger, format=lambda x: f"Error running saver {name}: {x}"):
saver(json.loads(pathlib.Path(path).read_bytes()))
def run_server_cli(host, port, publish_url):
with logging.log_exception(logging.get_module_logger(__file__),
to_suppress=(Exception, )):
server.run_server_with_url(host or "127.0.0.1", port or 8080,
publish_url)
