def main():
try:
settings = startup.read_settings()
with startup.SingleInstance(settings.args.filename):
constants.set(settings.constants)
Log.start(settings.debug)
extractor = Extract(settings)
def extract(please_stop):
with MySQL(**settings.snowflake.database) as db:
with db.transaction():
for kwargs in extractor.queue:
if please_stop:
break
try:
extractor.extract(db=db,
please_stop=please_stop,
**kwargs)
except Exception as e:
Log.warning("Could not extract", cause=e)
extractor.queue.add(kwargs)
for i in range(settings.extract.threads):
Thread.run("extract #" + text_type(i), extract)
please_stop = Signal()
Thread.wait_for_shutdown_signal(please_stop=please_stop,
allow_exit=True,
wait_forever=False)
except Exception as e:
Log.warning("Problem with data extraction", e)
finally:
Log.stop()
def main():
try:
config = startup.read_settings()
constants.set(config.constants)
Log.start(config.debug)
please_stop = Signal("main stop signal")
Thread.wait_for_shutdown_signal(please_stop)
except Exception, e:
Log.error("Problem with etl", cause=e)
result = http.post_json("http://localhost:5000/tuid",
json=request,
timeout=10000)
if not client.enabled:
Log.note("pausing consumer for {{num}}sec",
num=PAUSE_ON_FAILURE)
Till(seconds=PAUSE_ON_FAILURE).wait()
if result is None or len(result.data) != len(files):
Log.warning("expecting response for every file requested")
queue.commit()
if __name__ == '__main__':
try:
tmp_signal = Signal()
config = startup.read_settings()
constants.set(config.constants)
Log.start(config.debug)
queue_consumer(kwargs=config, please_stop=tmp_signal)
worker = Thread.run("sqs consumer", queue_consumer, kwargs=config)
Thread.wait_for_shutdown_signal(allow_exit=True,
please_stop=worker.stopped)
except BaseException as e: # MUST CATCH BaseException BECAUSE argparse LIKES TO EXIT THAT WAY, AND gunicorn WILL NOT REPORT
try:
Log.error("Serious problem with consumer construction! Shutdown!",
cause=e)
finally:
Log.stop()
# Intended to test exit behaviour from timeout, SIGINT (CTRL-C), or "exit"
###############################################################################
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from mo_logs import Log
from mo_threads import Thread, Signal
from mo_threads.till import Till
please_stop = Signal()
def timeout(please_stop):
(Till(seconds=20) | please_stop).wait()
please_stop.go()
Thread.run("timeout", target=timeout, please_stop=please_stop)
Log.note("you must type 'exit', and press Enter, or wait 20seconds")
Thread.wait_for_shutdown_signal(allow_exit=True, please_stop=please_stop)
if not please_stop:
Log.note("'exit' detected")
else:
Log.note("timeout detected")
please_stop.go()
# encoding: utf-8 # # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Author: Kyle Lahnakoski ([email protected]) # # THIS THREADING MODULE IS PERMEATED BY THE please_stop SIGNAL. # THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS # FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from mo_logs import Log from mo_threads import Thread Log.note("started") Thread.wait_for_shutdown_signal()
