def main(): try: settings = startup.read_settings() with startup.SingleInstance(settings.args.filename): constants.set(settings.constants) Log.start(settings.debug) extractor = Extract(settings) def extract(please_stop): with MySQL(**settings.snowflake.database) as db: with db.transaction(): for kwargs in extractor.queue: if please_stop: break try: extractor.extract(db=db, please_stop=please_stop, **kwargs) except Exception as e: Log.warning("Could not extract", cause=e) extractor.queue.add(kwargs) for i in range(settings.extract.threads): Thread.run("extract #" + text_type(i), extract) please_stop = Signal() Thread.wait_for_shutdown_signal(please_stop=please_stop, allow_exit=True, wait_forever=False) except Exception as e: Log.warning("Problem with data extraction", e) finally: Log.stop()
def main(): try: config = startup.read_settings() constants.set(config.constants) Log.start(config.debug) please_stop = Signal("main stop signal") Thread.wait_for_shutdown_signal(please_stop) except Exception, e: Log.error("Problem with etl", cause=e)
result = http.post_json("http://localhost:5000/tuid", json=request, timeout=10000) if not client.enabled: Log.note("pausing consumer for {{num}}sec", num=PAUSE_ON_FAILURE) Till(seconds=PAUSE_ON_FAILURE).wait() if result is None or len(result.data) != len(files): Log.warning("expecting response for every file requested") queue.commit() if __name__ == '__main__': try: tmp_signal = Signal() config = startup.read_settings() constants.set(config.constants) Log.start(config.debug) queue_consumer(kwargs=config, please_stop=tmp_signal) worker = Thread.run("sqs consumer", queue_consumer, kwargs=config) Thread.wait_for_shutdown_signal(allow_exit=True, please_stop=worker.stopped) except BaseException as e: # MUST CATCH BaseException BECAUSE argparse LIKES TO EXIT THAT WAY, AND gunicorn WILL NOT REPORT try: Log.error("Serious problem with consumer construction! Shutdown!", cause=e) finally: Log.stop()
# Intended to test exit behaviour from timeout, SIGINT (CTRL-C), or "exit" ############################################################################### from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from mo_logs import Log from mo_threads import Thread, Signal from mo_threads.till import Till please_stop = Signal() def timeout(please_stop): (Till(seconds=20) | please_stop).wait() please_stop.go() Thread.run("timeout", target=timeout, please_stop=please_stop) Log.note("you must type 'exit', and press Enter, or wait 20seconds") Thread.wait_for_shutdown_signal(allow_exit=True, please_stop=please_stop) if not please_stop: Log.note("'exit' detected") else: Log.note("timeout detected") please_stop.go()
# encoding: utf-8 # # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # # Author: Kyle Lahnakoski ([email protected]) # # THIS THREADING MODULE IS PERMEATED BY THE please_stop SIGNAL. # THIS SIGNAL IS IMPORTANT FOR PROPER SIGNALLING WHICH ALLOWS # FOR FAST AND PREDICTABLE SHUTDOWN AND CLEANUP OF THREADS from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from mo_logs import Log from mo_threads import Thread Log.note("started") Thread.wait_for_shutdown_signal()