def _init_infra(): operations.initialize_infrastructure() sqs.clear_all_messages()
def _init_infra(): from donomo.archive.utils import sqs operations.initialize_infrastructure() sqs.clear_all_messages()
def main(): """ Main. """ signal.signal(signal.SIGTERM, stop_process_driver) signal.signal(signal.SIGINT, stop_process_driver) parser = optparse.OptionParser() parser.add_option("--max-concurrency", default=10, type="int") parser.add_option("--daemonize", action="store_true", default=False) parser.add_option("--workdir") parser.add_option("--logfile") parser.add_option("--umask", type="int") parser.add_option("--pidfile") options, process_names = parser.parse_args() if options.daemonize: from django.utils.daemonize import become_daemon daemon_kwargs = {} if options.workdir: daemon_kwargs["our_home_dir"] = options.workdir if options.logfile: daemon_kwargs["out_log"] = options.logfile daemon_kwargs["err_log"] = options.logfile if options.umask: daemon_kwargs["umask"] = options.umask become_daemon(**daemon_kwargs) if options.pidfile: pidfile = open(options.pidfile, "w") try: pidfile.write("%d\n" % os.getpid()) finally: pidfile.close() logging.info("Starting processing queue %s " % settings.SQS_QUEUE_NAME) if len(process_names) == 0: process_names = DEFAULT_PROCESSORS options.processes = load_modules(process_names) options.queue = Queue.Queue() operations.initialize_infrastructure() # pylint: disable-msg=W0702 # -> no exception type given while not must_shut_down(): try: work_item = maybe_get_next(options) if work_item: handle_work_item(options, work_item) check_for_done(options) except: logging.exception("An exception occurred!") # pylint: enable-msg=W0702 while True: active_threads = threading.activeCount() - 1 if active_threads == 0: break logging.info("Waiting for %d workers to finish" % active_threads) time.sleep(2) logging.info("Stopped")
def _init_infra(): operations.initialize_infrastructure() sqs.clear_all_messages()
def main(): """ Main. """ signal.signal(signal.SIGTERM, stop_process_driver) signal.signal(signal.SIGINT, stop_process_driver) parser = optparse.OptionParser() parser.add_option('--max-concurrency', default=10, type='int') parser.add_option('--daemonize', action='store_true', default=False) parser.add_option('--workdir') parser.add_option('--logfile') parser.add_option('--umask', type='int') parser.add_option('--pidfile') options, process_names = parser.parse_args() if options.daemonize: from django.utils.daemonize import become_daemon daemon_kwargs = {} if options.workdir: daemon_kwargs['our_home_dir'] = options.workdir if options.logfile: daemon_kwargs['out_log'] = options.logfile daemon_kwargs['err_log'] = options.logfile if options.umask: daemon_kwargs['umask'] = options.umask become_daemon(**daemon_kwargs) if options.pidfile: pidfile = open(options.pidfile, 'w') try: pidfile.write('%d\n' % os.getpid()) finally: pidfile.close() logging.info("Starting processing queue %s " % settings.SQS_QUEUE_NAME) if len(process_names) == 0: process_names = DEFAULT_PROCESSORS options.processes = load_modules(process_names) options.queue = Queue.Queue() operations.initialize_infrastructure() # pylint: disable-msg=W0702 # -> no exception type given while not must_shut_down(): try: work_item = maybe_get_next(options) if work_item: handle_work_item(options, work_item) check_for_done(options) except: logging.exception('An exception occurred!') # pylint: enable-msg=W0702 while True: active_threads = threading.activeCount() - 1 if active_threads == 0: break logging.info("Waiting for %d workers to finish" % active_threads) time.sleep(2) logging.info("Stopped")