def _init_infra():
operations.initialize_infrastructure()
sqs.clear_all_messages()
def _init_infra():
from donomo.archive.utils import sqs
operations.initialize_infrastructure()
sqs.clear_all_messages()
def main():
""" Main.
"""
signal.signal(signal.SIGTERM, stop_process_driver)
signal.signal(signal.SIGINT, stop_process_driver)
parser = optparse.OptionParser()
parser.add_option("--max-concurrency", default=10, type="int")
parser.add_option("--daemonize", action="store_true", default=False)
parser.add_option("--workdir")
parser.add_option("--logfile")
parser.add_option("--umask", type="int")
parser.add_option("--pidfile")
options, process_names = parser.parse_args()
if options.daemonize:
from django.utils.daemonize import become_daemon
daemon_kwargs = {}
if options.workdir:
daemon_kwargs["our_home_dir"] = options.workdir
if options.logfile:
daemon_kwargs["out_log"] = options.logfile
daemon_kwargs["err_log"] = options.logfile
if options.umask:
daemon_kwargs["umask"] = options.umask
become_daemon(**daemon_kwargs)
if options.pidfile:
pidfile = open(options.pidfile, "w")
try:
pidfile.write("%d\n" % os.getpid())
finally:
pidfile.close()
logging.info("Starting processing queue %s " % settings.SQS_QUEUE_NAME)
if len(process_names) == 0:
process_names = DEFAULT_PROCESSORS
options.processes = load_modules(process_names)
options.queue = Queue.Queue()
operations.initialize_infrastructure()
# pylint: disable-msg=W0702
# -> no exception type given
while not must_shut_down():
try:
work_item = maybe_get_next(options)
if work_item:
handle_work_item(options, work_item)
check_for_done(options)
except:
logging.exception("An exception occurred!")
# pylint: enable-msg=W0702
while True:
active_threads = threading.activeCount() - 1
if active_threads == 0:
break
logging.info("Waiting for %d workers to finish" % active_threads)
time.sleep(2)
logging.info("Stopped")
def _init_infra():
operations.initialize_infrastructure()
sqs.clear_all_messages()
def main():
""" Main.
"""
signal.signal(signal.SIGTERM, stop_process_driver)
signal.signal(signal.SIGINT, stop_process_driver)
parser = optparse.OptionParser()
parser.add_option('--max-concurrency', default=10, type='int')
parser.add_option('--daemonize', action='store_true', default=False)
parser.add_option('--workdir')
parser.add_option('--logfile')
parser.add_option('--umask', type='int')
parser.add_option('--pidfile')
options, process_names = parser.parse_args()
if options.daemonize:
from django.utils.daemonize import become_daemon
daemon_kwargs = {}
if options.workdir:
daemon_kwargs['our_home_dir'] = options.workdir
if options.logfile:
daemon_kwargs['out_log'] = options.logfile
daemon_kwargs['err_log'] = options.logfile
if options.umask:
daemon_kwargs['umask'] = options.umask
become_daemon(**daemon_kwargs)
if options.pidfile:
pidfile = open(options.pidfile, 'w')
try:
pidfile.write('%d\n' % os.getpid())
finally:
pidfile.close()
logging.info("Starting processing queue %s " % settings.SQS_QUEUE_NAME)
if len(process_names) == 0:
process_names = DEFAULT_PROCESSORS
options.processes = load_modules(process_names)
options.queue = Queue.Queue()
operations.initialize_infrastructure()
# pylint: disable-msg=W0702
# -> no exception type given
while not must_shut_down():
try:
work_item = maybe_get_next(options)
if work_item:
handle_work_item(options, work_item)
check_for_done(options)
except:
logging.exception('An exception occurred!')
# pylint: enable-msg=W0702
while True:
active_threads = threading.activeCount() - 1
if active_threads == 0:
break
logging.info("Waiting for %d workers to finish" % active_threads)
time.sleep(2)
logging.info("Stopped")
