def main():
usage = "norc_scheduler [-e] [-d]"
def bad_args(message):
print message
print usage
sys.exit(2)
parser = OptionParser(usage)
parser.add_option("-e",
"--echo",
action="store_true",
default=False,
help="Echo log messages to stdout.")
parser.add_option("-d",
"--debug",
action="store_true",
default=False,
help="Enable debug messages.")
(options, args) = parser.parse_args()
if Scheduler.objects.alive().count() > 0:
print "Cannot run more than one scheduler at a time."
return
scheduler = Scheduler.objects.create()
scheduler.log = make_log(scheduler.log_path,
echo=options.echo,
debug=options.debug)
scheduler.start()
def main():
usage = "norc_executor <queue_name> -c <n> [-e] [-d]"
def bad_args(message):
print message
print usage
sys.exit(2)
parser = OptionParser(usage)
parser.add_option("-c", "--concurrent", type='int',
help="How many instances can be run concurrently.")
parser.add_option("-e", "--echo", action="store_true", default=False,
help="Echo log messages to stdout.")
parser.add_option("-d", "--debug", action="store_true", default=False,
help="Enable debug messages.")
(options, args) = parser.parse_args()
if len(args) != 1:
bad_args("A single queue name is required.")
if options.concurrent == None:
bad_args("You must give a maximum number of concurrent subprocesses.")
queue = Queue.get(args[0])
if not queue:
bad_args("Invalid queue name '%s'." % args[0])
executor = Executor.objects.create(queue=queue, concurrent=options.concurrent)
executor.log = make_log(executor.log_path,
echo=options.echo, debug=options.debug)
executor.start()
def start_instance(self, instance):
"""Starts a given instance in a new process."""
instance.executor = self
instance.save()
self.log.info("Starting %s..." % instance)
# p = Process(target=self.execute, args=[instance.start])
# p.start()
ct = ContentType.objects.get_for_model(instance)
f = make_log(instance.log_path).file
p = Popen('norc_taskrunner --ct_pk %s --target_pk %s' %
(ct.pk, instance.pk), stdout=f, stderr=STDOUT, shell=True)
p.instance = instance
self.processes[p.pid] = p
def main():
usage = "norc_executor <queue_name> -c <n> [-e] [-d]"
def bad_args(message):
print message
print usage
sys.exit(2)
parser = OptionParser(usage)
parser.add_option("-c",
"--concurrent",
type='int',
help="How many instances can be run concurrently.")
parser.add_option("-q",
"--create_queue",
action="store_true",
default=False,
help="Force creation of a DBQueue with this name.")
parser.add_option("-e",
"--echo",
action="store_true",
default=False,
help="Echo log messages to stdout.")
parser.add_option("-d",
"--debug",
action="store_true",
default=False,
help="Enable debug messages.")
(options, args) = parser.parse_args()
if len(args) != 1:
bad_args("A single queue name is required.")
if options.concurrent is None:
bad_args("You must give a maximum number of concurrent subprocesses.")
queue = Queue.get(args[0])
if not queue:
if options.create_queue:
queue = DBQueue.objects.create(name=args[0])
else:
bad_args("Invalid queue name '%s'." % args[0])
executor = Executor.objects.create(queue=queue,
concurrent=options.concurrent)
executor.log = make_log(executor.log_path,
echo=options.echo,
debug=options.debug)
executor.start()
def start_instance(self, instance):
"""Starts a given instance in a new process."""
instance.executor = self
instance.save()
self.log.info("Starting %s..." % instance)
# p = Process(target=self.execute, args=[instance.start])
# p.start()
ct = ContentType.objects.get_for_model(instance)
f = make_log(instance.log_path).file
p = Popen('norc_taskrunner --ct_pk %s --target_pk %s' %
(ct.pk, instance.pk),
stdout=f,
stderr=STDOUT,
shell=True)
p.instance = instance
self.processes[p.pid] = p
def start(self):
if not hasattr(self, 'log'):
self.log = make_log(self.log_path)
if self.status != Status.CREATED:
self.log.error("Can't start an instance more than once.")
return
try:
for signum in [signal.SIGINT, signal.SIGTERM]:
signal.signal(signum, self.kill_handler)
except ValueError:
pass
if self.timeout > 0:
signal.signal(signal.SIGALRM, self.timeout_handler)
signal.alarm(self.timeout)
self.log.info('Starting %s.' % self)
self.log.start_redirect()
self.status = Status.RUNNING
self.started = datetime.utcnow()
self.save()
try:
success = self.run()
except Exception:
self.log.error("Task failed with an exception!", trace=True)
self.status = Status.ERROR
except NorcInterruptException:
self.log.error("Interrupt signal received!")
self.status = Status.INTERRUPTED
except NorcTimeoutException:
self.log.info("Task timed out! Ceasing execution.")
self.status = Status.TIMEDOUT
else:
if success or success == None:
self.status = Status.SUCCESS
else:
self.status = Status.FAILURE
finally:
self.ended = datetime.utcnow()
self.save()
self.log.info("Task ended with status %s." %
Status.name(self.status))
self.log.stop_redirect()
self.log.close()
sys.exit(0 if self.status == Status.SUCCESS else 1)
def main():
usage = "norc_scheduler [-e] [-d]"
def bad_args(message):
print message
print usage
sys.exit(2)
parser = OptionParser(usage)
parser.add_option("-e", "--echo", action="store_true", default=False,
help="Echo log messages to stdout.")
parser.add_option("-d", "--debug", action="store_true", default=False,
help="Enable debug messages.")
(options, args) = parser.parse_args()
if Scheduler.objects.alive().count() > 0:
print "Cannot run more than one scheduler at a time."
return
scheduler = Scheduler.objects.create()
scheduler.log = make_log(scheduler.log_path,
echo=options.echo, debug=options.debug)
scheduler.start()
def start(self):
"""Performs initialization before calling run()."""
if not hasattr(self, 'log'):
self.log = make_log(self.log_path)
if self.status != Status.CREATED:
self.log.error("Can't start an instance more than once.")
return
try:
for signum in [signal.SIGINT, signal.SIGTERM]:
signal.signal(signum, self.kill_handler)
except ValueError:
pass
if self.timeout > 0:
signal.signal(signal.SIGALRM, self.timeout_handler)
signal.alarm(self.timeout)
self.log.info('Starting %s.' % self)
self.log.start_redirect()
self.status = Status.RUNNING
self.revision = self.get_revision()
self.started = datetime.utcnow()
self.save()
try:
success = self.run()
except Exception:
self.log.error("Task failed with an exception!", trace=True)
self.status = Status.FAILURE
else:
if success or success == None:
self.status = Status.SUCCESS
else:
self.status = Status.FAILURE
finally:
self.run_finally()
self.cleanup()
sys.exit(0 if self.status == Status.SUCCESS else 1)
def start(self):
"""Starts the daemon. Does initialization then calls run()."""
if self.status != Status.CREATED:
print "Can't start a %s that's already been run." \
% type(self).__name__
return
if not hasattr(self, 'id'):
self.save()
if not hasattr(self, 'log'):
self.log = make_log(self.log_path)
if settings.DEBUG:
self.log.info("WARNING, DEBUG is True, which means Django " +
"will gobble memory as it stores all database queries.")
# This try block is needed because the unit tests run daemons
# in threads, which breaks signals.
try:
for signum in (signal.SIGINT, signal.SIGTERM):
signal.signal(signum, self.signal_handler)
except ValueError:
pass
self.log.start_redirect()
self.log.info("%s initialized; starting..." % self)
self.status = Status.RUNNING
self.heartbeat = self.started = datetime.utcnow()
self.save()
self.heart.start()
try:
self.run()
except Exception:
self.set_status(Status.ERROR)
self.log.error("An internal error occured!", trace=True)
else:
if not Status.is_final(self.status):
self.set_status(Status.ENDED)
finally:
self.log.info("Shutting down...")
try:
self.clean_up()
except:
self.log.error("Clean up function failed.", trace=True)
if not Status.is_final(self.status):
self.set_status(Status.ERROR)
self.heart.flag.set()
self.heart.join()
self.ended = datetime.utcnow()
self.save()
if settings.BACKUP_SYSTEM:
self.log.info('Backing up log file...')
try:
if backup_log(self.log_path):
self.log.info('Completed log backup.')
else:
self.log.error('Failed to backup log.')
except:
self.log.error('Failed to backup log.', trace=True)
self.log.info('%s has been shut down successfully.' % self)
self.log.stop_redirect()
self.log.close()
def start(self):
"""Starts the daemon. Does initialization then calls run()."""
if self.status != Status.CREATED:
print "Can't start a %s that's already been run." \
% type(self).__name__
return
if not hasattr(self, 'id'):
self.save()
if not hasattr(self, 'log'):
self.log = make_log(self.log_path)
if settings.DEBUG:
self.log.info(
"WARNING, DEBUG is True, which means Django " +
"will gobble memory as it stores all database queries.")
# This try block is needed because the unit tests run daemons
# in threads, which breaks signals.
try:
for signum in (signal.SIGINT, signal.SIGTERM):
signal.signal(signum, self.signal_handler)
except ValueError:
pass
self.log.start_redirect()
self.log.info("%s initialized; starting..." % self)
self.status = Status.RUNNING
self.heartbeat = self.started = datetime.utcnow()
self.save()
self.heart.start()
try:
self.run()
except Exception:
self.set_status(Status.ERROR)
self.log.error("An internal error occured!", trace=True)
else:
if not Status.is_final(self.status):
self.set_status(Status.ENDED)
finally:
self.log.info("Shutting down...")
try:
self.clean_up()
except:
self.log.error("Clean up function failed.", trace=True)
if not Status.is_final(self.status):
self.set_status(Status.ERROR)
self.heart.flag.set()
self.heart.join()
self.ended = datetime.utcnow()
self.save()
if settings.BACKUP_SYSTEM:
self.log.info('Backing up log file...')
try:
if backup_log(self.log_path):
self.log.info('Completed log backup.')
else:
self.log.error('Failed to backup log.')
except:
self.log.error('Failed to backup log.', trace=True)
self.log.info('%s has been shut down successfully.' % self)
self.log.stop_redirect()
self.log.close()