def run_batch(self):
tasks_to_run = tms_manage.get_tasks_allowed_to_run(end_completed_iterations=True, max_to_return=10)
num_running_tasks = self.get_num_running_tasks()
log.debug("tmsd running %s task(s), at least %s task(s) due to run" % (num_running_tasks, len(tasks_to_run)))
need_resource_types = []
for (task, iteration) in tasks_to_run:
if self.__break_tasks_to_run_loop__:
# some other thread (request_stop) doesn't want me to continue. Stop here.
break
# check that there are currently sufficient resources to prevent
# erroneously thinking this task can be run when it cannot.
# There will be occasional cases where race conditions mean a task is not run when
# it could be, but there are many more cases when this will save threads.
if type(task) in need_resource_types:
# A Task of this type already returned unavailable resources; don't check again.
# This should be an efficiency gain for the running of Tasks to prevent
# excessive polling of the resources table when there are likely no new resources.
# log.info("Assuming no resources avail for Task type '%s'" % (type(task)))
pass
elif task.resources_available_to_run(self.get_daemon_status().get_region()):
try:
self.start_task(task, iteration)
except Exception, e:
log.error("Could not run Task '%s'" % (task), e)
else:
need_resource_types.append(type(task))
def main():
global task
parser = OptionParser(
"%prog --daemon_status_id <id> --queue_name <queue_name> \
[--nice <0>] [--stdout <file_name|DEFAULT>] [--stderr <file_name>|STDOUT>] [--debug]"
)
parser.add_option(
"--daemon_status_id", action="store", type="int", help="The id of the daemon status that launched this Task"
)
parser.add_option("--queue_name", action="store", type="string", help="The name of the queue from which to read")
parser.add_option("--nice", action="store", type="int", default=0, help="nice this process. defaults to 5.")
parser.add_option(
"--stdout",
action="store",
type="string",
help="Send stdout to this file, or special value 'DEFAULT' \
sends it a the stream unique to this Task request",
)
parser.add_option(
"--stderr",
action="store",
type="string",
help="Send stderr to this file, or special value 'STDOUT' sends it to stdout",
)
parser.add_option("--debug", action="store_true", help="more messages")
(options, args) = parser.parse_args()
# option parsing
if not options.daemon_status_id or not options.queue_name:
sys.exit(parser.get_usage())
log.set_logging_debug(options.debug)
if not options.nice == 0:
os.nice(options.nice)
console_stderr = None
try:
c = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = c.get_queue(options.queue_name)
boto_message = q.read()
task = __get_task__(boto_message, options.queue_name)
if task == None:
log.debug("No task in queue '%s' pid:%s" % (options.queue_name, os.getpid()))
sys.exit(133)
else:
log.debug("Starting SQS Queue '%s' Task:%s pid:%s" % (options.queue_name, task.get_id(), os.getpid()))
q.delete_message(boto_message)
console_stderr = __redirect_outputs__(task, options.stdout, options.stderr)
daemon_status = __get_daemon_status__(options.daemon_status_id)
__run_task__(task, daemon_status)
ending_status = task.get_current_run_status()
if ending_status == None:
sys.exit(134)
if not ending_status.was_successful():
sys.exit(1)
except SystemExit, se:
# in python 2.4, SystemExit extends Exception, this is changed in 2.5 to
# extend BaseException, specifically so this check isn't necessary. But
# we're using 2.4; upon upgrade, this check will be unecessary but ignorable.
sys.exit(se.code)
def close_all(self):
to_close = self.open_files.values()
log.debug("Closing %s log file(s)" % (len(to_close)))
for fh in to_close:
fh.close()