Beispiel #1
0
 def run_batch(self):
     tasks_to_run = tms_manage.get_tasks_allowed_to_run(end_completed_iterations=True, max_to_return=10)
     num_running_tasks = self.get_num_running_tasks()
     log.debug("tmsd running %s task(s), at least %s task(s) due to run" % (num_running_tasks, len(tasks_to_run)))
     need_resource_types = []
     for (task, iteration) in tasks_to_run:
         if self.__break_tasks_to_run_loop__:
             # some other thread (request_stop) doesn't want me to continue.  Stop here.
             break
         # check that there are currently sufficient resources to prevent
         # erroneously thinking this task can be run when it cannot.
         # There will be occasional cases where race conditions mean a task is not run when
         # it could be, but there are many more cases when this will save threads.
         if type(task) in need_resource_types:
             # A Task of this type already returned unavailable resources; don't check again.
             # This should be an efficiency gain for the running of Tasks to prevent
             # excessive polling of the resources table when there are likely no new resources.
             # log.info("Assuming no resources avail for Task type '%s'" % (type(task)))
             pass
         elif task.resources_available_to_run(self.get_daemon_status().get_region()):
             try:
                 self.start_task(task, iteration)
             except Exception, e:
                 log.error("Could not run Task '%s'" % (task), e)
         else:
             need_resource_types.append(type(task))
Beispiel #2
0
def main():
    global task
    parser = OptionParser(
        "%prog --daemon_status_id <id> --queue_name <queue_name> \
[--nice <0>] [--stdout <file_name|DEFAULT>] [--stderr <file_name>|STDOUT>] [--debug]"
    )
    parser.add_option(
        "--daemon_status_id", action="store", type="int", help="The id of the daemon status that launched this Task"
    )
    parser.add_option("--queue_name", action="store", type="string", help="The name of the queue from which to read")
    parser.add_option("--nice", action="store", type="int", default=0, help="nice this process. defaults to 5.")
    parser.add_option(
        "--stdout",
        action="store",
        type="string",
        help="Send stdout to this file, or special value 'DEFAULT' \
sends it a the stream unique to this Task request",
    )
    parser.add_option(
        "--stderr",
        action="store",
        type="string",
        help="Send stderr to this file, or special value 'STDOUT' sends it to stdout",
    )
    parser.add_option("--debug", action="store_true", help="more messages")
    (options, args) = parser.parse_args()

    # option parsing
    if not options.daemon_status_id or not options.queue_name:
        sys.exit(parser.get_usage())
    log.set_logging_debug(options.debug)

    if not options.nice == 0:
        os.nice(options.nice)

    console_stderr = None
    try:
        c = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
        q = c.get_queue(options.queue_name)
        boto_message = q.read()
        task = __get_task__(boto_message, options.queue_name)
        if task == None:
            log.debug("No task in queue '%s' pid:%s" % (options.queue_name, os.getpid()))
            sys.exit(133)
        else:
            log.debug("Starting SQS Queue '%s' Task:%s pid:%s" % (options.queue_name, task.get_id(), os.getpid()))
            q.delete_message(boto_message)
            console_stderr = __redirect_outputs__(task, options.stdout, options.stderr)
            daemon_status = __get_daemon_status__(options.daemon_status_id)
            __run_task__(task, daemon_status)
            ending_status = task.get_current_run_status()
            if ending_status == None:
                sys.exit(134)
            if not ending_status.was_successful():
                sys.exit(1)
    except SystemExit, se:
        # in python 2.4, SystemExit extends Exception, this is changed in 2.5 to
        # extend BaseException, specifically so this check isn't necessary. But
        # we're using 2.4; upon upgrade, this check will be unecessary but ignorable.
        sys.exit(se.code)
Beispiel #3
0
 def close_all(self):
     to_close = self.open_files.values()
     log.debug("Closing %s log file(s)" % (len(to_close)))
     for fh in to_close:
         fh.close()