Ejemplo n.º 1
0
def main():
    usage = "norc_scheduler [-e] [-d]"

    def bad_args(message):
        print message
        print usage
        sys.exit(2)

    parser = OptionParser(usage)
    parser.add_option("-e",
                      "--echo",
                      action="store_true",
                      default=False,
                      help="Echo log messages to stdout.")
    parser.add_option("-d",
                      "--debug",
                      action="store_true",
                      default=False,
                      help="Enable debug messages.")

    (options, args) = parser.parse_args()

    if Scheduler.objects.alive().count() > 0:
        print "Cannot run more than one scheduler at a time."
        return

    scheduler = Scheduler.objects.create()
    scheduler.log = make_log(scheduler.log_path,
                             echo=options.echo,
                             debug=options.debug)
    scheduler.start()
Ejemplo n.º 2
0
def main():
    usage = "norc_executor <queue_name> -c <n> [-e] [-d]"
    
    def bad_args(message):
        print message
        print usage
        sys.exit(2)
    
    parser = OptionParser(usage)
    parser.add_option("-c", "--concurrent", type='int',
        help="How many instances can be run concurrently.")
    parser.add_option("-e", "--echo", action="store_true", default=False,
        help="Echo log messages to stdout.")
    parser.add_option("-d", "--debug", action="store_true", default=False,
        help="Enable debug messages.")
    
    (options, args) = parser.parse_args()

    if len(args) != 1:
        bad_args("A single queue name is required.")
    
    if options.concurrent == None:
        bad_args("You must give a maximum number of concurrent subprocesses.")
    
    queue = Queue.get(args[0])
    if not queue:
        bad_args("Invalid queue name '%s'." % args[0])
    
    executor = Executor.objects.create(queue=queue, concurrent=options.concurrent)
    executor.log = make_log(executor.log_path,
        echo=options.echo, debug=options.debug)
    executor.start()
Ejemplo n.º 3
0
 def start_instance(self, instance):
     """Starts a given instance in a new process."""
     instance.executor = self
     instance.save()
     self.log.info("Starting %s..." % instance)
     # p = Process(target=self.execute, args=[instance.start])
     # p.start()
     ct = ContentType.objects.get_for_model(instance)
     f = make_log(instance.log_path).file
     p = Popen('norc_taskrunner --ct_pk %s --target_pk %s' %
         (ct.pk, instance.pk), stdout=f, stderr=STDOUT, shell=True)
     p.instance = instance
     self.processes[p.pid] = p
Ejemplo n.º 4
0
def main():
    usage = "norc_executor <queue_name> -c <n> [-e] [-d]"

    def bad_args(message):
        print message
        print usage
        sys.exit(2)

    parser = OptionParser(usage)
    parser.add_option("-c",
                      "--concurrent",
                      type='int',
                      help="How many instances can be run concurrently.")
    parser.add_option("-q",
                      "--create_queue",
                      action="store_true",
                      default=False,
                      help="Force creation of a DBQueue with this name.")
    parser.add_option("-e",
                      "--echo",
                      action="store_true",
                      default=False,
                      help="Echo log messages to stdout.")
    parser.add_option("-d",
                      "--debug",
                      action="store_true",
                      default=False,
                      help="Enable debug messages.")

    (options, args) = parser.parse_args()

    if len(args) != 1:
        bad_args("A single queue name is required.")

    if options.concurrent is None:
        bad_args("You must give a maximum number of concurrent subprocesses.")

    queue = Queue.get(args[0])
    if not queue:
        if options.create_queue:
            queue = DBQueue.objects.create(name=args[0])
        else:
            bad_args("Invalid queue name '%s'." % args[0])

    executor = Executor.objects.create(queue=queue,
                                       concurrent=options.concurrent)
    executor.log = make_log(executor.log_path,
                            echo=options.echo,
                            debug=options.debug)
    executor.start()
Ejemplo n.º 5
0
 def start_instance(self, instance):
     """Starts a given instance in a new process."""
     instance.executor = self
     instance.save()
     self.log.info("Starting %s..." % instance)
     # p = Process(target=self.execute, args=[instance.start])
     # p.start()
     ct = ContentType.objects.get_for_model(instance)
     f = make_log(instance.log_path).file
     p = Popen('norc_taskrunner --ct_pk %s --target_pk %s' %
               (ct.pk, instance.pk),
               stdout=f,
               stderr=STDOUT,
               shell=True)
     p.instance = instance
     self.processes[p.pid] = p
Ejemplo n.º 6
0
Archivo: task.py Proyecto: tml/norc
 def start(self):
     if not hasattr(self, 'log'):
         self.log = make_log(self.log_path)
     if self.status != Status.CREATED:
         self.log.error("Can't start an instance more than once.")
         return
     try:
         for signum in [signal.SIGINT, signal.SIGTERM]:
             signal.signal(signum, self.kill_handler)
     except ValueError:
         pass
     if self.timeout > 0:
         signal.signal(signal.SIGALRM, self.timeout_handler)
         signal.alarm(self.timeout)
     self.log.info('Starting %s.' % self)
     self.log.start_redirect()
     self.status = Status.RUNNING
     self.started = datetime.utcnow()
     self.save()
     try:
         success = self.run()
     except Exception:
         self.log.error("Task failed with an exception!", trace=True)
         self.status = Status.ERROR
     except NorcInterruptException:
         self.log.error("Interrupt signal received!")
         self.status = Status.INTERRUPTED
     except NorcTimeoutException:
         self.log.info("Task timed out!  Ceasing execution.")
         self.status = Status.TIMEDOUT
     else:
         if success or success == None:
             self.status = Status.SUCCESS
         else:
             self.status = Status.FAILURE
     finally:
         self.ended = datetime.utcnow()
         self.save()
         self.log.info("Task ended with status %s." %
             Status.name(self.status))
         self.log.stop_redirect()
         self.log.close()
         sys.exit(0 if self.status == Status.SUCCESS else 1)
Ejemplo n.º 7
0
def main():
    usage = "norc_scheduler [-e] [-d]"
    
    def bad_args(message):
        print message
        print usage
        sys.exit(2)
    
    parser = OptionParser(usage)
    parser.add_option("-e", "--echo", action="store_true", default=False,
        help="Echo log messages to stdout.")
    parser.add_option("-d", "--debug", action="store_true", default=False,
        help="Enable debug messages.")
    
    (options, args) = parser.parse_args()
    
    if Scheduler.objects.alive().count() > 0:
        print "Cannot run more than one scheduler at a time."
        return
    
    scheduler = Scheduler.objects.create()
    scheduler.log = make_log(scheduler.log_path,
        echo=options.echo, debug=options.debug)
    scheduler.start()
Ejemplo n.º 8
0
 def start(self):
     """Performs initialization before calling run()."""
     
     if not hasattr(self, 'log'):
         self.log = make_log(self.log_path)
     if self.status != Status.CREATED:
         self.log.error("Can't start an instance more than once.")
         return
     try:
         for signum in [signal.SIGINT, signal.SIGTERM]:
             signal.signal(signum, self.kill_handler)
     except ValueError:
         pass
     if self.timeout > 0:
         signal.signal(signal.SIGALRM, self.timeout_handler)
         signal.alarm(self.timeout)
     self.log.info('Starting %s.' % self)
     self.log.start_redirect()
     self.status = Status.RUNNING
     self.revision = self.get_revision()
     self.started = datetime.utcnow()
     self.save()
     try:
         success = self.run()
     except Exception:
         self.log.error("Task failed with an exception!", trace=True)
         self.status = Status.FAILURE
     else:
         if success or success == None:
             self.status = Status.SUCCESS
         else:
             self.status = Status.FAILURE
     finally:
         self.run_finally()
         self.cleanup()
         sys.exit(0 if self.status == Status.SUCCESS else 1)
Ejemplo n.º 9
0
 def start(self):
     """Starts the daemon.  Does initialization then calls run()."""
     
     if self.status != Status.CREATED:
         print "Can't start a %s that's already been run." \
             % type(self).__name__
         return
     
     if not hasattr(self, 'id'):
         self.save()
     if not hasattr(self, 'log'):
         self.log = make_log(self.log_path)
     
     if settings.DEBUG:
         self.log.info("WARNING, DEBUG is True, which means Django " +
             "will gobble memory as it stores all database queries.")
     
     # This try block is needed because the unit tests run daemons
     # in threads, which breaks signals.
     try:
         for signum in (signal.SIGINT, signal.SIGTERM):
             signal.signal(signum, self.signal_handler)
     except ValueError:
         pass
     
     self.log.start_redirect()
     self.log.info("%s initialized; starting..." % self)
     
     self.status = Status.RUNNING
     self.heartbeat = self.started = datetime.utcnow()
     self.save()
     self.heart.start()
     
     try:
         self.run()
     except Exception:
         self.set_status(Status.ERROR)
         self.log.error("An internal error occured!", trace=True)
     else:
         if not Status.is_final(self.status):
             self.set_status(Status.ENDED)
     finally:    
         self.log.info("Shutting down...")
         try:
             self.clean_up()
         except:
             self.log.error("Clean up function failed.", trace=True)
         if not Status.is_final(self.status):
             self.set_status(Status.ERROR)
         self.heart.flag.set()
         self.heart.join()
         self.ended = datetime.utcnow()
         self.save()
         if settings.BACKUP_SYSTEM:
             self.log.info('Backing up log file...')
             try:
                 if backup_log(self.log_path):
                     self.log.info('Completed log backup.')
                 else:
                     self.log.error('Failed to backup log.')
             except:
                 self.log.error('Failed to backup log.', trace=True)
         self.log.info('%s has been shut down successfully.' % self)
         self.log.stop_redirect()
         self.log.close()
Ejemplo n.º 10
0
    def start(self):
        """Starts the daemon.  Does initialization then calls run()."""

        if self.status != Status.CREATED:
            print "Can't start a %s that's already been run." \
                % type(self).__name__
            return

        if not hasattr(self, 'id'):
            self.save()
        if not hasattr(self, 'log'):
            self.log = make_log(self.log_path)

        if settings.DEBUG:
            self.log.info(
                "WARNING, DEBUG is True, which means Django " +
                "will gobble memory as it stores all database queries.")

        # This try block is needed because the unit tests run daemons
        # in threads, which breaks signals.
        try:
            for signum in (signal.SIGINT, signal.SIGTERM):
                signal.signal(signum, self.signal_handler)
        except ValueError:
            pass

        self.log.start_redirect()
        self.log.info("%s initialized; starting..." % self)

        self.status = Status.RUNNING
        self.heartbeat = self.started = datetime.utcnow()
        self.save()
        self.heart.start()

        try:
            self.run()
        except Exception:
            self.set_status(Status.ERROR)
            self.log.error("An internal error occured!", trace=True)
        else:
            if not Status.is_final(self.status):
                self.set_status(Status.ENDED)
        finally:
            self.log.info("Shutting down...")
            try:
                self.clean_up()
            except:
                self.log.error("Clean up function failed.", trace=True)
            if not Status.is_final(self.status):
                self.set_status(Status.ERROR)
            self.heart.flag.set()
            self.heart.join()
            self.ended = datetime.utcnow()
            self.save()
            if settings.BACKUP_SYSTEM:
                self.log.info('Backing up log file...')
                try:
                    if backup_log(self.log_path):
                        self.log.info('Completed log backup.')
                    else:
                        self.log.error('Failed to backup log.')
                except:
                    self.log.error('Failed to backup log.', trace=True)
            self.log.info('%s has been shut down successfully.' % self)
            self.log.stop_redirect()
            self.log.close()