Ejemplo n.º 1
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             log_extra = {'task_id': c.task_id, 'job_id': c.job_id}
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 cmd_logger.info(
                     'Command {}, pid {} is considered broken, will be marked as '
                     'finished'.format(c.uid, c.pid),
                     extra=log_extra,
                 )
             else:
                 cmd_logger.warn(
                     'Command {}, pid {} is considered broken, but process is running'
                     .format(c.uid, c.pid),
                     extra=log_extra,
                 )
         s.commit()
     except Exception:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Ejemplo n.º 2
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info(
                     'Command {}, pid {} is considered broken, will be marked as '
                     'finished'.format(
                         c.uid,
                         c.pid
                     )
                 )
             else:
                 logger.warn(
                     'Command {}, pid {} is considered broken, but process is running'.format(
                         c.uid,
                         c.pid
                     )
                 )
         s.commit()
     except Exception:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Ejemplo n.º 3
0
    def run(self):

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id')
        )
        # TODO: what about group_id, node, node_backend ?

        s.add(command)
        s.commit()

        try:
            self.execute()
        except Exception as e:
            self.error = e
            # TODO: raise?
        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            s.add(command)
            s.commit()
        except Exception as e:
            logger.exception('Failed to update db command')
            s.rollback()
Ejemplo n.º 4
0
    def run(self):

        self.start_ts = int(time.time())

        self.process = Subprocess(self.cmd,
                                  stdout=Subprocess.STREAM,
                                  stderr=Subprocess.STREAM,
                                  env=self.env,
                                  io_loop=self.io_loop)

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=self.process.pid,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id'),
        )

        s.update_ts = int(time.time())
        s.add(command)
        s.commit()

        self.watcher = self.watch(command)

        self.command = command
Ejemplo n.º 5
0
    def run(self):

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id'))
        # TODO: what about group_id, node, node_backend ?

        s.add(command)
        s.commit()

        try:
            self.execute()
        except Exception as e:
            self.error = e
            # TODO: raise?
        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            s.add(command)
            s.commit()
        except Exception as e:
            logger.exception('Failed to update db command')
            s.rollback()
Ejemplo n.º 6
0
    def run(self):

        self.start_ts = int(time.time())

        self.process = Subprocess(self.cmd,
                                  stdout=Subprocess.STREAM,
                                  stderr=Subprocess.STREAM,
                                  env=self.env,
                                  io_loop=self.io_loop)

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=self.process.pid,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id'),
            job_id=self.params.get('job_id'),
        )

        s.update_ts = int(time.time())
        s.add(command)
        s.commit()

        self.watcher = self.watch(command)

        self.command = command
Ejemplo n.º 7
0
 def update_db_command(self):
     s = Session()
     s.begin()
     try:
         command = self.command
         command.progress = self.progress
         command.exit_code = self.exit_code
         command.command_code = self.command_code
         command.finish_ts = self.finish_ts
         s.add(command)
         s.commit()
     except Exception as e:
         logger.exception('Failed to update db command: {0}'.format(e))
         s.rollback()
Ejemplo n.º 8
0
 def feed(self, s):
     prev_progress = self.progress
     self.output.append(s)
     if abs(self.progress - prev_progress) > 0.01:
         try:
             s = Session()
             s.begin()
             self.command.progress = self.progress
             s.add(self.command)
             s.commit()
         except Exception as e:
             logger.exception('pid {0}: failed to update db command'.format(
                 self.subprocess.pid))
             pass
Ejemplo n.º 9
0
 def on_update_progress(self):
     s = Session()
     s.begin()
     try:
         command = self.command
         command.progress = self.watcher.progress
         command.exit_code = self.watcher.exit_code
         command.command_code = self.command_code
         command.stdout = self.watcher.get_stdout()
         command.stderr = self.watcher.get_stderr()
         if command.exit_code is not None:
             command.artifacts = json.dumps(self.artifacts)
             command.finish_ts = self.finish_ts
         s.add(command)
         s.commit()
     except Exception:
         logger.exception('Failed to update db command')
         s.rollback()
Ejemplo n.º 10
0
 def on_update_progress(self):
     s = Session()
     s.begin()
     try:
         command = self.command
         command.progress = self.watcher.progress
         command.exit_code = self.watcher.exit_code
         command.command_code = self.command_code
         command.stdout = self.watcher.get_stdout()
         command.stderr = self.watcher.get_stderr()
         if command.exit_code is not None:
             command.artifacts = json.dumps(self.artifacts)
             command.finish_ts = self.finish_ts
         s.add(command)
         s.commit()
     except Exception:
         cmd_logger.exception('Failed to update db command', extra=self.log_extra)
         s.rollback()
Ejemplo n.º 11
0
    def run(self):

        self.start_ts = int(time.time())

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.COMMAND,
            start_ts=self.start_ts,
            task_id=self.params.get('task_id'),
            job_id=self.params.get('job_id'),
        )

        s.update_ts = int(time.time())
        s.add(command)
        s.commit()

        try:
            yield self.execute()
        except Exception as e:
            cmd_logger.exception('Command execution failed',
                                 extra=self.log_extra)
            self.error = e

        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            command.artifacts = json.dumps(self.artifacts)
            s.add(command)
            s.commit()
        except Exception as e:
            cmd_logger.exception('Failed to update db command',
                                 extra=self.log_extra)
            s.rollback()
Ejemplo n.º 12
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info('Command {0}, pid {1} is considered broken, '
                             'will be marked as finished'.format(
                                 c.uid, c.pid))
             else:
                 logger.warn('Command {0}, pid {1} is considered broken, '
                             'but process is running'.format(c.uid, c.pid))
         s.commit()
     except Exception as e:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Ejemplo n.º 13
0
    def run(self):

        self.start_ts = int(time.time())

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.COMMAND,
            start_ts=self.start_ts,
            task_id=self.params.get('task_id'),
            job_id=self.params.get('job_id'),
        )

        s.update_ts = int(time.time())
        s.add(command)
        s.commit()

        try:
            yield self.execute()
        except Exception as e:
            cmd_logger.exception('Command execution failed', extra=self.log_extra)
            self.error = e

        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            command.artifacts = json.dumps(self.artifacts)
            s.add(command)
            s.commit()
        except Exception as e:
            cmd_logger.exception('Failed to update db command', extra=self.log_extra)
            s.rollback()
Ejemplo n.º 14
0
    def run(self):
        self.process = Subprocess(self.cmd,
                                  stdout=Subprocess.STREAM,
                                  stderr=Subprocess.STREAM,
                                  env=self.env,
                                  io_loop=self.io_loop)

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(uid=self.uid,
                          pid=self.process.pid,
                          command=self.cmd_str,
                          start_ts=int(time.time()),
                          task_id=self.params.get('task_id'))
        #TODO:
        #what about group_id, node, node_backend ?

        s.add(command)
        s.commit()

        self.watcher = self.watch(command)