def update_broken_commands(self): s = Session() s.begin() try: for c in s.query(Command).filter_by(exit_code=None): log_extra = {'task_id': c.task_id, 'job_id': c.job_id} if not self.pid_exists(c.pid): c.progress = 1.0 c.exit_code = 666 c.finish_ts = int(time.time()) s.add(c) cmd_logger.info( 'Command {}, pid {} is considered broken, will be marked as ' 'finished'.format(c.uid, c.pid), extra=log_extra, ) else: cmd_logger.warn( 'Command {}, pid {} is considered broken, but process is running' .format(c.uid, c.pid), extra=log_extra, ) s.commit() except Exception: logger.exception('Failed to update broken commands') s.rollback() raise
def update_broken_commands(self): s = Session() s.begin() try: for c in s.query(Command).filter_by(exit_code=None): if not self.pid_exists(c.pid): c.progress = 1.0 c.exit_code = 666 c.finish_ts = int(time.time()) s.add(c) logger.info( 'Command {}, pid {} is considered broken, will be marked as ' 'finished'.format( c.uid, c.pid ) ) else: logger.warn( 'Command {}, pid {} is considered broken, but process is running'.format( c.uid, c.pid ) ) s.commit() except Exception: logger.exception('Failed to update broken commands') s.rollback() raise
def try_find_nonfailed_subprocess_and_status(self, job_id, task_id): if not task_id: return None, None for uid, subprocess in self.subprocesses.iteritems(): status = subprocess.status() # it is enough to check only task_id and ignore job_id if status.get('task_id') == task_id and (status['exit_code'] == 0 or status['progress'] < 1.0): return uid, status s = Session() # it is enough to check only task_id and ignore job_id query_start_ts = time.time() command = s.query(Command).filter(Command.task_id == task_id).order_by(desc(Command.start_ts)).first() logger.debug('find_nonfailed_subprocess query completed in {:.3f} seconds'.format(time.time() - query_start_ts)) if not command: return None, None if command.progress != 1.0: cmd_logger.warning( "Command {} ({}) was interrupted by minion restart, but not marked as finished".format( command.uid, command.command, ), extra={ 'job_id': job_id, 'task_id': task_id, }, ) return None, None if command.exit_code == 0: return command.uid, command.status return None, None
def status(self, uid): if uid in self.subprocesses: return self.subprocesses[uid].status() s = Session() command = s.query(Command).get(uid) if command is None: raise ValueError('Unknown command uid: {0}'.format(uid)) return command.status()
def unfinished_commands(self, finish_ts_gte=None): res = {} # 1. Getting unfinished commands from local db s = Session() criterias = [] if finish_ts_gte: criterias.append(Command.finish_ts == None) criterias.append(Command.finish_ts >= finish_ts_gte) for c in s.query(Command).filter(or_(*criterias)): res[c.uid] = c.status() # 2. Updating with in-memory commands for uid, sp in self.subprocesses.iteritems(): cmd_status = sp.status() if (finish_ts_gte and cmd_status['finish_ts'] and cmd_status['finish_ts'] < finish_ts_gte): continue res[uid] = cmd_status return res
def update_broken_commands(self): s = Session() s.begin() try: for c in s.query(Command).filter_by(exit_code=None): if not self.pid_exists(c.pid): c.progress = 1.0 c.exit_code = 666 c.finish_ts = int(time.time()) s.add(c) logger.info('Command {0}, pid {1} is considered broken, ' 'will be marked as finished'.format( c.uid, c.pid)) else: logger.warn('Command {0}, pid {1} is considered broken, ' 'but process is running'.format(c.uid, c.pid)) s.commit() except Exception as e: logger.exception('Failed to update broken commands') s.rollback() raise