def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
log_extra = {'task_id': c.task_id, 'job_id': c.job_id}
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
cmd_logger.info(
'Command {}, pid {} is considered broken, will be marked as '
'finished'.format(c.uid, c.pid),
extra=log_extra,
)
else:
cmd_logger.warn(
'Command {}, pid {} is considered broken, but process is running'
.format(c.uid, c.pid),
extra=log_extra,
)
s.commit()
except Exception:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
logger.info(
'Command {}, pid {} is considered broken, will be marked as '
'finished'.format(
c.uid,
c.pid
)
)
else:
logger.warn(
'Command {}, pid {} is considered broken, but process is running'.format(
c.uid,
c.pid
)
)
s.commit()
except Exception:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def run(self):
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id')
)
# TODO: what about group_id, node, node_backend ?
s.add(command)
s.commit()
try:
self.execute()
except Exception as e:
self.error = e
# TODO: raise?
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command')
s.rollback()
def run(self):
self.start_ts = int(time.time())
self.process = Subprocess(self.cmd,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=self.env,
io_loop=self.io_loop)
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=self.process.pid,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
self.watcher = self.watch(command)
self.command = command
def run(self):
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'))
# TODO: what about group_id, node, node_backend ?
s.add(command)
s.commit()
try:
self.execute()
except Exception as e:
self.error = e
# TODO: raise?
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command')
s.rollback()
def run(self):
self.start_ts = int(time.time())
self.process = Subprocess(self.cmd,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=self.env,
io_loop=self.io_loop)
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=self.process.pid,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
self.watcher = self.watch(command)
self.command = command
def update_db_command(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.progress
command.exit_code = self.exit_code
command.command_code = self.command_code
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command: {0}'.format(e))
s.rollback()
def feed(self, s):
prev_progress = self.progress
self.output.append(s)
if abs(self.progress - prev_progress) > 0.01:
try:
s = Session()
s.begin()
self.command.progress = self.progress
s.add(self.command)
s.commit()
except Exception as e:
logger.exception('pid {0}: failed to update db command'.format(
self.subprocess.pid))
pass
def on_update_progress(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.watcher.progress
command.exit_code = self.watcher.exit_code
command.command_code = self.command_code
command.stdout = self.watcher.get_stdout()
command.stderr = self.watcher.get_stderr()
if command.exit_code is not None:
command.artifacts = json.dumps(self.artifacts)
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception:
logger.exception('Failed to update db command')
s.rollback()
def on_update_progress(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.watcher.progress
command.exit_code = self.watcher.exit_code
command.command_code = self.command_code
command.stdout = self.watcher.get_stdout()
command.stderr = self.watcher.get_stderr()
if command.exit_code is not None:
command.artifacts = json.dumps(self.artifacts)
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception:
cmd_logger.exception('Failed to update db command', extra=self.log_extra)
s.rollback()
def run(self):
self.start_ts = int(time.time())
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.COMMAND,
start_ts=self.start_ts,
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
try:
yield self.execute()
except Exception as e:
cmd_logger.exception('Command execution failed',
extra=self.log_extra)
self.error = e
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
command.artifacts = json.dumps(self.artifacts)
s.add(command)
s.commit()
except Exception as e:
cmd_logger.exception('Failed to update db command',
extra=self.log_extra)
s.rollback()
def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
logger.info('Command {0}, pid {1} is considered broken, '
'will be marked as finished'.format(
c.uid, c.pid))
else:
logger.warn('Command {0}, pid {1} is considered broken, '
'but process is running'.format(c.uid, c.pid))
s.commit()
except Exception as e:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def run(self):
self.start_ts = int(time.time())
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.COMMAND,
start_ts=self.start_ts,
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
try:
yield self.execute()
except Exception as e:
cmd_logger.exception('Command execution failed', extra=self.log_extra)
self.error = e
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
command.artifacts = json.dumps(self.artifacts)
s.add(command)
s.commit()
except Exception as e:
cmd_logger.exception('Failed to update db command', extra=self.log_extra)
s.rollback()
def run(self):
self.process = Subprocess(self.cmd,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM,
env=self.env,
io_loop=self.io_loop)
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(uid=self.uid,
pid=self.process.pid,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'))
#TODO:
#what about group_id, node, node_backend ?
s.add(command)
s.commit()
self.watcher = self.watch(command)