def establish_connection(self):
if self.task.state != Task.CREATED:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.ESTABLISHING_CONNECTION
self.task.save()
try:
self.command_runner = RemoteCommand(
self.task.host, self.task.username, self.task.task_runner.task_group.ssh_user.private_key
)
except:
self.fail_on_error()
class TaskCoordinator(object):
WAIT = 1
def __init__(self, task):
self.task = task
self.task_runner = task.task_runner
self.remote_files = task.task_runner.task_group.command.remote_files.all()
@property
def run_failed(self):
return self.task_runner.state == TaskRunner.FAILED
def fail_on_error(self):
e = traceback.format_exc()
self.task.error = e
self.task.state = Task.FAILED
self.task.save()
raise TaskGuardException("Task failed with error %s" % repr(e))
def _task_wait(self):
time.sleep(self.WAIT)
self.task_runner.refresh_from_db()
if self.run_failed:
raise TaskGuardException("Task runner failed")
def establish_connection(self):
if self.task.state != Task.CREATED:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.ESTABLISHING_CONNECTION
self.task.save()
try:
self.command_runner = RemoteCommand(
self.task.host, self.task.username, self.task.task_runner.task_group.ssh_user.private_key
)
except:
self.fail_on_error()
def wait_all_tasks_established_connections(self):
if self.task.state != Task.ESTABLISHING_CONNECTION:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.CONNECTION_ESTABLISHED
self.task.save()
while self.task_runner.state < TaskRunner.CONNECTIONS_ESTABLISHED:
self._task_wait()
self.task.state = Task.PREPARING_ASSETS
self.task.save()
def prepare_assets(self):
if self.task.state != Task.PREPARING_ASSETS:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
for f in self.remote_files:
try:
self.command_runner.write_file(f.filename, f.content, f.permissions)
except:
self.fail_on_error()
def wait_all_tasks_prepared_assets(self):
if self.task.state != Task.PREPARING_ASSETS:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.ASSETS_PREPARED
self.task.save()
while self.task_runner.state < TaskRunner.ASSETS_PREPARED:
self._task_wait()
self.task.state = Task.RUNNING_COMMAND
self.task.save()
def run_command(self):
if self.task.state != Task.RUNNING_COMMAND:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
command_start = datetime.now()
try:
self.result = self.command_runner.run(self.task.command)
except:
self.fail_on_error()
self.command_runtime = datetime.now() - command_start
def wait_all_commands_run(self):
if self.task.state != Task.RUNNING_COMMAND:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.COMMAND_RUN
self.task.save()
while self.task_runner.state < TaskRunner.COMMANDS_RUN:
self._task_wait()
self.task.state = Task.CLEANING_UP
self.task.save()
def cleanup_assets(self):
if self.task.state != Task.CLEANING_UP:
raise TaskGuardException("Invalid task state %s" % self.task.state)
for f in self.remote_files:
try:
self.command_runner.delete_file(f.filename)
except:
self.fail_on_error()
def wait_all_tasks_completed(self):
if self.task.state != Task.CLEANING_UP:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
self.task.state = Task.COMPLETE
self.task.save()
while self.task_runner.state < TaskRunner.COMPLETE:
self._task_wait()
def save_task_result(self):
if self.task.state != Task.COMPLETE:
raise TaskGuardException("Invlaid task state %s" % self.task.state)
if self.result.succeeded:
self.task.state = Task.SUCCEEDED
elif self.result.failed:
self.task.state = Task.FAILED
self.task.time = self.command_runtime.total_seconds()
self.task.stderr = self.result.stderr
self.task.stdout = self.result.stdout
self.task.status = self.result.status
self.task.save()