def delete_task(self, task_name):
""" Deletes the named Task in this Job. """
logger.debug('Deleting task {0}'.format(task_name))
if not self.state.allow_change_graph:
raise DagobahError(
"job's graph is immutable in its current state: %s" %
self.state.status)
if task_name not in self.tasks:
raise DagobahError('task %s does not exist' % task_name)
self.tasks.pop(task_name)
self.delete_node(task_name)
self.commit()
def start(self):
""" Begins the job by kicking off all tasks with no dependencies. """
logger.info('Job {0} starting job run'.format(self.name))
if not self.state.allow_start:
raise DagobahError('job cannot be started in its current state; ' +
'it is probably already running')
self.initialize_snapshot()
# don't increment if the job was run manually
if self.cron_iter and datetime.utcnow() > self.next_run:
self.next_run = self.cron_iter.get_next(datetime)
self.run_log = {
'job_id': self.job_id,
'name': self.name,
'parent_id': self.parent.dagobah_id,
'log_id': self.backend.get_new_log_id(),
'start_time': datetime.utcnow(),
'tasks': {}
}
self._set_status('running')
logger.debug('Job {0} resetting all tasks prior to start'.format(
self.name))
for task in self.tasks.values():
task.reset()
logger.debug('Job {0} seeding run logs'.format(self.name))
for task_name in self.ind_nodes(self.snapshot):
self._put_task_in_run_log(task_name)
self.tasks[task_name].start()
self._commit_run_log()
def schedule(self, cron_schedule, base_datetime=None):
""" Schedules the job to run periodically using Cron syntax. """
logger.debug('Scheduling job {0} with cron schedule {1}'.format(
self.name, cron_schedule))
if not self.state.allow_change_schedule:
raise DagobahError(
"job's schedule cannot be changed in state: %s" %
self.state.status)
if cron_schedule is None:
self.cron_schedule = None
self.cron_iter = None
self.next_run = None
else:
if base_datetime is None:
base_datetime = datetime.utcnow()
self.cron_schedule = cron_schedule
self.cron_iter = croniter(cron_schedule, base_datetime)
self.next_run = self.cron_iter.get_next(datetime)
logger.debug('Determined job {0} next run of {1}'.format(
self.name, self.next_run))
self.commit()
def _tail_temp_file(self, temp_file, num_lines, seek_offset=10000):
""" Returns a list of the last num_lines lines from a temp file.
This works by first moving seek_offset chars back from the end of
the file, then attempting to tail the file from there. It is
possible that fewer than num_lines will be returned, even if the
file has more total lines than num_lines.
"""
if not isinstance(num_lines, int):
raise DagobahError('num_lines must be an integer')
temp_file.seek(0, os.SEEK_END)
size = temp_file.tell()
temp_file.seek(-1 * min(size, seek_offset), os.SEEK_END)
result = []
while True:
this_line = temp_file.readline()
if this_line == '':
break
result.append(this_line.strip())
if len(result) > num_lines:
result.pop(0)
return result
def from_backend(self, dagobah_id):
""" Reconstruct this Dagobah instance from the backend. """
logger.debug('Reconstructing Dagobah instance from backend with ID {0}'.format(dagobah_id))
rec = self.backend.get_dagobah_json(dagobah_id)
if not rec:
raise DagobahError('dagobah with id %s does not exist '
'in backend' % dagobah_id)
self._construct_from_json(rec)
def update_job_notes(self, notes):
logger.debug('Job {0} updating notes'.format(self.name))
if not self.state.allow_edit_job:
raise DagobahError('job cannot be edited in its current state')
setattr(self, 'notes', notes)
self.parent.commit(cascade=True)
def delete_job(self, job_name):
""" Delete a job by name, or error out if no such job exists. """
logger.debug('Deleting job {0}'.format(job_name))
for idx, job in enumerate(self.jobs):
if job.name == job_name:
self.backend.delete_job(job.job_id)
del self.jobs[idx]
self.commit()
return
raise DagobahError('no job with name %s exists' % job_name)
def kill(self):
""" Send SIGKILL to the task's process. """
logger.info('Sending SIGKILL to task {0}'.format(self.name))
if hasattr(self, 'remote_client') and self.remote_client is not None:
self.kill_sent = True
self.remote_client.close()
return
if not self.process:
raise DagobahError('task does not have a running process')
self.kill_sent = True
self.process.kill()
def add_task_to_job(self, job_or_job_name, task_command, task_name=None,
**kwargs):
""" Add a task to a job owned by the Dagobah instance. """
if isinstance(job_or_job_name, Job):
job = job_or_job_name
else:
job = self.get_job(job_or_job_name)
if not job:
raise DagobahError('job %s does not exist' % job_or_job_name)
logger.debug('Adding task with command {0} to job {1}'.format(task_command, job.name))
if not job.state.allow_change_graph:
raise DagobahError("job's graph is immutable in its current " +
"state: %s"
% job.state.status)
job.add_task(task_command, task_name, **kwargs)
job.commit()
def _head_temp_file(self, temp_file, num_lines):
""" Returns a list of the first num_lines lines from a temp file. """
if not isinstance(num_lines, int):
raise DagobahError('num_lines must be an integer')
temp_file.seek(0)
result, curr_line = [], 0
for line in temp_file:
curr_line += 1
result.append(line.strip())
if curr_line >= num_lines:
break
return result
def add_dependency(self, from_task_name, to_task_name):
""" Add a dependency between two tasks. """
logger.debug('Adding dependency from {0} to {1}'.format(
from_task_name, to_task_name))
if not self.state.allow_change_graph:
raise DagobahError(
"job's graph is immutable in its current state: %s" %
self.state.status)
self.add_edge(from_task_name, to_task_name)
self.commit()
def edit_task(self, task_name, **kwargs):
""" Change the name of a Task owned by this Job.
This will affect the historical data available for this
Task, e.g. past run logs will no longer be accessible.
"""
logger.debug('Job {0} editing task {1}'.format(self.name, task_name))
if not self.state.allow_edit_task:
raise DagobahError("tasks cannot be edited in this job's " +
"current state")
if task_name not in self.tasks:
raise DagobahError('task %s not found' % task_name)
if 'name' in kwargs and isinstance(kwargs['name'], str):
if kwargs['name'] in self.tasks:
raise DagobahError('task name %s is unavailable' %
kwargs['name'])
task = self.tasks[task_name]
for key in ['name', 'command']:
if key in kwargs and isinstance(kwargs[key], str):
setattr(task, key, kwargs[key])
if 'soft_timeout' in kwargs:
task.set_soft_timeout(kwargs['soft_timeout'])
if 'hard_timeout' in kwargs:
task.set_hard_timeout(kwargs['hard_timeout'])
if 'hostname' in kwargs:
task.set_hostname(kwargs['hostname'])
if 'name' in kwargs and isinstance(kwargs['name'], str):
self.rename_edges(task_name, kwargs['name'])
self.tasks[kwargs['name']] = task
del self.tasks[task_name]
self.parent.commit(cascade=True)
def edit(self, **kwargs):
""" Change this Job's name.
This will affect the historical data available for this
Job, e.g. past run logs will no longer be accessible.
"""
logger.debug('Job {0} changing name to {1}'.format(
self.name, kwargs.get('name')))
if not self.state.allow_edit_job:
raise DagobahError('job cannot be edited in its current state')
if 'name' in kwargs and isinstance(kwargs['name'], str):
if not self.parent._name_is_available(kwargs['name']):
raise DagobahError('new job name %s is not available' %
kwargs['name'])
for key in ['name']:
if key in kwargs and isinstance(kwargs[key], str):
setattr(self, key, kwargs[key])
self.parent.commit(cascade=True)
def add_job(self, job_name, job_id=None):
""" Create a new, empty Job. """
logger.debug('Creating a new job named {0}'.format(job_name))
if not self._name_is_available(job_name):
raise DagobahError('name %s is not available' % job_name)
if not job_id:
job_id = self.backend.get_new_job_id()
self.created_jobs += 1
self.jobs.append(Job(self, self.backend, job_id, job_name))
job = self.get_job(job_name)
job.commit()
def initialize_snapshot(self):
""" Copy the DAG and validate """
logger.debug('Initializing DAG snapshot for job {0}'.format(self.name))
if self.snapshot is not None:
logger.warn("Attempting to initialize DAG snapshot without " +
"first destroying old snapshot.")
snapshot_to_validate = deepcopy(self.graph)
is_valid, reason = self.validate(snapshot_to_validate)
if not is_valid:
raise DagobahError(reason)
self.snapshot = snapshot_to_validate
def add_task(self, command, name=None, **kwargs):
""" Adds a new Task to the graph with no edges. """
logger.debug('Adding task with command {0} to job {1}'.format(
command, self.name))
if not self.state.allow_change_graph:
raise DagobahError(
"job's graph is immutable in its current state: %s" %
self.state.status)
if name is None:
name = command
new_task = Task(self, command, name, **kwargs)
self.tasks[name] = new_task
self.add_node(name)
self.commit()
def retry(self):
""" Restarts failed tasks of a job. """
logger.info('Job {0} retrying all failed tasks'.format(self.name))
self.initialize_snapshot()
failed_task_names = []
for task_name, log in self.run_log['tasks'].items():
if log.get('success', True) == False:
failed_task_names.append(task_name)
if len(failed_task_names) == 0:
raise DagobahError('no failed tasks to retry')
self._set_status('running')
self.run_log['last_retry_time'] = datetime.utcnow()
logger.debug('Job {0} seeding run logs'.format(self.name))
for task_name in failed_task_names:
self._put_task_in_run_log(task_name)
self.tasks[task_name].start()
self._commit_run_log()
def _set_status(self, status):
""" Enforces enum-like behavior on the status field. """
try:
self.state.set_status(status)
except:
raise DagobahError('could not set status %s' % status)
def _map_string_to_file(self, stream):
if stream not in ['stdout', 'stderr']:
raise DagobahError('stream must be stdout or stderr')
return self.stdout_file if stream == 'stdout' else self.stderr_file