def delete_task(self, task_name): """ Deletes the named Task in this Job. """ logger.debug('Deleting task {0}'.format(task_name)) if not self.state.allow_change_graph: raise DagobahError( "job's graph is immutable in its current state: %s" % self.state.status) if task_name not in self.tasks: raise DagobahError('task %s does not exist' % task_name) self.tasks.pop(task_name) self.delete_node(task_name) self.commit()
def start(self): """ Begins the job by kicking off all tasks with no dependencies. """ logger.info('Job {0} starting job run'.format(self.name)) if not self.state.allow_start: raise DagobahError('job cannot be started in its current state; ' + 'it is probably already running') self.initialize_snapshot() # don't increment if the job was run manually if self.cron_iter and datetime.utcnow() > self.next_run: self.next_run = self.cron_iter.get_next(datetime) self.run_log = { 'job_id': self.job_id, 'name': self.name, 'parent_id': self.parent.dagobah_id, 'log_id': self.backend.get_new_log_id(), 'start_time': datetime.utcnow(), 'tasks': {} } self._set_status('running') logger.debug('Job {0} resetting all tasks prior to start'.format( self.name)) for task in self.tasks.values(): task.reset() logger.debug('Job {0} seeding run logs'.format(self.name)) for task_name in self.ind_nodes(self.snapshot): self._put_task_in_run_log(task_name) self.tasks[task_name].start() self._commit_run_log()
def schedule(self, cron_schedule, base_datetime=None): """ Schedules the job to run periodically using Cron syntax. """ logger.debug('Scheduling job {0} with cron schedule {1}'.format( self.name, cron_schedule)) if not self.state.allow_change_schedule: raise DagobahError( "job's schedule cannot be changed in state: %s" % self.state.status) if cron_schedule is None: self.cron_schedule = None self.cron_iter = None self.next_run = None else: if base_datetime is None: base_datetime = datetime.utcnow() self.cron_schedule = cron_schedule self.cron_iter = croniter(cron_schedule, base_datetime) self.next_run = self.cron_iter.get_next(datetime) logger.debug('Determined job {0} next run of {1}'.format( self.name, self.next_run)) self.commit()
def _tail_temp_file(self, temp_file, num_lines, seek_offset=10000): """ Returns a list of the last num_lines lines from a temp file. This works by first moving seek_offset chars back from the end of the file, then attempting to tail the file from there. It is possible that fewer than num_lines will be returned, even if the file has more total lines than num_lines. """ if not isinstance(num_lines, int): raise DagobahError('num_lines must be an integer') temp_file.seek(0, os.SEEK_END) size = temp_file.tell() temp_file.seek(-1 * min(size, seek_offset), os.SEEK_END) result = [] while True: this_line = temp_file.readline() if this_line == '': break result.append(this_line.strip()) if len(result) > num_lines: result.pop(0) return result
def from_backend(self, dagobah_id): """ Reconstruct this Dagobah instance from the backend. """ logger.debug('Reconstructing Dagobah instance from backend with ID {0}'.format(dagobah_id)) rec = self.backend.get_dagobah_json(dagobah_id) if not rec: raise DagobahError('dagobah with id %s does not exist ' 'in backend' % dagobah_id) self._construct_from_json(rec)
def update_job_notes(self, notes): logger.debug('Job {0} updating notes'.format(self.name)) if not self.state.allow_edit_job: raise DagobahError('job cannot be edited in its current state') setattr(self, 'notes', notes) self.parent.commit(cascade=True)
def delete_job(self, job_name): """ Delete a job by name, or error out if no such job exists. """ logger.debug('Deleting job {0}'.format(job_name)) for idx, job in enumerate(self.jobs): if job.name == job_name: self.backend.delete_job(job.job_id) del self.jobs[idx] self.commit() return raise DagobahError('no job with name %s exists' % job_name)
def kill(self): """ Send SIGKILL to the task's process. """ logger.info('Sending SIGKILL to task {0}'.format(self.name)) if hasattr(self, 'remote_client') and self.remote_client is not None: self.kill_sent = True self.remote_client.close() return if not self.process: raise DagobahError('task does not have a running process') self.kill_sent = True self.process.kill()
def add_task_to_job(self, job_or_job_name, task_command, task_name=None, **kwargs): """ Add a task to a job owned by the Dagobah instance. """ if isinstance(job_or_job_name, Job): job = job_or_job_name else: job = self.get_job(job_or_job_name) if not job: raise DagobahError('job %s does not exist' % job_or_job_name) logger.debug('Adding task with command {0} to job {1}'.format(task_command, job.name)) if not job.state.allow_change_graph: raise DagobahError("job's graph is immutable in its current " + "state: %s" % job.state.status) job.add_task(task_command, task_name, **kwargs) job.commit()
def _head_temp_file(self, temp_file, num_lines): """ Returns a list of the first num_lines lines from a temp file. """ if not isinstance(num_lines, int): raise DagobahError('num_lines must be an integer') temp_file.seek(0) result, curr_line = [], 0 for line in temp_file: curr_line += 1 result.append(line.strip()) if curr_line >= num_lines: break return result
def add_dependency(self, from_task_name, to_task_name): """ Add a dependency between two tasks. """ logger.debug('Adding dependency from {0} to {1}'.format( from_task_name, to_task_name)) if not self.state.allow_change_graph: raise DagobahError( "job's graph is immutable in its current state: %s" % self.state.status) self.add_edge(from_task_name, to_task_name) self.commit()
def edit_task(self, task_name, **kwargs): """ Change the name of a Task owned by this Job. This will affect the historical data available for this Task, e.g. past run logs will no longer be accessible. """ logger.debug('Job {0} editing task {1}'.format(self.name, task_name)) if not self.state.allow_edit_task: raise DagobahError("tasks cannot be edited in this job's " + "current state") if task_name not in self.tasks: raise DagobahError('task %s not found' % task_name) if 'name' in kwargs and isinstance(kwargs['name'], str): if kwargs['name'] in self.tasks: raise DagobahError('task name %s is unavailable' % kwargs['name']) task = self.tasks[task_name] for key in ['name', 'command']: if key in kwargs and isinstance(kwargs[key], str): setattr(task, key, kwargs[key]) if 'soft_timeout' in kwargs: task.set_soft_timeout(kwargs['soft_timeout']) if 'hard_timeout' in kwargs: task.set_hard_timeout(kwargs['hard_timeout']) if 'hostname' in kwargs: task.set_hostname(kwargs['hostname']) if 'name' in kwargs and isinstance(kwargs['name'], str): self.rename_edges(task_name, kwargs['name']) self.tasks[kwargs['name']] = task del self.tasks[task_name] self.parent.commit(cascade=True)
def edit(self, **kwargs): """ Change this Job's name. This will affect the historical data available for this Job, e.g. past run logs will no longer be accessible. """ logger.debug('Job {0} changing name to {1}'.format( self.name, kwargs.get('name'))) if not self.state.allow_edit_job: raise DagobahError('job cannot be edited in its current state') if 'name' in kwargs and isinstance(kwargs['name'], str): if not self.parent._name_is_available(kwargs['name']): raise DagobahError('new job name %s is not available' % kwargs['name']) for key in ['name']: if key in kwargs and isinstance(kwargs[key], str): setattr(self, key, kwargs[key]) self.parent.commit(cascade=True)
def add_job(self, job_name, job_id=None): """ Create a new, empty Job. """ logger.debug('Creating a new job named {0}'.format(job_name)) if not self._name_is_available(job_name): raise DagobahError('name %s is not available' % job_name) if not job_id: job_id = self.backend.get_new_job_id() self.created_jobs += 1 self.jobs.append(Job(self, self.backend, job_id, job_name)) job = self.get_job(job_name) job.commit()
def initialize_snapshot(self): """ Copy the DAG and validate """ logger.debug('Initializing DAG snapshot for job {0}'.format(self.name)) if self.snapshot is not None: logger.warn("Attempting to initialize DAG snapshot without " + "first destroying old snapshot.") snapshot_to_validate = deepcopy(self.graph) is_valid, reason = self.validate(snapshot_to_validate) if not is_valid: raise DagobahError(reason) self.snapshot = snapshot_to_validate
def add_task(self, command, name=None, **kwargs): """ Adds a new Task to the graph with no edges. """ logger.debug('Adding task with command {0} to job {1}'.format( command, self.name)) if not self.state.allow_change_graph: raise DagobahError( "job's graph is immutable in its current state: %s" % self.state.status) if name is None: name = command new_task = Task(self, command, name, **kwargs) self.tasks[name] = new_task self.add_node(name) self.commit()
def retry(self): """ Restarts failed tasks of a job. """ logger.info('Job {0} retrying all failed tasks'.format(self.name)) self.initialize_snapshot() failed_task_names = [] for task_name, log in self.run_log['tasks'].items(): if log.get('success', True) == False: failed_task_names.append(task_name) if len(failed_task_names) == 0: raise DagobahError('no failed tasks to retry') self._set_status('running') self.run_log['last_retry_time'] = datetime.utcnow() logger.debug('Job {0} seeding run logs'.format(self.name)) for task_name in failed_task_names: self._put_task_in_run_log(task_name) self.tasks[task_name].start() self._commit_run_log()
def _set_status(self, status): """ Enforces enum-like behavior on the status field. """ try: self.state.set_status(status) except: raise DagobahError('could not set status %s' % status)
def _map_string_to_file(self, stream): if stream not in ['stdout', 'stderr']: raise DagobahError('stream must be stdout or stderr') return self.stdout_file if stream == 'stdout' else self.stderr_file