def check(self):
common.print_header('Checking for database prep (dir: {0})'.format(self.directory),
level=2)
dep_manager = Dependency(SqliteDB, self.doit_config['dep_file'])
missing = False
for task in self.tasks:
status = dep_manager.get_status(task, self.tasks)
self.logger.debug('{0}:{1}'.format(task.name, status.status))
if status.status != 'up-to-date':
missing = True
self.logger.warning('[ ] {0}'.format(task.name))
else:
self.logger.info('[x] {0}'.format(task.name))
common.print_header('Database results', level=2)
if missing:
self.logger.warning('Database prep incomplete')
common.print_header('to prepare databases, run: dammit databases'\
' --install', level=2)
else:
self.logger.info('All databases prepared!')
return missing
def check(self):
common.print_header('Checking for database prep (dir: {0})'.format(
self.directory),
level=2)
dep_manager = Dependency(SqliteDB, self.doit_config['dep_file'])
missing = False
for task in self.tasks:
status = dep_manager.get_status(task, self.tasks)
self.logger.debug('{0}:{1}'.format(task.name, status.status))
if status.status != 'up-to-date':
missing = True
self.logger.warning('[ ] {0}'.format(task.name))
else:
self.logger.info('[x] {0}'.format(task.name))
common.print_header('Database results', level=2)
if missing:
self.logger.warning('Database prep incomplete...')
else:
self.logger.info('All databases prepared!')
return missing
def check(self):
common.print_header("Checking for database prep (dir: {0})".format(self.directory), level=2)
dep_manager = Dependency(SqliteDB, self.doit_config["dep_file"])
missing = False
for task in self.tasks:
status = dep_manager.get_status(task, self.tasks)
self.logger.debug("{0}:{1}".format(task.name, status.status))
if status.status != "up-to-date":
missing = True
self.logger.warning("[ ] {0}".format(task.name))
else:
self.logger.info("[x] {0}".format(task.name))
common.print_header("Database results", level=2)
if missing:
self.logger.warning("Database prep incomplete")
common.print_header("to prepare databases, run: dammit databases" " --install", level=2)
else:
self.logger.info("All databases prepared!")
return missing
def check_status(task, tasks=None, dep_file='.doit.db'):
if tasks is None:
tasks = [task]
mgr = Dependency(DbmDB, os.path.abspath(dep_file))
status = mgr.get_status(task, tasks)
return status
def check_status(task, dep_file='.doit.db'):
mgr = Dependency(DbmDB, os.path.abspath(dep_file))
status = mgr.get_status(task, [task])
return status
class TaskHandler(TaskLoader):
def __init__(self, directory, logger, files=None,
profile=False, db=None, n_threads=1, **doit_config_kwds):
'''Stores tasks and the files they operate on, along with
doit config and other metadata. This is the core of the pipelines:
it passes its tasks along to doit for execution, and can check task
and pipeline completion status.
Args:
directory (str): The directory in which to run the tasks. Will be
created it it doesn't exist.
logger (logging.Logger): Logger to record to.
files (dict): Files used by the handler. Starts empty if omitted.
profile (bool): If True, profile task execution.
db (str): Name of the doit database.
**doit_config_kwds: Keyword arguments passed to doit.
Attributes:
files (dict): Files used by the tasks.
directory (str): Working directory for execution.
tasks (OrderedDict): The tasks to execute.
dep_file (str): Path of the doit database.
doit_config (dict): The doit configuration given to the task runner.
doit_dep_mgr (doit.dependency.Dependency): Doit object to track task
status.
profile (bool): Whether to run the profiler on tasks.
logger (logging.Logger): Logger to use.
'''
super(TaskHandler, self).__init__()
if files is None:
self.files = {}
elif type(files) is not dict:
raise TypeError('files must be of type dict')
else:
self.files = files
self.tasks = OrderedDict()
self.directory = directory
try:
mkdir(directory)
except OSError:
pass
if db is None:
dep_file = path.join(self.directory, 'doit.db')
else:
dep_file = path.join(self.directory, '{0}.doit.db'.format(db))
self.dep_file = dep_file
logger.debug('Dependency Database File: {0}'.format(dep_file))
self.doit_config = dict(dep_file=self.dep_file,
reporter=ui.GithubMarkdownReporter,
**doit_config_kwds)
self.doit_dep_mgr = Dependency(SqliteDB, dep_file)
self.n_threads = n_threads
self.profile = profile
self.logger = logger
def register_task(self, name, task, files=None):
'''Register a new task and its files with the handler.
It may seem redundant or confusing to give the tasks a name different
than their internal doit name. I do this because doit tasks need to have
names as unique as possible, so that they can be reused in different
projects. A particular TaskHandler instance is only used for one
pipeline run, and allowing different names makes it easier to reference
tasks from elsewhere.
Args:
name (str): Name of the task. Does not have to correspond to doit's
internal task name.
task (:obj:): Either a dictionary or Task object.
files (dict): Dictionary of files used.
'''
if files is None:
files = {}
if type(files) is not dict:
raise TypeError('files must be of type dict')
self.tasks[name] = task
self.files.update(files)
self.logger.debug('registered task {0}: {1}\n'
' with files {2}'.format(name, task, files))
def clear_tasks(self):
'''Empty the task dictionary.'''
self.logger.debug('Clearing {0} tasks'.format(len(self.tasks)))
self.tasks = {}
def get_status(self, task, move=False):
'''Get the up-to-date status of a single task.
Args:
task (str): The task name to look up.
move (bool): If True, move to the handler's directory before
checking. Whether this is necessary depends mostly on whether
the task uses relative or absolute paths.
Returns:
str: The string represenation of the status. Either "run" or
"uptodate".
'''
if type(task) is str:
try:
task = self.tasks[task]
except KeyError:
self.logger.error('Task not found:{0}'.format(task))
raise
self.logger.debug('Getting status for task {0}'.format(task.name))
if move:
with Move(self.directory):
status = self.doit_dep_mgr.get_status(task, self.tasks.values(),
get_log=True)
else:
status = self.doit_dep_mgr.get_status(task, self.tasks.values(),
get_log=True)
self.logger.debug('Task {0} had status {1}'.format(task, status.status))
try:
self.logger.debug('Task {0} had reasons {1}'.format(task, status.reasons))
except AttributeError:
pass
return status.status
def print_statuses(self, uptodate_msg='All tasks up-to-date!',
outofdate_msg='Some tasks out of date!'):
'''Print the up-to-date status of all tasks.
Args:
uptodate_msg (str): The message to print if all tasks are up to
date.
Returns:
tuple: A bool (True if all up to date) and a dictionary of statuses.
'''
uptodate, statuses = self.check_uptodate()
if uptodate:
print(ui.paragraph(uptodate_msg))
else:
print(ui.paragraph(outofdate_msg))
uptodate_list = [t for t,s in statuses.items() if s is True]
outofdate_list = [t for t,s in statuses.items() if s is False]
if uptodate_list:
print('\nUp-to-date tasks:')
print(ui.listing(uptodate_list))
if outofdate_list:
print('\nOut-of-date tasks:')
print(ui.listing(outofdate_list))
return uptodate, statuses
def check_uptodate(self):
'''Check if all tasks are up-to-date, ie if the pipeline is complete.
Note that this moves to the handler's directory to lessen issues with
relative versus absolute paths.
Returns:
bool: True if all are up to date.
'''
with Move(self.directory):
statuses = {}
outofdate = False
for task_name, task in self.tasks.items():
status = self.get_status(task)
statuses[task_name] = status == 'up-to-date'
return all(statuses.values()), statuses
def load_tasks(self, cmd, opt_values, pos_args):
'''Internal to doit -- triggered by the TaskLoader.'''
self.logger.debug('loading {0} tasks'.format(len(self.tasks)))
return self.tasks.values(), self.doit_config
def run(self, doit_args=None, verbose=True):
'''Run the pipeline. Movees to the directory, loads the tasks into doit,
and executes that tasks that are not up-to-date.
Args:
doit_args (list): Args that would be passed to the doit shell
command. By default, just run.
verbose (bool): If True, print UI stuff.
Returns:
int: Exit status of the doit command.
'''
if verbose:
print(ui.header('Run Tasks', level=4))
if doit_args is None:
doit_args = ['run']
if self.n_threads > 1:
doit_args.extend(['-n', str(self.n_threads)])
runner = DoitMain(self)
with Move(self.directory):
if self.profile is True:
profile_fn = path.join(self.directory, 'profile.csv')
with StartProfiler(filename=profile_fn):
return runner.run(doit_args)
else:
return runner.run(doit_args)
class Runner(object):
"""Task runner
run_all()
run_tasks():
for each task:
select_task()
execute_task()
process_task_result()
finish()
"""
def __init__(self, dependency_file, reporter, continue_=False,
always_execute=False, verbosity=0):
"""@param dependency_file: (string) file path of the db file
@param reporter: reporter to be used. It can be a class or an object
@param continue_: (bool) execute all tasks even after a task failure
@param always_execute: (bool) execute even if up-to-date or ignored
@param verbosity: (int) 0,1,2 see Task.execute
"""
self.dependency_manager = Dependency(dependency_file)
self.reporter = reporter
self.continue_ = continue_
self.always_execute = always_execute
self.verbosity = verbosity
self.teardown_list = [] # list of tasks to be teardown
self.final_result = SUCCESS # until something fails
self._stop_running = False
def _handle_task_error(self, task, catched_excp):
"""handle all task failures/errors
called whenever there is an error before executing a task or
its execution is not successful.
"""
assert isinstance(catched_excp, CatchedException)
self.dependency_manager.remove_success(task)
self.reporter.add_failure(task, catched_excp)
# only return FAILURE if no errors happened.
if isinstance(catched_excp, TaskFailed):
self.final_result = FAILURE
else:
self.final_result = ERROR
if not self.continue_:
self._stop_running = True
def select_task(self, task):
"""Returns bool, task should be executed
* side-effect: set task.options
Tasks should be executed if they are not up-to-date.
Tasks that cointains setup-tasks must be selected twice,
so it gives chance for dependency tasks to be executed after
checking it is not up-to-date.
"""
# if run_status is not None, it was already calculated
if task.run_status is None:
self.reporter.get_status(task)
# check if task is up-to-date
try:
task.run_status = self.dependency_manager.get_status(task)
except Exception, exception:
msg = "ERROR: Task '%s' checking dependencies" % task.name
dep_error = DependencyError(msg, exception)
self._handle_task_error(task, dep_error)
return False
if not self.always_execute:
# if task is up-to-date skip it
if task.run_status == 'up-to-date':
self.reporter.skip_uptodate(task)
task.values = self.dependency_manager.get_values(task.name)
return False
# check if task should be ignored (user controlled)
if task.run_status == 'ignore':
self.reporter.skip_ignore(task)
return False
if task.setup_tasks:
# dont execute now, execute setup first...
return False
else:
def check_status(task, dep_file='.doit.db'):
mgr = Dependency(DbmDB, os.path.abspath(dep_file))
status = mgr.get_status(task, [task])
return status
def check_status(task, tasks=None, dep_file='.doit.db'):
if tasks is None:
tasks = [task]
mgr = Dependency(DbmDB, os.path.abspath(dep_file))
status = mgr.get_status(task, tasks)
return status
