def execute_train_task_with_dependencies(self, task_cls, **kwargs):
"""
Run the training, as well as any dependencies of the training
task_cls - class of a task
"""
log.info("Task {0}".format(get_task_name(task_cls)))
#Instantiate the task
task_inst = task_cls()
#Grab arguments from the task instance and set them
for arg in task_inst.args:
if arg not in kwargs:
kwargs[arg] = task_inst.args[arg]
#Check for dependencies defined by the task
if hasattr(task_inst, "dependencies"):
deps = task_inst.dependencies
dep_results = []
#Run the dependencies through recursion (in case of dependencies of dependencies, etc)
for dep in deps:
log.info("Dependency {0}".format(get_task_name(dep)))
dep_results.append(self.execute_train_task_with_dependencies(dep.cls, **dep.args))
trained_dependencies = []
#Add executed dependency to trained_dependencies list on the task
for i in xrange(0,len(deps)):
dep = deps[i]
dep_result = dep_results[i]
name = dep.name
namespace = dep.namespace
category = dep.category
trained_dependencies.append(TrainedDependency(category=category, namespace=namespace, name = name, inst = dep))
task_inst.trained_dependencies = trained_dependencies
#Finally, run the task
task_inst.train(**kwargs)
return task_inst
def predict(self, **kwargs):
"""
Do the workflow prediction (done after training, with new data)
"""
reformatted_predict = self.reformat_predict_data()
results = {}
for task_inst in self.trained_tasks:
predict = reformatted_predict[task_inst.data_format]['predict']
kwargs['predict']=predict
results.update({get_task_name(task_inst) : self.execute_predict_task(task_inst, predict, **kwargs)})
return results
def command(self, *args, **options):
headers = ["Name", "Help"]
help_table = []
for entry in registry:
cls = entry.cls
name = get_task_name(cls)
help_text = getattr(cls, "help_text", "")
help_table.append([name, help_text])
width = max([len(h[0]) for h in help_table]) + 5
row_format = ("{:<" + str(width) + "}") * (len(help_table[0]) + 1)
print row_format.format("", *headers)
for row in help_table:
print row_format.format("", *row)
def execute_train_task_with_dependencies(self, task_cls, **kwargs):
"""
Run the training, as well as any dependencies of the training
task_cls - class of a task
"""
log.info("Task {0}".format(get_task_name(task_cls)))
#Instantiate the task
task_inst = task_cls()
#Grab arguments from the task instance and set them
for arg in task_inst.args:
if arg not in kwargs:
kwargs[arg] = task_inst.args[arg]
#Check for dependencies defined by the task
if hasattr(task_inst, "dependencies"):
deps = task_inst.dependencies
dep_results = []
#Run the dependencies through recursion (in case of dependencies of dependencies, etc)
for dep in deps:
log.info("Dependency {0}".format(get_task_name(dep)))
dep_results.append(
self.execute_train_task_with_dependencies(
dep.cls, **dep.args))
trained_dependencies = []
#Add executed dependency to trained_dependencies list on the task
for i in xrange(0, len(deps)):
dep = deps[i]
dep_result = dep_results[i]
name = dep.name
namespace = dep.namespace
category = dep.category
trained_dependencies.append(
TrainedDependency(category=category,
namespace=namespace,
name=name,
inst=dep))
task_inst.trained_dependencies = trained_dependencies
#Finally, run the task
task_inst.train(**kwargs)
return task_inst
def command(self, *args, **options):
headers = ["Name", "Help"]
help_table = []
for entry in registry:
cls = entry.cls
name = get_task_name(cls)
help_text = getattr(cls, "help_text", "")
help_table.append([name, help_text])
width = max([len(h[0]) for h in help_table])+5
row_format =("{:<" +str(width) + "}") * (len(help_table[0]) + 1)
print row_format.format("", *headers)
for row in help_table:
print row_format.format("", *row)
def predict(self, **kwargs):
"""
Do the workflow prediction (done after training, with new data)
"""
reformatted_predict = self.reformat_predict_data()
results = {}
for task_inst in self.trained_tasks:
predict = reformatted_predict[task_inst.data_format]['predict']
kwargs['predict'] = predict
results.update({
get_task_name(task_inst):
self.execute_predict_task(task_inst, predict, **kwargs)
})
return results
