def execute_train_task_with_dependencies(self, task_cls, **kwargs): """ Run the training, as well as any dependencies of the training task_cls - class of a task """ log.info("Task {0}".format(get_task_name(task_cls))) #Instantiate the task task_inst = task_cls() #Grab arguments from the task instance and set them for arg in task_inst.args: if arg not in kwargs: kwargs[arg] = task_inst.args[arg] #Check for dependencies defined by the task if hasattr(task_inst, "dependencies"): deps = task_inst.dependencies dep_results = [] #Run the dependencies through recursion (in case of dependencies of dependencies, etc) for dep in deps: log.info("Dependency {0}".format(get_task_name(dep))) dep_results.append(self.execute_train_task_with_dependencies(dep.cls, **dep.args)) trained_dependencies = [] #Add executed dependency to trained_dependencies list on the task for i in xrange(0,len(deps)): dep = deps[i] dep_result = dep_results[i] name = dep.name namespace = dep.namespace category = dep.category trained_dependencies.append(TrainedDependency(category=category, namespace=namespace, name = name, inst = dep)) task_inst.trained_dependencies = trained_dependencies #Finally, run the task task_inst.train(**kwargs) return task_inst
def predict(self, **kwargs): """ Do the workflow prediction (done after training, with new data) """ reformatted_predict = self.reformat_predict_data() results = {} for task_inst in self.trained_tasks: predict = reformatted_predict[task_inst.data_format]['predict'] kwargs['predict']=predict results.update({get_task_name(task_inst) : self.execute_predict_task(task_inst, predict, **kwargs)}) return results
def command(self, *args, **options): headers = ["Name", "Help"] help_table = [] for entry in registry: cls = entry.cls name = get_task_name(cls) help_text = getattr(cls, "help_text", "") help_table.append([name, help_text]) width = max([len(h[0]) for h in help_table]) + 5 row_format = ("{:<" + str(width) + "}") * (len(help_table[0]) + 1) print row_format.format("", *headers) for row in help_table: print row_format.format("", *row)
def execute_train_task_with_dependencies(self, task_cls, **kwargs): """ Run the training, as well as any dependencies of the training task_cls - class of a task """ log.info("Task {0}".format(get_task_name(task_cls))) #Instantiate the task task_inst = task_cls() #Grab arguments from the task instance and set them for arg in task_inst.args: if arg not in kwargs: kwargs[arg] = task_inst.args[arg] #Check for dependencies defined by the task if hasattr(task_inst, "dependencies"): deps = task_inst.dependencies dep_results = [] #Run the dependencies through recursion (in case of dependencies of dependencies, etc) for dep in deps: log.info("Dependency {0}".format(get_task_name(dep))) dep_results.append( self.execute_train_task_with_dependencies( dep.cls, **dep.args)) trained_dependencies = [] #Add executed dependency to trained_dependencies list on the task for i in xrange(0, len(deps)): dep = deps[i] dep_result = dep_results[i] name = dep.name namespace = dep.namespace category = dep.category trained_dependencies.append( TrainedDependency(category=category, namespace=namespace, name=name, inst=dep)) task_inst.trained_dependencies = trained_dependencies #Finally, run the task task_inst.train(**kwargs) return task_inst
def command(self, *args, **options): headers = ["Name", "Help"] help_table = [] for entry in registry: cls = entry.cls name = get_task_name(cls) help_text = getattr(cls, "help_text", "") help_table.append([name, help_text]) width = max([len(h[0]) for h in help_table])+5 row_format =("{:<" +str(width) + "}") * (len(help_table[0]) + 1) print row_format.format("", *headers) for row in help_table: print row_format.format("", *row)
def predict(self, **kwargs): """ Do the workflow prediction (done after training, with new data) """ reformatted_predict = self.reformat_predict_data() results = {} for task_inst in self.trained_tasks: predict = reformatted_predict[task_inst.data_format]['predict'] kwargs['predict'] = predict results.update({ get_task_name(task_inst): self.execute_predict_task(task_inst, predict, **kwargs) }) return results