def create_puppet_workflow(regression_class=_RegressionWorkflowMixin, base_class=workflow.WorkflowBase, result_processor_class=NoOutputRP): puppet_parent = workflow._factory_build_inferelator(regression=regression_class, workflow=base_class) class PuppetClass(puppet_parent): """ Standard workflow except it takes all the data as references to __init__ instead of as filenames on disk or as environment variables, and returns the model AUPR and edge counts without writing files (unless told to) """ write_network = True network_file_name = None pr_curve_file_name = None initialize_mp = False def __init__(self, data, prior_data, gs_data): self.data = data self.priors_data = prior_data self.gold_standard = gs_data super(PuppetClass, self).__init__() def startup_run(self): # Skip all of the data loading self.process_priors_and_gold_standard() def create_output_dir(self, *args, **kwargs): pass return PuppetClass
def setUp(self): self.workflow = workflow._factory_build_inferelator( regression=FakeRegression, workflow=tfa_workflow.TFAWorkFlow)() self.workflow.input_dir = os.path.join(my_dir, "../../data/dream4") self.workflow.expression_matrix_file = default.DEFAULT_EXPRESSION_FILE self.workflow.tf_names_file = default.DEFAULT_TFNAMES_FILE self.workflow.meta_data_file = default.DEFAULT_METADATA_FILE self.workflow.priors_file = default.DEFAULT_PRIORS_FILE self.workflow.gold_standard_file = default.DEFAULT_GOLDSTANDARD_FILE self.workflow.get_data()
class DownsampleDataWorkflow(workflow._factory_build_inferelator(regression="stars", workflow="single-cell")): sample_ratio = None sample_seed = 1000 def startup_run(self): super(DownsampleDataWorkflow, self).startup_run() if self.sample_ratio == 1.: return rgen = np.random.default_rng(self.sample_seed) n_keep = int(self.data.num_obs * self.sample_ratio) n_keep = 1 if n_keep < 1 else n_keep self.data.get_random_samples(n_keep, random_gen=rgen, inplace=True, with_replacement=False)
class DownsampleDataWorkflow( workflow._factory_build_inferelator(regression="amusr", workflow="multitask")): sample_ratio = None sample_seed = 1000 def startup_run(self): super(DownsampleDataWorkflow, self).startup_run() if self.sample_ratio == 1.: return rgen = np.random.default_rng(self.sample_seed) for tobj in self._task_objects: n_keep = int(tobj.data.num_obs * self.sample_ratio) n_keep = 1 if n_keep < 1 else n_keep tobj.data.get_random_samples(n_keep, random_gen=rgen, inplace=True, with_replacement=False)
def test_abstractness(self): self.workflow = workflow._factory_build_inferelator(regression='base', workflow=tfa_workflow.TFAWorkFlow)() with self.assertRaises(NotImplementedError): self.workflow.run_bootstrap([])
def create_task_data_class(workflow_class="single-cell"): task_parent = workflow._factory_build_inferelator(regression="base", workflow=workflow_class) class TaskData(task_parent): """ TaskData is a workflow object which only loads and preprocesses data from files. """ task_name = None tasks_from_metadata = False meta_data_task_column = None task_workflow_class = str(workflow_class) str_attrs = ["input_dir", "expression_matrix_file", "tf_names_file", "meta_data_file", "priors_file"] def __str__(self): """ Create a printable report of the settings in this TaskData object :return: Settings in str_attrs in a printable string :rtype: str """ task_str = "{n}:\n\tWorkflow Class: {cl}\n".format(n=self.task_name, cl=self.task_workflow_class) for attr in self.str_attrs: try: task_str += "\t{attr}: {val}\n".format(attr=attr, val=getattr(self, attr)) except AttributeError: task_str += "\t{attr}: Nonexistant\n".format(attr=attr) return task_str def __init__(self): if self._file_format_settings is None: self._file_format_settings = dict() def initialize_multiprocessing(self): """ Don't do anything with multiprocessing in this object """ pass def startup(self): raise NotImplementedError def startup_run(self): raise NotImplementedError def get_data(self): """ Load all the data and then return a list of references to TaskData objects There will be multiple objects returned if tasks_from_metadata is set. If tasks_from_metadata is not set, the list contains only this task (self) :return: List of TaskData objects with loaded data :rtype: list(TaskData) """ Debug.vprint("Loading data for task {task_name}".format(task_name=self.task_name)) super(TaskData, self).get_data() if self.tasks_from_metadata: return self.separate_tasks_by_metadata() else: return [self] def validate_data(self): """ Don't validate data in TaskData. The parent workflow will check. """ pass def set_run_parameters(self): """ Set parameters used during runtime """ warnings.warn("Task-specific `num_bootstraps` and `random_seed` is not supported. Set on parent workflow.") def process_priors_and_gold_standard(self, gold_standard=None, cv_flag=None, cv_axis=None, shuffle_priors=None, add_prior_noise=None): """ Make sure that the priors for this task are correct This will remove circularity from the task priors based on the parent gold standard """ gold_standard = self.gold_standard if gold_standard is None else gold_standard cv_flag = self.split_gold_standard_for_crossvalidation if cv_flag is None else cv_flag cv_axis = self.cv_split_axis if cv_axis is None else cv_axis shuffle_priors = self.shuffle_prior_axis if shuffle_priors is None else shuffle_priors add_prior_noise = self.add_prior_noise if add_prior_noise is None else add_prior_noise # Remove circularity from the gold standard if cv_flag: self.priors_data, _ = self.prior_manager._remove_prior_circularity(self.priors_data, gold_standard, split_axis=cv_axis) if self.tf_names is not None: self.priors_data = self.prior_manager.filter_to_tf_names_list(self.priors_data, self.tf_names) # Filter priors and expression to a list of genes self.filter_to_gene_list() # Shuffle prior labels if shuffle_priors is not None: self.priors_data = self.prior_manager.shuffle_priors(self.priors_data, shuffle_priors, self.random_seed) if add_prior_noise is not None: self.priors_data = self.prior_manager.add_prior_noise(self.priors_data, add_prior_noise, self.random_seed) if min(self.priors_data.shape) == 0: raise ValueError("Priors for task {n} have an axis of length 0".format(n=self.task_name)) def separate_tasks_by_metadata(self, meta_data_column=None): """ Take a single expression matrix and break it into multiple dataframes based on meta_data. Return a list of TaskData objects which have the task-specific data loaded into them :param meta_data_column: Meta_data column which corresponds to task ID :type meta_data_column: str :return new_task_objects: List of the TaskData objects with only one task's data each :rtype: list(TaskData) """ if self.data is None: raise ValueError("No data has been loaded prior to `separate_tasks_by_metadata`") meta_data_column = meta_data_column if meta_data_column is not None else self.meta_data_task_column if meta_data_column is None: raise ValueError("tasks_from_metadata is set but meta_data_task_column is not") elif meta_data_column not in self.data.meta_data: msg = "meta_data_task_column is not found in task {t}".format(t=str(self)) raise ValueError(msg) new_task_objects = list() tasks = self.data.meta_data[meta_data_column].unique().tolist() Debug.vprint("Creating {n} tasks from metadata column {col}".format(n=len(tasks), col=meta_data_column), level=0) # Remove data references from self data = self.data self.data = None for task in tasks: # Copy this object task_obj = copy.deepcopy(self) # Get an index of the stuff to keep task_idx = data.meta_data[meta_data_column] == task # Reset expression matrix, metadata, and task_name in the copy task_obj.data = data.subset_copy(row_index=task_idx) task_obj.data.name = task task_obj.task_name = task new_task_objects.append(task_obj) Debug.vprint("Separated data into {ntask} tasks".format(ntask=len(new_task_objects)), level=0) return new_task_objects return TaskData