class FilterByInteraction(GenericBlock): block_base_name = "FILTER_BY_BI" name = "Filter ES by interaction" block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _mRNA_es = InputBlockField(name="mRNA_es", order_num=10, required_data_type="ExpressionSet", required=True) _miRNA_es = InputBlockField(name="miRNA_es", order_num=20, required_data_type="ExpressionSet", required=True) _interaction = InputBlockField(name="interaction", order_num=30, required_data_type="BinaryInteraction", required=True) m_rna_filtered_es = OutputBlockField(name="m_rna_filtered_es", provided_data_type="ExpressionSet") mi_rna_filtered_es = OutputBlockField(name="mi_rna_filtered_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(FilterByInteraction, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() mRNA_es = self.get_input_var("mRNA_es") miRNA_es = self.get_input_var("miRNA_es") interaction_matrix = self.get_input_var("interaction") self.celery_task = wrapper_task.s( filter_by_bi, exp, self, m_rna_es=mRNA_es, mi_rna_es=miRNA_es, interaction_matrix=interaction_matrix, base_filename="%s_filtered_by_BI" % self.uuid, ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, m_rna_filtered_es, mi_rna_filtered_es): self.set_out_var("m_rna_filtered_es", m_rna_filtered_es) self.set_out_var("mi_rna_filtered_es", mi_rna_filtered_es) exp.store_block(self)
class UploadGeneSets(GenericBlock): block_base_name = "GENE_SETS_UPLOAD" block_group = GroupType.INPUT_DATA name = "Upload Gene Sets" _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "done"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) upload_gs = ParamField("upload_gs", title="Gene sets in .gmt format", order_num=10, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM) _gene_sets = OutputBlockField(name="gene_sets", provided_data_type="GeneSets") def on_params_is_valid(self, exp, *args, **kwargs): try: gmt_file = self.upload_gs.get_file() gs = GmtStorage.read_inp(gmt_file, "\t") gene_sets = GeneSets(exp.get_data_folder(), str(self.uuid)) gene_sets.store_gs(gs) self.set_out_var("gene_sets", gene_sets) except Exception as e: exp.log(self.uuid, e, severity="CRITICAL") log.error(e) exp.store_block(self)
class EnrichmentNoTBlock(GenericBlock): block_base_name = "ENRICHMENT_COM" name = "Comodule Enrichment" is_abstract = False block_group = GroupType.TESTING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _cs_1 = InputBlockField(name="gs", order_num=10, required_data_type="GeneSets", required=True) H = InputBlockField(name="patterns", order_num=11, required_data_type="GeneSets", required=True) _t = ParamField(name="T", order_num=12, title="Enrichment threshold", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val="0.05") dict = OutputBlockField(name="dictionary_set", provided_data_type="DictionarySet") def __init__(self, *args, **kwargs): super(EnrichmentNoTBlock, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() gs = self.get_input_var("gs") cs = self.get_input_var("patterns") self.celery_task = wrapper_task.s(enrichment_no_t_task, exp, self, T=self.T, gs=gs, patterns=cs, base_filename="%s_%s_enrich" % (self.uuid, 'enrichment_cont')) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, flt_es): self.set_out_var("dictionary_set", flt_es) exp.store_block(self)
class MergeGeneSetWithPlatformAnnotation(GenericBlock): block_base_name = "MERGE_GS_GPL_ANN" name = "Merge gene set with platform" block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _input_gs = InputBlockField(name="gs", order_num=10, required_data_type="GeneSets", required=True) _input_ann = InputBlockField(name="ann", order_num=20, required_data_type="PlatformAnnotation", required=True) _gs = OutputBlockField(name="gs", field_type=FieldType.HIDDEN, init_val=None, provided_data_type="GeneSets") def __init__(self, *args, **kwargs): super(MergeGeneSetWithPlatformAnnotation, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() gs, ann = self.get_input_var("gs"), self.get_input_var("ann") # import ipdb; ipdb.set_trace() self.celery_task = wrapper_task.s(map_gene_sets_to_probes, exp, self, base_dir=exp.get_data_folder(), base_filename="%s_merged" % self.uuid, ann_gene_sets=ann.gene_sets, src_gene_sets=gs) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, gs): self.set_out_var("gs", gs) exp.store_block(self)
class SvdSubAgg(GenericBlock): is_abstract = True block_group = GroupType.AGGREGATION is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _mRNA_es = InputBlockField(name="mRNA_es", order_num=10, required_data_type="ExpressionSet", required=True) _miRNA_es = InputBlockField(name="miRNA_es", order_num=20, required_data_type="ExpressionSet", required=True) _interaction = InputBlockField(name="interaction", order_num=30, required_data_type="BinaryInteraction", required=True) c = ParamField(name="c", title="Constant c", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=1.0) agg_es = OutputBlockField(name="agg_es", provided_data_type="ExpressionSet") mode = "" def __init__(self, *args, **kwargs): super(SvdSubAgg, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() mRNA_es = self.get_input_var("mRNA_es") miRNA_es = self.get_input_var("miRNA_es") interaction_matrix = self.get_input_var("interaction") self.celery_task = wrapper_task.s( aggregation_task, exp, self, mode=self.mode, c=self.c, m_rna_es=mRNA_es, mi_rna_es=miRNA_es, interaction_matrix=interaction_matrix, base_filename="%s_%s_agg" % (self.uuid, self.mode) ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, agg_es): self.set_out_var("agg_es", agg_es) exp.store_block(self)
class GlobalTest(GenericBlock): block_base_name = "GLOBAL_TEST" name = "Goeman global test" block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _input_es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) _input_gs = InputBlockField(name="gs", order_num=20, required_data_type="GeneSets", required=True) _result = OutputBlockField(name="result", field_type=FieldType.STR, provided_data_type="TableResult", init_val=None) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=[ "gt_result.html" ]) def __init__(self, *args, **kwargs): super(GlobalTest, self).__init__(*args, **kwargs) self.celery_task = None exp = Experiment.get_exp_by_id(self.exp_id) self.result = TableResult( base_dir=exp.get_data_folder(), base_filename="%s_gt_result" % self.uuid, ) self.result.headers = ['p-value', 'Statistic', 'Expected', 'Std.dev', '#Cov'] def execute(self, exp, *args, **kwargs): self.clean_errors() self.celery_task = wrapper_task.s( global_test_task, exp, self, es=self.get_input_var("es"), gene_sets=self.get_input_var("gs"), table_result=self.result ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, result, *args, **kwargs): self.result = result self.set_out_var("result", self.result) exp.store_block(self)
class ThresholdBlock(GenericBlock): block_base_name = "THRESHOLD" name = "Threshold" is_abstract = False block_group = GroupType.SNMNMF is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) t = ParamField(name="T", title="Threshold", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=0.1) flt_es = OutputBlockField(name="gene_sets", provided_data_type="GeneSets") def __init__(self, *args, **kwargs): super(ThresholdBlock, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() es = self.get_input_var("es") # T = self.get_input_var("T") self.celery_task = wrapper_task.s(threshold_task, exp, self, es=es, T=self.T, base_filename="%s_%s_thr" % (self.uuid, 'threshold')) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, flt_es): self.set_out_var("gene_sets", flt_es) exp.store_block(self)
class MergeExpressionSets(GenericBlock): block_base_name = "MergeES" name = "Merge ES by concatenation" block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es_1 = InputBlockField(name="es_1", title="Set 1", order_num=10, required_data_type="ExpressionSet", required=True) _es_2 = InputBlockField(name="es_2", title="Set 2", order_num=20, required_data_type="ExpressionSet", required=True) merged_es = OutputBlockField(name="merged_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(MergeExpressionSets, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() # import ipdb; ipdb.set_trace() self.celery_task = wrapper_task.s( merge_two_es, exp, self, es_1=self.get_input_var("es_1"), es_2=self.get_input_var("es_2"), base_filename="%s_merged" % self.uuid, ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, es): self.set_out_var("merged_es", es) exp.store_block(self)
class RcVisualizer(GenericBlock): block_base_name = "RC_VIZUALIZER" is_block_supports_auto_execution = False block_group = GroupType.VISUALIZE is_abstract = True _block_actions = ActionsList([ ActionRecord( "save_params", ["created", "valid_params", "done", "ready", "input_bound"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "input_bound"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("configure_table", ["input_bound", "ready"], "ready"), ]) results_container = InputBlockField(name="results_container", required_data_type="ResultsContainer", required=True, field_type=FieldType.CUSTOM) _rc = BlockField(name="rc", field_type=FieldType.CUSTOM, is_a_property=True) _available_metrics = BlockField(name="available_metrics", field_type=FieldType.RAW, is_a_property=True) metric = ParamField(name="metric", title="Metric", field_type=FieldType.STR, input_type=InputType.SELECT, select_provider="available_metrics") def __init__(self, *args, **kwargs): super(RcVisualizer, self).__init__(*args, **kwargs) @property @log_timing def available_metrics(self): try: return [{ "pk": metric_name, "str": metric.title } for metric_name, metric in metrics_dict.iteritems() if metric.produce_single_number] except Exception, e: log.exception(e) return []
class UploadInteraction(GenericBlock): block_base_name = "GENE_INTERACTION" block_group = GroupType.INPUT_DATA name = "Upload gene interaction" _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "done"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) upload_interaction = ParamField("upload_interaction", title="Interaction matrix", order_num=10, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM) row_units = ParamField("row_units", title="Row units", order_num=11, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) col_units = ParamField("col_units", title="Column units", order_num=12, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) _interaction = OutputBlockField(name="interaction", provided_data_type="BinaryInteraction") def on_params_is_valid(self, exp, *args, **kwargs): # Convert to BinaryInteraction interaction_df = self.upload_interaction.get_as_data_frame() interaction = BinaryInteraction(exp.get_data_folder(), str(self.uuid)) interaction.store_matrix(interaction_df) interaction.row_units = self.row_units interaction.col_units = self.col_units self.set_out_var("interaction", interaction) exp.store_block(self)
class ZScoreBlock(GenericBlock): block_base_name = "ZSCORE_NORM" name = "Z-score Normalization" is_abstract = False block_group = GroupType.NORMALIZATION is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) flt_es = OutputBlockField(name="flt_zscore_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(ZScoreBlock, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() es = self.get_input_var("es") self.celery_task = wrapper_task.s(zscore_task, exp, self, es=es, base_filename="%s_%s_flt" % (self.uuid, 'zscore')) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, flt_es): self.set_out_var("flt_zscore_es", flt_es) exp.store_block(self)
class GetBroadInstituteGeneSet(GenericBlock): block_base_name = "BI_GENE_SET" block_group = GroupType.INPUT_DATA name = "Get MSigDB Gene Set" _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "done", reload_block_in_client=True), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) # TODO: maybe create more general solution ? _all_gene_sets = BlockField("all_gene_sets", title="", input_type=InputType.HIDDEN, field_type=FieldType.RAW, is_a_property=True) msigdb_id = ParamField( name="msigdb_id", title="MSigDB gene set", input_type=InputType.SELECT, field_type=FieldType.INT, init_val=0, # TODO: fix hardcoded value select_provider="all_gene_sets") _gs = OutputBlockField(name="gs", field_type=FieldType.HIDDEN, provided_data_type="GeneSets") @property def all_gene_sets(self): return BroadInstituteGeneSet.get_all_meta() def on_params_is_valid(self, exp): gs = BroadInstituteGeneSet.objects.get( pk=self.msigdb_id).get_gene_sets() self.set_out_var("gs", gs) super(GetBroadInstituteGeneSet, self).on_params_is_valid(exp)
class UserUploadComplex(GenericBlock): # unit_options = block_base_name = "UPLOAD_CMPLX" block_group = GroupType.INPUT_DATA name = "Upload mRna/miRna/methyl" _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("process_upload", ["valid_params", "processing_upload"], "processing_upload", "Process uploaded data"), ActionRecord("success", ["processing_upload"], "done", reload_block_in_client=True), ActionRecord("error", ["processing_upload"], "valid_params"), ]) m_rna_matrix = ParamField("m_rna_matrix", title="mRNA expression", order_num=10, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM) m_rna_platform = ParamField("m_rna_platform", title="Platform ID", order_num=11, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) m_rna_unit = ParamField( "m_rna_unit", title="Working unit [used when platform is unknown]", order_num=12, input_type=InputType.SELECT, field_type=FieldType.STR, required=False, init_val="RefSeq", options={ "inline_select_provider": True, "select_options": [["RefSeq", "RefSeq"], ["Entrez", "EntrezID"], ["Symbol", "Symbol"]] }) m_rna_matrix_ori = ParamField("m_rna_matrix_ori", title="Matrix orientation", order_num=13, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [["SxG", "Samples x Genes"], ["GxS", "Genes x Samples"]] }) csv_sep_m_rna = ParamField("csv_sep_m_rna", title="CSV separator symbol", order_num=14, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] }) mi_rna_matrix = ParamField("mi_rna_matrix", title=u"μRNA expression", order_num=20, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, required=False) mi_rna_platform = ParamField("mi_rna_platform", title="Platform ID", order_num=21, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) mi_rna_unit = ParamField( "mi_rna_unit", title="Working unit [used when platform is unknown]", order_num=22, input_type=InputType.SELECT, field_type=FieldType.STR, required=False, init_val="RefSeq", options={ "inline_select_provider": True, "select_options": [["RefSeq", "RefSeq"], ["mirbase", "miRBase ID"]] }) mi_rna_matrix_ori = ParamField("mi_rna_matrix_ori", title="Matrix orientation", order_num=23, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [["SxG", "Samples x Genes"], ["GxS", "Genes x Samples"]] }) csv_sep_mi_rna = ParamField("csv_sep_mi_rna", title="CSV separator symbol", order_num=24, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] }) methyl_matrix = ParamField("methyl_matrix", title="Methylation expression", order_num=30, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, required=False) methyl_platform = ParamField("methyl_platform", title="Platform ID", order_num=31, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) # methyl_unit = ParamField("methyl_unit", title="Working unit [used when platform is unknown]", init_val=None, # order_num=32, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) methyl_matrix_ori = ParamField("methyl_matrix_ori", title="Matrix orientation", order_num=33, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [["SxG", "Samples x Genes"], ["GxS", "Genes x Samples"]] }) csv_sep_methyl = ParamField("csv_sep_methyl", title="CSV separator symbol", order_num=34, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] }) pheno_matrix = ParamField("pheno_matrix", title="Phenotype matrix", order_num=40, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, required=False) csv_sep_pheno = ParamField("csv_sep_pheno", title="CSV separator symbol", order_num=50, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] }) _is_sub_pages_visible = BlockField("is_sub_pages_visible", FieldType.RAW, is_a_property=True) _m_rna_es = OutputBlockField(name="m_rna_es", field_type=FieldType.HIDDEN, provided_data_type="ExpressionSet") # _m_rna_annotation = OutputBlockField(name="m_rna_annotation", field_type=FieldType.HIDDEN, # provided_data_type="PlatformAnnotation") _mi_rna_es = OutputBlockField(name="mi_rna_es", field_type=FieldType.HIDDEN, provided_data_type="ExpressionSet") _methyl_es = OutputBlockField(name="methyl_es", field_type=FieldType.HIDDEN, provided_data_type="ExpressionSet") mrna_gpl_file = BlockField("mrna_gpl_file", FieldType.CUSTOM, None) mirna_gpl_file = BlockField("mirna_gpl_file", FieldType.CUSTOM, None) methyl_gpl_file = BlockField("methyl_gpl_file", FieldType.CUSTOM, None) pages = BlockField("pages", FieldType.RAW, init_val={ "assign_phenotype_classes": { "title": "Assign phenotype classes", "resource": "assign_phenotype_classes", "widget": "widgets/assign_phenotype_classes.html" }, }) @property def is_sub_pages_visible(self): if self.state in [ 'source_was_preprocessed', 'sample_classes_assigned', 'ready', 'done' ]: return True return False def __init__(self, *args, **kwargs): super(UserUploadComplex, self).__init__(*args, **kwargs) self.celery_task = None def process_upload(self, exp, *args, **kwargs): self.clean_errors() self.celery_task = wrapper_task.s(user_upload_complex_task, exp, self) exp.store_block(self) self.celery_task.apply_async() def phenotype_for_js(self, exp, *args, **kwargs): m_rna_es = self.get_out_var("m_rna_es") mi_rna_es = self.get_out_var("mi_rna_es") methyl_es = self.get_out_var("methyl_es") es = None if m_rna_es is not None: es = m_rna_es elif mi_rna_es is not None: es = mi_rna_es elif methyl_es is not None: es = methyl_es if es is None: raise Exception("No data was stored before") return prepare_phenotype_for_js_from_es(es) def update_user_classes_assignment(self, exp, request, *args, **kwargs): m_rna_es = self.get_out_var("m_rna_es") mi_rna_es = self.get_out_var("mi_rna_es") methyl_es = self.get_out_var("methyl_es") es = None if m_rna_es is not None: es = m_rna_es elif mi_rna_es is not None: es = mi_rna_es elif methyl_es is not None: es = methyl_es if es is None: raise Exception("No data was stored before") pheno_df = es.get_pheno_data_frame() received = json.loads(request.body) pheno_df[received["user_class_title"]] = received["classes"] for work_es in [m_rna_es, mi_rna_es, methyl_es]: if work_es is not None: work_es.pheno_metadata["user_class_title"] = received[ "user_class_title"] work_es.store_pheno_data_frame(pheno_df) # import ipdb; ipdb.set_trace() exp.store_block(self) def success(self, exp, m_rna_es, mi_rna_es, methyl_es): if m_rna_es: self.set_out_var("m_rna_es", m_rna_es) if mi_rna_es: self.set_out_var("mi_rna_es", mi_rna_es) if methyl_es: self.set_out_var("methyl_es", methyl_es) exp.store_block(self)
if isinstance(new_errors, collections.Iterable): self.errors.extend(new_errors) elif new_errors: self.errors.append(new_errors) exp.store_block(self) def reset_execution(self, exp, *args, **kwargs): self.clean_errors() exp.store_block(self) save_params_actions_list = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) execute_block_actions_list = ActionsList([ ActionRecord("execute", ["ready"], "working", user_title="Run block"), ActionRecord("success", ["working"], "done", propagate_auto_execution=True), ActionRecord("error", ["*", "ready", "working"], "execution_error"), ActionRecord("reset_execution", ["*", "done", "execution_error", "ready", "working"], "ready", user_title="Reset execution") ])
class MergeComoduleSets(GenericBlock): block_base_name = "MERGE_COMODULE_SETS" name = "Merge Comodule Sets" is_abstract = False block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _cs_1 = InputBlockField(name="cs_1", order_num=10, required_data_type="ComoduleSet", required=True) _cs_1_name = ParamField(name="cs_1_name", order_num=11, title="Comodule 1 name", input_type=InputType.TEXT, field_type=FieldType.STR, init_val="genes") _cs_2 = InputBlockField(name="cs_2", order_num=20, required_data_type="ComoduleSet", required=True) _cs_2_name = ParamField(name="cs_2_name", order_num=21, title="Comodule 2 name", input_type=InputType.TEXT, field_type=FieldType.STR, init_val="genes") flt_es = OutputBlockField(name="comodule_set", provided_data_type="ComoduleSet") def __init__(self, *args, **kwargs): super(MergeComoduleSets, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() cs_1 = self.get_input_var("cs_1") cs_2 = self.get_input_var("cs_2") self.celery_task = wrapper_task.s(merge_comodules_task, exp, self, cs_1=cs_1, cs_2=cs_2, cs_1_name=self.cs_1_name, cs_2_name=self.cs_2_name, base_filename="%s_%s_thr" % (self.uuid, 'merge_cs')) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, flt_es): self.set_out_var("comodule_set", flt_es) exp.store_block(self)
class NIMFASNMNMFBlock(GenericBlock): block_base_name = "NIMFA_SNMNMF" name = "NIMFA SNMNMF" is_abstract = False block_group = GroupType.SNMNMF is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _m_rna = InputBlockField(name="mRNA", order_num=10, required_data_type="ExpressionSet", required=True) _mi_rna = InputBlockField(name="miRNA", order_num=20, required_data_type="ExpressionSet", required=True) # _dna_methyl = InputBlockField(name="DNAmethyl", order_num=30, required_data_type="ExpressionSet", required=False) _gene2gene = InputBlockField(name="Gene2Gene", order_num=40, required_data_type="BinaryInteraction", required=True) _mirna2gene = InputBlockField(name="miRNA2gene", order_num=50, required_data_type="BinaryInteraction", required=True) # _gene2DNAmethylation = InputBlockField(name="Gene2DNAmethyl", order_num=60, required_data_type="BinaryInteraction", required=False) l1 = ParamField(name="l1", order_num=70, title="l1", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=0.1) l2 = ParamField(name="l2", order_num=80, title="l2", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=0.1) g1 = ParamField(name="g1", order_num=90, title="g1", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=0.1) g2 = ParamField(name="g2", order_num=100, title="g2", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=0.1) rank = ParamField(name="rank", order_num=110, title="rank", input_type=InputType.TEXT, field_type=FieldType.INT, init_val=50) w = OutputBlockField(name="W", provided_data_type="ExpressionSet") H1_miRNA = OutputBlockField(name="H1_miRNA", provided_data_type="ExpressionSet") H2_genes = OutputBlockField(name="H2_genes", provided_data_type="ExpressionSet") # H3_DNAmethyl = OutputBlockField(name="H3_DNAmethyl", provided_data_type="ExpressionSet") #H1_perf = OutputBlockField(name="H1_perf", provided_data_type="ExpressionSet") #H2_perf = OutputBlockField(name="H2_perf", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(NIMFASNMNMFBlock, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() mRNA = self.get_input_var("mRNA") miRNA = self.get_input_var("miRNA") #DNAmethyl = self.get_input_var("DNAmethyl") Gene2Gene = self.get_input_var("Gene2Gene") miRNA2gene = self.get_input_var("miRNA2gene") #Gene2DNAmethyl = self.get_input_var("Gene2DNAmethyl") self.celery_task = wrapper_task.s( nimfa_snmnmf_task, exp, self, mRNA=mRNA, miRNA=miRNA, #DNAmethyl = DNAmethyl, gene2gene=Gene2Gene, miRNA2gene=miRNA2gene, #gene2DNAmethylation = Gene2DNAmethyl, params={'l1': self.l1, 'l2': self.l2, 'g1': self.g1, 'g2': self.g2, 'rank': self.rank}, base_filename="%s_nimfa_snmnmf" % self.uuid ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, W, H1, H2): self.set_out_var("W", W) self.set_out_var("H1_miRNA", H1) self.set_out_var("H2_genes", H2) #self.set_out_var("H1_perf", matrices[3]) #self.set_out_var("H2_perf", matrices[4]) exp.store_block(self)
class GeneSetAgg(GenericBlock): block_base_name = "GENE_SET_AGG" name = "Gene sets aggregation" block_group = GroupType.PROCESSING is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) _gs = InputBlockField(name="gs", order_num=20, required_data_type="GeneSets", required=True) agg_method = ParamField("agg_method", title="Aggregate method", order_num=50, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="mean", options={ "inline_select_provider": True, "select_options": [["mean", "Mean"], ["media", "Median"]] }) agg_es = OutputBlockField(name="agg_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(GeneSetAgg, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() es = self.get_input_var("es") gs = self.get_input_var("gs") base_filename = "%s_gs_agg" % (self.uuid, ) self.celery_task = wrapper_task.s(do_gs_agg, exp, self, es, gs, self.agg_method, base_filename) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, agg_es): self.set_out_var("agg_es", agg_es) exp.store_block(self)
class GenericClassifier(GenericBlock): block_group = GroupType.CLASSIFIER is_abstract = True is_block_supports_auto_execution = True classifier_name = "" # Block behavior _block_actions = ActionsList([]) _block_actions.extend(save_params_actions_list) _block_actions.extend(execute_block_actions_list) # User defined parameters # Input ports definition _train_es = InputBlockField(name="train_es", order_num=10, required_data_type="ExpressionSet", required=True) _test_es = InputBlockField(name="test_es", order_num=20, required_data_type="ExpressionSet", required=True) # Provided outputs _result = OutputBlockField(name="result", field_type=FieldType.CUSTOM, provided_data_type="ClassifierResult", init_val=None) def __init__(self, *args, **kwargs): super(GenericClassifier, self).__init__(*args, **kwargs) self.celery_task = None self.classifier_options = {} self.fit_options = {} @abstractmethod def collect_options(self): """ Should populate `self.classifier_options` and `self.fit_options` from block parameters. """ pass def get_option_safe(self, name, target_type=None): if hasattr(self, name): raw = getattr(self, name) if raw: if target_type: try: return target_type(raw) except: pass else: return raw return None def collect_option_safe(self, name, target_type=None, target_name=None): value = self.get_option_safe(name, target_type) # from celery.contrib import rdb; rdb.set_trace() if value: if target_name: self.classifier_options[target_name] = value else: self.classifier_options[name] = value return value def execute(self, exp, *args, **kwargs): self.set_out_var("result", None) self.collect_options() train_es = self.get_input_var("train_es") test_es = self.get_input_var("test_es") self.celery_task = wrapper_task.s( apply_classifier, exp=exp, block=self, train_es=train_es, test_es=test_es, classifier_name=self.classifier_name, classifier_options=self.classifier_options, fit_options=self.fit_options, base_folder=exp.get_data_folder(), base_filename="%s_%s" % (self.uuid, self.classifier_name), ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, result, *args, **kwargs): # We store obtained result as an output variable self.set_out_var("result", result) exp.store_block(self) def reset_execution(self, exp, *args, **kwargs): self.clean_errors() # self.get_scope().remove_temp_vars() self.set_out_var("result", None) exp.store_block(self)
class FeatureSelectionByCut(GenericBlock): block_base_name = "FS_BY_CUT" block_group = GroupType.FILTER name = "Feature Selection by Ranking" is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) _rank_table = InputBlockField(name="rank_table", order_num=20, required_data_type="TableResult", required=True) _cut_property_options = BlockField(name="cut_property_options", field_type=FieldType.RAW, is_a_property=True) cut_property = ParamField( name="cut_property", title="Ranking property to use", # input_type=InputType.SELECT, input_type=InputType.TEXT, field_type=FieldType.STR, #select_provider="cut_property_options", order_num=10, ) threshold = ParamField( name="threshold", title="Threshold for cut", order_num=20, input_type=InputType.TEXT, field_type=FieldType.INT, ) _cut_direction_options = BlockField(name="cut_direction_options", field_type=FieldType.RAW) cut_direction_options = ["<", "<=", ">=", ">"] cut_direction = ParamField(name="cut_direction", title="Direction of cut", input_type=InputType.SELECT, field_type=FieldType.STR, select_provider="cut_direction_options", order_num=30, options={ "inline_select_provider": True, "select_options": [[op, op] for op in ["<", "<=", ">=", ">"]] }) es = OutputBlockField(name="es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(FeatureSelectionByCut, self).__init__(*args, **kwargs) self.celery_task = None @property def cut_property_options(self): # import ipdb; ipdb.set_trace() rank_table = self.get_input_var("rank_table") if rank_table and hasattr(rank_table, "headers"): return [{ "pk": header, "str": header } for header in rank_table.headers] def execute(self, exp, *args, **kwargs): self.clean_errors() self.celery_task = wrapper_task.s( feature_selection_by_cut, exp=exp, block=self, src_es=self.get_input_var("es"), rank_table=self.get_input_var("rank_table"), cut_property=self.cut_property, threshold=self.threshold, cut_direction=self.cut_direction, base_filename="%s_feature_selection" % self.uuid, ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, es): self.set_out_var("es", es) exp.store_block(self)
class GeneSetAggCV(GenericBlock): block_group = GroupType.AGGREGATION block_base_name = "CV_GS_A" name = "CV Gene Sets Aggregation" is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _input_train_es = InputBlockField(name="train_es", order_num=10, required_data_type="ExpressionSet", required=True) _input_test_es = InputBlockField(name="test_es", order_num=20, required_data_type="ExpressionSet", required=True) _input_gs = InputBlockField(name="gs", order_num=30, required_data_type="GeneSets", required=True) agg_method = ParamField( "agg_method", title="Aggregate method", order_num=50, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="mean", options={ "inline_select_provider": True, "select_options": [ ["mean", "Mean"], ["median", "Median"], ["pca", "PCA"] ] } ) out_train_es = OutputBlockField(name="out_train_es", provided_data_type="ExpressionSet") out_test_es = OutputBlockField(name="out_test_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(GeneSetAggCV, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() train_es = self.get_input_var("train_es") test_es = self.get_input_var("test_es") gene_sets = self.get_input_var("gs") self.celery_task = wrapper_task.s( agg_task_cv, exp, self, train_es=train_es, test_es=test_es, gene_sets=gene_sets, method=self.agg_method, base_filename="%s_%s_agg" % (self.uuid, "pca_cv") ) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, out_train_es, out_test_es): self.set_out_var("out_train_es", out_train_es) self.set_out_var("out_test_es", out_test_es) exp.store_block(self)
class EnrichmentVisualize(GenericBlock): block_base_name = "EV_VIEW" block_group = GroupType.VISUALIZE name = "Enrichment Visualize" is_block_supports_auto_execution = False _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready", "input_bound"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params") ]) _input_dictionary_set = InputBlockField(name="ds", order_num=10, required_data_type="DictionarySet", required=True) _table_for_js = BlockField(name="table_js", field_type=FieldType.RAW, is_a_property=True) _export_raw_results_url = BlockField(name="export_raw_results_url", field_type=FieldType.STR, is_a_property=True) _export_results_csv_url = BlockField(name="export_results_csv_url", field_type=FieldType.STR, is_a_property=True) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=[ "enrichment_view.html" ]) def map_to_symbols(self, gene_set): genes = [gi.name for gi in GeneIdentifier.objects.filter(refseq__refseq__in=gene_set)] return genes @property def export_results_csv_url(self): return reverse("block_field_formatted", kwargs={ "exp_id": self.exp_id, "block_uuid": self.uuid, "field": "export_csv", "format": "csv" }) @property def export_raw_results_url(self): return reverse("block_field_formatted", kwargs={ "exp_id": self.exp_id, "block_uuid": self.uuid, "field": "export_json", "format": "json" }) @property def table_js(self): cs = self.get_input_var("ds") """:type :DictionarySet""" if cs: table = cs.load_dict() table_headers = ['comodule','(term, p-val)','genes'] column_title_to_code_name = { title: "_" + hashlib.md5(title).hexdigest()[:8] for title in table_headers } fields_list = [column_title_to_code_name[title] for title in table_headers] return { "columns": [ { "title": title, "field": column_title_to_code_name[title], "visible": True } for title in table_headers ], "rows": [ dict(zip(fields_list, row)) for row in [(k, v[1], set(self.map_to_symbols(v[0]))) for k, v in table.iteritems()] #table.to_records().tolist() #[:100] ] } else: return None def export_json(self, exp, *args, **kwargs): ds = self.get_input_var("ds") dic = ds.load_dict() return dic def export_csv(self, exp, *args, **kwargs): import csv import StringIO ds = self.get_input_var("ds") dic = ds.load_dict() out = StringIO.StringIO() w = csv.writer(out) w.writerows(dic.items()) out.seek(0) return out.read()
class GeneSetsView(GenericBlock): block_base_name = "GS_VIEW" block_group = GroupType.VISUALIZE name = "Gene Sets view" is_block_supports_auto_execution = False _block_actions = ActionsList([ ActionRecord( "save_params", ["created", "valid_params", "done", "ready", "input_bound"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params"), ]) _input_dictionary_set = InputBlockField(name="gs", order_num=10, required_data_type="GeneSets", required=True) _table_for_js = BlockField(name="table_js", field_type=FieldType.RAW, is_a_property=True) _export_raw_results_url = BlockField(name="export_raw_results_url", field_type=FieldType.STR, is_a_property=True) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=["dictionary_set_view.html"]) @property def export_raw_results_url(self): return reverse("block_field_formatted", kwargs={ "exp_id": self.exp_id, "block_uuid": self.uuid, "field": "export_json", "format": "json" }) @property def table_js(self): cs = self.get_input_var("gs") """:type :GeneSets""" if cs: table = cs.get_gs(conv=False).genes table_headers = ['key', 'value'] column_title_to_code_name = { title: "_" + hashlib.md5(title).hexdigest()[:8] for title in table_headers } fields_list = [ column_title_to_code_name[title] for title in table_headers ] return { "columns": [{ "title": title, "field": column_title_to_code_name[title], "visible": True } for title in table_headers], "rows": [ dict(zip(fields_list, row)) for row in [(k, list(v)) for k, v in table.iteritems()] # table.to_records().tolist() #[:100] ] } else: return None def export_json(self, exp, *args, **kwargs): ds = self.get_input_var("gs") dic = ds.get_gs().genes return dic
class MultiFeature(UniformMetaBlock): block_base_name = "MULTI_FEATURE" name = "Multi Feature Validation" _mf_block_actions = ActionsList([ ActionRecord("on_feature_selection_updated", ["valid_params", "ready", "done"], "ready"), ]) _input_es_dyn = InputBlockField(name="es_inputs", order_num=-10, required_data_type="ExpressionSet", required=True, multiply_extensible=True) _is_sub_pages_visible = BlockField("is_sub_pages_visible", FieldType.RAW, init_val=False, is_a_property=True) pages = BlockField("pages", FieldType.RAW, init_val={ "select_feature": { "title": "Select features to examine", "resource": "select_feature", "widget": "widgets/select_feature.html" }, }) def __init__(self, *args, **kwargs): super(MultiFeature, self).__init__(*args, **kwargs) self.features = [] @property def is_sub_pages_visible(self): if self.state in ['valid_params', 'done', 'ready']: return True return False def get_fold_labels(self): return self.features def add_dyn_input_hook(self, exp, dyn_port, new_port): """ @type new_port: InputBlockField """ new_inner_output = InnerOutputField( name="%s_i" % new_port.name, provided_data_type=new_port.required_data_type) self.inner_output_es_names_map[new_port.name] = new_inner_output.name self.register_inner_output_variables([new_inner_output]) def execute(self, exp, *args, **kwargs): # self.celery_task = wrapper_task.s( # # ) self.inner_output_manager.reset() es_dict = { inp_name: self.get_input_var(inp_name) for inp_name in self.es_inputs } self.celery_task = wrapper_task.s( prepare_folds, exp, self, features=self.features, es_dict=es_dict, inner_output_es_names_map=self.inner_output_es_names_map, success_action="on_folds_generation_success") exp.store_block(self) self.celery_task.apply_async() def phenotype_for_js(self, exp, *args, **kwargs): es = None for input_name in self.es_inputs: es = self.get_input_var(input_name) if es is not None: break res = prepare_phenotype_for_js_from_es(es) res["features"] = self.features return res def update_feature_selection(self, exp, request, *args, **kwargs): req = json.loads(request.body) self.features = req["features"] if self.features: self.do_action("on_feature_selection_updated", exp) def on_feature_selection_updated(self, *args, **kwargs): pass
class UserUpload(GenericBlock): block_base_name = "UPLOAD" block_group = GroupType.INPUT_DATA is_abstract = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("process_upload", ["valid_params", "processing_upload"], "processing_upload", "Process uploaded data", reload_block_in_client=True), ActionRecord("success", ["processing_upload"], "done", reload_block_in_client=True), ActionRecord("error", ["processing_upload"], "valid_params", reload_block_in_client=True), ]) es_matrix = ParamField("es_matrix", title="Expression set matrix", order_num=0, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM) es_matrix_ori = ParamField( "es_matrix_ori", title="Matrix orientation", order_num=1, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [ ["SxG", "Samples x Genes"], ["GxS", "Genes x Samples"] ] } ) pheno_matrix = ParamField("pheno_matrix", title="Phenotype matrix", order_num=10, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM) gpl_platform = ParamField("gpl_platform", title="Platform ID", order_num=20, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) working_unit = ParamField("working_unit", title="Working unit [used when platform is unknown]", order_num=3, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) # TODO: add sub page field # pages = BlockField("pages", FieldType.RAW, init_val={ # "assign_sample_classes": { # "title": "Assign sample classes", # "resource": "assign_sample_classes", # "widget": "widgets/fetch_gse/assign_sample_classes.html" # }, # }) _is_sub_pages_visible = BlockField("is_sub_pages_visible", FieldType.RAW, is_a_property=True) ### PARAMETERS _expression_set = OutputBlockField(name="expression_set", field_type=FieldType.HIDDEN, provided_data_type="ExpressionSet") _gpl_annotation = OutputBlockField(name="gpl_annotation", field_type=FieldType.HIDDEN, provided_data_type="PlatformAnnotation") # TODO: COPY PASTE from fetch_gse block pages = BlockField("pages", FieldType.RAW, init_val={ "assign_phenotype_classes": { "title": "Assign phenotype classes", "resource": "assign_phenotype_classes", "widget": "widgets/assign_phenotype_classes.html" }, }) def __init__(self, *args, **kwargs): super(UserUpload, self).__init__("User upload", *args, **kwargs) @property def is_sub_pages_visible(self): if self.state in ['source_was_preprocessed', 'sample_classes_assigned', 'ready', 'done']: return True return False def phenotype_for_js(self, exp, *args, **kwargs): return prepare_phenotype_for_js_from_es(self.get_out_var("expression_set")) def update_user_classes_assignment(self, exp, request, *args, **kwargs): es = self.get_out_var("expression_set") pheno_df = es.get_pheno_data_frame() received = json.loads(request.body) es.pheno_metadata["user_class_title"] = received["user_class_title"] pheno_df[received["user_class_title"]] = received["classes"] es.store_pheno_data_frame(pheno_df) exp.store_block(self) def process_upload(self, exp, *args, **kwargs): """ @param exp: Experiment """ self.clean_errors() assay_df = pd.DataFrame.from_csv(self.es_matrix.get_file()) es = ExpressionSet(base_dir=exp.get_data_folder(), base_filename="%s_annotation" % self.uuid) pheno_df = pd.DataFrame.from_csv(self.pheno_matrix.get_file()) pheno_df.set_index(pheno_df.columns[0]) user_class_title = es.pheno_metadata["user_class_title"] if user_class_title not in pheno_df.columns: pheno_df[es.pheno_metadata["user_class_title"]] = "" # if matrix is bad oriented, then do transposition if self.es_matrix_ori == "GxS": assay_df = assay_df.T es.store_assay_data_frame(assay_df) es.store_pheno_data_frame(pheno_df) if self.working_unit: es.working_unit = self.working_unit self.set_out_var("expression_set", es) exp.store_block(self) self.do_action("success", exp) # self.celery_task_fetch.apply_async() def success(self, exp, *args, **kwargs): pass
class MassUpload(UniformMetaBlock): block_base_name = "BunchUpload" name = "Mass Upload Expression Sets" _bu_block_actions = ActionsList([ ActionRecord("process_upload", ["valid_params", "processing_upload"], "processing_upload", "Process uploaded data"), ActionRecord("error_on_processing", ["processing_upload"], "valid_params"), ActionRecord("processing_done", ["processing_upload"], "ready") # ActionRecord("success", ["processing_upload"], "done", reload_block_in_client=True), # ActionRecord("error", ["processing_upload"], "valid_params"), ]) es_mRNA_matrices = ParamField( "es_mRNA_matrices", title="mRNA Expression sets", order_num=10, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, options={"multiple": True}, ) es_mRNA_matrices_ori = ParamField( "es_mRNA_matrices_ori", title="Matrices orientation", order_num=11, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [ ["SxG", "Samples x Features"], ["GxS", "Features x Samples"] ] } ) m_rna_platform = ParamField("m_rna_platform", title="Platform ID", order_num=12, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) m_rna_unit = ParamField("m_rna_unit", title="Working unit [used when platform is unknown]", order_num=13, input_type=InputType.SELECT, field_type=FieldType.STR, required=False, init_val="RefSeq", options={ "inline_select_provider": True, "select_options": [ ["RefSeq", "RefSeq"], ["Entrez", "EntrezID"], ["Symbol", "Symbol"] ] }) csv_sep_m_rna = ParamField( "csv_sep_m_rna", title="CSV separator symbol", order_num=14, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] } ) es_miRNA_matrices = ParamField( "es_miRNA_matrices", title="miRNA Expression sets", order_num=15, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, options={"multiple": True}, ) mi_rna_platform = ParamField("mi_rna_platform", title="Platform ID", order_num=21, input_type=InputType.TEXT, field_type=FieldType.STR, required=False) mi_rna_unit = ParamField("mi_rna_unit", title="Working unit [used when platform is unknown]", order_num=22, input_type=InputType.SELECT, field_type=FieldType.STR, required=False, init_val="RefSeq", options={ "inline_select_provider": True, "select_options": [ ["RefSeq", "RefSeq"], ["mirbase", "miRBase ID"] ] }) es_miRNA_matrices_ori = ParamField( "es_miRNA_matrices_ori", title="Matrices orientation", order_num=23, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="SxG", options={ "inline_select_provider": True, "select_options": [ ["SxG", "Samples x Genes"], ["GxS", "Genes x Samples"] ] } ) csv_sep_mi_rna = ParamField( "csv_sep_mi_rna", title="CSV separator symbol", order_num=24, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] } ) pheno_matrices = ParamField( "pheno_matrices", title="Phenotypes", order_num=40, input_type=InputType.FILE_INPUT, field_type=FieldType.CUSTOM, options={"multiple": True}, ) csv_sep = ParamField( "csv_sep", title="CSV separator symbol", order_num=50, input_type=InputType.SELECT, field_type=FieldType.STR, init_val=",", options={ "inline_select_provider": True, "select_options": [ [" ", "space ( )"], [",", "comma (,)"], ["\t", "tab (\\t)"], [";", "semicolon (;)"], [":", "colon (:)"], ] } ) # cells = BlockField(name="cells", field_type=FieldType.CUSTOM, init_val=None) # elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=[ # "mass_upload/data_spec.html" # ]) def __init__(self, *args, **kwargs): super(MassUpload, self).__init__(*args, **kwargs) self.es_mRNA_matrices = MultiUploadField() self.es_miRNA_matrices = MultiUploadField() self.pheno_matrices = MultiUploadField() self.pheno_by_es_names = {} self.labels = [] self.seq = [] self.register_inner_output_variables([InnerOutputField( name="mRNA_es", provided_data_type="ExpressionSet" ), InnerOutputField( name="miRNA_es", provided_data_type="ExpressionSet" )]) @property def is_sub_pages_visible(self): if self.state in ['source_was_preprocessed', 'sample_classes_assigned', 'ready', 'done']: return True return False def get_fold_labels(self): return self.labels def error_on_processing(self, *args, **kwargs): pass def processing_done(self, exp, block): exp.store_block(block) def process_upload(self, exp, *args, **kwargs): """ @param exp: Experiment """ self.clean_errors() try: if len(self.pheno_matrices) != len(self.es_mRNA_matrices): raise RuntimeError("Different number of phenotypes and mRNA expression sets") if self.es_miRNA_matrices: if len(self.pheno_matrices) != len(self.es_miRNA_matrices): raise RuntimeError("Different number of phenotypes and miRNA expression sets") self.labels = es_mRNA_matrix_names = sorted(self.es_mRNA_matrices) es_miRNA_matrix_names = sorted(self.es_miRNA_matrices) pheno_matrix_names = sorted(self.pheno_matrices) if len(es_miRNA_matrix_names) == 0: es_miRNA_matrix_names = len(es_mRNA_matrix_names) * [None] self.pheno_by_es_names = { pheno_name: es_name for es_name, pheno_name in zip(zip(es_mRNA_matrix_names, es_miRNA_matrix_names), pheno_matrix_names) } self.clean_errors() self.celery_task = wrapper_task.s( bunch_upload_task, exp, self, success_action="processing_done", error_action="error_on_processing" ) exp.store_block(self) self.celery_task.apply_async() except Exception as e: exp.log(self.uuid, e, severity="CRITICAL") log.exception(e) self.errors.append(e) self.do_action("error_on_processing", exp, e) # self.celery_task_fetch.apply_async() def execute(self, exp, *args, **kwargs): self.inner_output_manager.reset() self.do_action("on_folds_generation_success", exp, self.seq) def get_repeat_labels(self): pass
class PcaVisualize(GenericBlock): block_base_name = "PCA_VISUALIZE" name = "2D PCA Plot" block_group = GroupType.VISUALIZE is_block_supports_auto_execution = False _block_actions = ActionsList([ ActionRecord( "save_params", ["created", "valid_params", "done", "ready", "input_bound"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params"), ActionRecord("compute_pca", ["valid_params"], "computing_pca", user_title="Compute PCA"), ActionRecord( "pca_done", ["computing_pca"], "done", ), ActionRecord("reset_execution", ["*", "done", "execution_error", "ready", "working"], "ready", user_title="Reset execution") #ActionRecord("update", ["input_bound", "ready"], "ready"), ]) input_es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) chart_series = BlockField(name="chart_series", field_type=FieldType.RAW, init_val=[]) chart_categories = BlockField(name="chart_categories", field_type=FieldType.SIMPLE_LIST, init_val=[]) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=["pca.html"]) def __init__(self, *args, **kwargs): super(PcaVisualize, self).__init__("PCA visualise", *args, **kwargs) def on_params_is_valid(self, exp, *args, **kwargs): super(PcaVisualize, self).on_params_is_valid(exp, *args, **kwargs) self.do_action("compute_pca", exp) def compute_pca(self, exp, *args, **kwargs): log.info("compute pca invoked") es = self.get_input_var("es") """:type :ExpressionSet""" df = es.get_assay_data_frame() pheno_df = es.get_pheno_data_frame() target_column = es.pheno_metadata['user_class_title'] X = df.as_matrix().transpose() pca_model = decomposition.PCA(n_components=2) pca_model.fit(X) Xp = pca_model.transform(X).tolist() names = [x.strip() for x in pheno_df[target_column].tolist()] series_by_names = defaultdict(list) for x, name in zip(Xp, names): series_by_names[name].append(x) self.chart_series = [{ "name": name, "data": points } for name, points in series_by_names.iteritems()] self.do_action("pca_done", exp) def pca_done(self, exp, *args, **kwargs): log.info("pca done")
class FilterBlock(GenericBlock): block_base_name = "FILTER" name = "Var/Val Filter" is_abstract = False block_group = GroupType.FILTER is_block_supports_auto_execution = True _block_actions = ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "ready"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ]) _block_actions.extend(execute_block_actions_list) _es = InputBlockField(name="es", order_num=10, required_data_type="ExpressionSet", required=True) filter_method = ParamField("filter_method", title="Filter method", order_num=50, input_type=InputType.SELECT, field_type=FieldType.STR, init_val="LOW_VAL", options={ "inline_select_provider": True, "select_options": [["LOW_VAL", "Low Val Filter"], ["VAR", "Var Filter"]] }) q = ParamField(name="q", title="Threshold", input_type=InputType.TEXT, field_type=FieldType.FLOAT, init_val=30.0) flt_es = OutputBlockField(name="flt_es", provided_data_type="ExpressionSet") def __init__(self, *args, **kwargs): super(FilterBlock, self).__init__(*args, **kwargs) self.celery_task = None def execute(self, exp, *args, **kwargs): self.clean_errors() es = self.get_input_var("es") self.celery_task = wrapper_task.s(filter_task, exp, self, filter_type=self.filter_method, q=self.q, es=es, base_filename="%s_%s_flt" % (self.uuid, self.filter_method)) exp.store_block(self) self.celery_task.apply_async() def success(self, exp, flt_es): self.set_out_var("flt_es", flt_es) exp.store_block(self)
class CrossValidation(UniformMetaBlock): block_base_name = "CROSS_VALID" name = "Cross Validation K-fold" _cv_actions = ActionsList( [ActionRecord("become_ready", ["valid_params"], "ready")]) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=["cv_info.html"]) _input_es_dyn = InputBlockField(name="es_inputs", required_data_type="ExpressionSet", required=True, multiply_extensible=True, order_num=-1) folds_num = ParamField(name="folds_num", title="Folds number", order_num=10, input_type=InputType.TEXT, field_type=FieldType.INT, init_val=5) repeats_num = ParamField(name="repeats_num", title="Repeats number", order_num=20, input_type=InputType.TEXT, field_type=FieldType.INT, init_val=1) def get_fold_labels(self): out = [] for repeat in range(self.repeats_num): for num in range(self.folds_num): out.append("fold_%s_%s" % (repeat + 1, num + 1)) return out # ["fold_%s_%s" % (repeat + 1, num + 1) for num in range(self.folds_num) for repeat in range(self.repeats_num)] def get_repeat_labels(self): return [ "repeat_%s" % (repeat + 1) for repeat in range(self.repeats_num) ] def add_dyn_input_hook(self, exp, dyn_port, new_port): """ @type new_port: InputBlockField """ new_inner_output_train = InnerOutputField( name="%s_train_i" % new_port.name, provided_data_type=new_port.required_data_type) new_inner_output_test = InnerOutputField( name="%s_test_i" % new_port.name, provided_data_type=new_port.required_data_type) self.inner_output_es_names_map[new_port.name] = \ (new_inner_output_train.name, new_inner_output_test.name) self.register_inner_output_variables( [new_inner_output_train, new_inner_output_test]) def execute(self, exp, *args, **kwargs): self.clean_errors() self.inner_output_manager.reset() es_dict = { inp_name: self.get_input_var(inp_name) for inp_name in self.es_inputs } self.celery_task = wrapper_task.s( generate_cv_folds, exp, self, folds_num=self.folds_num, repeats_num=self.repeats_num, es_dict=es_dict, inner_output_es_names_map=self.inner_output_es_names_map, success_action="on_folds_generation_success", ) exp.store_block(self) self.celery_task.apply_async() def on_params_is_valid(self, exp, *args, **kwargs): super(CrossValidation, self).on_params_is_valid(exp, *args, **kwargs) self.do_action("become_ready", exp) def become_ready(self, *args, **kwargs): pass def build_result_collection(self, exp): if settings.CELERY_DEBUG: import sys sys.path.append( '/Migration/skola/phd/projects/miXGENE/mixgene_project/wrappers/pycharm-debug.egg' ) import pydevd pydevd.settrace('localhost', port=6901, stdoutToServer=True, stderrToServer=True) rc = ResultsContainer(base_dir=exp.get_data_folder(), base_filename="%s" % self.uuid) res_seq = self.res_seq def create_new_dim_rc(local_rc, axis_meta_block, axis_meta_block_labels): local_rc.axis_list = [axis_meta_block] local_rc.labels_dict[axis_meta_block] = axis_meta_block_labels local_rc.init_ar() local_rc.update_label_index() # WARNING: We only support homogeneous results, so we only check first element res_seq_field_name, data_type = res_seq.fields.iteritems().next() if data_type == "ClassifierResult": fold_labels = self.get_fold_labels() single_rc_list = [] for field_name in res_seq.fields: run_num = 0 loc_list = [] for idx, res_seq_cell in enumerate(res_seq.sequence): if (idx % self.folds_num) == 0: rc_run = ResultsContainer("", "") create_new_dim_rc(rc_run, self.base_name + "_folds", [ "fold_%s" % fold_num for fold_num in range(self.folds_num) ]) loc_list.append(rc_run) run_num += 1 rc_run.ar[idx % self.folds_num] = res_seq_cell[field_name] rc_single = ResultsContainer("", "") rc_single.add_dim_layer(loc_list, self.base_name, self.get_repeat_labels()) single_rc_list.append(rc_single) rc.add_dim_layer(single_rc_list, self.collector_spec.label, res_seq.fields.keys()) elif data_type == "ResultsContainer": if len(res_seq.fields) > 1: raise Exception( "Meta block only support single output of type ResultsContainer" ) else: rc_list = [] for cell in res_seq.sequence: sub_rc = cell[res_seq_field_name] sub_rc.load() rc_list.append(sub_rc) rc.add_dim_layer(rc_list, self.base_name, self.get_fold_labels()) elif data_type == "SequenceContainer": # TODO remove this check pass else: raise Exception("Meta blocks only support ClassifierResult " "or ResultsContainer in the output collection. " " Instead got: %s" % data_type) rc.store() rc.ar = None self.set_out_var("results_container", rc)
class UniformMetaBlock(GenericBlock): is_abstract = True block_group = GroupType.META_PLUGIN create_new_scope = True is_block_supports_auto_execution = True _block_actions = ActionsList([]) _block_actions.extend(ActionsList([ ActionRecord("save_params", ["created", "valid_params", "done", "ready"], "validating_params", user_title="Save parameters"), ActionRecord("on_params_is_valid", ["validating_params"], "valid_params"), ActionRecord("on_params_not_valid", ["validating_params"], "created"), ActionRecord("add_collector_var", ["created", "ready", "done", "valid_params"], "validating_params"), ActionRecord("remove_collector_var", ["created", "ready", "done", "valid_params"], "validating_params"), ActionRecord("execute", ["ready"], "generating_folds", user_title="Run block"), ActionRecord("on_folds_generation_success", ["generating_folds"], "ready_to_run_sub_scope", reload_block_in_client=True), ActionRecord("continue_collecting_sub_scope", ["ready_to_run_sub_scope"], "sub_scope_executing"), ActionRecord("run_sub_scope", ["ready_to_run_sub_scope"], "sub_scope_executing"), ActionRecord("on_sub_scope_done", ["sub_scope_executing"], "ready_to_run_sub_scope"), ActionRecord("success", ["working", "ready_to_run_sub_scope"], "done", propagate_auto_execution=True, reload_block_in_client=True), ActionRecord("error", ["*", "ready", "working", "sub_scope_executing", "generating_folds", "ready_to_run_sub_scope"], "execution_error", reload_block_in_client=True), ActionRecord("reset_execution", ["*", "done", "sub_scope_executing", "ready_to_run_sub_scope", "generating_folds", "execution_error"], "ready", user_title="Reset execution"), ])) _collector_spec = ParamField(name="collector_spec", title="", field_type=FieldType.CUSTOM, input_type=InputType.HIDDEN, init_val=None, required=False ) res_seq = BlockField(name="res_seq", provided_data_type="SequenceContainer", field_type=FieldType.HIDDEN, init_val=None) _results_container = OutputBlockField( name="results_container", provided_data_type="ResultsContainer", field_type=FieldType.HIDDEN, init_val=None ) def __init__(self, *args, **kwargs): super(UniformMetaBlock, self).__init__(*args, **kwargs) self.auto_exec_status_working.update(["sub_scope_executing", "ready_to_run_sub_scope", "generating_folds"]) self.inner_output_manager = IteratedInnerFieldManager() self.collector_spec = CollectorSpecification() self.collector_spec.label = self.block_base_name + "_collection" self.inner_output_es_names_map = {} self.celery_task = None self.set_out_var("results_container", None) self.res_seq = SequenceContainer() def remap_inputs(self, mapping): for var in self.bound_inputs.itervalues(): var.change_block(mapping) for var in self.collector_spec.bound.itervalues(): var.change_block(mapping) @property def is_sub_pages_visible(self): if self.state in ['valid_params', 'done', 'ready']: return True return False @abstractmethod def get_fold_labels(self): pass @abstractmethod def get_repeat_labels(self): pass def get_inner_out_var(self, name): return self.inner_output_manager.get_var(name) def run_sub_scope(self, exp, *args, **kwargs): self.reset_execution_for_sub_blocks() cell = self.res_seq.sequence[self.inner_output_manager.iterator] log.debug("Cell!!!!!!!! %s", str(cell)) act = self.inner_output_manager.sequence[self.inner_output_manager.iterator] log.debug("Cell!!!!!!!! %s", str(act)) exp.store_block(self) sr = ScopeRunner(exp, self.sub_scope_name) sr.execute() def on_sub_scope_done(self, exp, *args, **kwargs): """ @type exp: Experiment This action should be called by ScopeRunner when all blocks in sub-scope have exec status == done """ r = get_redis_instance() with redis_lock.Lock(r, ExpKeys.get_block_global_lock_key(self.exp_id, self.uuid)): cell = self.res_seq.sequence[self.inner_output_manager.iterator] for name, scope_var in self.collector_spec.bound.iteritems(): var = exp.get_scope_var_value(scope_var) exp.log(self.uuid, "Collected %s from %s" % (var, scope_var.title), severity="CRITICAL") log.debug("Collected %s from %s", var, scope_var.title) if var is not None: if hasattr(var, "clone"): cell[name] = var.clone("%s_%s" % (self.uuid, self.inner_output_manager.iterator)) else: cell[name] = deepcopy(var) self.res_seq.sequence[self.inner_output_manager.iterator] = cell exp.store_block(self) if len(cell) < len(self.res_seq.fields): self.do_action("continue_collecting_sub_scope", exp) else: try: self.inner_output_manager.next() self.do_action("run_sub_scope", exp) except StopIteration, e: # All folds were processed without errors self.build_result_collection(exp) self.do_action("success", exp)
class CustomIterator(UniformMetaBlock): block_base_name = "CUSTOM_ITERATOR" name = "Custom Iterator" has_custom_layout = True _ci_block_actions = ActionsList([ ActionRecord("become_ready", ["valid_params"], "ready"), ActionRecord("reset_settings", ["*", "done", "sub_scope_executing", "ready_to_run_sub_scope", "generating_folds", "execution_error"], "ready", user_title="Reset to initial state", reload_block_in_client=True) ]) cells_prototype = BlockField(name="cells_prototype", field_type=FieldType.CUSTOM, init_val=None) cells = BlockField(name="cells", field_type=FieldType.CUSTOM, init_val=None) is_cells_prototype_defined = BlockField(name="is_cells_prototype_defined", field_type=FieldType.BOOLEAN, init_val=False) elements = BlockField(name="elements", field_type=FieldType.SIMPLE_LIST, init_val=[ "custom_iterator/cell_prototype_definition.html", "custom_iterator/cell_dyn_inputs.html" ]) def __init__(self, *args, **kwargs): super(CustomIterator, self).__init__(*args, **kwargs) self.cells_prototype = CellsPrototype() self.cells = CellInfoList() def add_cell_prototype_field(self, exp, received_block, *args, **kwargs): new_field_dict = received_block.get("cells_prototype", {}).get("new_cell_field") if new_field_dict: cf = CellField(**new_field_dict) cf.update_name_from_label() self.cells_prototype.add_cell(cf) exp.store_block(self) def finish_cells_prototype_definition(self, exp, *args, **kwargs): self.is_cells_prototype_defined = True for field_prototype in self.cells_prototype.cells_list: new_inner_output = InnerOutputField( name=field_prototype.name, provided_data_type=field_prototype.data_type ) self.register_inner_output_variables([new_inner_output]) exp.store_block(self) def add_cell(self, exp, received_block, *args, **kwargs): new_cell_dict = received_block.get("cells", {}).get("new") if new_cell_dict: cell = CellInfo(new_cell_dict["label"]) for field_prototype in self.cells_prototype.cells_list: new_name = "%s_%s" % (field_prototype.name, len(self.cells.cells)) cell.inputs_list.append((field_prototype.name, new_name)) # TODO: add input port to block new_port = InputBlockField( name=new_name, required_data_type=field_prototype.data_type, required=True ) self.add_input_port(new_port) self.cells.cells.append(cell) exp.store_block(self) def remove_cell(self, exp, cell_json, *args, **kwargs): try: cell = json.loads(cell_json) self.cells.remove_by_label(cell["label"]) exp.store_block(self) except: pass def become_ready(self, *args, **kwargs): pass def on_params_is_valid(self, exp, *args, **kwargs): super(CustomIterator, self).on_params_is_valid(exp, *args, **kwargs) self.do_action("become_ready", exp, *args, **kwargs) def get_fold_labels(self): return [cell.label for cell in self.cells.cells] def execute(self, exp, *args, **kwargs): self.inner_output_manager.reset() seq = [] for cell_def in self.cells.cells: cell = {} for name, input_var_name in cell_def.inputs_list: # TODO: hmm maybe we should create deepcopy? cell[name] = self.get_input_var(input_var_name) seq.append(cell) exp.store_block(self) self.do_action("on_folds_generation_success", exp, seq) def reset_settings(self, exp, *args, **kwargs): self.cells_prototype = CellsPrototype() self.cells = CellInfoList() self.is_cells_prototype_defined = False exp.store_block(self)