def test_bidsgrab_output_dn3sub(self):

        # Create correct output
        subjects = [f'm{sub:02}' for sub in list(range(2, 5))]
        common_prefix = [
            join(dirname(__file__), 'dn3sub_fmriprep_dummy', 'derivatives',
                 'fmriprep', f'sub-{sub}/', 'func', f'sub-{sub}_task-{task}')
            for sub in subjects for task in ['prlpun', 'prlrew']
        ]
        conf_json = [
            path_prefix + '_desc-confounds_regressors.json'
            for path_prefix in common_prefix
        ]
        conf_raw = [
            path_prefix + '_desc-confounds_regressors.tsv'
            for path_prefix in common_prefix
        ]
        fmri_prep = [
            path_prefix + '_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'
            for path_prefix in common_prefix
        ]
        entities = [{
            'datatype': 'func',
            'subject': sub,
            'task': task
        } for sub in subjects for task in ['prlpun', 'prlrew']]
        tr_dict = {task: 2 for task in ['prlpun', 'prlrew']}

        # No flags
        grabber = BIDSGrab(bids_dir=dn_dataset['bids_dir'])
        results = grabber.run()
        self.assertEqual(fmri_prep, results.outputs.fmri_prep)
        self.assertEqual(conf_json, results.outputs.conf_json)
        self.assertEqual(conf_raw, results.outputs.conf_raw)
        self.assertEqual(entities, results.outputs.entities)
        self.assertEqual(tr_dict, results.outputs.tr_dict)

        # Participant flag
        grabber = BIDSGrab(bids_dir=dn_dataset['bids_dir'], subject=['m03'])
        results = grabber.run()
        subfilter = lambda l: list(filter(lambda x: 'm03' in x, l))
        self.assertEqual(subfilter(fmri_prep), results.outputs.fmri_prep)
        self.assertEqual(subfilter(conf_json), results.outputs.conf_json)
        self.assertEqual(subfilter(conf_raw), results.outputs.conf_raw)
        self.assertEqual(
            [entity for entity in entities if entity['subject'] == 'm03'],
            results.outputs.entities)
        self.assertEqual(tr_dict, results.outputs.tr_dict)

        # Task flag
        grabber = BIDSGrab(bids_dir=dn_dataset['bids_dir'], task=['prlpun'])
        results = grabber.run()
        taskfilter = lambda l: list(filter(lambda x: 'prlpun' in x, l))
        self.assertEqual(taskfilter(fmri_prep), results.outputs.fmri_prep)
        self.assertEqual(taskfilter(conf_json), results.outputs.conf_json)
        self.assertEqual(taskfilter(conf_raw), results.outputs.conf_raw)
        self.assertEqual(
            [entity for entity in entities if entity['task'] == 'prlpun'],
            results.outputs.entities)
        self.assertEqual({'prlpun': 2}, results.outputs.tr_dict)
Beispiel #2
0
 def test_select_one_multiple_finds(self):
     paths = self.generatePaths(['01', '02'], ['1', '2'], ['rest'],
                                ['test', 'experiment'])
     with self.assertRaises(ValueError):
         BIDSGrab.select_one(paths,
                             subject='01',
                             session='1',
                             task='rest',
                             run=None)
     with self.assertRaises(ValueError):
         BIDSGrab.select_one(paths,
                             subject='01',
                             session='1',
                             task='rest',
                             run=Undefined)
Beispiel #3
0
 def test_select_one_none(self):
     paths = self.generatePaths(['01', '02'], ['1', '2'], ['rest'],
                                ['test', 'experiment'])
     file = BIDSGrab.select_one(paths,
                                subject='01',
                                session='1',
                                task='rest',
                                run='nonexists')
     self.assertEqual('', file)
Beispiel #4
0
 def test_select_one_valid(self):
     paths = self.generatePaths(['01', '02'], ['1', '2'], ['rest'],
                                ['test', 'experiment'])
     file = BIDSGrab.select_one(paths,
                                subject='01',
                                session='1',
                                task='rest',
                                run='test')
     self.assertEqual('sub-01_ses-1_task-rest_run-test_fileName.extension',
                      file)
Beispiel #5
0
    def test_grab_entities_subjects(self):
        subject_outputs = [{'subject': '01'}, {'subject': '02'}, {'subject': '03'}, {'subject': '04'}, {'subject': '05'}, {'subject': '06'}, {'subject': '07'}, {'subject': '08'}, {'subject': '09'}, {'subject': '10'}, {'subject': '11'}, {'subject': '12'}, {'subject': '13'}]
        bl = Node(BIDSGrab(bids_dir=TEST_DATASETS['NEW_FMRIPREP_DUMMY']), name="TestBIDSGrab")
        outs = bl.run()
        bl_subjects = [{'subject': element['subject']} for element in outs.outputs.entities]
        self.assertEqual(bl_subjects, subject_outputs)


# if __name__ == '__main__':
#     import cProfile
#     bl = Node(BIDSGrab(bids_dir=TEST_DATASETS['NEW_FMRIPREP_DUMMY']), name="TestBIDSGrab")
#     bl.run()
    #   cProfile.run('bl.run()')
Beispiel #6
0
def init_fmridenoise_wf(bids_dir,
                        derivatives='fmriprep',
                        task=[],
                        session=[],
                        subject=[],
                        pipelines_paths=get_pipelines_paths(),
                        smoothing=True,
                        ica_aroma=False,
                        high_pass=0.008,
                        low_pass=0.08,
                        # desc=None,
                        # ignore=None, force_index=None,
                        base_dir='/tmp/fmridenoise/', name='fmridenoise_wf'
                        ):
    workflow = pe.Workflow(name=name, base_dir=base_dir)
    temps.base_dir = base_dir

    # 1) --- Selecting pipeline

    # Inputs: fulfilled
    pipelineselector = pe.Node(
       PipelineSelector(),
       name="PipelineSelector")
    pipelineselector.iterables = ('pipeline_path', pipelines_paths)
    # Outputs: pipeline, pipeline_name, low_pass, high_pass

    # 2) --- Loading BIDS structure

    # Inputs: directory, task, derivatives
    grabbing_bids = pe.Node(
        BIDSGrab(
            bids_dir=bids_dir,
            derivatives=derivatives,
            task=task,
            session=session,
            subject=subject,
            ica_aroma=ica_aroma
        ),
        name="BidsGrabber")
    # Outputs: fmri_prep, conf_raw, conf_json, entities, tr_dict

    # 3) --- Confounds preprocessing

    # Inputs: pipeline, conf_raw, conf_json
    temppath = os.path.join(base_dir, 'prep_conf')
    prep_conf = pe.MapNode(
        Confounds(
            output_dir=temps.mkdtemp(temppath)
        ),
        iterfield=['conf_raw', 'conf_json', 'entities'],
        name="ConfPrep")
    # Outputs: conf_prep, low_pass, high_pass

    # 4) --- Denoising

    # Inputs: conf_prep, low_pass, high_pass
    iterate = ['fmri_prep', 'conf_prep', 'entities']
    if not ica_aroma:
        iterate = iterate
    else:
        iterate.append('fmri_prep_aroma')

    temppath = os.path.join(base_dir, 'denoise')
    denoise = pe.MapNode(
        Denoise(
            smoothing=smoothing,
            high_pass=high_pass,
            low_pass=low_pass,
            ica_aroma=ica_aroma,
            output_dir=temps.mkdtemp(temppath)
        ),
        iterfield=iterate,
        name="Denoiser", mem_gb=6)
    # Outputs: fmri_denoised

    # 5) --- Connectivity estimation

    # Inputs: fmri_denoised
    temppath = os.path.join(base_dir, 'connectivity')
    parcellation_path = get_parcelation_file_path()
    connectivity = pe.MapNode(
        Connectivity(
            output_dir=temps.mkdtemp(temppath),
            parcellation=parcellation_path
        ),
        iterfield=['fmri_denoised'],
        name='ConnCalc')
    # Outputs: conn_mat, carpet_plot

    # 6) --- Group confounds

    # Inputs: conf_summary, pipeline_name
    # FIXME BEGIN
    # This is part of temporary solution.
    # Group nodes write to bids dir insted of tmp and let files be grabbed by datasink
    os.makedirs(os.path.join(bids_dir, 'derivatives', 'fmridenoise'), exist_ok=True)
    # FIXME END
    group_conf_summary = pe.Node(
        GroupConfounds(
            output_dir=os.path.join(bids_dir, 'derivatives', 'fmridenoise'),
        ),
        name="GroupConf")

    # Outputs: group_conf_summary

    # 7) --- Group connectivity

    # Inputs: corr_mat, pipeline_name

    group_connectivity = pe.Node(
        GroupConnectivity(
            output_dir=os.path.join(bids_dir, 'derivatives', 'fmridenoise'),
        ),
        name="GroupConn")

    # Outputs: group_corr_mat

    # 8) --- Quality measures

    # Inputs: group_corr_mat, group_conf_summary, pipeline_name

    quality_measures = pe.MapNode(
        QualityMeasures(
            output_dir=os.path.join(bids_dir, 'derivatives', 'fmridenoise'),
            distance_matrix=get_distance_matrix_file_path()
        ),
        iterfield=['group_corr_mat', 'group_conf_summary'],
        name="QualityMeasures")
    # Outputs: fc_fd_summary, edges_weight, edges_weight_clean

    # 9) --- Merge quality measures into lists for further processing

    # Inputs: fc_fd_summary, edges_weight, edges_weight_clean

    merge_quality_measures = pe.JoinNode(MergeGroupQualityMeasures(),
                                         joinsource=pipelineselector,
                                         name="Merge")

    # Outputs: fc_fd_summary, edges_weight

    # 10) --- Quality measures across pipelines

    # Inputs: fc_fd_summary, edges_weight

    pipelines_quality_measures = pe.Node(
        PipelinesQualityMeasures(
            output_dir=os.path.join(bids_dir, 'derivatives', 'fmridenoise'),
        ),
        name="PipelinesQC")

    # Outputs: pipelines_fc_fd_summary, pipelines_edges_weight

    # 11) --- Report from data

    report_creator = pe.JoinNode(
        ReportCreator(
            group_data_dir=os.path.join(bids_dir, 'derivatives', 'fmridenoise')
        ),
        joinsource=pipelineselector,
        joinfield=['pipelines', 'pipelines_names'],
        name='ReportCreator')

    # 12) --- Save derivatives
    # TODO: Fill missing in/out
    ds_confounds = pe.MapNode(BIDSDataSink(base_directory=bids_dir),
                    iterfield=['in_file', 'entities'],
                    name="ds_confounds")

    ds_denoise = pe.MapNode(BIDSDataSink(base_directory=bids_dir),
                    iterfield=['in_file', 'entities'],
                    name="ds_denoise")

    ds_connectivity = pe.MapNode(BIDSDataSink(base_directory=bids_dir),
                    iterfield=['in_file', 'entities'],
                    name="ds_connectivity")

    ds_carpet_plot = pe.MapNode(BIDSDataSink(base_directory=bids_dir),
                                 iterfield=['in_file', 'entities'],
                                 name="ds_carpet_plot")

    ds_matrix_plot = pe.MapNode(BIDSDataSink(base_directory=bids_dir),
                                 iterfield=['in_file', 'entities'],
                                 name="ds_matrix_plot")


# --- Connecting nodes

    workflow.connect([
        (grabbing_bids, denoise, [('tr_dict', 'tr_dict')]),
        (grabbing_bids, denoise, [('fmri_prep', 'fmri_prep'),
                                  ('fmri_prep_aroma', 'fmri_prep_aroma')]),
        (grabbing_bids, denoise, [('entities', 'entities')]),
        (grabbing_bids, prep_conf, [('conf_raw', 'conf_raw'),
                                    ('conf_json', 'conf_json'),
                                    ('entities', 'entities')]),
        (grabbing_bids, ds_confounds, [('entities', 'entities')]),
        (grabbing_bids, ds_denoise, [('entities', 'entities')]),
        (grabbing_bids, ds_connectivity, [('entities', 'entities')]),
        (grabbing_bids, ds_carpet_plot, [('entities', 'entities')]),
        (grabbing_bids, ds_matrix_plot, [('entities', 'entities')]),

        (pipelineselector, prep_conf, [('pipeline', 'pipeline')]),
        (pipelineselector, denoise, [('pipeline', 'pipeline')]),
        (prep_conf, group_conf_summary, [('conf_summary', 'conf_summary'),
                                        ('pipeline_name', 'pipeline_name')]),

        (pipelineselector, ds_denoise, [('pipeline_name', 'pipeline_name')]),
        (pipelineselector, ds_connectivity, [('pipeline_name', 'pipeline_name')]),
        (pipelineselector, ds_confounds, [('pipeline_name', 'pipeline_name')]),
        (pipelineselector, ds_carpet_plot, [('pipeline_name', 'pipeline_name')]),
        (pipelineselector, ds_matrix_plot, [('pipeline_name', 'pipeline_name')]),

        (prep_conf, denoise, [('conf_prep', 'conf_prep')]),
        (denoise, connectivity, [('fmri_denoised', 'fmri_denoised')]),

        (prep_conf, group_connectivity, [('pipeline_name', 'pipeline_name')]),
        (connectivity, group_connectivity, [('corr_mat', 'corr_mat')]),

        (prep_conf, ds_confounds, [('conf_prep', 'in_file')]),
        (denoise, ds_denoise, [('fmri_denoised', 'in_file')]),
        (connectivity, ds_connectivity, [('corr_mat', 'in_file')]),
        (connectivity, ds_carpet_plot, [('carpet_plot', 'in_file')]),
        (connectivity, ds_matrix_plot, [('matrix_plot', 'in_file')]),

        (group_connectivity, quality_measures, [('pipeline_name', 'pipeline_name'),
                                                ('group_corr_mat', 'group_corr_mat')]),
        (group_conf_summary, quality_measures, [('group_conf_summary', 'group_conf_summary')]),
        (quality_measures, merge_quality_measures, [('fc_fd_summary', 'fc_fd_summary'),
                                                    ('edges_weight', 'edges_weight'),
                                                    ('edges_weight_clean', 'edges_weight_clean'),
                                                    ('exclude_list', 'exclude_list')]),
        (merge_quality_measures, pipelines_quality_measures,
            [('fc_fd_summary', 'fc_fd_summary'),
             ('edges_weight', 'edges_weight'),
             ('edges_weight_clean', 'edges_weight_clean')]),
        (merge_quality_measures, report_creator,
            [('exclude_list', 'excluded_subjects')]),
        (pipelines_quality_measures, report_creator,
            [('plot_pipeline_edges_density', 'plot_pipeline_edges_density'),
             ('plot_pipelines_edges_density_no_high_motion', 'plot_pipelines_edges_density_no_high_motion'),
             ('plot_pipelines_fc_fd_pearson', 'plot_pipelines_fc_fd_pearson'),
             ('plot_pipelines_fc_fd_uncorr', 'plot_pipelines_fc_fd_uncorr'),
             ('plot_pipelines_distance_dependence', 'plot_pipelines_distance_dependence')]),
        (pipelineselector, report_creator,
            [('pipeline', 'pipelines'),
             ('pipeline_name', 'pipelines_names')])
    ])

    return workflow
Beispiel #7
0
    def __init__(self, bids_dir: str, subjects: t.List[str],
                 tasks: t.List[str], conf_raw: t.List[str],
                 conf_json: t.List[str], tr_dic: dict,
                 pipelines_paths: t.List[str], high_pass: float,
                 low_pass: float):
        self.fmri_prep_aroma_files = []
        self.fmri_prep_files = []
        # 1) --- Itersources for all further processing
        # Inputs: fulfilled
        self.pipelineselector = Node(PipelineSelector(),
                                     name="PipelineSelector")
        self.pipelineselector.iterables = ('pipeline_path', pipelines_paths)
        # Outputs: pipeline, pipeline_name, low_pass, high_pass

        # Inputs: fulfilled
        self.subjectselector = Node(IdentityInterface(fields=['subject']),
                                    name="SubjectSelector")
        self.subjectselector.iterables = ('subject', subjects)
        # Outputs: subject

        # Inputs: fulfilled
        self.taskselector = Node(IdentityInterface(fields=['task']),
                                 name="TaskSelector")
        self.taskselector.iterables = ('task', tasks)
        # Outputs: task

        # 2) --- Loading BIDS files

        # Inputs: subject, session, task
        self.bidsgrabber = Node(BIDSGrab(conf_raw_files=conf_raw,
                                         conf_json_files=conf_json),
                                name="BidsGrabber")
        # Outputs: fmri_prep, fmri_prep_aroma, conf_raw, conf_json

        # 3) --- Confounds preprocessing

        # Inputs: pipeline, conf_raw, conf_json
        self.prep_conf = Node(Confounds(output_dir=temps.mkdtemp('prep_conf')),
                              name="ConfPrep")
        # Outputs: conf_prep, conf_summary

        # 4) --- Denoising
        # Inputs: fmri_prep, fmri_prep_aroma, conf_prep, pipeline, entity, tr_dict
        self.denoise = Node(Denoise(high_pass=high_pass,
                                    low_pass=low_pass,
                                    tr_dict=tr_dic,
                                    output_dir=temps.mkdtemp('denoise')),
                            name="Denoiser",
                            mem_gb=12)
        # Outputs: fmri_denoised

        # 5) --- Connectivity estimation

        # Inputs: fmri_denoised
        self.connectivity = Node(
            Connectivity(output_dir=temps.mkdtemp('connectivity')),
            name='ConnCalc')
        # Outputs: conn_mat, carpet_plot

        # 6) --- Group confounds

        # Inputs: conf_summary, pipeline_name

        self.group_conf_summary = JoinNode(
            GroupConfounds(output_dir=temps.mkdtemp('group_conf_summary'), ),
            joinfield=["conf_summary_json_files"],
            joinsource=self.subjectselector,
            name="GroupConf")

        # Outputs: group_conf_summary

        # 7) --- Group connectivity

        # Inputs: corr_mat, pipeline_name

        self.group_connectivity = JoinNode(GroupConnectivity(
            output_dir=temps.mkdtemp('group_connectivity'), ),
                                           joinfield=["corr_mat"],
                                           joinsource=self.subjectselector,
                                           name="GroupConn")

        # Outputs: group_corr_mat

        # 8) --- Quality measures

        # Inputs: group_corr_mat, group_conf_summary, pipeline_name

        self.quality_measures = Node(QualityMeasures(
            output_dir=temps.mkdtemp('quality_measures'),
            distance_matrix=get_distance_matrix_file_path()),
                                     name="QualityMeasures")
        # Outputs: fc_fd_summary, edges_weight, edges_weight_clean
        self.quality_measures_join = create_flatten_identity_join_node(
            name='JoinQualityMeasuresOverPipeline',
            joinsource=self.pipelineselector,
            fields=[
                'excluded_subjects', 'warnings', 'corr_matrix_plot',
                'corr_matrix_no_high_motion_plot'
            ],
            flatten_fields=['warnings'])
        # 10) --- Quality measures across pipelines

        # Inputs: fc_fd_summary, edges_weight
        self.pipelines_join = JoinNode(IdentityInterface(fields=['pipelines']),
                                       name='JoinPipelines',
                                       joinsource=self.pipelineselector,
                                       joinfield=['pipelines'])
        self.pipelines_quality_measures = JoinNode(
            PipelinesQualityMeasures(
                output_dir=temps.mkdtemp('pipelines_quality_measures'),
                # TODO: Replace with datasinks for needed output
            ),
            joinsource=self.pipelineselector,
            joinfield=[
                'fc_fd_summary', 'edges_weight', 'edges_weight_clean',
                'fc_fd_corr_values', 'fc_fd_corr_values_clean'
            ],
            name="PipelinesQualityMeasures")
        self.pipeline_quality_measures_join_tasks = create_flatten_identity_join_node(
            name="JoinPipelinesQualityMeasuresOverTasks",
            joinsource=self.taskselector,
            fields=[
                'warnings', 'excluded_subjects',
                'plot_pipelines_edges_density',
                'plot_pipelines_edges_density_no_high_motion',
                'plot_pipelines_fc_fd_pearson',
                'plot_pipelines_fc_fd_pearson_no_high_motion',
                'plot_pipelines_fc_fd_uncorr',
                'plot_pipelines_distance_dependence',
                'plot_pipelines_distance_dependence_no_high_motion',
                'plot_pipelines_tdof_loss', 'corr_matrix_plot',
                'corr_matrix_no_high_motion_plot'
            ],
            flatten_fields=[
                'warnings', 'excluded_subjects', 'corr_matrix_plot',
                'corr_matrix_no_high_motion_plot'
            ])
        # Outputs: pipelines_fc_fd_summary, pipelines_edges_weight
        # 11) --- Report from data
        report_dir = os.path.join(bids_dir, 'derivatives', 'fmridenoise',
                                  'report')
        os.makedirs(report_dir, exist_ok=True)
        self.report_creator = Node(ReportCreator(runtime_info=RuntimeInfo(
            input_args=str(reduce(lambda x, y: f"{x} {y}", sys.argv)),
            version=get_versions().get('version')),
                                                 output_dir=report_dir),
                                   name='ReportCreator')
        self.report_creator.inputs.tasks = tasks
        # 12) --- Save derivatives
        base_entities = {'bids_dir': bids_dir, 'derivative': 'fmridenoise'}
        self.ds_confounds = Node(BIDSDataSink(base_entities=base_entities),
                                 name="ds_confounds")
        self.ds_denoise = Node(BIDSDataSink(base_entities=base_entities),
                               name="ds_denoise")
        self.ds_connectivity_corr_mat = Node(
            BIDSDataSink(base_entities=base_entities), name="ds_connectivity")
        self.ds_connectivity_carpet_plot = Node(
            BIDSDataSink(base_entities=base_entities), name="ds_carpet_plot")
        self.ds_connectivity_matrix_plot = Node(
            BIDSDataSink(base_entities=base_entities), name="ds_matrix_plot")
        self.ds_group_conf_summary = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_group_conf_summary")
        self.ds_group_connectivity = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_group_connectivity")
        self.ds_qm_motion_plot = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_quality_measures_motion_plot")
        self.ds_qm_corr_matrix_plot_no_high = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_quality_measures_corr_matrix_plot_no_high")
        self.ds_qm_corr_matrix_plot = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_quality_measures_corr_matrix_plot")
        self.ds_pqm_fc_fd_summary = Node(
            BIDSDataSink(base_entities=base_entities),
            name="ds_pipeline_qm_fc_fd_summery")
        self.ds_pqm_edges_weight = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_edges_weight')
        self.ds_pqm_edges_weight_clean = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_edges_weight_clean')
        self.ds_pqm_plot_edges_density = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_edges_density')
        self.ds_pqm_plot_edges_density_no_high = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_edges_density_no_high')
        self.ds_pqm_plot_fc_fd = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_fc_fd')
        self.ds_pqm_plot_fc_fd_no_high = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_fc_fd_no_high')
        self.ds_pqm_plot_fc_fd_uncorr = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_fc_fd_uncorr')
        self.ds_pqm_plot_distance_dependence = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_distance_dependence')
        self.ds_pqm_plot_distance_dependence_no_high = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_distance_dependence_no_high')
        self.ds_pqm_plot_tdof_loss = Node(
            BIDSDataSink(base_entities=base_entities),
            name='ds_pipeline_qm_plot_tdof_loss')

        self.connections = [
            # bidsgrabber
            (self.subjectselector, self.bidsgrabber, [('subject', 'subject')]),
            (self.taskselector, self.bidsgrabber, [('task', 'task')]),
            # prep_conf
            (self.pipelineselector, self.prep_conf, [('pipeline', 'pipeline')]
             ),
            (self.bidsgrabber, self.prep_conf, [('conf_raw', 'conf_raw'),
                                                ('conf_json', 'conf_json')]),
            # denoise
            (self.prep_conf, self.denoise, [('conf_prep', 'conf_prep')]),
            (self.pipelineselector, self.denoise, [('pipeline', 'pipeline')]),
            # group conf summary
            (self.prep_conf, self.group_conf_summary,
             [('conf_summary', 'conf_summary_json_files')]),
            # connectivity
            (self.denoise, self.connectivity, [('fmri_denoised',
                                                'fmri_denoised')]),
            # group connectivity
            (self.connectivity, self.group_connectivity, [("corr_mat",
                                                           "corr_mat")]),
            # quality measures
            (self.pipelineselector, self.quality_measures, [('pipeline',
                                                             'pipeline')]),
            (self.group_connectivity, self.quality_measures,
             [('group_corr_mat', 'group_corr_mat')]),
            (self.group_conf_summary, self.quality_measures,
             [('group_conf_summary', 'group_conf_summary')]),
            # quality measure join over pipelines
            (self.quality_measures, self.quality_measures_join, [
                ('excluded_subjects', 'excluded_subjects'),
                ('warnings', 'warnings'),
                ('corr_matrix_plot', 'corr_matrix_plot'),
                ('corr_matrix_no_high_motion_plot',
                 'corr_matrix_no_high_motion_plot')
            ]),
            # pipeline quality measures
            (self.quality_measures, self.pipelines_quality_measures,
             [('fc_fd_summary', 'fc_fd_summary'),
              ('edges_weight', 'edges_weight'),
              ('edges_weight_clean', 'edges_weight_clean'),
              ('fc_fd_corr_values', 'fc_fd_corr_values'),
              ('fc_fd_corr_values_clean', 'fc_fd_corr_values_clean')]),
            (self.taskselector, self.pipelines_quality_measures, [('task',
                                                                   'task')]),
            # pipelines_join
            (self.pipelineselector, self.pipelines_join, [('pipeline',
                                                           'pipelines')]),
            # pipeline_quality_measures_join
            (self.pipelines_quality_measures,
             self.pipeline_quality_measures_join_tasks, [
                 ('pipelines_fc_fd_summary', 'pipelines_fc_fd_summary'),
                 ('plot_pipelines_edges_density',
                  'plot_pipelines_edges_density'),
                 ('plot_pipelines_edges_density_no_high_motion',
                  'plot_pipelines_edges_density_no_high_motion'),
                 ('plot_pipelines_fc_fd_pearson',
                  'plot_pipelines_fc_fd_pearson'),
                 ('plot_pipelines_fc_fd_pearson_no_high_motion',
                  'plot_pipelines_fc_fd_pearson_no_high_motion'),
                 ('plot_pipelines_fc_fd_uncorr',
                  'plot_pipelines_fc_fd_uncorr'),
                 ('plot_pipelines_distance_dependence',
                  'plot_pipelines_distance_dependence'),
                 ('plot_pipelines_distance_dependence_no_high_motion',
                  'plot_pipelines_distance_dependence_no_high_motion'),
                 ('plot_pipelines_tdof_loss', 'plot_pipelines_tdof_loss'),
             ]),
            (self.quality_measures_join,
             self.pipeline_quality_measures_join_tasks, [
                 ('excluded_subjects', 'excluded_subjects'),
                 ('warnings', 'warnings'),
                 ('corr_matrix_plot', 'corr_matrix_plot'),
                 ('corr_matrix_no_high_motion_plot',
                  'corr_matrix_no_high_motion_plot')
             ]),
            # report creator
            (self.pipelines_join, self.report_creator, [('pipelines',
                                                         'pipelines')]),
            # all datasinks
            # # ds_denoise
            (self.denoise, self.ds_denoise, [("fmri_denoised", "in_file")]),
            # # ds_connectivity
            (self.connectivity, self.ds_connectivity_corr_mat, [("corr_mat",
                                                                 "in_file")]),
            (self.connectivity, self.ds_connectivity_matrix_plot,
             [("matrix_plot", "in_file")]),
            (self.connectivity, self.ds_connectivity_carpet_plot,
             [("carpet_plot", "in_file")]),
            # # ds_confounds
            (self.prep_conf, self.ds_confounds, [("conf_prep", "in_file")]),
            # # ds_group_conf
            (self.group_conf_summary, self.ds_group_conf_summary,
             [('group_conf_summary', 'in_file')]),
            # # ds_group_connectivity
            (self.group_connectivity, self.ds_group_connectivity,
             [('group_corr_mat', 'in_file')]),
            # # ds_quality_measures
            (self.quality_measures, self.ds_qm_motion_plot, [('motion_plot',
                                                              'in_file')]),
            (self.quality_measures, self.ds_qm_corr_matrix_plot,
             [('corr_matrix_plot', 'in_file')]),
            (self.quality_measures, self.ds_qm_corr_matrix_plot_no_high,
             [('corr_matrix_no_high_motion_plot', 'in_file')]),
            # # ds_pipelines_quality_measures
            (self.pipelines_quality_measures, self.ds_pqm_fc_fd_summary,
             [('pipelines_fc_fd_summary', 'in_file')]),
            (self.pipelines_quality_measures, self.ds_pqm_edges_weight,
             [('pipelines_edges_weight', 'in_file')]),
            (self.pipelines_quality_measures, self.ds_pqm_edges_weight_clean,
             [('pipelines_edges_weight_clean', 'in_file')]),
            (self.pipelines_quality_measures, self.ds_pqm_plot_edges_density,
             [('plot_pipelines_edges_density', 'in_file')]),
            (self.pipelines_quality_measures,
             self.ds_pqm_plot_edges_density_no_high, [
                 ('plot_pipelines_edges_density_no_high_motion', 'in_file')
             ]),
            (self.pipelines_quality_measures, self.ds_pqm_plot_fc_fd,
             [('plot_pipelines_fc_fd_pearson', 'in_file')]),
            (self.pipelines_quality_measures, self.ds_pqm_plot_fc_fd_no_high,
             [('plot_pipelines_fc_fd_pearson_no_high_motion', 'in_file')]),
            (self.pipelines_quality_measures, self.ds_pqm_plot_fc_fd_uncorr,
             [('plot_pipelines_fc_fd_uncorr', 'in_file')]),
            (self.pipelines_quality_measures,
             self.ds_pqm_plot_distance_dependence, [
                 ('plot_pipelines_distance_dependence', 'in_file')
             ]),
            (self.pipelines_quality_measures,
             self.ds_pqm_plot_distance_dependence_no_high, [
                 ('plot_pipelines_distance_dependence_no_high_motion',
                  'in_file')
             ]),
            (self.pipelines_quality_measures, self.ds_pqm_plot_tdof_loss,
             [('plot_pipelines_tdof_loss', 'in_file')])
        ]
        self.last_join = self.pipeline_quality_measures_join_tasks