def test_load_heuristic(): by_name = load_heuristic('reproin') from_file = load_heuristic(op.join(HEURISTICS_PATH, 'reproin.py')) assert by_name assert by_name.filename == from_file.filename with pytest.raises(ImportError): load_heuristic('unknownsomething') with pytest.raises(ImportError): load_heuristic(op.join(HEURISTICS_PATH, 'unknownsomething.py'))
def convert_to_bids(client, project_label, heuristic_path, subject_labels=None, session_labels=None, dry_run=False): """Converts a project to bids by reading the file entries from flywheel and using the heuristics to write back to the BIDS namespace of the flywheel containers Args: client (Client): The flywheel sdk client project_label (str): The label of the project heuristic_path (str): The path to the heuristic file or the name of a known heuristic subject_code (str): The subject code session_label (str): The session label dry_run (bool): Print the changes, don't apply them on flywheel """ # Make sure we can find the heuristic logger.info("Loading heuristic file...") try: if os.path.isfile(heuristic_path): heuristic = utils.load_heuristic(heuristic_path) elif "github" in heuristic_path and validators.url(heuristic_path): # read from github try: response = requests.get(heuristic_path) if response.ok: name = 'heuristic' spec = importlib.util.spec_from_loader(name, loader=None) heuristic = importlib.util.module_from_spec(spec) code = response.text exec(code, heuristic.__dict__) else: logger.error( "Couldn't find a valid URL for this heuristic at:\n\n" + heuristic_path + "\n") raise ModuleNotFoundError except: logger.error("Trouble retrieving the URL!") raise ModuleNotFoundError( "Is this a valid URL to a heuristic on Github? Please check spelling!" ) else: heuristic = importlib.import_module( 'fw_heudiconv.example_heuristics.{}'.format(heuristic_path)) except ModuleNotFoundError as e: logger.error("Couldn't load the specified heuristic file!") logger.error(e) sys.exit(1) logger.info("Heuristic loaded successfully!") if dry_run: logger.setLevel(logging.DEBUG) logger.info("Querying Flywheel server...") project_obj = client.projects.find_first( 'label="{}"'.format(project_label)) assert project_obj, "Project not found! Maybe check spelling...?" logger.debug('Found project: %s (%s)', project_obj['label'], project_obj.id) project_obj = confirm_bids_namespace(project_obj, dry_run) sessions = client.get_project_sessions(project_obj.id) # filters if subject_labels: sessions = [ s for s in sessions if s.subject['label'] in subject_labels ] if session_labels: sessions = [s for s in sessions if s.label in session_labels] assert sessions, "No sessions found!" logger.debug( 'Found sessions:\n\t%s', "\n\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions])) # try subject/session label functions if hasattr(heuristic, "ReplaceSubject"): subject_rename = heuristic.ReplaceSubject else: subject_rename = None if hasattr(heuristic, "ReplaceSession"): session_rename = heuristic.ReplaceSession else: session_rename = None # try attachments if hasattr(heuristic, "AttachToProject"): logger.info("Processing project attachments based on heuristic file") attachments = heuristic.AttachToProject() if not isinstance(attachments, list): attachments = [attachments] for at in attachments: upload_attachment(client, project_obj, level='project', attachment_dict=at, subject_rename=subject_rename, session_rename=session_rename, folders=['anat', 'dwi', 'func', 'fmap', 'perf'], dry_run=dry_run) '''if hasattr(heuristic, "AttachToSubject"): logger.info("Processing subject attachments based on heuristic file") attachments = heuristic.AttachToSubject() if not isinstance(attachments, list): attachments = [attachments] for at in attachments: logger.debug( "\tFilename: {}\n\tData: {}\n\tMIMEType: {}".format( at['name'], at['data'], at['type'] ) ) verify_name, verify_data, verify_type = verify_attachment(at['name'], at['data'], at['type']) if not all([verify_name, verify_data, verify_type]): logger.warning("Attachments may not be valid for upload!") logger.debug( "\tFilename valid: {}\n\tData valid: {}\n\tMIMEType valid: {}".format( verify_name, verify_data, verify_type ) ) if not dry_run: subjects = [x.subject for x in sessions] file_spec = flywheel.FileSpec(at['name'], at['data'], at['type']) [sub.upload_file(file_spec) for sub in subjects]''' num_sessions = len(sessions) for sesnum, session in enumerate(sessions): # Find SeqInfos to apply the heuristic to logger.info("Applying heuristic to %s (%d/%d)...", session.label, sesnum + 1, num_sessions) seq_infos = get_seq_info(client, project_label, [session]) logger.debug( "Found SeqInfos:\n%s", "\n\t".join([pretty_string_seqinfo(seq) for seq in seq_infos])) # apply heuristic to seqinfos to_rename = heuristic.infotodict(seq_infos) if not to_rename: logger.debug("No changes to apply!") continue # try intendedfors intention_map = defaultdict(list) if hasattr(heuristic, "IntendedFor"): logger.info( "Processing IntendedFor fields based on heuristic file") intention_map.update(heuristic.IntendedFor) logger.debug( "Intention map: %s", pprint.pformat([(k[0], v) for k, v in dict(intention_map).items()])) # try metadataextras metadata_extras = defaultdict(list) if hasattr(heuristic, "MetadataExtras"): logger.info("Processing Medatata fields based on heuristic file") metadata_extras.update(heuristic.MetadataExtras) logger.debug("Metadata extras: %s", metadata_extras) # try subject/session label functions if hasattr(heuristic, "ReplaceSubject"): subject_rename = heuristic.ReplaceSubject else: subject_rename = None if hasattr(heuristic, "ReplaceSession"): session_rename = heuristic.ReplaceSession else: session_rename = None # try attachments if hasattr(heuristic, "AttachToSession"): logger.info( "Processing session attachments based on heuristic file") attachments = heuristic.AttachToSession() if not isinstance(attachments, list): attachments = [attachments] for at in attachments: upload_attachment( client, session, level='session', attachment_dict=at, subject_rename=subject_rename, session_rename=session_rename, folders=['anat', 'dwi', 'func', 'fmap', 'perf'], dry_run=dry_run) # final prep if not dry_run: logger.info("Applying changes to files...") for key, val in to_rename.items(): # assert val is list if not isinstance(val, set): val = set(val) for seqitem, value in enumerate(val): apply_heuristic(client, key, value, dry_run, intention_map[key], metadata_extras[key], subject_rename, session_rename, seqitem + 1) confirm_intentions(client, session, dry_run) print("\n")
def test_populate_intended_for(tmpdir, monkeypatch, capfd, subjects, sesID, expected_session_folder, heuristic): """ Test convert For now, I'm just going to test that the call to populate_intended_for is done with the correct argument. More tests can be added here. """ def mock_populate_intended_for(session, matching_parameters='Shims', criterion='Closest'): """ Pretend we run populate_intended_for, but just print out the arguments. """ print('session: {}'.format(session)) print('matching_parameters: {}'.format(matching_parameters)) print('criterion: {}'.format(criterion)) return # mock the "populate_intended_for": monkeypatch.setattr(heudiconv.convert, "populate_intended_for", mock_populate_intended_for) outdir = op.join(str(tmpdir), 'foo') outfolder = op.join(outdir, 'sub-{sID}', 'ses-{ses}') if sesID else op.join( outdir, 'sub-{sID}') sub_ses = 'sub-{sID}' + ('_ses-{ses}' if sesID else '') # items are a list of tuples, with each tuple having three elements: # prefix, outtypes, item_dicoms items = [(op.join(outfolder, 'anat', sub_ses + '_T1w').format(sID=s, ses=sesID), ('', ), []) for s in subjects] heuristic = load_heuristic(heuristic) if heuristic else None heudiconv.convert.convert(items, converter='', scaninfo_suffix='.json', custom_callable=None, populate_intended_for_opts=getattr( heuristic, 'POPULATE_INTENDED_FOR_OPTS', None), with_prov=None, bids_options=[], outdir=outdir, min_meta=True, overwrite=False) output = capfd.readouterr() # if the heuristic module has a 'POPULATE_INTENDED_FOR_OPTS' field, we expect # to get the output of the mock_populate_intended_for, otherwise, no output: pif_cfg = getattr(heuristic, 'POPULATE_INTENDED_FOR_OPTS', None) if pif_cfg: assert all([ "\n".join([ "session: " + outfolder.format(sID=s, ses=sesID), # "ImagingVolume" is defined in heuristic file; "Shims" is the default f"matching_parameters: {pif_cfg['matching_parameters']}", f"criterion: {pif_cfg['criterion']}" ]) in output.out for s in subjects ]) else: # If there was no heuristic, make sure populate_intended_for was not called assert not output.out
def convert_to_bids(client, project_label, heuristic_path, subject_labels=None, session_labels=None, dry_run=False): """Converts a project to bids by reading the file entries from flywheel and using the heuristics to write back to the BIDS namespace of the flywheel containers Args: client (Client): The flywheel sdk client project_label (str): The label of the project heuristic_path (str): The path to the heuristic file or the name of a known heuristic subject_code (str): The subject code session_label (str): The session label dry_run (bool): Print the changes, don't apply them on flywheel """ if dry_run: logger.setLevel(logging.DEBUG) logger.info("Querying Flywheel server...") project_obj = client.projects.find_first( 'label="{}"'.format(project_label)) assert project_obj, "Project not found! Maybe check spelling...?" logger.debug('Found project: %s (%s)', project_obj['label'], project_obj.id) project_obj = confirm_bids_namespace(project_obj, dry_run) sessions = client.get_project_sessions(project_obj.id) # filters if subject_labels: sessions = [ s for s in sessions if s.subject['label'] in subject_labels ] if session_labels: sessions = [s for s in sessions if s.label in session_labels] assert sessions, "No sessions found!" logger.debug( 'Found sessions:\n\t%s', "\n\t".join(['%s (%s)' % (ses['label'], ses.id) for ses in sessions])) # Find SeqInfos to apply the heuristic to seq_infos = get_seq_info(client, project_label, sessions) logger.debug( "Found SeqInfos:\n%s", "\n\t".join([pretty_string_seqinfo(seq) for seq in seq_infos])) logger.info("Loading heuristic file...") try: if os.path.isfile(heuristic_path): heuristic = utils.load_heuristic(heuristic_path) else: heuristic = importlib.import_module( 'heudiconv.heuristics.{}'.format(heuristic_path)) except ModuleNotFoundError as e: logger.error("Couldn't load the specified heuristic file!") logger.error(e) sys.exit(1) logger.info("Applying heuristic to query results...") to_rename = heuristic.infotodict(seq_infos) if not to_rename: logger.debug("No changes to apply!") sys.exit(1) intention_map = defaultdict(list) if hasattr(heuristic, "IntendedFor"): logger.info("Processing IntendedFor fields based on heuristic file") intention_map.update(heuristic.IntendedFor) logger.debug("Intention map: %s", intention_map) metadata_extras = defaultdict(list) if hasattr(heuristic, "MetadataExtras"): logger.info("Processing Medatata fields based on heuristic file") metadata_extras.update(heuristic.MetadataExtras) logger.debug("Metadata extras: %s", metadata_extras) if not dry_run: logger.info("Applying changes to files...") if hasattr(heuristic, "ReplaceSubject"): subject_rename = heuristic.ReplaceSubject else: subject_rename = None if hasattr(heuristic, "ReplaceSession"): session_rename = heuristic.ReplaceSession else: session_rename = None for key, val in to_rename.items(): # assert val is list if not isinstance(val, set): val = set(val) for seqitem, value in enumerate(val): apply_heuristic(client, key, value, dry_run, intention_map[key], metadata_extras[key], subject_rename, session_rename, seqitem + 1) if not dry_run: for ses in sessions: confirm_intentions(client, ses)