def test_read_phoenix_metadata(): args = LochnessArgs() args.source = ['xnat', 'box', 'redcap'] args.studies = ['StudyA'] args.dry = [False] config_string, fp = create_config() Lochness = mock_load(args.config, args.archive_base) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): # Subject(active=1, # study='StudyA', id='EXAMPLE', consent='1979-01-01', # beiwe=defaultdict(<class 'list'>, # {'beiwe': [('5432', 'abcde')]}), # icognition={}, # saliva={}, # xnat=defaultdict(<class 'list'>, # {'xnat.hcpep': [('HCPEP-BWH', '1001')]}), # redcap=defaultd ict(<class 'list'>, # {'redcap.hcpep': ['1001_1']}), # dropbox={}, # box=defaultdict(<class 'list'>, # {'box.mclean': ['O1234']}), # general_folder='./PHOENIX/GENERAL/StudyA/EXAMPLE', # protected_folder='./PHOENIX/PROTECTED/StudyA/EXAMPLE') assert subject.study == 'StudyA' assert subject.general_folder == './PHOENIX/GENERAL/StudyA/EXAMPLE' assert list(subject.xnat.keys())[0] == 'xnat.StudyA' assert list(subject.xnat.values())[0][0][0] == 'HCPEP-BWH'
def test_box_sync_module_protect_processed(args_and_Lochness): args, Lochness = args_and_Lochness # change protect to true for all actigraphy for study in args.studies: new_list = [] for i in Lochness['box'][study]['file_patterns']['actigraphy']: i['protect'] = True i['processed'] = True new_list.append(i) Lochness['box'][study]['file_patterns']['actigraphy'] = new_list for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False) for study in args.studies: subject_dir = protected_root / study / 'processed' / '1001' assert (subject_dir / 'actigraphy').is_dir() assert len(list((subject_dir / 'actigraphy').glob('*csv'))) > 1 subject_dir = general_root / study / 'processed' / '1001' assert not (subject_dir / 'actigraphy').is_dir() assert len(list((subject_dir / 'actigraphy/processed/').glob('*csv'))) == 0 show_tree_then_delete('tmp_lochness')
def do(args): # reload config every time Lochness = config.load(args.config, args.archive_base) # Lochness to Lochness transfer on the receiving side if args.lochness_sync_receive: lochness_to_lochness_transfer_receive(Lochness) return True # break the do function here for the receiving side # initialize (overwrite) metadata.csv using either REDCap or RPMS database if 'redcap' in args.source or 'rpms' in args.source: lochness.initialize_metadata(Lochness, args) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): if not subject.active and args.skip_inactive: logger.info(f'skipping inactive subject={subject.id}, ' f'study={subject.study}') continue if args.hdd: for Module in args.hdd: lochness.attempt(Module.sync, Lochness, subject, dry=args.dry) else: for Module in args.source: lochness.attempt(Module.sync, Lochness, subject, dry=args.dry) # transfer new files after all sync attempts are done if args.lochness_sync_send: lochness_to_lochness_transfer(Lochness)
def test_read_phoenix_data(): args = LochnessArgs() args.source = ['xnat', 'box'] args.studies = ['StudyA'] args.dry = [False] Lochness = mock_load(args.config, args.archive_base) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): print(subject)
def test_sync_from_empty(Lochness): dry=False study_name = 'StudyA' initialize_metadata(Lochness, study_name) for subject in lochness.read_phoenix_metadata(Lochness, studies=[study_name]): sync(Lochness, subject, dry) show_tree_then_delete('tmp_lochness')
def lochness_subject_raw_json(LochnessMetadataInitialized): for subject in lochness.read_phoenix_metadata(LochnessMetadataInitialized, studies=['StudyA']): if subject.id != 'subject_1': continue phoenix_path = Path(LochnessMetadataInitialized['phoenix_root']) subject_proc_p = phoenix_path / 'PROTECTED' / 'StudyA' / 'raw' / 'subject_1' raw_json = subject_proc_p / 'surveys' / f"{subject.id}.StudyA.json" return LochnessMetadataInitialized, subject, raw_json
def test_read_phoenix_data(args): syncArgs = SyncArgs('tmp_lochness') create_lochness_template(args) _ = KeyringAndEncrypt(args.outdir) args.source = ['xnat', 'box'] args.studies = ['mclean'] args.dry = [False] Lochness = config.load(syncArgs.config, syncArgs.archive_base) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): print(subject)
def test_box_sync_module_no_redownload(args_and_Lochness): args, Lochness = args_and_Lochness # change subject name for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False) a_file_path = general_root / 'StudyA' / 'raw' / '1001' / 'actigraphy' / \ 'BLS-F6VVM-GENEActivQC-day22to51.csv' init_time = a_file_path.stat().st_mtime # change subject name for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False) post_time = a_file_path.stat().st_mtime assert init_time == post_time show_tree_then_delete('tmp_lochness')
def do(args): # reload config every time Lochness = config.load(args.config, args.archive_base) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): if not subject.active and args.skip_inactive: logger.info('skipping inactive subject={0}, study={1}'.format(subject.id, subject.study)) continue if args.hdd: for Module in args.hdd: lochness.attempt(Module.sync, Lochness, subject, dry=args.dry) else: for Module in args.source: lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
def test_mindlamp_module(): args = LochnessArgs() args.source = ['xnat', 'mindlamp'] args.studies = ['StudyA'] args.dry = [False] config_string, fp = create_config() Lochness = mock_load(args.config, args.archive_base) for subject in lochness.read_phoenix_metadata(Lochness, args.studies): print(subject.study) for module in subject.mindlamp: assert module == 'mindlamp.StudyA' mindlamp.sync(Lochness, subject, dry=True)
def test_box_sync_module_missing_root(args_and_Lochness): args, Lochness = args_and_Lochness # change base for StudyA to missing path Lochness['box']['StudyA']['base'] = 'hahah' for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False) study = 'StudyA' subject_dir = protected_root / study / 'raw' / '1001' assert (subject_dir / 'actigraphy').is_dir() == False show_tree_then_delete('tmp_lochness')
def test_initialize_metadata_then_sync(args_and_Lochness): args, Lochness = args_and_Lochness # before initializing metadata for study in args.studies: phoenix_path = Path(Lochness['phoenix_root']) general_path = phoenix_path / 'GENERAL' metadata = general_path / study / f"{study}_metadata.csv" initialize_metadata(Lochness, study, 'record_id1', 'cons_date') for subject in lochness.read_phoenix_metadata(Lochness, studies=['StudyA']): sync(Lochness, subject, False) show_tree_then_delete('tmp_lochness')
def test_create_lochness_template(Lochness): create_fake_rpms_repo() # create_lochness_template(args) study_name = 'StudyA' initialize_metadata(Lochness, study_name, 'record_id1', 'Consent') for subject in lochness.read_phoenix_metadata(Lochness, studies=['StudyA']): # print(subject) for module in subject.rpms: print(module) print(module) print(module) # break # break show_tree_then_delete('tmp_lochness')
def test_sync_from_empty(args): outdir = 'tmp_lochness' args.outdir = outdir create_lochness_template(args) KeyringAndEncryptRPMS(args.outdir) create_fake_rpms_repo() dry = False study_name = 'StudyA' Lochness = config_load_test(f'{args.outdir}/config.yml', '') initialize_metadata(Lochness, study_name, 'record_id1', 'Consent') for subject in lochness.read_phoenix_metadata(Lochness, studies=['StudyA']): sync(Lochness, subject, dry) # print the structure show_tree_then_delete('tmp_lochness')
def test_box_module(): args = LochnessArgs() args.source = ['xnat', 'box', 'redcap'] args.studies = ['StudyA'] args.dry = [False] config_string, fp = create_config() Lochness = mock_load(args.config, args.archive_base) Lochness['keyring']['lochness']['SECRETS'] = {} Lochness['keyring']['lochness']['SECRETS']['StudyA'] = 'ha' for subject in lochness.read_phoenix_metadata(Lochness, args.studies): print(subject.study) for module in subject.box: print(module) print(module) try: box.sync_module(Lochness, subject, module, dry=True) except boxsdk.BoxAPIException as e: print('ha')
def test_box_sync_module_missing_subject(args_and_Lochness): args, Lochness = args_and_Lochness # change subject name keyring = KeyringAndEncrypt(args.outdir) information_to_add_to_metadata = {'box': { 'subject_id': '1001', 'source_id': 'O12341234'}} for study in args.studies: keyring.update_for_box(study) # update box metadata initialize_metadata_test('tmp_lochness/PHOENIX', study, information_to_add_to_metadata) for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False) show_tree_then_delete('tmp_lochness')
def test_sync(Lochness): for subject in lochness.read_phoenix_metadata(Lochness, studies=['StudyA']): sync(Lochness, subject, False)
def test_xnat_sync_module_default(args_and_Lochness): args, Lochness = args_and_Lochness for subject in lochness.read_phoenix_metadata(Lochness): sync(Lochness, subject, dry=False)