def test_read_phoenix_metadata():
args = LochnessArgs()
args.source = ['xnat', 'box', 'redcap']
args.studies = ['StudyA']
args.dry = [False]
config_string, fp = create_config()
Lochness = mock_load(args.config, args.archive_base)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
# Subject(active=1,
# study='StudyA', id='EXAMPLE', consent='1979-01-01',
# beiwe=defaultdict(<class 'list'>,
# {'beiwe': [('5432', 'abcde')]}),
# icognition={},
# saliva={},
# xnat=defaultdict(<class 'list'>,
# {'xnat.hcpep': [('HCPEP-BWH', '1001')]}),
# redcap=defaultd ict(<class 'list'>,
# {'redcap.hcpep': ['1001_1']}),
# dropbox={},
# box=defaultdict(<class 'list'>,
# {'box.mclean': ['O1234']}),
# general_folder='./PHOENIX/GENERAL/StudyA/EXAMPLE',
# protected_folder='./PHOENIX/PROTECTED/StudyA/EXAMPLE')
assert subject.study == 'StudyA'
assert subject.general_folder == './PHOENIX/GENERAL/StudyA/EXAMPLE'
assert list(subject.xnat.keys())[0] == 'xnat.StudyA'
assert list(subject.xnat.values())[0][0][0] == 'HCPEP-BWH'
def test_box_sync_module_protect_processed(args_and_Lochness):
args, Lochness = args_and_Lochness
# change protect to true for all actigraphy
for study in args.studies:
new_list = []
for i in Lochness['box'][study]['file_patterns']['actigraphy']:
i['protect'] = True
i['processed'] = True
new_list.append(i)
Lochness['box'][study]['file_patterns']['actigraphy'] = new_list
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
for study in args.studies:
subject_dir = protected_root / study / 'processed' / '1001'
assert (subject_dir / 'actigraphy').is_dir()
assert len(list((subject_dir /
'actigraphy').glob('*csv'))) > 1
subject_dir = general_root / study / 'processed' / '1001'
assert not (subject_dir / 'actigraphy').is_dir()
assert len(list((subject_dir /
'actigraphy/processed/').glob('*csv'))) == 0
show_tree_then_delete('tmp_lochness')
def do(args):
# reload config every time
Lochness = config.load(args.config, args.archive_base)
# Lochness to Lochness transfer on the receiving side
if args.lochness_sync_receive:
lochness_to_lochness_transfer_receive(Lochness)
return True # break the do function here for the receiving side
# initialize (overwrite) metadata.csv using either REDCap or RPMS database
if 'redcap' in args.source or 'rpms' in args.source:
lochness.initialize_metadata(Lochness, args)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
if not subject.active and args.skip_inactive:
logger.info(f'skipping inactive subject={subject.id}, '
f'study={subject.study}')
continue
if args.hdd:
for Module in args.hdd:
lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
else:
for Module in args.source:
lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
# transfer new files after all sync attempts are done
if args.lochness_sync_send:
lochness_to_lochness_transfer(Lochness)
def test_read_phoenix_data():
args = LochnessArgs()
args.source = ['xnat', 'box']
args.studies = ['StudyA']
args.dry = [False]
Lochness = mock_load(args.config, args.archive_base)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
print(subject)
def test_sync_from_empty(Lochness):
dry=False
study_name = 'StudyA'
initialize_metadata(Lochness, study_name)
for subject in lochness.read_phoenix_metadata(Lochness,
studies=[study_name]):
sync(Lochness, subject, dry)
show_tree_then_delete('tmp_lochness')
def lochness_subject_raw_json(LochnessMetadataInitialized):
for subject in lochness.read_phoenix_metadata(LochnessMetadataInitialized,
studies=['StudyA']):
if subject.id != 'subject_1':
continue
phoenix_path = Path(LochnessMetadataInitialized['phoenix_root'])
subject_proc_p = phoenix_path / 'PROTECTED' / 'StudyA' / 'raw' / 'subject_1'
raw_json = subject_proc_p / 'surveys' / f"{subject.id}.StudyA.json"
return LochnessMetadataInitialized, subject, raw_json
def test_read_phoenix_data(args):
syncArgs = SyncArgs('tmp_lochness')
create_lochness_template(args)
_ = KeyringAndEncrypt(args.outdir)
args.source = ['xnat', 'box']
args.studies = ['mclean']
args.dry = [False]
Lochness = config.load(syncArgs.config, syncArgs.archive_base)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
print(subject)
def test_box_sync_module_no_redownload(args_and_Lochness):
args, Lochness = args_and_Lochness
# change subject name
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
a_file_path = general_root / 'StudyA' / 'raw' / '1001' / 'actigraphy' / \
'BLS-F6VVM-GENEActivQC-day22to51.csv'
init_time = a_file_path.stat().st_mtime
# change subject name
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
post_time = a_file_path.stat().st_mtime
assert init_time == post_time
show_tree_then_delete('tmp_lochness')
def do(args):
# reload config every time
Lochness = config.load(args.config, args.archive_base)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
if not subject.active and args.skip_inactive:
logger.info('skipping inactive subject={0}, study={1}'.format(subject.id, subject.study))
continue
if args.hdd:
for Module in args.hdd:
lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
else:
for Module in args.source:
lochness.attempt(Module.sync, Lochness, subject, dry=args.dry)
def test_mindlamp_module():
args = LochnessArgs()
args.source = ['xnat', 'mindlamp']
args.studies = ['StudyA']
args.dry = [False]
config_string, fp = create_config()
Lochness = mock_load(args.config, args.archive_base)
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
print(subject.study)
for module in subject.mindlamp:
assert module == 'mindlamp.StudyA'
mindlamp.sync(Lochness, subject, dry=True)
def test_box_sync_module_missing_root(args_and_Lochness):
args, Lochness = args_and_Lochness
# change base for StudyA to missing path
Lochness['box']['StudyA']['base'] = 'hahah'
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
study = 'StudyA'
subject_dir = protected_root / study / 'raw' / '1001'
assert (subject_dir / 'actigraphy').is_dir() == False
show_tree_then_delete('tmp_lochness')
def test_initialize_metadata_then_sync(args_and_Lochness):
args, Lochness = args_and_Lochness
# before initializing metadata
for study in args.studies:
phoenix_path = Path(Lochness['phoenix_root'])
general_path = phoenix_path / 'GENERAL'
metadata = general_path / study / f"{study}_metadata.csv"
initialize_metadata(Lochness, study, 'record_id1', 'cons_date')
for subject in lochness.read_phoenix_metadata(Lochness,
studies=['StudyA']):
sync(Lochness, subject, False)
show_tree_then_delete('tmp_lochness')
def test_create_lochness_template(Lochness):
create_fake_rpms_repo()
# create_lochness_template(args)
study_name = 'StudyA'
initialize_metadata(Lochness, study_name, 'record_id1', 'Consent')
for subject in lochness.read_phoenix_metadata(Lochness,
studies=['StudyA']):
# print(subject)
for module in subject.rpms:
print(module)
print(module)
print(module)
# break
# break
show_tree_then_delete('tmp_lochness')
def test_sync_from_empty(args):
outdir = 'tmp_lochness'
args.outdir = outdir
create_lochness_template(args)
KeyringAndEncryptRPMS(args.outdir)
create_fake_rpms_repo()
dry = False
study_name = 'StudyA'
Lochness = config_load_test(f'{args.outdir}/config.yml', '')
initialize_metadata(Lochness, study_name, 'record_id1', 'Consent')
for subject in lochness.read_phoenix_metadata(Lochness,
studies=['StudyA']):
sync(Lochness, subject, dry)
# print the structure
show_tree_then_delete('tmp_lochness')
def test_box_module():
args = LochnessArgs()
args.source = ['xnat', 'box', 'redcap']
args.studies = ['StudyA']
args.dry = [False]
config_string, fp = create_config()
Lochness = mock_load(args.config, args.archive_base)
Lochness['keyring']['lochness']['SECRETS'] = {}
Lochness['keyring']['lochness']['SECRETS']['StudyA'] = 'ha'
for subject in lochness.read_phoenix_metadata(Lochness, args.studies):
print(subject.study)
for module in subject.box:
print(module)
print(module)
try:
box.sync_module(Lochness, subject, module, dry=True)
except boxsdk.BoxAPIException as e:
print('ha')
def test_box_sync_module_missing_subject(args_and_Lochness):
args, Lochness = args_and_Lochness
# change subject name
keyring = KeyringAndEncrypt(args.outdir)
information_to_add_to_metadata = {'box': {
'subject_id': '1001',
'source_id': 'O12341234'}}
for study in args.studies:
keyring.update_for_box(study)
# update box metadata
initialize_metadata_test('tmp_lochness/PHOENIX', study,
information_to_add_to_metadata)
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
show_tree_then_delete('tmp_lochness')
def test_sync(Lochness):
for subject in lochness.read_phoenix_metadata(Lochness,
studies=['StudyA']):
sync(Lochness, subject, False)
def test_xnat_sync_module_default(args_and_Lochness):
args, Lochness = args_and_Lochness
for subject in lochness.read_phoenix_metadata(Lochness):
sync(Lochness, subject, dry=False)
