def test_expand_study_id_conflict():
cassette = os.path.join(CASSETTES, 'studies.v1.yaml')
filter_params = [('access_key', Keyring['ACCESS_KEY']),
('secret_key', Keyring['SECRET_KEY']),
('study_id', 'STUDY_ID')]
with vcr.use_cassette(cassette,
decode_compressed_response=True,
filter_post_data_parameters=filter_params) as cass:
with pytest.raises(mano.AmbiguousStudyIDError):
_ = mano.expand_study_id(Keyring, '123')
def test_expand_study_id_nomatch():
cassette = os.path.join(CASSETTES, 'studies.v1.yaml')
filter_params = [('access_key', Keyring['ACCESS_KEY']),
('secret_key', Keyring['SECRET_KEY']),
('study_id', 'STUDY_ID')]
with vcr.use_cassette(cassette,
decode_compressed_response=True,
filter_post_data_parameters=filter_params) as cass:
ans = mano.expand_study_id(Keyring, '321')
assert ans == None
def test_expand_study_id():
cassette = os.path.join(CASSETTES, 'studies.v1.yaml')
filter_params = [('access_key', Keyring['ACCESS_KEY']),
('secret_key', Keyring['SECRET_KEY']),
('study_id', 'STUDY_ID')]
with vcr.use_cassette(cassette,
decode_compressed_response=True,
filter_post_data_parameters=filter_params) as cass:
study = ('Project A', '123lrVdb0g6tf3PeJr5ZtZC8')
ans = mano.expand_study_id(Keyring, '123lrVdb0g6tf3PeJr5ZtZC8')
assert ans == study
def sync(Lochness, subject, dry=False):
'''
Sync beiwe data
'''
logger.debug('exploring {0}/{1}'.format(subject.study, subject.id))
backfill_start = Lochness['beiwe'].get('backfill_start',
BACKFILL_START_FALLBACK)
if str(backfill_start).strip().lower() == 'consent':
backfill_start = subject.consent
for alias, beiwe_uids in iter(subject.beiwe.items()):
logger.debug('getting {0} from keyring'.format(alias))
Keyring = mano.keyring(alias,
keyring_file=Lochness['keyring_file'],
passphrase=config.load.passphrase)
base_url = Keyring['URL']
for uid in beiwe_uids:
study_frag, beiwe_id = uid
study_name, study_id = mano.expand_study_id(Keyring, study_frag)
study_name_enc = study_name.encode('utf-8')
dst_general_folder = tree.get('phone',
subject.general_folder,
beiwe_id=beiwe_id,
BIDS=Lochness['BIDS'])
dst_protected_folder = tree.get('phone',
subject.protected_folder,
beiwe_id=beiwe_id,
BIDS=Lochness['BIDS'])
# save a hidden file with the study id, original name, and sanitized name
save_study_file(dst_general_folder, study_id, study_name)
protected_streams = set(PROTECT)
general_streams = set(mano.DATA_STREAMS) - protected_streams
# begin backfill download of all GENERAL data streams
logger.info(
'backfill general streams for subject={0}, study={1}, url={2}'.
format(beiwe_id, study_name_enc, base_url))
mano.sync.backfill(Keyring,
study_id,
beiwe_id,
os.path.dirname(dst_general_folder),
start_date=str(backfill_start),
data_streams=general_streams)
# begin backfill download of all PROTECTED data streams
passphrase = Lochness['keyring']['lochness']['SECRETS'].get(
subject.study, None)
logger.info(
'backfill protected streams for subject={0}, study={1}, url={2}'
.format(beiwe_id, study_name_enc, base_url))
mano.sync.backfill(Keyring,
study_id,
beiwe_id,
os.path.dirname(dst_protected_folder),
start_date=str(backfill_start),
data_streams=protected_streams,
lock=protected_streams,
passphrase=passphrase)
# begin delta download of all GENERAL data streams
registry = None
registry_file = os.path.join(dst_general_folder, '.registry')
if os.path.exists(registry_file):
logger.debug(
'reading in registry file {0}'.format(registry_file))
with open(registry_file) as fo:
registry = fo.read()
else:
logger.warn(
'no registry file on disk {0}'.format(registry_file))
logger.info(
'delta download of general data streams for subject={0}, study={1}, url={2}'
.format(beiwe_id, study_name_enc, base_url))
archive = mano.sync.download(Keyring,
study_id,
beiwe_id,
data_streams=general_streams,
registry=registry)
mano.sync.save(Keyring, archive, beiwe_id,
os.path.dirname(dst_general_folder))
# begin delta download of all PROTECTED streams
registry = None
registry_file = os.path.join(dst_protected_folder, '.registry')
if os.path.exists(registry_file):
logger.debug('reading registry file {0}'.format(registry_file))
with open(registry_file) as fo:
registry = fo.read()
else:
logger.warn(
'no registry file on disk {0}'.format(registry_file))
logger.info(
'delta download of protected data streams for subject={0}, study={1}, url={2}'
.format(beiwe_id, study_name_enc, base_url))
archive = mano.sync.download(Keyring,
study_id,
beiwe_id,
data_streams=protected_streams,
registry=registry)
mano.sync.save(Keyring,
archive,
beiwe_id,
os.path.dirname(dst_protected_folder),
lock=protected_streams,
passphrase=passphrase)