def test_expand_study_id_conflict(): cassette = os.path.join(CASSETTES, 'studies.v1.yaml') filter_params = [('access_key', Keyring['ACCESS_KEY']), ('secret_key', Keyring['SECRET_KEY']), ('study_id', 'STUDY_ID')] with vcr.use_cassette(cassette, decode_compressed_response=True, filter_post_data_parameters=filter_params) as cass: with pytest.raises(mano.AmbiguousStudyIDError): _ = mano.expand_study_id(Keyring, '123')
def test_expand_study_id_nomatch(): cassette = os.path.join(CASSETTES, 'studies.v1.yaml') filter_params = [('access_key', Keyring['ACCESS_KEY']), ('secret_key', Keyring['SECRET_KEY']), ('study_id', 'STUDY_ID')] with vcr.use_cassette(cassette, decode_compressed_response=True, filter_post_data_parameters=filter_params) as cass: ans = mano.expand_study_id(Keyring, '321') assert ans == None
def test_expand_study_id(): cassette = os.path.join(CASSETTES, 'studies.v1.yaml') filter_params = [('access_key', Keyring['ACCESS_KEY']), ('secret_key', Keyring['SECRET_KEY']), ('study_id', 'STUDY_ID')] with vcr.use_cassette(cassette, decode_compressed_response=True, filter_post_data_parameters=filter_params) as cass: study = ('Project A', '123lrVdb0g6tf3PeJr5ZtZC8') ans = mano.expand_study_id(Keyring, '123lrVdb0g6tf3PeJr5ZtZC8') assert ans == study
def sync(Lochness, subject, dry=False): ''' Sync beiwe data ''' logger.debug('exploring {0}/{1}'.format(subject.study, subject.id)) backfill_start = Lochness['beiwe'].get('backfill_start', BACKFILL_START_FALLBACK) if str(backfill_start).strip().lower() == 'consent': backfill_start = subject.consent for alias, beiwe_uids in iter(subject.beiwe.items()): logger.debug('getting {0} from keyring'.format(alias)) Keyring = mano.keyring(alias, keyring_file=Lochness['keyring_file'], passphrase=config.load.passphrase) base_url = Keyring['URL'] for uid in beiwe_uids: study_frag, beiwe_id = uid study_name, study_id = mano.expand_study_id(Keyring, study_frag) study_name_enc = study_name.encode('utf-8') dst_general_folder = tree.get('phone', subject.general_folder, beiwe_id=beiwe_id, BIDS=Lochness['BIDS']) dst_protected_folder = tree.get('phone', subject.protected_folder, beiwe_id=beiwe_id, BIDS=Lochness['BIDS']) # save a hidden file with the study id, original name, and sanitized name save_study_file(dst_general_folder, study_id, study_name) protected_streams = set(PROTECT) general_streams = set(mano.DATA_STREAMS) - protected_streams # begin backfill download of all GENERAL data streams logger.info( 'backfill general streams for subject={0}, study={1}, url={2}'. format(beiwe_id, study_name_enc, base_url)) mano.sync.backfill(Keyring, study_id, beiwe_id, os.path.dirname(dst_general_folder), start_date=str(backfill_start), data_streams=general_streams) # begin backfill download of all PROTECTED data streams passphrase = Lochness['keyring']['lochness']['SECRETS'].get( subject.study, None) logger.info( 'backfill protected streams for subject={0}, study={1}, url={2}' .format(beiwe_id, study_name_enc, base_url)) mano.sync.backfill(Keyring, study_id, beiwe_id, os.path.dirname(dst_protected_folder), start_date=str(backfill_start), data_streams=protected_streams, lock=protected_streams, passphrase=passphrase) # begin delta download of all GENERAL data streams registry = None registry_file = os.path.join(dst_general_folder, '.registry') if os.path.exists(registry_file): logger.debug( 'reading in registry file {0}'.format(registry_file)) with open(registry_file) as fo: registry = fo.read() else: logger.warn( 'no registry file on disk {0}'.format(registry_file)) logger.info( 'delta download of general data streams for subject={0}, study={1}, url={2}' .format(beiwe_id, study_name_enc, base_url)) archive = mano.sync.download(Keyring, study_id, beiwe_id, data_streams=general_streams, registry=registry) mano.sync.save(Keyring, archive, beiwe_id, os.path.dirname(dst_general_folder)) # begin delta download of all PROTECTED streams registry = None registry_file = os.path.join(dst_protected_folder, '.registry') if os.path.exists(registry_file): logger.debug('reading registry file {0}'.format(registry_file)) with open(registry_file) as fo: registry = fo.read() else: logger.warn( 'no registry file on disk {0}'.format(registry_file)) logger.info( 'delta download of protected data streams for subject={0}, study={1}, url={2}' .format(beiwe_id, study_name_enc, base_url)) archive = mano.sync.download(Keyring, study_id, beiwe_id, data_streams=protected_streams, registry=registry) mano.sync.save(Keyring, archive, beiwe_id, os.path.dirname(dst_protected_folder), lock=protected_streams, passphrase=passphrase)