def test_anonymize(): """Test that sensitive information can be anonymized.""" pytest.raises(TypeError, anonymize_info, 'foo') # Fake some subject data raw = read_raw_fif(raw_fname) raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar', first_name='bar', birthday=(1987, 4, 8), sex=0, hand=1) orig_file_id = raw.info['file_id']['secs'] orig_meas_id = raw.info['meas_id']['secs'] # Test instance method events = read_events(event_name) epochs = Epochs(raw, events[:1], 2, 0., 0.1) for inst in [raw, epochs]: assert 'subject_info' in inst.info.keys() assert inst.info['subject_info'] is not None for key in ('file_id', 'meas_id'): assert inst.info[key]['secs'] != DATE_NONE[0] assert inst.info[key]['usecs'] != DATE_NONE[1] assert tuple(inst.info['meas_date']) != DATE_NONE inst.anonymize() assert 'subject_info' not in inst.info.keys() for key in ('file_id', 'meas_id'): assert inst.info[key]['secs'] == DATE_NONE[0] assert inst.info[key]['usecs'] == DATE_NONE[1] assert inst.info['meas_date'] is None # When we write out with raw.save, these get overwritten with the # new save time tempdir = _TempDir() out_fname = op.join(tempdir, 'test_subj_info_raw.fif') raw.save(out_fname, overwrite=True) raw = read_raw_fif(out_fname) assert raw.info.get('subject_info') is None assert raw.info['meas_date'] is None # XXX mne.io.write.write_id necessarily writes secs assert raw.info['file_id']['secs'] != orig_file_id assert raw.info['meas_id']['secs'] != orig_meas_id # Test no error for incomplete info info = raw.info.copy() info.pop('file_id') anonymize_info(info)
def test_anonymize(): """Test that sensitive information can be anonymized.""" assert_raises(ValueError, anonymize_info, 'foo') # Fake some subject data raw = read_raw_fif(raw_fname) raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar', first_name='bar', birthday=(1987, 4, 8), sex=0, hand=1) orig_file_id = raw.info['file_id']['secs'] orig_meas_id = raw.info['meas_id']['secs'] # Test instance method events = read_events(event_name) epochs = Epochs(raw, events[:1], 2, 0., 0.1) for inst in [raw, epochs]: assert_true('subject_info' in inst.info.keys()) assert_true(inst.info['subject_info'] is not None) assert_true(inst.info['file_id']['secs'] != 0) assert_true(inst.info['meas_id']['secs'] != 0) assert_true(np.any(inst.info['meas_date'] != [0, 0])) inst.anonymize() assert_true('subject_info' not in inst.info.keys()) assert_equal(inst.info['file_id']['secs'], 0) assert_equal(inst.info['meas_id']['secs'], 0) assert_array_equal(inst.info['meas_date'], [0, 0]) # When we write out with raw.save, these get overwritten with the # new save time tempdir = _TempDir() out_fname = op.join(tempdir, 'test_subj_info_raw.fif') raw.save(out_fname, overwrite=True) raw = read_raw_fif(out_fname) assert_true(raw.info.get('subject_info') is None) assert_array_equal(raw.info['meas_date'], [0, 0]) # XXX mne.io.write.write_id necessarily writes secs assert_true(raw.info['file_id']['secs'] != orig_file_id) assert_true(raw.info['meas_id']['secs'] != orig_meas_id) # Test no error for incomplete info info = raw.info.copy() info.pop('file_id') anonymize_info(info)
def test_anonymize(): """Checks that sensitive information can be anonymized.""" assert_raises(ValueError, anonymize_info, 'foo') # Fake some subject data raw = read_raw_fif(raw_fname) raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar', first_name='bar', birthday=(1987, 4, 8), sex=0, hand=1) orig_file_id = raw.info['file_id']['secs'] orig_meas_id = raw.info['meas_id']['secs'] # Test instance method events = read_events(event_name) epochs = Epochs(raw, events[:1], 2, 0., 0.1) for inst in [raw, epochs]: assert_true('subject_info' in inst.info.keys()) assert_true(inst.info['subject_info'] is not None) assert_true(inst.info['file_id']['secs'] != 0) assert_true(inst.info['meas_id']['secs'] != 0) assert_true(np.any(inst.info['meas_date'] != [0, 0])) inst.anonymize() assert_true('subject_info' not in inst.info.keys()) assert_equal(inst.info['file_id']['secs'], 0) assert_equal(inst.info['meas_id']['secs'], 0) assert_equal(inst.info['meas_date'], [0, 0]) # When we write out with raw.save, these get overwritten with the # new save time tempdir = _TempDir() out_fname = op.join(tempdir, 'test_subj_info_raw.fif') raw.save(out_fname, overwrite=True) raw = read_raw_fif(out_fname) assert_true(raw.info.get('subject_info') is None) assert_array_equal(raw.info['meas_date'], [0, 0]) # XXX mne.io.write.write_id necessarily writes secs assert_true(raw.info['file_id']['secs'] != orig_file_id) assert_true(raw.info['meas_id']['secs'] != orig_meas_id) # Test no error for incomplete info info = raw.info.copy() info.pop('file_id') anonymize_info(info)
def test_anonymize(): """Test that sensitive information can be anonymized.""" pytest.raises(TypeError, anonymize_info, 'foo') # Fake some subject data raw = read_raw_fif(raw_fname) raw.set_annotations( Annotations(onset=[0, 1], duration=[1, 1], description='dummy', orig_time=None)) raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar', first_name='bar', birthday=(1987, 4, 8), sex=0, hand=1) # Test no error for incomplete info info = raw.info.copy() info.pop('file_id') anonymize_info(info) # Test instance method events = read_events(event_name) epochs = Epochs(raw, events[:1], 2, 0., 0.1) assert not any(_is_anonymous(raw)) raw.anonymize() assert all(_is_anonymous(raw)) assert not any(_is_anonymous(epochs)[:-1]) # epochs has no annotations epochs.anonymize() assert all(_is_anonymous(epochs)[:-1]) # When we write out with raw.save, these get overwritten with the # new save time tempdir = _TempDir() out_fname = op.join(tempdir, 'test_subj_info_raw.fif') raw.save(out_fname, overwrite=True) assert all(_is_anonymous(read_raw_fif(out_fname)))
def test_anonymize(): """Test that sensitive information can be anonymized.""" pytest.raises(TypeError, anonymize_info, 'foo') # Fake some subject data raw = read_raw_fif(raw_fname) raw.set_annotations(Annotations(onset=[0, 1], duration=[1, 1], description='dummy', orig_time=None)) raw.info['subject_info'] = dict(id=1, his_id='foobar', last_name='bar', first_name='bar', birthday=(1987, 4, 8), sex=0, hand=1) # Test no error for incomplete info info = raw.info.copy() info.pop('file_id') anonymize_info(info) # Test instance method events = read_events(event_name) epochs = Epochs(raw, events[:1], 2, 0., 0.1) assert not any(_is_anonymous(raw)) raw.anonymize() assert all(_is_anonymous(raw)) assert not any(_is_anonymous(epochs)[:-1]) # epochs has no annotations epochs.anonymize() assert all(_is_anonymous(epochs)[:-1]) # When we write out with raw.save, these get overwritten with the # new save time tempdir = _TempDir() out_fname = op.join(tempdir, 'test_subj_info_raw.fif') raw.save(out_fname, overwrite=True) assert all(_is_anonymous(read_raw_fif(out_fname)))
def test_get_matched_empty_room(): """Test reading of empty room data.""" bids_root = _TempDir() raw = mne.io.read_raw_fif(raw_fname) bids_basename = make_bids_basename(subject='01', session='01', task='audiovisual', run='01') write_raw_bids(raw, bids_basename, bids_root, overwrite=True) er_basename = get_matched_empty_room(bids_basename=bids_basename, bids_root=bids_root) assert er_basename is None # testing data has no noise recording, so save the actual data # as if it were noise er_raw_fname = op.join(data_path, 'MEG', 'sample', 'ernoise_raw.fif') raw.crop(0, 10).save(er_raw_fname, overwrite=True) er_raw = mne.io.read_raw_fif(er_raw_fname) er_date = er_raw.info['meas_date'] if not isinstance(er_date, datetime): # mne < v0.20 er_date = datetime.fromtimestamp(er_raw.info['meas_date'][0]) er_date = er_date.strftime('%Y%m%d') er_bids_basename = make_bids_basename(subject='emptyroom', task='noise', session=er_date) write_raw_bids(er_raw, er_bids_basename, bids_root, overwrite=True) recovered_er_basename = get_matched_empty_room(bids_basename=bids_basename, bids_root=bids_root) assert er_bids_basename == recovered_er_basename # assert that we get best emptyroom if there are multiple available sh.rmtree(op.join(bids_root, 'sub-emptyroom')) dates = ['20021204', '20021201', '20021001'] for date in dates: er_bids_basename.update(session=date) er_meas_date = datetime.strptime(date, '%Y%m%d') er_meas_date = er_meas_date.replace(tzinfo=timezone.utc) if check_version('mne', '0.20'): er_raw.set_meas_date(er_meas_date) else: er_raw.info['meas_date'] = (er_meas_date.timestamp(), 0) write_raw_bids(er_raw, er_bids_basename, bids_root) best_er_basename = get_matched_empty_room(bids_basename=bids_basename, bids_root=bids_root) assert '20021204' in best_er_basename # assert that we get error if meas_date is not available. raw = read_raw_bids(bids_basename=bids_basename, bids_root=bids_root, kind='meg') if check_version('mne', '0.20'): raw.set_meas_date(None) else: raw.info['meas_date'] = None raw.annotations.orig_time = None anonymize_info(raw.info) write_raw_bids(raw, bids_basename, bids_root, overwrite=True) with pytest.raises(ValueError, match='The provided recording does not ' 'have a measurement date set'): get_matched_empty_room(bids_basename=bids_basename, bids_root=bids_root)
def test_find_empty_room(return_bids_test_dir): """Test reading of empty room data.""" data_path = testing.data_path() raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw.fif') bids_root = _TempDir() tmp_dir = _TempDir() raw = _read_raw_fif(raw_fname) bids_path = BIDSPath(subject='01', session='01', task='audiovisual', run='01', root=bids_root, suffix='meg') write_raw_bids(raw, bids_path, overwrite=True) # No empty-room data present. er_basename = bids_path.find_empty_room() assert er_basename is None # Now create data resembling an empty-room recording. # The testing data has no "noise" recording, so save the actual data # as named as if it were noise. We first need to write the FIFF file # before reading it back in. er_raw_fname = op.join(tmp_dir, 'ernoise_raw.fif') raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True) er_raw = _read_raw_fif(er_raw_fname) if not isinstance(er_raw.info['meas_date'], datetime): # mne < v0.20 er_date = datetime.fromtimestamp(er_raw.info['meas_date'][0]) else: er_date = er_raw.info['meas_date'] er_date = er_date.strftime('%Y%m%d') er_bids_path = BIDSPath(subject='emptyroom', task='noise', session=er_date, suffix='meg', root=bids_root) write_raw_bids(er_raw, er_bids_path, overwrite=True) recovered_er_bids_path = bids_path.find_empty_room() assert er_bids_path == recovered_er_bids_path # assert that we get best emptyroom if there are multiple available sh.rmtree(op.join(bids_root, 'sub-emptyroom')) dates = ['20021204', '20021201', '20021001'] for date in dates: er_bids_path.update(session=date) er_meas_date = datetime.strptime(date, '%Y%m%d') er_meas_date = er_meas_date.replace(tzinfo=timezone.utc) if check_version('mne', '0.20'): er_raw.set_meas_date(er_meas_date) else: er_raw.info['meas_date'] = (er_meas_date.timestamp(), 0) write_raw_bids(er_raw, er_bids_path) best_er_basename = bids_path.find_empty_room() assert best_er_basename.session == '20021204' with pytest.raises(ValueError, match='The root of the "bids_path" must be set'): bids_path.copy().update(root=None).find_empty_room() # assert that we get error if meas_date is not available. raw = read_raw_bids(bids_path=bids_path) if check_version('mne', '0.20'): raw.set_meas_date(None) else: raw.info['meas_date'] = None raw.annotations.orig_time = None anonymize_info(raw.info) write_raw_bids(raw, bids_path, overwrite=True) with pytest.raises(ValueError, match='The provided recording does not ' 'have a measurement date set'): bids_path.find_empty_room()
def test_find_empty_room(return_bids_test_dir, tmpdir): """Test reading of empty room data.""" data_path = testing.data_path() raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw.fif') bids_root = tmpdir.mkdir("bids") tmp_dir = tmpdir.mkdir("tmp") raw = _read_raw_fif(raw_fname) bids_path = BIDSPath(subject='01', session='01', task='audiovisual', run='01', root=bids_root, suffix='meg') write_raw_bids(raw, bids_path, overwrite=True) # No empty-room data present. er_basename = bids_path.find_empty_room() assert er_basename is None # Now create data resembling an empty-room recording. # The testing data has no "noise" recording, so save the actual data # as named as if it were noise. We first need to write the FIFF file # before reading it back in. er_raw_fname = op.join(tmp_dir, 'ernoise_raw.fif') raw.copy().crop(0, 10).save(er_raw_fname, overwrite=True) er_raw = _read_raw_fif(er_raw_fname) if not isinstance(er_raw.info['meas_date'], datetime): # pragma: no cover # mne < v0.20 er_date = datetime.fromtimestamp(er_raw.info['meas_date'][0]) else: er_date = er_raw.info['meas_date'] er_date = er_date.strftime('%Y%m%d') er_bids_path = BIDSPath(subject='emptyroom', task='noise', session=er_date, suffix='meg', root=bids_root) write_raw_bids(er_raw, er_bids_path, overwrite=True) recovered_er_bids_path = bids_path.find_empty_room() assert er_bids_path == recovered_er_bids_path # assert that we get best emptyroom if there are multiple available sh.rmtree(op.join(bids_root, 'sub-emptyroom')) dates = ['20021204', '20021201', '20021001'] for date in dates: er_bids_path.update(session=date) er_meas_date = datetime.strptime(date, '%Y%m%d') er_meas_date = er_meas_date.replace(tzinfo=timezone.utc) er_raw.set_meas_date(er_meas_date) write_raw_bids(er_raw, er_bids_path) best_er_basename = bids_path.find_empty_room() assert best_er_basename.session == '20021204' with pytest.raises(ValueError, match='The root of the "bids_path" must be set'): bids_path.copy().update(root=None).find_empty_room() # assert that we get an error if meas_date is not available. raw = read_raw_bids(bids_path=bids_path) raw.set_meas_date(None) anonymize_info(raw.info) write_raw_bids(raw, bids_path, overwrite=True) with pytest.raises(ValueError, match='The provided recording does not ' 'have a measurement date set'): bids_path.find_empty_room() # test that the `AssociatedEmptyRoom` key in MEG sidecar is respected bids_root = tmpdir.mkdir('associated-empty-room') raw = _read_raw_fif(raw_fname) meas_date = datetime(year=2020, month=1, day=10, tzinfo=timezone.utc) er_date = datetime(year=2010, month=1, day=1, tzinfo=timezone.utc) raw.set_meas_date(meas_date) er_raw_matching_date = er_raw.copy().set_meas_date(meas_date) er_raw_associated = er_raw.copy().set_meas_date(er_date) # First write empty-room data # We write two empty-room recordings: one with a date matching exactly the # experimental measurement date, and one dated approx. 10 years earlier # We will want to enforce using the older recording via # `AssociatedEmptyRoom` (without AssociatedEmptyRoom, find_empty_room() # would return the recording with the matching date instead) er_matching_date_bids_path = BIDSPath(subject='emptyroom', session='20200110', task='noise', root=bids_root, datatype='meg', suffix='meg', extension='.fif') write_raw_bids(er_raw_matching_date, bids_path=er_matching_date_bids_path) er_associated_bids_path = (er_matching_date_bids_path.copy().update( session='20100101')) write_raw_bids(er_raw_associated, bids_path=er_associated_bids_path) # Now we write experimental data and associate it with the earlier # empty-room recording bids_path = (er_matching_date_bids_path.copy().update(subject='01', session=None, task='task')) write_raw_bids(raw, bids_path=bids_path, empty_room=er_associated_bids_path) # Retrieve empty-room BIDSPath assert bids_path.find_empty_room() == er_associated_bids_path # Should only work for MEG with pytest.raises(ValueError, match='only supported for MEG'): bids_path.copy().update(datatype='eeg').find_empty_room() # Don't create `AssociatedEmptyRoom` entry in sidecar – we should now # retrieve the empty-room recording closer in time write_raw_bids(raw, bids_path=bids_path, empty_room=None, overwrite=True) assert bids_path.find_empty_room() == er_matching_date_bids_path
def copyfile_brainvision(vhdr_src, vhdr_dest, anonymize=None, verbose=False): """Copy a BrainVision file triplet to a new location and repair links. The BrainVision file format consists of three files: .vhdr, .eeg, and .vmrk The .eeg and .vmrk files associated with the .vhdr file will be given names as in `vhdr_dest` with adjusted extensions. Internal file pointers will be fixed. Parameters ---------- vhdr_src : str The src path of the .vhdr file to be copied. vhdr_dest : str The destination path of the .vhdr file. anonymize : dict | None If None (default), no anonymization is performed. If dict, data will be anonymized depending on the keys provided with the dict: `daysback` is a required key, `keep_his` is an optional key. `daysback` : int Number of days by which to move back the recording date in time. In studies with multiple subjects the relative recording date differences between subjects can be kept by using the same number of `daysback` for all subject anonymizations. `daysback` should be great enough to shift the date prior to 1925 to conform with BIDS anonymization rules. `keep_his` : bool By default (False), all subject information next to the recording date will be overwritten as well. If True, keep subject information apart from the recording date. verbose : bool Determine whether results should be logged. Defaults to False. See Also -------- mne.io.anonymize_info """ # Get extenstion of the brainvision file fname_src, ext_src = _parse_ext(vhdr_src) fname_dest, ext_dest = _parse_ext(vhdr_dest) if ext_src != ext_dest: raise ValueError(f'Need to move data with same extension' f' but got "{ext_src}", "{ext_dest}"') eeg_file_path, vmrk_file_path = _get_brainvision_paths(vhdr_src) # extract encoding from brainvision header file, or default to utf-8 enc = _get_brainvision_encoding(vhdr_src, verbose) # Copy data .eeg ... no links to repair sh.copyfile(eeg_file_path, fname_dest + '.eeg') # Write new header and marker files, fixing the file pointer links # For that, we need to replace an old "basename" with a new one # assuming that all .eeg, .vhdr, .vmrk share one basename __, basename_src = op.split(fname_src) assert basename_src + '.eeg' == op.split(eeg_file_path)[-1] assert basename_src + '.vmrk' == op.split(vmrk_file_path)[-1] __, basename_dest = op.split(fname_dest) search_lines = ['DataFile=' + basename_src + '.eeg', 'MarkerFile=' + basename_src + '.vmrk'] with open(vhdr_src, 'r', encoding=enc) as fin: with open(vhdr_dest, 'w', encoding=enc) as fout: for line in fin.readlines(): if line.strip() in search_lines: line = line.replace(basename_src, basename_dest) fout.write(line) with open(vmrk_file_path, 'r', encoding=enc) as fin: with open(fname_dest + '.vmrk', 'w', encoding=enc) as fout: for line in fin.readlines(): if line.strip() in search_lines: line = line.replace(basename_src, basename_dest) fout.write(line) if anonymize is not None: raw = read_raw_brainvision(vhdr_src, preload=False, verbose=0) daysback, keep_his = _check_anonymize(anonymize, raw, '.vhdr') raw.info = anonymize_info(raw.info, daysback=daysback, keep_his=keep_his) _anonymize_brainvision(fname_dest + '.vhdr', date=raw.info['meas_date']) if verbose: for ext in ['.eeg', '.vhdr', '.vmrk']: _, fname = os.path.split(fname_dest + ext) dirname = op.dirname(op.realpath(vhdr_dest)) print(f'Created "{fname}" in "{dirname}".') if anonymize: print('Anonymized all dates in VHDR and VMRK.')
def copyfile_edf(src, dest, anonymize=None): """Copy an EDF, EDF+, or BDF file to a new location, optionally anonymize. .. warning:: EDF/EDF+/BDF files contain two fields for recording dates: A generic "startdate" field that supports only 2-digit years, and a "Startdate" field as part of the "local recording identification", which supports 4-digit years. If you want to anonymize your file, MNE-BIDS will set the "startdate" field to 85 (i.e., 1985), the earliest possible date for that field. However, the "Startdate" field in the file's "local recording identification" and the date in the session's corresponding ``scans.tsv`` will be set correctly according to the argument provided to the ``anonymize`` parameter. Note that it is possible that not all EDF/EDF+/BDF reading software parses the accurate recording date, and that for some reading software, the wrong year (1985) may be parsed. Parameters ---------- src : str | pathlib.Path The source path of the .edf or .bdf file to be copied. dest : str | pathlib.Path The destination path of the .edf or .bdf file. anonymize : dict | None If None (default), no anonymization is performed. If dict, data will be anonymized depending on the keys provided with the dict: `daysback` is a required key, `keep_his` is an optional key. `daysback` : int Number of days by which to move back the recording date in time. In studies with multiple subjects the relative recording date differences between subjects can be kept by using the same number of `daysback` for all subject anonymizations. `daysback` should be great enough to shift the date prior to 1925 to conform with BIDS anonymization rules. Due to limitations of the EDF/BDF format, the year of the anonymized date will always be set to 1985 in the 'startdate' field of the file. The correctly-shifted year will be written to the 'local recording identification' region of the file header, which may not be parsed by all EDF/EDF+/BDF reader softwares. `keep_his` : bool By default (False), all subject information next to the recording date will be overwritten as well. If True, keep subject information apart from the recording date. Participant names and birthdates will always be anonymized if present, regardless of this setting. See Also -------- mne.io.anonymize_info copyfile_brainvision copyfile_bti copyfile_ctf copyfile_eeglab copyfile_kit """ # Ensure source & destination extensions are the same fname_src, ext_src = _parse_ext(src) fname_dest, ext_dest = _parse_ext(dest) if ext_src != ext_dest: raise ValueError(f'Need to move data with same extension, ' f' but got "{ext_src}" and "{ext_dest}"') # Copy data prior to any anonymization sh.copyfile(src, dest) # Anonymize EDF/BDF data, if requested if anonymize is not None: if ext_src == '.bdf': raw = read_raw_bdf(dest, preload=False, verbose=0) elif ext_src == '.edf': raw = read_raw_edf(dest, preload=False, verbose=0) else: raise ValueError('Unsupported file type ({0})'.format(ext_src)) # Get subject info, recording info, and recording date with open(dest, 'rb') as f: f.seek(8) # id_info field starts 8 bytes in id_info = f.read(80).decode('ascii').rstrip() rec_info = f.read(80).decode('ascii').rstrip() # Parse metadata from file if len(id_info) == 0 or len(id_info.split(' ')) != 4: id_info = "X X X X" if len(rec_info) == 0 or len(rec_info.split(' ')) != 5: rec_info = "Startdate X X X X" pid, sex, birthdate, name = id_info.split(' ') start_date, admin_code, tech, equip = rec_info.split(' ')[1:5] # Try to anonymize the recording date daysback, keep_his = _check_anonymize(anonymize, raw, '.edf') anonymize_info(raw.info, daysback=daysback, keep_his=keep_his) start_date = '01-JAN-1985' meas_date = '01.01.85' # Anonymize ID info and write to file if keep_his: # Always remove participant birthdate and name to be safe id_info = [pid, sex, "X", "X"] rec_info = ["Startdate", start_date, admin_code, tech, equip] else: id_info = ["0", "X", "X", "X"] rec_info = ["Startdate", start_date, "X", "mne-bids_anonymize", "X"] with open(dest, 'r+b') as f: f.seek(8) # id_info field starts 8 bytes in f.write(bytes(" ".join(id_info).ljust(80), 'ascii')) f.write(bytes(" ".join(rec_info).ljust(80), 'ascii')) f.write(bytes(meas_date, 'ascii'))