def get_model_file(setname, type, steps=None): filename = type + ".model" if steps is None: filename = type else: filename = "%s-%s" % (type, str(steps)) return os.path.join(utils.set_directory(setname), filename + ".model")
}; var tf = new TableFilter('results', filtersConfig); tf.init(); for (div of document.getElementsByClassName("div_checklist")) { div.style.height = 100; } </script> </body> </html> ''' html_table = '' set_directory('html/dics') image_folder = 'dics' image_path = op.join('html', image_folder) set_directory(image_path) for i, setting in enumerate(dics_settings): # construct query setting = tuple(['none' if s is None else s for s in setting]) reg, sensor_type, pick_ori, inversion, weight_norm, normalize_fwd, real_filter, use_noise_cov, reduce_rank = setting q = ( f"reg=={reg:.2f} and sensor_type=='{sensor_type}' and pick_ori=='{pick_ori}' and inversion=='{inversion}' and real_filter=={real_filter} and " f"weight_norm=='{weight_norm}' and normalize_fwd=={normalize_fwd} and use_noise_cov=={use_noise_cov} and reduce_rank=={reduce_rank}" )
def create_epochs(raw, title='Simulated evoked', fn_simulated_epochs=None, fn_report_h5=None): """ Create epochs based on the raw object with the baseline going from config.tmin to config.tmax Parameters: ----------- raw : instance of Raw Simulated raw file. fn_simulated_raw : None | string Path where the epochs file is to be saved. If None the file is not saved. fn_report_h5 : None | string Path where the .h5 file for the report is to be saved. Returns: -------- epochs : instance of Epochs Epochs created from the simulated raw. """ sfreq = raw.info['sfreq'] trial_length = int((config.tmax - config.tmin) * sfreq) events = np.hstack(( (np.arange(config.n_trials) * trial_length)[:, np.newaxis] - int(config.tmin * sfreq), np.zeros((config.n_trials, 1)), np.ones((config.n_trials, 1)), )).astype(np.int) # Use tmin=0.1 and tmax=trial_length - 0.1 to avoid edge artifacts epochs = mne.Epochs(raw=raw, events=events, event_id=1, tmin=config.tmin + 1 / sfreq, tmax=config.tmax - 1 / sfreq, baseline=(None, 0), preload=True) ############################################################################### # Save everything ############################################################################### if fn_simulated_epochs is not None: set_directory(op.dirname(fn_simulated_epochs)) epochs.save(fn_simulated_epochs, overwrite=True) # Plot the simulated epochs in the report if fn_report_h5 is not None: set_directory(op.dirname(fn_report_h5)) fn_report_html = fn_report_h5.rsplit('.h5')[0] + '.html' with mne.open_report(fn_report_h5) as report: fig = epochs.average().plot_joint(picks='mag', show=False) report.add_figs_to_section(fig, title, section='Sensor-level', replace=True) report.save(fn_report_html, overwrite=True, open_browser=False) return epochs
def simulate_raw(info, fwd_disc_true, signal_vertex, signal_freq, n_trials, noise_multiplier, random_state, n_noise_dipoles, er_raw, fn_stc_signal=None, fn_simulated_raw=None, fn_report_h5=None): """ Simulate raw time courses for two dipoles with frequencies given by signal_freq1 and signal_freq2. Noise dipoles are placed randomly in the whole cortex. Parameters: ----------- info : instance of Info | instance of Raw The channel information to use for simulation. fwd_disc_true : instance of mne.Forward The forward operator for the discrete source space created with the true transformation file. signal_vertex : int The vertex where signal dipole is placed. signal_freq : float The frequency of the signal. n_trials : int Number of trials to create. noise_multiplier : float Multiplier for the noise dipoles. For noise_multiplier equal to one the signal and noise dipoles have the same magnitude. random_state : None | int | instance of RandomState If random_state is an int, it will be used as a seed for RandomState. If None, the seed will be obtained from the operating system (see RandomState for details). Default is None. n_noise_dipoles : int The number of noise dipoles to place within the volume. er_raw : instance of Raw Empty room measurement to be used as sensor noise. fn_stc_signal : None | string Path where the signal source time courses are to be saved. If None the file is not saved. fn_simulated_raw : None | string Path where the raw data is to be saved. If None the file is not saved. fn_report_h5 : None | string Path where the .h5 file for the report is to be saved. Returns: -------- raw : instance of Raw Simulated raw file. stc_signal : instance of SourceEstimate Source time courses of the signal. """ sfreq = info['sfreq'] trial_length = int((config.tmax - config.tmin) * sfreq) times = np.arange(trial_length) / sfreq + config.tmin ############################################################################### # Simulate a single signal dipole source as signal ############################################################################### # TODO: I think a discrete source space was used because mne.simulate_raw did not take volume source spaces -> test src = fwd_disc_true['src'] signal_vert = src[0]['vertno'][signal_vertex] data = np.asarray([generate_signal(times, freq=signal_freq)]) vertices = np.array([signal_vert]) stc_signal = mne.VolSourceEstimate(data=data, vertices=[vertices], tmin=times[0], tstep=np.diff(times[:2])[0], subject='sample') if fn_stc_signal is not None: set_directory(op.dirname(fn_stc_signal)) stc_signal.save(fn_stc_signal) ############################################################################### # Create trials of simulated data ############################################################################### # select n_noise_dipoles entries from rr and their corresponding entries from nn raw_list = [] for i in range(n_trials): # Simulate random noise dipoles stc_noise = simulate_sparse_stc(src, n_noise_dipoles, times, data_fun=generate_random, random_state=random_state, labels=None) # Project to sensor space stc = add_volume_stcs(stc_signal, noise_multiplier * stc_noise) raw = simulate_raw_mne(info, stc, trans=None, src=None, bem=None, forward=fwd_disc_true) raw_list.append(raw) print('%02d/%02d' % (i + 1, n_trials)) raw = mne.concatenate_raws(raw_list) # Use empty room noise as sensor noise raw_picks = mne.pick_types(raw.info, meg=True, eeg=False) er_raw_picks = mne.pick_types(er_raw.info, meg=True, eeg=False) raw._data[raw_picks] += er_raw._data[er_raw_picks, :len(raw.times)] ############################################################################### # Save everything ############################################################################### if fn_simulated_raw is not None: set_directory(op.dirname(fn_simulated_raw)) raw.save(fn_simulated_raw, overwrite=True) # Plot the simulated raw data in the report if fn_report_h5 is not None: from matplotlib import pyplot as plt set_directory(op.dirname(fn_report_h5)) fn_report_html = fn_report_h5.rsplit('.h5')[0] + '.html' now = datetime.now() with mne.open_report(fn_report_h5) as report: fig = plt.figure() plt.plot(times, generate_signal(times, freq=10)) plt.xlabel('Time (s)') ax = fig.axes[0] add_text_next_to_xlabel(fig, ax, now.strftime('%m/%d/%Y, %H:%M:%S')) report.add_figs_to_section(fig, now.strftime('Signal time course'), section='Sensor-level', replace=True) fig = raw.plot() # axis 1 contains the xlabel ax = fig.axes[1] add_text_next_to_xlabel(fig, ax, now.strftime('%m/%d/%Y, %H:%M:%S')) report.add_figs_to_section(fig, now.strftime('Simulated raw'), section='Sensor-level', replace=True) report.save(fn_report_html, overwrite=True, open_browser=False) raw._annotations = mne.annotations.Annotations([], [], []) return raw, stc_signal
parser.add_argument('--n_folds', type=int, default=3) parser.add_argument('--preprocess', type=bool, default=True) parser.add_arguemtn('--stain_norm', type=bool, default=True) parser.add_argument('--seed', type=int, default=42) parser.add_argument('--ckpt_dir', type=str, default='/data/volume/model/') parser.add_argument('--model_name', type=str, default='fpn_model/') args = parser.parse_args() TRAIN_DIR, LABEL_PATH = '/data/train', '/data/train/label.csv' CKPT_DIR, MODEL_NAME = args.ckpt_dir, args.model_name random.seed(args.seed) np.random.seed(args.seed) # check isdir set_directory(CKPT_DIR, MODEL_NAME) # preprocessing if PREPROCESS: preprocess = Preprocess( patch_size=args.patch_size, is_norm=args.stain_norm, target_norm_path='./preprocess/target_norm.png', mode='train', server='kakao') preprocess.save_patches() else: print('Already Preprocessed.') # set dataset patch_loader = PatchLoader(n_kfold=args.n_folds,
} </script> </body> </html> ''' html_table = '' ############################################################################### # Set up directories ############################################################################### img_folder = op.join('somato', 'dip_vs_lcmv') html_path = op.join('html') image_path = op.join(html_path, img_folder) set_directory(image_path) ############################################################################### # Compute LCMV solution and plot stc at dipole location ############################################################################### dists = [] focs = [] ori_errors = [] for ii, setting in enumerate(lcmv_settings): reg, sensor_type, pick_ori, inversion, weight_norm, normalize_fwd, use_noise_cov, reduce_rank = setting try: if sensor_type == 'grad':
def train_mnli_gradient_reversal(**kwargs): dir = set_directory(name=kwargs['type'], type_net=kwargs['type']) writer = SummaryWriter(dir) train, dev_matched_train, test, dev_matched_test, dev_mismatched_test, vocab = prepare_mnli_split(root='datasets/data', urls=['https://www.nyu.edu/projects/bowman/multinli/multinli_1.0.zip'], dir='MultiNLI', name='MultiNLI', data_path='datasets/data/MultiNLI/multinli_1.0', train_genres=[['government', 'telephone', 'slate', 'travel']], test_genres=[['fiction']], max_len=60) weight_matrix = prepare_glove(glove_path="datasets/GloVe/glove.840B.300d.txt", vocab=vocab) train_loader = DataLoader( MultiNLIDataset(dataset=train[0]), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) test_loader = DataLoader( MultiNLIDataset(dataset=test[0]), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) val_loader = DataLoader( MultiNLIDataset(dataset=dev_matched_train[0]), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) model = construct_model_r(model_type=kwargs['type'], weight_matrix=weight_matrix) num_parameters = sum([p.data.nelement() for p in model.parameters()]) print(f'Number of model parameters: {num_parameters}') if torch.cuda.is_available(): torch.cuda.set_device(kwargs['device']) if torch.cuda.is_available(): model = model.cuda() loss_function = torch.nn.CrossEntropyLoss().cuda() else: loss_function = torch.nn.CrossEntropyLoss() optimizer = construct_optimizer(optimizer=kwargs['optim'], model=model, lr=kwargs['lr']) cudnn.benchmark = True total_steps = 0 for epoch in tqdm(range(kwargs['epochs'])): total_steps = train_single_epoch_with_gradient_reversal(train_loader=train_loader, val_loader=test_loader, model=model, criterion=loss_function, optimizer=optimizer, epoch=epoch, alpha=1e-2, total_steps=total_steps, print_freq=kwargs['print_freq'], writer=writer) validate(val_loader=val_loader, model=model, criterion=loss_function, epoch=epoch, print_freq=kwargs['print_freq'], writer=writer) print('Zero Shot Performance') train_loader = DataLoader( MultiNLIDataset(dataset=test[0]), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) val_loader = [DataLoader( MultiNLIDataset(dataset=dataset[0]), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) for dataset in [dev_matched_test, dev_mismatched_test]] validate(val_loader=train_loader, model=model, criterion=loss_function, epoch=epoch, print_freq=kwargs['print_freq'], writer=writer) for loader in val_loader: validate(val_loader=loader, model=model, criterion=loss_function, epoch=epoch, print_freq=kwargs['print_freq'], writer=writer)
def train_mnli(**kwargs): dir = set_directory(name=kwargs['type'], type_net=kwargs['type']) writer = SummaryWriter(dir) train, dev_matched, dev_mismatched, vocab = prepare_mnli(root='datasets/data', urls=['https://www.nyu.edu/projects/bowman/multinli/multinli_1.0.zip'], dir='MultiNLI', name='MultiNLI', data_path='datasets/data/MultiNLI/multinli_1.0', max_len=60) weight_matrix = prepare_glove(glove_path="datasets/GloVe/glove.840B.300d.txt", vocab=vocab) train_loader = DataLoader( MultiNLIDataset(dataset=train), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) val_loader = [DataLoader( MultiNLIDataset(dataset=loader), batch_size=kwargs['batch_size'], shuffle=True, num_workers=1, pin_memory=torch.cuda.is_available()) for loader in [dev_matched, dev_mismatched]] model = construct_model(model_type=kwargs['type'], weight_matrix=weight_matrix) num_parameters = sum([p.data.nelement() for p in model.parameters()]) print(f'Number of model parameters: {num_parameters}') if torch.cuda.is_available(): torch.cuda.set_device(kwargs['device']) if torch.cuda.is_available(): model = model.cuda() loss_function = torch.nn.CrossEntropyLoss().cuda() else: loss_function = torch.nn.CrossEntropyLoss() optimizer = construct_optimizer(optimizer=kwargs['optim'], model=model, lr=kwargs['lr']) total_steps = 0 cudnn.benchmark = True for epoch in tqdm(range(kwargs['epochs'])): total_steps = train_single_epoch(train_loader=train_loader, model=model, criterion=loss_function, optimizer=optimizer, epoch=epoch, total_steps=total_steps, print_freq=kwargs['print_freq'], writer=writer) for loader in val_loader: validate(val_loader=loader, model=model, criterion=loss_function, epoch=epoch, print_freq=kwargs['print_freq'], writer=writer)
<td><input type="text" onkeyup="filter(0, this)" placeholder="reg"></td> <td><input type="text" onkeyup="filter(1, this)" placeholder="sensor type"></td> <td><input type="text" onkeyup="filter(2, this)" placeholder="pick_ori"></td> <td><input type="text" onkeyup="filter(3, this)" placeholder="weight_norm"></td> <td><input type="text" onkeyup="filter(4, this)" placeholder="use_noise_doc"></td> <td><input type="text" onkeyup="filter(5, this)" placeholder="depth"></td> <td></td> <td></td> </tr> ''' html_footer = '</body></table>' html_table = '' set_directory('html/lcmv') for i, setting in enumerate(settings): # construct query q = ("reg==%.1f and sensor_type=='%s' and pick_ori=='%s' and " "weight_norm=='%s' and use_noise_cov==%s and depth==%s" % setting) print(q) sel = lcmv.query(q).dropna() if len(sel) < 1000: continue reg, sensor_type, pick_ori, weight_norm, use_noise_cov, depth = setting
<td><input type="text" onkeyup="filter(1, this)" placeholder="sensor type"></td> <td><input type="text" onkeyup="filter(2, this)" placeholder="pick_ori"></td> <td><input type="text" onkeyup="filter(3, this)" placeholder="inversion"></td> <td><input type="text" onkeyup="filter(4, this)" placeholder="weight_norm"></td> <td><input type="text" onkeyup="filter(5, this)" placeholder="normalize_fwd"></td> <td><input type="text" onkeyup="filter(6, this)" placeholder="real_filter"></td> <td></td> <td></td> </tr> ''' html_footer = '</body></table>' html_table = '' set_directory('html/dics') for i, setting in enumerate(settings): # construct query q = ( "reg==%.1f and sensor_type=='%s' and pick_ori=='%s' and " "inversion=='%s' and weight_norm=='%s' and normalize_fwd==%s and real_filter==%s" % setting) print(q) sel = dics.query(q).dropna() if len(sel) < 1000: continue
def chop_and_apply_ica(raw_filt_fname, ica_cfg): """ Read raw file, chop it into smaller segments and apply ica on the chops. Save the ICA objects plus cleaned raw chops. Plot overview of the artifact rejection. Parameters: ----------- raw_filt_fname : str The filtered raw file to clean. ica_cfg : dict Dict containing the ica specific settings from the config file. Returns: -------- clean_filtered : mne.io.Raw instances Cleaned, filtered raw object. clean_unfiltered : mne.io.Raw instances or None Cleaned, unfiltered raw object or None if ica is not to be applied on unfiltered data. """ raw_chop_clean_filtered_list = [] raw_chop_clean_unfiltered_list = [] print('Running chop_and_apply_ica on ', raw_filt_fname) ########################################################################### # load settings from ica config ########################################################################### chop_length = ica_cfg['chop_length'] ecg_ch = ica_cfg['ecg_ch'] eog_hor = ica_cfg['eog_hor_ch'] eog_ver = ica_cfg['eog_ver_ch'] flow_ecg = ica_cfg['flow_ecg'] fhigh_ecg = ica_cfg['fhigh_ecg'] flow_eog = ica_cfg['flow_eog'] fhigh_eog = ica_cfg['fhigh_eog'] ecg_thresh = ica_cfg['ecg_thresh'] eog_thresh = ica_cfg['eog_thresh'] use_jumeg = ica_cfg['use_jumeg'] random_state = ica_cfg['random_state'] unfiltered = ica_cfg['unfiltered'] reject = ica_cfg['reject'] exclude = ica_cfg['exclude'] save = ica_cfg['save'] # start cleaning raw_filt = mne.io.Raw(raw_filt_fname, preload=True, verbose=True) if unfiltered: raw_unfilt_fname = raw_filt_fname.replace(',fibp', '') raw_unfilt = mne.io.Raw(raw_unfilt_fname, preload=True, verbose=True) picks = mne.pick_types(raw_filt.info, meg=True, exclude=exclude) # you might want to determine the chop time in a more sophisticated way # to avoid accidentally chopping in the middle of a trial chop_times = determine_chop_times_every_x_s(raw_filt.n_times / raw_filt.info["sfreq"], chop_length=chop_length) # chop the data and apply filtering # avoid double counting of data point at chop: tmax = chop_times[i] - 1./raw.info["sfreq"] for i in range(0, len(chop_times) + 1): # get chop interval tmin, tmax = get_tmin_tmax(ct_idx=i, chop_times=chop_times, sfreq=raw_filt.info["sfreq"]) ####################################################################### # building the file names here ####################################################################### info_filt = "fibp" if tmax is not None: tmaxi = int(tmax) else: tmaxi = tmax dirname = op.join(op.dirname(raw_filt_fname), 'chops') set_directory(dirname) prefix_filt = raw_filt_fname.rsplit('/')[-1].rsplit('-raw.fif')[0] ica_fname = op.join( dirname, prefix_filt + ',{}-{}-ica.fif'.format(int(tmin), tmaxi)) # make sure to copy because the original is lost raw_filt_chop = raw_filt.copy().crop(tmin=tmin, tmax=tmax) clean_filt_fname = op.join( dirname, prefix_filt + ',{},ar,{}-{}-raw.fif'.format(info_filt, int(tmin), tmaxi)) raw_filt_chop_fname = op.join( dirname, prefix_filt + ',{},{}-{}-raw.fif'.format(info_filt, int(tmin), tmaxi)) if unfiltered: prefix_unfilt = prefix_filt.replace(',fibp', '') raw_unfilt_chop = raw_unfilt.copy().crop(tmin=tmin, tmax=tmax) clean_unfilt_fname = op.join( dirname, prefix_unfilt + ',ar,{}-{}-raw.fif'.format(int(tmin), tmaxi)) raw_unfilt_chop_fname = op.join( dirname, prefix_unfilt + ',{}-{}-raw.fif'.format(int(tmin), tmaxi)) ####################################################################### # run the ICA on the chops ####################################################################### print('Starting ICA...') if op.isfile(ica_fname): ica = read_ica(ica_fname) else: ica = fit_ica(raw=raw_filt_chop, picks=picks, reject=reject, ecg_ch=ecg_ch, eog_hor=eog_hor, eog_ver=eog_ver, flow_ecg=flow_ecg, fhigh_ecg=fhigh_ecg, flow_eog=flow_eog, fhigh_eog=fhigh_eog, ecg_thresh=ecg_thresh, eog_thresh=eog_thresh, use_jumeg=use_jumeg, random_state=random_state) # plot topo-plots first because sometimes components are hard to identify # ica.plot_components() # do the most important manual check ica.plot_sources(raw_filt_chop, block=True) # save ica object ica.save(ica_fname) print('ICA components excluded: ', ica.exclude) ####################################################################### # apply the ICA to data and save the resulting files ####################################################################### print('Running cleaning on filtered data...') clean_filt_chop = apply_ica_and_plot_performance( raw_filt_chop, ica, ecg_ch, eog_ver, raw_filt_chop_fname, clean_fname=clean_filt_fname, picks=picks, replace_pre_whitener=True, reject=reject, save=save) raw_chop_clean_filtered_list.append(clean_filt_chop) if unfiltered: print('Running cleaning on unfiltered data...') clean_unfilt_chop = apply_ica_and_plot_performance( raw_unfilt_chop, ica, ecg_ch, eog_ver, raw_unfilt_chop_fname, clean_fname=clean_unfilt_fname, picks=picks, replace_pre_whitener=True, reject=reject, save=save) raw_chop_clean_unfiltered_list.append(clean_unfilt_chop) # if tmax is None, last chop is reached if tmax is None: break clean_filt_concat = mne.concatenate_raws(raw_chop_clean_filtered_list) if unfiltered: clean_unfilt_concat = mne.concatenate_raws( raw_chop_clean_unfiltered_list) else: clean_unfilt_concat = None return clean_filt_concat, clean_unfilt_concat