def save_data_model_duration_classify(participator, timesteps, stride, max_freq, min_freq): logger_name = str(participator) + str(timesteps) + str(stride) logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data f_time = datetime.datetime.today() gal = GAL_data() gal.set_logger(logger) gal.load_data(load_list=['eeg', 'info', 'kin']) gal.preprocess_filter(max_freq=max_freq, min_freq=min_freq, low_pass=False) # gal.preprocess_filter_multiple() data_description = gal.get_data_description() participator = participator logger.info('participator : {0}'.format(participator)) event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase'] logger.info( 'running model data') data_len=gal.part_data_count[participator] data_split_ratio = [0.8, 0.1, 0.1] data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32) data_name = ['train', 'validation', 'test'] if not os.path.exists(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq))): os.makedirs(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq))) for i in range(3): x = data[i * 2] y = data[i * 2 + 1] np.save(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), data_name[i]+'_X_ts_{}_st_{}.npy'.format(timesteps, stride)), x) np.save(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), data_name[i]+'_y_ts_{}_st_{}.npy'.format(timesteps, stride)), y)
def run_model_duration_classify_predict(model_name, participator, timesteps, stride, batch_size, save_by, load_weight_from): logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(load_weight_from) logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data f_time = datetime.datetime.today() output_dir = os.path.join('output', 'predict_{0}_P{1}_ts{2}_stride{3}_bs_{4}_weight_{5}_{6}'.format(model_name, participator, timesteps, stride, batch_size,load_weight_from)) if not os.path.exists(output_dir): os.makedirs(output_dir) else: print('same configuation already exists!') return hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log')) logger.addHandler(hdlr) console_handler = logging.StreamHandler() logger.addHandler(console_handler) logger.setLevel(logging.INFO) gal = GAL_data() gal.set_logger(logger) gal.load_data(load_list=['eeg', 'info', 'kin']) gal.preprocess_filter(max_freq=50, min_freq=0.2, low_pass=False) data_description = gal.get_data_description() participator = participator logger.info('participator : {0}'.format(participator)) event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase'] rnn = EEG_model(event_list) rnn.set_logger(logger) rnn.select_model(model_name) rnn.load_model_weight(model_name, load_weight_from) logger.info('loaded weight') logger.info( 'running model data') data_len=gal.part_data_count[participator] data_split_ratio = [0.8, 0.1, 0.1] data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32) rnn.set_data_description(data_description) rnn.save_event_classify(data=data, event_list=event_list, output_dir=output_dir)
def run_model_duration_classify(model_name, participator, timesteps, stride, max_freq, min_freq, nb_epoch, batch_size, save_by, multi_filter = False, load_weight_from = None, load_data_from= None): logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(nb_epoch) + str(load_weight_from) logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data f_time = datetime.datetime.today() if load_weight_from != None: load_weight_from_str = load_weight_from.split('/') output_dir = os.path.join('output', '{0}_P{1}_ts{2}_stride{3}_ep{4}_bs_{5}_weight_{6}_maxmin_{7}_saveby_{8}'.format(model_name, participator, timesteps, stride, nb_epoch, batch_size, load_weight_from_str, '{}_{}'.format(max_freq, min_freq), save_by)) else: output_dir = os.path.join('output', '{0}_P{1}_ts{2}_stride{3}_ep{4}_bs_{5}_maxmin_{6}_saveby_{7}'.format(model_name, participator, timesteps, stride, nb_epoch, batch_size, '{}_{}'.format(max_freq, min_freq), save_by)) if not os.path.exists(output_dir): os.makedirs(output_dir) else: print('same configuation already exists!') return if load_data_from != None: assert timesteps == load_data_from[0], 'timesteps does not match' assert stride == load_data_from[1], 'stride does not match' hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log')) logger.addHandler(hdlr) console_handler = logging.StreamHandler() logger.addHandler(console_handler) logger.setLevel(logging.INFO) event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase'] rnn = EEG_model(event_list) rnn.set_logger(logger) rnn.select_model(model_name) if load_weight_from != None: rnn.load_model_weight(model_name, load_weight_from) logger.info('loaded weight') logger.info( 'running model data') data_split_ratio = [0.7, 0.2, 0.1] if load_data_from == None: gal = GAL_data() gal.set_logger(logger) gal.load_data(load_list=['eeg', 'info', 'kin']) if multi_filter ==True: gal.preprocess_filter_multiple() else: gal.preprocess_filter(max_freq=max_freq, min_freq=min_freq, low_pass=False) data_description = gal.get_data_description() participator = participator logger.info('participator : {0}'.format(participator)) data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32) else: data_description = dict() data_description['participator'] = participator data_description['timesteps'] = timesteps data_description['stride'] = stride data_description['event_list'] = event_list data_description['preprocess_filter'] = '{}_{}'.format(max_freq, min_freq) ts = load_data_from[0] st = load_data_from[1] data = list() data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'train_X_ts_{}_st_{}.npy'.format(ts, st)))) data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'train_y_ts_{}_st_{}.npy'.format(ts, st)))) data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'validation_X_ts_{}_st_{}.npy'.format(ts, st)))) data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'validation_y_ts_{}_st_{}.npy'.format(ts, st)))) data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'test_X_ts_{}_st_{}.npy'.format(ts, st)))) data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'test_y_ts_{}_st_{}.npy'.format(ts, st)))) for i in range(nb_epoch/save_by): loss_train, loss_test = rnn.run_model_event(data=data, nb_epoch = save_by, batch_size=batch_size) output_dir_temp = os.path.join(output_dir, str(i)) os.makedirs(output_dir_temp) df_loss = pd.DataFrame() df_loss['acc'] = loss_train.history['acc'] df_loss['loss'] = loss_train.history['loss'] df_loss.to_csv(os.path.join(output_dir_temp, 'train_loss_acc.csv')) rnn.set_data_description(data_description) rnn.set_model_config('epoch', nb_epoch/save_by * i) rnn.save_event_classify(data=data, event_list=event_list, output_dir=output_dir_temp)