def save_data_model_duration_classify(participator, timesteps, stride, max_freq, min_freq):
logger_name = str(participator) + str(timesteps) + str(stride)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info', 'kin'])
gal.preprocess_filter(max_freq=max_freq, min_freq=min_freq, low_pass=False)
# gal.preprocess_filter_multiple()
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase']
logger.info( 'running model data')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.1, 0.1]
data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32)
data_name = ['train', 'validation', 'test']
if not os.path.exists(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq))):
os.makedirs(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq)))
for i in range(3):
x = data[i * 2]
y = data[i * 2 + 1]
np.save(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), data_name[i]+'_X_ts_{}_st_{}.npy'.format(timesteps, stride)), x)
np.save(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), data_name[i]+'_y_ts_{}_st_{}.npy'.format(timesteps, stride)), y)
def predict(model_name, participator, load_weight_from):
logger_name = model_name + str(participator) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', 'predict_'+str(f_time))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'kin'])
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
rnn = EEG_model(None)
rnn.set_logger(logger)
rnn.select_model(model_name)
rnn.load_model_weight(model_name, load_weight_from)
logger.info( 'running model data from a generator')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.2]
train_list = np.arange(int(data_len * data_split_ratio[0]))
test_list = np.arange(data_len - int(data_len * data_split_ratio[0]))
rnn.set_data_description(data_description)
generator = gal.data_generator_kin(part=participator, timesteps=10, stride=10)
rnn.save_kin_generator(generator=generator,train_list=train_list, test_list=test_list, output_dir=output_dir)
def run_model_duration_generator(model_name, participator, timesteps, stride, nb_epoch, load_weight_from = None):
logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(nb_epoch) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', 'dur_'+str(f_time))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info'])
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
event_list=['Dur_Reach', 'Dur_LoadReach', 'Dur_LoadMaintain', 'Dur_LoadRetract', 'Dur_Retract']
rnn = EEG_model(event_list)
rnn.set_logger(logger)
rnn.select_model(model_name)
if load_weight_from != None:
rnn.load_model_weight(model_name, load_weight_from)
logger.info( 'running model data from a generator')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.1, 0.1]
train_list = np.arange(int(data_len * data_split_ratio[0]))
validate_list = np.arange(int(data_len * data_split_ratio[1]))
test_list = np.arange(data_len - int(data_len * data_split_ratio[0]) - int(data_len * data_split_ratio[1]))
for epoch in range(nb_epoch):
generator = gal.data_generator_event(part=participator, timesteps=timesteps, stride=stride, event_list=event_list)
logger.info( 'epoch : {0}'.format(epoch))
start = time.clock()
rnn.run_model_with_generator_event(generator=generator, train_list=train_list, validate_list=validate_list, test_list=test_list)
logger.info( 'epoch {0} ran for {1} minutes'.format(epoch, (time.clock() - start)/60))
rnn.set_data_description(data_description)
rnn.set_model_config('epoch', nb_epoch)
generator = gal.data_generator_event(part=participator, timesteps=timesteps, stride=stride, event_list=event_list)
rnn.save_event_generator_classify(generator=generator,train_list=train_list, validate_list=validate_list,test_list=test_list, event_list=event_list, output_dir=output_dir)
def run_model_duration_classify_predict(model_name, participator, timesteps, stride, batch_size, save_by, load_weight_from):
logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', 'predict_{0}_P{1}_ts{2}_stride{3}_bs_{4}_weight_{5}_{6}'.format(model_name, participator, timesteps, stride, batch_size,load_weight_from))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
else:
print('same configuation already exists!')
return
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info', 'kin'])
gal.preprocess_filter(max_freq=50, min_freq=0.2, low_pass=False)
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase']
rnn = EEG_model(event_list)
rnn.set_logger(logger)
rnn.select_model(model_name)
rnn.load_model_weight(model_name, load_weight_from)
logger.info('loaded weight')
logger.info( 'running model data')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.1, 0.1]
data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32)
rnn.set_data_description(data_description)
rnn.save_event_classify(data=data, event_list=event_list, output_dir=output_dir)
def get_data_multiple_filter(participator, timesteps, stride):
logger_name = 'data' + str(participator) + str(timesteps) + str(stride)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', '{0}_P{1}_ts{2}_stride{3}'.format('data', participator, timesteps, stride))
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info', 'kin'])
gal.preprocess_filter_multiple()
gal.save_individual_eeg()
def run_model_event_range_generator(model_name, participator, timesteps, stride, nb_epoch, event_range, load_weight_from = None):
logger = logging.getLogger()
f_time = datetime.datetime.today()
output_dir = os.path.join('output', str(f_time))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data()
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
event_list = ['tHandStart', 'tFirstDigitTouch', 'tBothStartLoadPhase', 'tLiftOff', 'tReplace', 'tBothReleased', 'tHandStop']
rnn = EEG_model(event_list)
rnn.set_logger(logger)
rnn.select_model(model_name)
if load_weight_from != None:
rnn.load_model_weight(model_name, load_weight_from)
logger.info( 'running model data from a generator')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.1, 0.1]
train_list = np.arange(int(data_len * data_split_ratio[0]))
validate_list = np.arange(int(data_len * data_split_ratio[1]))
test_list = np.arange(data_len - int(data_len * data_split_ratio[0]) - int(data_len * data_split_ratio[1]))
for epoch in range(nb_epoch):
generator = gal.X_y_part_generator(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, event_range=event_range)
logger.info( 'epoch : {0}'.format(epoch))
start = time.clock()
rnn.run_model_with_generator_event(generator=generator, train_list=train_list, validate_list=validate_list, test_list=test_list)
logger.info( 'epoch {0} ran for {1} minutes'.format(epoch, (time.clock() - start)/60))
rnn.set_data_description(data_description)
rnn.set_model_config('epoch', nb_epoch)
generator = gal.X_y_part_generator(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, event_range=event_range)
rnn.save_event(generator=generator,train_list=train_list, validate_list=validate_list,test_list=test_list, event_list=event_list, output_dir=output_dir)
def run_model_duration(model_name, participator, timesteps, stride, nb_epoch, load_weight_from = None):
logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(nb_epoch) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', 'dur_'+str(f_time))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info'])
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
#event_list=['Dur_Reach', 'Dur_Preload', 'Dur_LoadPhase', 'Dur_Release', 'Dur_Retract']
event_list=['Dur_Reach', 'Dur_LoadReach', 'Dur_LoadMaintain', 'Dur_LoadRetract', 'Dur_Retract']
rnn = EEG_model(event_list)
rnn.set_logger(logger)
rnn.select_model(model_name)
if load_weight_from != None:
rnn.load_model_weight(model_name, load_weight_from)
logger.info( 'running model data as a whole')
data_split_ratio = [0.8, 0.1, 0.1]
data = gal.data_event(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32)
loss_train, loss_val, loss_test = rnn.run_model_event(data=data, nb_epoch = nb_epoch)
with open(os.path.join(output_dir, 'train_loss.json'), 'w') as f:
json.dump(loss_train, f)
with open(os.path.join(output_dir, 'validate_loss.json'), 'w') as f:
json.dump(loss_val, f)
with open(os.path.join(output_dir, 'test_loss.json'), 'w') as f:
json.dump(loss_test, f)
rnn.set_data_description(data_description)
rnn.set_model_config('epoch', nb_epoch)
generator = gal.data_generator_event(part=participator, timesteps=timesteps, stride=stride, event_list=event_list)
rnn.save_event(data=data, event_list=event_list, output_dir=output_dir)
def evaluate(self, X, y):
pred = self.clf.predict(X)
accuracy = accuracy_score(y, pred)
cm = confusion_matrix(y, pred)
cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
return accuracy, cm_normalized
nb_epoch = 100
participator = 1
gal = GAL_data()
gal.load_data(load_list=['eeg', 'info'])
data_description = gal.get_data_description()
event_list=['Idle', 'Recognition_Phase', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase']
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8, 0.1, 0.1]
train_list = np.arange(int(data_len * data_split_ratio[0]))
validate_list = np.arange(int(data_len * data_split_ratio[1]))
test_list = np.arange(data_len - int(data_len * data_split_ratio[0]) - int(data_len * data_split_ratio[1]))
for epoch in range(nb_epoch):
generator = gal.data_generator_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list)
def run(data_list):
for _ in data_list:
def run_model_kin_generator(model_name, participator, timesteps, stride, nb_epoch, patience_limit, loss_delta_limit, load_weight_from = None):
logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(nb_epoch) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
output_dir = os.path.join('output', 'kin_'+str(f_time))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'kin'])
gal.preprocess_kin()
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
rnn = EEG_model(None)
rnn.set_logger(logger)
rnn.select_model(model_name)
if load_weight_from != None:
rnn.load_model_weight(model_name, load_weight_from)
logger.info( 'running model data from a generator')
data_len=gal.part_data_count[participator]
data_split_ratio = [0.8,0.2]
train_list = np.arange(int(data_len * data_split_ratio[0]))
test_list = np.arange(data_len - int(data_len * data_split_ratio[0]))
loss_train_df = pd.DataFrame(columns = ['epoch', 'loss'])
loss_test_df = pd.DataFrame(columns = ['epoch', 'loss'])
patience = 0
for epoch in range(nb_epoch):
generator = gal.data_generator_kin(part=participator, timesteps=timesteps, stride=stride)
logger.info( 'epoch : {0}'.format(epoch))
start = time.clock()
train_loss, test_loss = rnn.run_model_with_generator_kin(generator=generator, train_list=train_list, test_list=test_list)
loss_train_df.loc[epoch, ['epoch', 'loss']] = [epoch+1, train_loss]
loss_test_df.loc[epoch, ['epoch', 'loss']] = [epoch+1, test_loss]
if epoch == 0:
prev_train_loss = train_loss
logger.info( 'epoch {0} ran for {1} minutes'.format(epoch, (time.clock() - start)/60))
loss_delta = abs(prev_train_loss - train_loss) / prev_train_loss * 100
if loss_delta < loss_delta_limit:
patience = patience + 1
if patience > patience_limit:
logger.info('training stopped at epoch {0} due to patience threshold'.format(epoch))
break
else:
patience = patience - 1
loss_train_df.to_csv(os.path.join(output_dir, 'train_loss.csv'), index=False)
loss_test_df.to_csv(os.path.join(output_dir, 'test_loss.csv'), index=False)
rnn.set_data_description(data_description)
rnn.set_model_config('epoch', nb_epoch)
generator = gal.data_generator_kin(part=participator, timesteps=timesteps, stride = stride)
rnn.save_kin_generator(generator=generator,train_list=train_list, test_list=test_list, output_dir=output_dir)
def check_time():
gal = GAL_data()
gal.load_data()
gal.examine_time()
def run_model_duration_classify(model_name, participator, timesteps, stride, max_freq, min_freq, nb_epoch, batch_size, save_by, multi_filter = False, load_weight_from = None, load_data_from= None):
logger_name = model_name + str(participator) + str(timesteps) + str(stride) + str(nb_epoch) + str(load_weight_from)
logger = logging.getLogger(logger_name) # so that no multiple loggers input the same data
f_time = datetime.datetime.today()
if load_weight_from != None:
load_weight_from_str = load_weight_from.split('/')
output_dir = os.path.join('output', '{0}_P{1}_ts{2}_stride{3}_ep{4}_bs_{5}_weight_{6}_maxmin_{7}_saveby_{8}'.format(model_name, participator, timesteps, stride, nb_epoch, batch_size, load_weight_from_str, '{}_{}'.format(max_freq, min_freq), save_by))
else:
output_dir = os.path.join('output', '{0}_P{1}_ts{2}_stride{3}_ep{4}_bs_{5}_maxmin_{6}_saveby_{7}'.format(model_name, participator, timesteps, stride, nb_epoch, batch_size, '{}_{}'.format(max_freq, min_freq), save_by))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
else:
print('same configuation already exists!')
return
if load_data_from != None:
assert timesteps == load_data_from[0], 'timesteps does not match'
assert stride == load_data_from[1], 'stride does not match'
hdlr = logging.FileHandler(os.path.join(output_dir, 'rnn.log'))
logger.addHandler(hdlr)
console_handler = logging.StreamHandler()
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)
event_list=['Idle', 'Reach_Phase', 'LoadReach_Phase', 'LoadMaintain_Phase', 'LoadRetract_Phase', 'Retract_Phase']
rnn = EEG_model(event_list)
rnn.set_logger(logger)
rnn.select_model(model_name)
if load_weight_from != None:
rnn.load_model_weight(model_name, load_weight_from)
logger.info('loaded weight')
logger.info( 'running model data')
data_split_ratio = [0.7, 0.2, 0.1]
if load_data_from == None:
gal = GAL_data()
gal.set_logger(logger)
gal.load_data(load_list=['eeg', 'info', 'kin'])
if multi_filter ==True:
gal.preprocess_filter_multiple()
else:
gal.preprocess_filter(max_freq=max_freq, min_freq=min_freq, low_pass=False)
data_description = gal.get_data_description()
participator = participator
logger.info('participator : {0}'.format(participator))
data = gal.data_event_classify(part=participator, timesteps=timesteps, stride=stride, event_list=event_list, partition_ratio=data_split_ratio, input_dim=32)
else:
data_description = dict()
data_description['participator'] = participator
data_description['timesteps'] = timesteps
data_description['stride'] = stride
data_description['event_list'] = event_list
data_description['preprocess_filter'] = '{}_{}'.format(max_freq, min_freq)
ts = load_data_from[0]
st = load_data_from[1]
data = list()
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'train_X_ts_{}_st_{}.npy'.format(ts, st))))
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'train_y_ts_{}_st_{}.npy'.format(ts, st))))
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'validation_X_ts_{}_st_{}.npy'.format(ts, st))))
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'validation_y_ts_{}_st_{}.npy'.format(ts, st))))
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'test_X_ts_{}_st_{}.npy'.format(ts, st))))
data.append(np.load(os.path.join('data', 'numpy_binary', 'maxfreq_{}_minfreq_{}'.format(max_freq, min_freq), 'test_y_ts_{}_st_{}.npy'.format(ts, st))))
for i in range(nb_epoch/save_by):
loss_train, loss_test = rnn.run_model_event(data=data, nb_epoch = save_by, batch_size=batch_size)
output_dir_temp = os.path.join(output_dir, str(i))
os.makedirs(output_dir_temp)
df_loss = pd.DataFrame()
df_loss['acc'] = loss_train.history['acc']
df_loss['loss'] = loss_train.history['loss']
df_loss.to_csv(os.path.join(output_dir_temp, 'train_loss_acc.csv'))
rnn.set_data_description(data_description)
rnn.set_model_config('epoch', nb_epoch/save_by * i)
rnn.save_event_classify(data=data, event_list=event_list, output_dir=output_dir_temp)
def read_save_raw():
gal = GAL_data()
gal.set_logger(logging.getLogger())
gal.read_raw_data()
gal.save_raw_data()