def main(chords, tempo, duration, omit_display, omit_play): """Play a sequence of chords.\n e.g. python picardy.py Cm Ddim G7 Cm """ for chord in chords: notes = utils.get_indices(lut.lut[utils.get_canonical_name(chord)]) if not omit_display: display.clear() display.draw_keyboard(notes) print(utils.get_canonical_name(chord)) if not omit_play: play.play_chord(notes, tempo, duration) time.sleep(duration * 60 / tempo)
def fit(self, A, b, c): """set matrix and vectors and solve LP by updatind tableau table Parameters ---------- A : sparse matrix shape = [n_inequalities, n_variables] Coefficient matrix of inequality system b : array-like, shape = [n_inequalities] Intercept vector of inequality system c : array-like, shape = [n_variables] Coefficient vector of objective function Returns ------- self : object """ self.tableau = make_tableau(A, b) i, j = get_indices(self.tableau) while i is not None and j is not None: self.tableau = sweep_out(self.tableau, i, j) i, j = get_indices(self.tableau) self.tableau = renew(A, b, c, self.tableau) assert self.tableau[-1][-1] == sum([-i for i in b if i < 0]), \ "LP has no solution (infeasible)" i, j = get_indices(self.tableau) while i is not None and j is not None: self.tableau = sweep_out(self.tableau, i, j) i, j = get_indices(self.tableau) if i is not None: assert j is not None, "LP has no solution (unbounded)" return self
def get_peak_latencies(df, co, min_heights, dt, used_inds, min_height, win_start=0, win_stop=0.5): ''' Parameters: df: pandas dataframe containing experiment data Returns latencies: list of lists of latency from nearest target appearance in window. Each list gives the latencies for a different controller output all_peak_counts: count of all controller peaks, for each controller output target_peak_count: count of all peaks within window of target, for each controller ouput ''' dt = utils.get_dt(h5file, input_info) used_inds = utils.get_indices(input_info, trial_type) targets = df.loc[used_inds].kinematic.query('hit_target') trial_len = co.shape[1] * dt t_lfads = np.arange(co.shape[1]) * dt #time labels of lfads input all_peak_count = 0 # count of all controller peaks target_peak_count = 0 # count of all peaks within window of target latencies = [] # latency for i in used_inds: peaks = [] for input_idx in range(1): #range(co.shape[2]): input_peaks, _ = signal.find_peaks(np.abs(co[i, :, input_idx]), height=min_heights[input_idx]) peaks.append(input_peaks) peaks = np.concatenate(peaks) t_peaks = t_lfads[peaks] t_targets = targets.loc[i].index all_peak_count += len(t_peaks) for tp in t_peaks: if any((tp - t_targets >= win_start) & (tp - t_targets < win_stop)): diff_targets = tp - t_targets latency = np.min( diff_targets[diff_targets > 0]) #latency to closest target latencies.append(latency) target_peak_count += 1
def get_wall(obj): children = [] mins = [] maxs = [] for child in obj.children: location = {"x": child.location.x, "y": child.location.y} texture_path = utils.get_texture(child) indices = utils.get_indices(child) vertices = utils.get_vertices(child, indices) left_edge = min(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"] bottom_edge = min(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"] right_edge = max(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"] top_edge = max(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"] mins.append(child.location.y + bottom_edge) maxs.append(child.location.y + top_edge) bulbs = utils.get_bulbs(child) child_ref = { "type": child['type'], "location": location, "texture_path": texture_path, "indices": indices, "vertices": vertices, "bulbs": bulbs } children.append(child_ref) total_bottom = min(mins) total_top = max(maxs) total_height = total_top - total_bottom obj_ref = { "children": children, "total_height": total_height, } return obj_ref
def get_column(obj): texture_path = utils.get_texture(obj) indices = utils.get_indices(obj) vertices = utils.get_vertices(obj, indices) bulbs = utils.get_bulbs(obj) left_edge = min(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"] bottom_edge = min(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"] right_edge = max(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"] top_edge = max(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"] original_size = {"x": right_edge - left_edge, "y": top_edge - bottom_edge} for v in vertices: v["pos"]["x"] -= left_edge v["pos"]["y"] -= bottom_edge print(v["pos"]["x"], v["pos"]["y"]) children = [] for child in obj.children: location = { "x": child.location.x - left_edge, "y": child.location.y - bottom_edge } child_ref = {"type": child['type'], "location": location} children.append(child_ref) obj_ref = { "type": obj['type'], "original_size": original_size, "vertices": vertices, "indices": indices, "texture_path": texture_path, "bulbs": bulbs, "children": children } return obj_ref
frames=frames, init_func=init, blit=True) return anim, frames if __name__ == '__main__': lfads_file = snakemake.input[0] filename = snakemake.input[1] input_info_file = snakemake.input[2] out_directory = os.path.dirname(snakemake.output[0]) os.makedirs(out_directory, exist_ok=True) input_info = io.loadmat(input_info_file) used_inds = get_indices(input_info, snakemake.wildcards.trial_type) with h5py.File(lfads_file) as h5file: dt = 0.01 trial_t = np.arange(h5file['controller_outputs'].shape[1]) * dt trial_len = trial_t[-1] + dt data = io.loadmat(filename) used_trials = np.where( np.diff(data['cpl_st_trial_rew'].real, axis=1) > trial_len)[0] #TODO: replace with animate_trial function and test for video_idx, plotted_trial in enumerate(used_inds): t_start = data['cpl_st_trial_rew'][used_trials[plotted_trial], 0].real t_end = t_start + trial_len
def main(argv): draw_keyboard(utils.get_indices(lut.lut[utils.get_canonical_name( argv[1])]))
if __name__=='__main__': # data_filename = snakemake.input[0] # lfads_filename = snakemake.input[1] # inputInfo_filename = snakemake.input[2] # output_filename = snakemake.output[0] #trial_type = snakemake.wildcards.trial_type data_filename = "../data/intermediate/mk08011M1m-mack-RTP.p" inputInfo_filename = "../data/model_output/mk08011M1m-mack-RTP_inputInfo.mat" lfads_filename = "../data/model_output/mk08011M1m-mack-RTP_2OLS24_all.h5" output_filename = "~/process_test.p" trial_type = "all" input_info = io.loadmat(inputInfo_filename) used_inds = get_indices(input_info, trial_type) df = pd.read_pickle(data_filename) n_neurons = df.loc[0].neural.shape[1] dt = 0.010 #TODO read from lfads file kin_dt = 0.001 if cfg['align_peaks']: n_win = int((cfg['post_peak_win']*2)/dt) else: n_win = int((cfg['post_target_win_stop'] - cfg['post_target_win_start'])/dt) n_spikes = n_win * int(dt/kin_dt) #length of window of spikes to extract kinematic_vars = ['x', 'y'] with h5py.File(lfads_filename,'r') as h5_file: trial_len = h5_file['controller_outputs'].shape[1] * dt
def main(): global logging parser = argparse.ArgumentParser() parser.add_argument("-c", default="dopey.yaml", help="yaml config file") parser.add_argument("--eshost", default="", help="eshost here will overwrite that in config file") parser.add_argument( "--base-day", default="0", help= "number 0(today), 1(tommorow), -1(yestoday), or string line 2011-11-11" ) parser.add_argument( "--action-filters", default="", help= "comma splited. d:delete, c:close, u:update settings, f:forcemerge, fr:freeze. \ leaving blank means do all the actions configuared in config file") parser.add_argument("-l", default="-", help="log file") parser.add_argument("--level", default="info") args = parser.parse_args() global config config = yaml.load(open(args.c)) if args.eshost: config['eshost'] = args.eshost initlog(level=args.level, log=config["l"] if "log" in config else args.l) all_indices = utils.get_indices(config['eshost']) logging.debug(u"all_indices: {}".format(all_indices)) for action in config.get("setup", []): settings = action.values()[0] eval(action.keys()[0])(settings) base_day = _get_base_day(args.base_day) logging.info("base day is %s" % base_day) action_filters = _get_action_filters(args.action_filters) if 'delete_indices' in action_filters: to_delete_indices = utils.get_to_delete_indices( config, all_indices, base_day) logging.info('try to delete `{}`'.format(' '.join( e[0] for e in to_delete_indices))) utils.delete_indices(config, to_delete_indices) if 'close_indices' in action_filters: to_close_indices = utils.get_to_close_indices(config, all_indices, base_day) logging.info('try to close `{}`'.format(' '.join( e[0] for e in to_close_indices))) utils.close_indices(config, to_close_indices) if 'freeze_indices' in action_filters: to_freeze_indices = utils.get_to_freeze_indices( config, all_indices, base_day) logging.info('try to freeze `{}`'.format(' '.join( e[0] for e in to_freeze_indices))) utils.freeze_indices(config, to_freeze_indices) if 'update_settings' in action_filters: to_update_indices = utils.get_to_update_indices( config, all_indices, base_day) logging.info('(before settings diff filter)try to update `{}`'.format( ' '.join(e[0] for e in to_update_indices))) utils.update_settings(config, to_update_indices) if 'optimize_indices' in action_filters: to_optimize_indices = utils.get_to_optimize_indices( config, all_indices, base_day) logging.info('try to forcemerge `{}`'.format(' '.join( e[0] for e in to_optimize_indices))) utils.optimize_indices(config, to_optimize_indices) # dopey_summary.add( # u"未处理:\n{}\n删除:\n{}\n关闭:\n{}\n优化:{}\n更新索配置:{}".format( # "\n".join(sorted(not_dealt)), # "\n".join(sorted(_delete)), # "\n".join(sorted(_close)), # "\n".join(sorted(_optimize)), # "\n".join(sorted(_update_settings)))) for action in config.get("teardown", []): settings = action.values()[0] eval(action.keys()[0])(settings) sumary_config = config.get("sumary") for action, kargs in sumary_config.items(): if kargs: getattr(dopey_summary, action)(**kargs) else: getattr(dopey_summary, action)()
import pickle import numpy as np from blending import build_matrix, train_als from helpers import load_data from utils import get_indices # load training set print('Loading training set') train = load_data('../data/data_train.csv') nnz_train = get_indices(train) """ After some tests, the best score is achieved by only blending the als predictions (that means dilating them). Another good performance was achieved by blending ALS, user-based and item based collaborative filtering. Details about the tests can be found in the notebook 'src/notebook/blending.ipynb'. """ methods = ['als'] # Train the model (actually if the files are present, nothing is done. print('Training the model') train_als(train) # Load trained model and predictions print('Now loading the pre trained model') with open(b'../data/pickle/data_train_als.pickle', 'rb') as f: als_train = pickle.load(f) als_predictions = load_data('../data/predictions/als_prediction.csv')
def train_for_epoch(self, epoch, args, train_data_with_time, train_rnn_inp): """Train DETM on data for one epoch. """ train_data, train_times = data.get_time_columns(train_data_with_time) self.train() acc_loss = 0 acc_nll = 0 acc_kl_theta_loss = 0 acc_kl_eta_loss = 0 acc_kl_alpha_loss = 0 cnt = 0 indices = torch.randperm(train_data.shape[0]) indices = torch.split(indices, args.batch_size) optimizer = self.get_optimizer(args) for idx, ind in enumerate(indices): optimizer.zero_grad() self.zero_grad() data_batch, times_batch = data.get_batch(train_data, ind, self.device, train_times) times_batch = get_indices(train_times, times_batch) sums = data_batch.sum(1).unsqueeze(1) times_batch = torch.from_numpy(times_batch) if args.bow_norm: normalized_data_batch = data_batch / sums else: normalized_data_batch = data_batch loss, nll, kl_alpha, kl_eta, kl_theta = self.forward( data_batch, normalized_data_batch, times_batch, train_rnn_inp, train_data.shape[0]) loss.backward() if args.clip > 0: torch.nn.utils.clip_grad_norm_(self.parameters(), args.clip) optimizer.step() acc_loss += torch.sum(loss).item() acc_nll += torch.sum(nll).item() acc_kl_theta_loss += torch.sum(kl_theta).item() acc_kl_eta_loss += torch.sum(kl_eta).item() acc_kl_alpha_loss += torch.sum(kl_alpha).item() cnt += 1 if idx % args.log_interval == 0 and idx > 0: cur_loss = round(acc_loss / cnt, 2) cur_nll = round(acc_nll / cnt, 2) cur_kl_theta = round(acc_kl_theta_loss / cnt, 2) cur_kl_eta = round(acc_kl_eta_loss / cnt, 2) cur_kl_alpha = round(acc_kl_alpha_loss / cnt, 2) lr = optimizer.param_groups[0]['lr'] print( 'Epoch: {} .. batch: {}/{} .. LR: {} .. KL_theta: {} .. KL_eta: {} .. KL_alpha: {} .. Rec_loss: {} .. NELBO: {}' .format(epoch, idx, len(indices), lr, cur_kl_theta, cur_kl_eta, cur_kl_alpha, cur_nll, cur_loss)) cur_loss = round(acc_loss / cnt, 2) cur_nll = round(acc_nll / cnt, 2) cur_kl_theta = round(acc_kl_theta_loss / cnt, 2) cur_kl_eta = round(acc_kl_eta_loss / cnt, 2) cur_kl_alpha = round(acc_kl_alpha_loss / cnt, 2) lr = optimizer.param_groups[0]['lr'] print('*' * 100) print( 'Epoch----->{} .. LR: {} .. KL_theta: {} .. KL_eta: {} .. KL_alpha: {} .. Rec_loss: {} .. NELBO: {}' .format(epoch, lr, cur_kl_theta, cur_kl_eta, cur_kl_alpha, cur_nll, cur_loss)) print('*' * 100)
from utils import get_indices indices, index_in_clips = get_indices(46, 2, 8, 2, 'mid') print(indices) print(index_in_clips)
input_path, sep='\t', header=None, )) mf = [] for i, k in enumerate(nnz_train): item = int(k[0]) - 1 user = int(k[1]) - 1 mf.append(user_features[:, user].T.dot(item_features[:, item])) return mf # load training set print('Loading training set') train = load_data('../data/data_train.csv') nnz_train = get_indices(train) """ After some tests, the best score is achieved by only blending the als predictions (that means dilating them). Another good performance was achieved by blending ALS, user-based and item based collaborative filtering. Details about the tests can be found in the notebook 'src/notebook/blending.ipynb'. """ # Train the model (actually if the files are present, nothing is done. print('Training the model') _, user_features, item_features = matrix_factorization_als( ratings, None, verbose=True, stop_criterion=0.00001, lambda_user=0.014, lambda_item=0.575,
def problem_3_and_4(train_mb_size, epochs, alg_type, see_mb_loss, save_epoch, load_dir, predict_only): # Print all arguments print("Batch size: {}".format(train_mb_size)) print("Epoch: {}".format(epochs)) print("Algorithm type: {}".format(alg_type)) print("See mini-batch loss: {}".format(see_mb_loss)) print("Save epoch: {}".format(save_epoch)) print("Load model: {}".format(load_dir)) print("Predict only: {}".format(predict_only)) # Build model dim = 8 B = train_mb_size B_valid = 1 init_W = np.random.normal(0.0, 0.4, (dim, dim)) init_A = np.random.normal(0.0, 0.4, (dim)) gnn = GNN(dim, init_W=init_W, init_A=init_A) # Get and partition indices into training and validation indices train_indices = utils.get_indices(TRAIN_PATH) random.shuffle(train_indices) split_index = int(len(train_indices) * 0.8) # Save data or load data if os.path.isfile('./train_ids.npy'): print("Load data") train_data = np.load('train_ids.npy') valid_data = np.load('valid_ids.npy') else: print("Save data") train_data = np.array(train_indices[:split_index]) valid_data = np.array(train_indices[split_index:]) np.save('train_ids.npy', train_data) np.save('valid_ids.npy', valid_data) # Records train_accs = [] train_losses = [] valid_accs = [] valid_losses = [] # Load model if load_dir is not None: gnn.load_model(load_dir) # Adam time if alg_type == 'adam': t = 0 # Train model if not predict_only: for epoch in range(1, epochs + 1): print("Epoch {}".format(epoch)) # Make generators train_gen = utils.graph_generator(TRAIN_PATH, train_data, dim, mb_size=B, shuffle=True) valid_gen = utils.graph_generator(TRAIN_PATH, valid_data, dim, mb_size=B_valid) train_mb_size = np.ceil(len(train_data) / B) valid_mb_size = np.ceil(len(valid_data) / B_valid) # -------------------- # Training # -------------------- total_data = 0 # Initialize running mb loss running_loss = [] # Training loop for graphs, labels in tqdm.tqdm(train_gen, total=train_mb_size): len_data = len(labels) total_data += len_data if alg_type == 'sgd': running_loss.append(gnn.train_sgd(graphs, labels)) elif alg_type == 'msgd': running_loss.append(gnn.train_momentum_sgd(graphs, labels)) elif alg_type == 'adam': t += 1 running_loss.append(gnn.train_adam(graphs, labels, t)) if see_mb_loss: avg_mb_loss = running_loss[-1] / len_data print("Average mb training loss: {}".format(avg_mb_loss)) # Calculate running mb loss if see_mb_loss: avg_loss = np.sum(running_loss) / total_data print("Average whole training loss: {}".format(avg_loss)) # -------------------- # Training Predictions # -------------------- # Reinit train generator train_gen = utils.graph_generator(TRAIN_PATH, train_data, dim, mb_size=1) train_mb_size = len(train_data) # Initialize running loss and number of correct predictions correct = 0 pred_run_loss = 0 for graphs, labels in tqdm.tqdm(train_gen, total=train_mb_size): assert len(graphs) == len(labels) assert len(graphs) == 1 pred_label, pred_loss = gnn.prediction(graphs[0], labels[0]) if pred_label == labels[0]: correct += 1 pred_run_loss += pred_loss # Training Accuracy train_acc = correct * 100.0 / len(train_data) print("Correct data : {}".format(correct)) print("Train data : {}".format(len(train_data))) print("Accuracy (Training): {}".format(train_acc)) train_accs.append(train_acc) # Training Loss train_loss = pred_run_loss / len(train_data) print("Loss (Training): {}".format(train_loss)) train_losses.append(train_loss) # -------------------- # Validation Predictions # -------------------- # Initialize running loss and number of correct predictions correct = 0 pred_run_loss = 0 for graphs, labels in tqdm.tqdm(valid_gen, total=valid_mb_size): assert len(graphs) == len(labels) assert len(graphs) == 1 pred_label, pred_loss = gnn.prediction(graphs[0], labels[0]) if pred_label == labels[0]: correct += 1 pred_run_loss += pred_loss # Validation Accuracy valid_acc = correct * 100.0 / len(valid_data) print("Accuracy (Validation): {}".format(valid_acc)) valid_accs.append(valid_acc) # Validation Loss valid_loss = pred_run_loss / len(valid_data) print("Loss (Training): {}".format(valid_loss)) valid_losses.append(valid_loss) # Save model if (save_epoch != 0) and (epoch % save_epoch == 0): parent_msave_dir = './models/gnn_{}/'.format(alg_type) if not os.path.exists(parent_msave_dir): os.makedirs(parent_msave_dir) save_dir = parent_msave_dir + 'e{}_b{}_va{:.2f}.pkl'.format( epoch, B, valid_acc) print("Saving model to {}".format(save_dir)) gnn.save_model(save_dir) # Save records parent_save_dir = './train_' + alg_type if not os.path.exists(parent_save_dir): os.makedirs(parent_save_dir) save_list(parent_save_dir + '/train_losses_b{}.pkl'.format(B), train_losses) save_list(parent_save_dir + '/train_accs_b{}.pkl'.format(B), train_accs) save_list(parent_save_dir + '/valid_losses_b{}.pkl'.format(B), valid_losses) save_list(parent_save_dir + '/valid_accs_b{}.pkl'.format(B), valid_accs) # Predict model only else: # Check validation accuracy print("Check validation accuracy") valid_gen = utils.graph_generator(TRAIN_PATH, valid_data, dim, mb_size=B_valid) valid_mb_size = np.ceil(len(valid_data) / B_valid) # -------------------- # Validation Predictions # -------------------- # Initialize running loss and number of correct predictions correct = 0 pred_run_loss = 0 for graphs, labels in tqdm.tqdm(valid_gen, total=valid_mb_size): assert len(graphs) == len(labels) assert len(graphs) == 1 pred_label, pred_loss = gnn.prediction(graphs[0], labels[0]) if pred_label == labels[0]: correct += 1 pred_run_loss += pred_loss # Validation Accuracy valid_acc = correct * 100.0 / len(valid_data) print("Accuracy (Validation): {}".format(valid_acc)) valid_accs.append(valid_acc) # Validation Loss valid_loss = pred_run_loss / len(valid_data) print("Loss (Training): {}".format(valid_loss)) valid_losses.append(valid_loss) # -------------------- # Test Predictions # -------------------- print("Running test predictions") test_data = utils.get_indices(TEST_PATH) test_data.sort(key=int) test_gen = utils.graph_generator(TEST_PATH, test_data, dim, mb_size=1, train=False) test_mb_size = np.ceil(len(test_data) / 1) out_file = open("../prediction.txt", "w") for graphs, _ in tqdm.tqdm(test_gen, total=test_mb_size): assert len(graphs) == 1 pred_label, _ = gnn.prediction(graphs[0]) out_file.write(str(pred_label)) out_file.write("\n") out_file.close()
win_start=0, win_stop=0.5): '''Chaining above function above to get useful dataframe''' targets = get_targets(df) peaks = get_peaks(co, dt, min_heights, min_distance=cfg['min_peak_spacing']) peak_df, _ = get_latencies(targets, peaks, win_start, win_stop, trial_len=trial_len) return peak_df if __name__ == '__main__': trial_type = 'all' lfads_filename = "/home/pmalonis/lfads_analysis/data/model_output/rockstar_2OLS24_%s.h5" % trial_type data_filename = "/home/pmalonis/lfads_analysis/data/intermediate/rockstar.p" inputInfo_filename = "/home/pmalonis/lfads_analysis/data/model_output/rockstar_inputInfo.mat" df = pd.read_pickle(data_filename) input_info = io.loadmat(inputInfo_filename) with h5py.File(lfads_filename) as h5file: co = h5file['controller_outputs'][:] used_inds = utils.get_indices(input_info, trial_type)
def main(argv): play_chord(utils.get_indices(lut.lut[utils.get_canonical_name(argv[1])]), 60, 1) time.sleep(1)