def main(chords, tempo, duration, omit_display, omit_play):
"""Play a sequence of chords.\n
e.g. python picardy.py Cm Ddim G7 Cm
"""
for chord in chords:
notes = utils.get_indices(lut.lut[utils.get_canonical_name(chord)])
if not omit_display:
display.clear()
display.draw_keyboard(notes)
print(utils.get_canonical_name(chord))
if not omit_play:
play.play_chord(notes, tempo, duration)
time.sleep(duration * 60 / tempo)
def fit(self, A, b, c):
"""set matrix and vectors and solve LP by updatind tableau table
Parameters
----------
A : sparse matrix shape = [n_inequalities, n_variables]
Coefficient matrix of inequality system
b : array-like, shape = [n_inequalities]
Intercept vector of inequality system
c : array-like, shape = [n_variables]
Coefficient vector of objective function
Returns
-------
self : object
"""
self.tableau = make_tableau(A, b)
i, j = get_indices(self.tableau)
while i is not None and j is not None:
self.tableau = sweep_out(self.tableau, i, j)
i, j = get_indices(self.tableau)
self.tableau = renew(A, b, c, self.tableau)
assert self.tableau[-1][-1] == sum([-i for i in b if i < 0]), \
"LP has no solution (infeasible)"
i, j = get_indices(self.tableau)
while i is not None and j is not None:
self.tableau = sweep_out(self.tableau, i, j)
i, j = get_indices(self.tableau)
if i is not None:
assert j is not None, "LP has no solution (unbounded)"
return self
def get_peak_latencies(df,
co,
min_heights,
dt,
used_inds,
min_height,
win_start=0,
win_stop=0.5):
'''
Parameters:
df: pandas dataframe containing experiment data
Returns
latencies: list of lists of latency from nearest target appearance in window.
Each list gives the latencies for a different controller output
all_peak_counts: count of all controller peaks, for each controller output
target_peak_count: count of all peaks within window of target, for each controller ouput
'''
dt = utils.get_dt(h5file, input_info)
used_inds = utils.get_indices(input_info, trial_type)
targets = df.loc[used_inds].kinematic.query('hit_target')
trial_len = co.shape[1] * dt
t_lfads = np.arange(co.shape[1]) * dt #time labels of lfads input
all_peak_count = 0 # count of all controller peaks
target_peak_count = 0 # count of all peaks within window of target
latencies = [] # latency
for i in used_inds:
peaks = []
for input_idx in range(1): #range(co.shape[2]):
input_peaks, _ = signal.find_peaks(np.abs(co[i, :, input_idx]),
height=min_heights[input_idx])
peaks.append(input_peaks)
peaks = np.concatenate(peaks)
t_peaks = t_lfads[peaks]
t_targets = targets.loc[i].index
all_peak_count += len(t_peaks)
for tp in t_peaks:
if any((tp - t_targets >= win_start)
& (tp - t_targets < win_stop)):
diff_targets = tp - t_targets
latency = np.min(
diff_targets[diff_targets > 0]) #latency to closest target
latencies.append(latency)
target_peak_count += 1
def get_wall(obj):
children = []
mins = []
maxs = []
for child in obj.children:
location = {"x": child.location.x, "y": child.location.y}
texture_path = utils.get_texture(child)
indices = utils.get_indices(child)
vertices = utils.get_vertices(child, indices)
left_edge = min(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"]
bottom_edge = min(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"]
right_edge = max(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"]
top_edge = max(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"]
mins.append(child.location.y + bottom_edge)
maxs.append(child.location.y + top_edge)
bulbs = utils.get_bulbs(child)
child_ref = {
"type": child['type'],
"location": location,
"texture_path": texture_path,
"indices": indices,
"vertices": vertices,
"bulbs": bulbs
}
children.append(child_ref)
total_bottom = min(mins)
total_top = max(maxs)
total_height = total_top - total_bottom
obj_ref = {
"children": children,
"total_height": total_height,
}
return obj_ref
def get_column(obj):
texture_path = utils.get_texture(obj)
indices = utils.get_indices(obj)
vertices = utils.get_vertices(obj, indices)
bulbs = utils.get_bulbs(obj)
left_edge = min(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"]
bottom_edge = min(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"]
right_edge = max(vertices, key=lambda v: v["pos"]["x"])["pos"]["x"]
top_edge = max(vertices, key=lambda v: v["pos"]["y"])["pos"]["y"]
original_size = {"x": right_edge - left_edge, "y": top_edge - bottom_edge}
for v in vertices:
v["pos"]["x"] -= left_edge
v["pos"]["y"] -= bottom_edge
print(v["pos"]["x"], v["pos"]["y"])
children = []
for child in obj.children:
location = {
"x": child.location.x - left_edge,
"y": child.location.y - bottom_edge
}
child_ref = {"type": child['type'], "location": location}
children.append(child_ref)
obj_ref = {
"type": obj['type'],
"original_size": original_size,
"vertices": vertices,
"indices": indices,
"texture_path": texture_path,
"bulbs": bulbs,
"children": children
}
return obj_ref
frames=frames,
init_func=init,
blit=True)
return anim, frames
if __name__ == '__main__':
lfads_file = snakemake.input[0]
filename = snakemake.input[1]
input_info_file = snakemake.input[2]
out_directory = os.path.dirname(snakemake.output[0])
os.makedirs(out_directory, exist_ok=True)
input_info = io.loadmat(input_info_file)
used_inds = get_indices(input_info, snakemake.wildcards.trial_type)
with h5py.File(lfads_file) as h5file:
dt = 0.01
trial_t = np.arange(h5file['controller_outputs'].shape[1]) * dt
trial_len = trial_t[-1] + dt
data = io.loadmat(filename)
used_trials = np.where(
np.diff(data['cpl_st_trial_rew'].real, axis=1) > trial_len)[0]
#TODO: replace with animate_trial function and test
for video_idx, plotted_trial in enumerate(used_inds):
t_start = data['cpl_st_trial_rew'][used_trials[plotted_trial],
0].real
t_end = t_start + trial_len
def main(argv):
draw_keyboard(utils.get_indices(lut.lut[utils.get_canonical_name(
argv[1])]))
if __name__=='__main__':
# data_filename = snakemake.input[0]
# lfads_filename = snakemake.input[1]
# inputInfo_filename = snakemake.input[2]
# output_filename = snakemake.output[0]
#trial_type = snakemake.wildcards.trial_type
data_filename = "../data/intermediate/mk08011M1m-mack-RTP.p"
inputInfo_filename = "../data/model_output/mk08011M1m-mack-RTP_inputInfo.mat"
lfads_filename = "../data/model_output/mk08011M1m-mack-RTP_2OLS24_all.h5"
output_filename = "~/process_test.p"
trial_type = "all"
input_info = io.loadmat(inputInfo_filename)
used_inds = get_indices(input_info, trial_type)
df = pd.read_pickle(data_filename)
n_neurons = df.loc[0].neural.shape[1]
dt = 0.010 #TODO read from lfads file
kin_dt = 0.001
if cfg['align_peaks']:
n_win = int((cfg['post_peak_win']*2)/dt)
else:
n_win = int((cfg['post_target_win_stop'] - cfg['post_target_win_start'])/dt)
n_spikes = n_win * int(dt/kin_dt) #length of window of spikes to extract
kinematic_vars = ['x', 'y']
with h5py.File(lfads_filename,'r') as h5_file:
trial_len = h5_file['controller_outputs'].shape[1] * dt
def main():
global logging
parser = argparse.ArgumentParser()
parser.add_argument("-c", default="dopey.yaml", help="yaml config file")
parser.add_argument("--eshost",
default="",
help="eshost here will overwrite that in config file")
parser.add_argument(
"--base-day",
default="0",
help=
"number 0(today), 1(tommorow), -1(yestoday), or string line 2011-11-11"
)
parser.add_argument(
"--action-filters",
default="",
help=
"comma splited. d:delete, c:close, u:update settings, f:forcemerge, fr:freeze. \
leaving blank means do all the actions configuared in config file")
parser.add_argument("-l", default="-", help="log file")
parser.add_argument("--level", default="info")
args = parser.parse_args()
global config
config = yaml.load(open(args.c))
if args.eshost:
config['eshost'] = args.eshost
initlog(level=args.level, log=config["l"] if "log" in config else args.l)
all_indices = utils.get_indices(config['eshost'])
logging.debug(u"all_indices: {}".format(all_indices))
for action in config.get("setup", []):
settings = action.values()[0]
eval(action.keys()[0])(settings)
base_day = _get_base_day(args.base_day)
logging.info("base day is %s" % base_day)
action_filters = _get_action_filters(args.action_filters)
if 'delete_indices' in action_filters:
to_delete_indices = utils.get_to_delete_indices(
config, all_indices, base_day)
logging.info('try to delete `{}`'.format(' '.join(
e[0] for e in to_delete_indices)))
utils.delete_indices(config, to_delete_indices)
if 'close_indices' in action_filters:
to_close_indices = utils.get_to_close_indices(config, all_indices,
base_day)
logging.info('try to close `{}`'.format(' '.join(
e[0] for e in to_close_indices)))
utils.close_indices(config, to_close_indices)
if 'freeze_indices' in action_filters:
to_freeze_indices = utils.get_to_freeze_indices(
config, all_indices, base_day)
logging.info('try to freeze `{}`'.format(' '.join(
e[0] for e in to_freeze_indices)))
utils.freeze_indices(config, to_freeze_indices)
if 'update_settings' in action_filters:
to_update_indices = utils.get_to_update_indices(
config, all_indices, base_day)
logging.info('(before settings diff filter)try to update `{}`'.format(
' '.join(e[0] for e in to_update_indices)))
utils.update_settings(config, to_update_indices)
if 'optimize_indices' in action_filters:
to_optimize_indices = utils.get_to_optimize_indices(
config, all_indices, base_day)
logging.info('try to forcemerge `{}`'.format(' '.join(
e[0] for e in to_optimize_indices)))
utils.optimize_indices(config, to_optimize_indices)
# dopey_summary.add(
# u"未处理:\n{}\n删除:\n{}\n关闭:\n{}\n优化:{}\n更新索配置:{}".format(
# "\n".join(sorted(not_dealt)),
# "\n".join(sorted(_delete)),
# "\n".join(sorted(_close)),
# "\n".join(sorted(_optimize)),
# "\n".join(sorted(_update_settings))))
for action in config.get("teardown", []):
settings = action.values()[0]
eval(action.keys()[0])(settings)
sumary_config = config.get("sumary")
for action, kargs in sumary_config.items():
if kargs:
getattr(dopey_summary, action)(**kargs)
else:
getattr(dopey_summary, action)()
import pickle
import numpy as np
from blending import build_matrix, train_als
from helpers import load_data
from utils import get_indices
# load training set
print('Loading training set')
train = load_data('../data/data_train.csv')
nnz_train = get_indices(train)
"""
After some tests, the best score is achieved by only blending the als predictions (that means dilating them).
Another good performance was achieved by blending ALS, user-based and item based collaborative filtering. Details about
the tests can be found in the notebook 'src/notebook/blending.ipynb'.
"""
methods = ['als']
# Train the model (actually if the files are present, nothing is done.
print('Training the model')
train_als(train)
# Load trained model and predictions
print('Now loading the pre trained model')
with open(b'../data/pickle/data_train_als.pickle', 'rb') as f:
als_train = pickle.load(f)
als_predictions = load_data('../data/predictions/als_prediction.csv')
def train_for_epoch(self, epoch, args, train_data_with_time,
train_rnn_inp):
"""Train DETM on data for one epoch.
"""
train_data, train_times = data.get_time_columns(train_data_with_time)
self.train()
acc_loss = 0
acc_nll = 0
acc_kl_theta_loss = 0
acc_kl_eta_loss = 0
acc_kl_alpha_loss = 0
cnt = 0
indices = torch.randperm(train_data.shape[0])
indices = torch.split(indices, args.batch_size)
optimizer = self.get_optimizer(args)
for idx, ind in enumerate(indices):
optimizer.zero_grad()
self.zero_grad()
data_batch, times_batch = data.get_batch(train_data, ind,
self.device, train_times)
times_batch = get_indices(train_times, times_batch)
sums = data_batch.sum(1).unsqueeze(1)
times_batch = torch.from_numpy(times_batch)
if args.bow_norm:
normalized_data_batch = data_batch / sums
else:
normalized_data_batch = data_batch
loss, nll, kl_alpha, kl_eta, kl_theta = self.forward(
data_batch, normalized_data_batch, times_batch, train_rnn_inp,
train_data.shape[0])
loss.backward()
if args.clip > 0:
torch.nn.utils.clip_grad_norm_(self.parameters(), args.clip)
optimizer.step()
acc_loss += torch.sum(loss).item()
acc_nll += torch.sum(nll).item()
acc_kl_theta_loss += torch.sum(kl_theta).item()
acc_kl_eta_loss += torch.sum(kl_eta).item()
acc_kl_alpha_loss += torch.sum(kl_alpha).item()
cnt += 1
if idx % args.log_interval == 0 and idx > 0:
cur_loss = round(acc_loss / cnt, 2)
cur_nll = round(acc_nll / cnt, 2)
cur_kl_theta = round(acc_kl_theta_loss / cnt, 2)
cur_kl_eta = round(acc_kl_eta_loss / cnt, 2)
cur_kl_alpha = round(acc_kl_alpha_loss / cnt, 2)
lr = optimizer.param_groups[0]['lr']
print(
'Epoch: {} .. batch: {}/{} .. LR: {} .. KL_theta: {} .. KL_eta: {} .. KL_alpha: {} .. Rec_loss: {} .. NELBO: {}'
.format(epoch, idx, len(indices), lr, cur_kl_theta,
cur_kl_eta, cur_kl_alpha, cur_nll, cur_loss))
cur_loss = round(acc_loss / cnt, 2)
cur_nll = round(acc_nll / cnt, 2)
cur_kl_theta = round(acc_kl_theta_loss / cnt, 2)
cur_kl_eta = round(acc_kl_eta_loss / cnt, 2)
cur_kl_alpha = round(acc_kl_alpha_loss / cnt, 2)
lr = optimizer.param_groups[0]['lr']
print('*' * 100)
print(
'Epoch----->{} .. LR: {} .. KL_theta: {} .. KL_eta: {} .. KL_alpha: {} .. Rec_loss: {} .. NELBO: {}'
.format(epoch, lr, cur_kl_theta, cur_kl_eta, cur_kl_alpha, cur_nll,
cur_loss))
print('*' * 100)
from utils import get_indices indices, index_in_clips = get_indices(46, 2, 8, 2, 'mid') print(indices) print(index_in_clips)
input_path,
sep='\t',
header=None,
))
mf = []
for i, k in enumerate(nnz_train):
item = int(k[0]) - 1
user = int(k[1]) - 1
mf.append(user_features[:, user].T.dot(item_features[:, item]))
return mf
# load training set
print('Loading training set')
train = load_data('../data/data_train.csv')
nnz_train = get_indices(train)
"""
After some tests, the best score is achieved by only blending the als predictions (that means dilating them).
Another good performance was achieved by blending ALS, user-based and item based collaborative filtering. Details about
the tests can be found in the notebook 'src/notebook/blending.ipynb'.
"""
# Train the model (actually if the files are present, nothing is done.
print('Training the model')
_, user_features, item_features = matrix_factorization_als(
ratings,
None,
verbose=True,
stop_criterion=0.00001,
lambda_user=0.014,
lambda_item=0.575,
def problem_3_and_4(train_mb_size, epochs, alg_type, see_mb_loss, save_epoch,
load_dir, predict_only):
# Print all arguments
print("Batch size: {}".format(train_mb_size))
print("Epoch: {}".format(epochs))
print("Algorithm type: {}".format(alg_type))
print("See mini-batch loss: {}".format(see_mb_loss))
print("Save epoch: {}".format(save_epoch))
print("Load model: {}".format(load_dir))
print("Predict only: {}".format(predict_only))
# Build model
dim = 8
B = train_mb_size
B_valid = 1
init_W = np.random.normal(0.0, 0.4, (dim, dim))
init_A = np.random.normal(0.0, 0.4, (dim))
gnn = GNN(dim, init_W=init_W, init_A=init_A)
# Get and partition indices into training and validation indices
train_indices = utils.get_indices(TRAIN_PATH)
random.shuffle(train_indices)
split_index = int(len(train_indices) * 0.8)
# Save data or load data
if os.path.isfile('./train_ids.npy'):
print("Load data")
train_data = np.load('train_ids.npy')
valid_data = np.load('valid_ids.npy')
else:
print("Save data")
train_data = np.array(train_indices[:split_index])
valid_data = np.array(train_indices[split_index:])
np.save('train_ids.npy', train_data)
np.save('valid_ids.npy', valid_data)
# Records
train_accs = []
train_losses = []
valid_accs = []
valid_losses = []
# Load model
if load_dir is not None:
gnn.load_model(load_dir)
# Adam time
if alg_type == 'adam':
t = 0
# Train model
if not predict_only:
for epoch in range(1, epochs + 1):
print("Epoch {}".format(epoch))
# Make generators
train_gen = utils.graph_generator(TRAIN_PATH,
train_data,
dim,
mb_size=B,
shuffle=True)
valid_gen = utils.graph_generator(TRAIN_PATH,
valid_data,
dim,
mb_size=B_valid)
train_mb_size = np.ceil(len(train_data) / B)
valid_mb_size = np.ceil(len(valid_data) / B_valid)
# --------------------
# Training
# --------------------
total_data = 0
# Initialize running mb loss
running_loss = []
# Training loop
for graphs, labels in tqdm.tqdm(train_gen, total=train_mb_size):
len_data = len(labels)
total_data += len_data
if alg_type == 'sgd':
running_loss.append(gnn.train_sgd(graphs, labels))
elif alg_type == 'msgd':
running_loss.append(gnn.train_momentum_sgd(graphs, labels))
elif alg_type == 'adam':
t += 1
running_loss.append(gnn.train_adam(graphs, labels, t))
if see_mb_loss:
avg_mb_loss = running_loss[-1] / len_data
print("Average mb training loss: {}".format(avg_mb_loss))
# Calculate running mb loss
if see_mb_loss:
avg_loss = np.sum(running_loss) / total_data
print("Average whole training loss: {}".format(avg_loss))
# --------------------
# Training Predictions
# --------------------
# Reinit train generator
train_gen = utils.graph_generator(TRAIN_PATH,
train_data,
dim,
mb_size=1)
train_mb_size = len(train_data)
# Initialize running loss and number of correct predictions
correct = 0
pred_run_loss = 0
for graphs, labels in tqdm.tqdm(train_gen, total=train_mb_size):
assert len(graphs) == len(labels)
assert len(graphs) == 1
pred_label, pred_loss = gnn.prediction(graphs[0], labels[0])
if pred_label == labels[0]:
correct += 1
pred_run_loss += pred_loss
# Training Accuracy
train_acc = correct * 100.0 / len(train_data)
print("Correct data : {}".format(correct))
print("Train data : {}".format(len(train_data)))
print("Accuracy (Training): {}".format(train_acc))
train_accs.append(train_acc)
# Training Loss
train_loss = pred_run_loss / len(train_data)
print("Loss (Training): {}".format(train_loss))
train_losses.append(train_loss)
# --------------------
# Validation Predictions
# --------------------
# Initialize running loss and number of correct predictions
correct = 0
pred_run_loss = 0
for graphs, labels in tqdm.tqdm(valid_gen, total=valid_mb_size):
assert len(graphs) == len(labels)
assert len(graphs) == 1
pred_label, pred_loss = gnn.prediction(graphs[0], labels[0])
if pred_label == labels[0]:
correct += 1
pred_run_loss += pred_loss
# Validation Accuracy
valid_acc = correct * 100.0 / len(valid_data)
print("Accuracy (Validation): {}".format(valid_acc))
valid_accs.append(valid_acc)
# Validation Loss
valid_loss = pred_run_loss / len(valid_data)
print("Loss (Training): {}".format(valid_loss))
valid_losses.append(valid_loss)
# Save model
if (save_epoch != 0) and (epoch % save_epoch == 0):
parent_msave_dir = './models/gnn_{}/'.format(alg_type)
if not os.path.exists(parent_msave_dir):
os.makedirs(parent_msave_dir)
save_dir = parent_msave_dir + 'e{}_b{}_va{:.2f}.pkl'.format(
epoch, B, valid_acc)
print("Saving model to {}".format(save_dir))
gnn.save_model(save_dir)
# Save records
parent_save_dir = './train_' + alg_type
if not os.path.exists(parent_save_dir):
os.makedirs(parent_save_dir)
save_list(parent_save_dir + '/train_losses_b{}.pkl'.format(B),
train_losses)
save_list(parent_save_dir + '/train_accs_b{}.pkl'.format(B),
train_accs)
save_list(parent_save_dir + '/valid_losses_b{}.pkl'.format(B),
valid_losses)
save_list(parent_save_dir + '/valid_accs_b{}.pkl'.format(B),
valid_accs)
# Predict model only
else:
# Check validation accuracy
print("Check validation accuracy")
valid_gen = utils.graph_generator(TRAIN_PATH,
valid_data,
dim,
mb_size=B_valid)
valid_mb_size = np.ceil(len(valid_data) / B_valid)
# --------------------
# Validation Predictions
# --------------------
# Initialize running loss and number of correct predictions
correct = 0
pred_run_loss = 0
for graphs, labels in tqdm.tqdm(valid_gen, total=valid_mb_size):
assert len(graphs) == len(labels)
assert len(graphs) == 1
pred_label, pred_loss = gnn.prediction(graphs[0], labels[0])
if pred_label == labels[0]:
correct += 1
pred_run_loss += pred_loss
# Validation Accuracy
valid_acc = correct * 100.0 / len(valid_data)
print("Accuracy (Validation): {}".format(valid_acc))
valid_accs.append(valid_acc)
# Validation Loss
valid_loss = pred_run_loss / len(valid_data)
print("Loss (Training): {}".format(valid_loss))
valid_losses.append(valid_loss)
# --------------------
# Test Predictions
# --------------------
print("Running test predictions")
test_data = utils.get_indices(TEST_PATH)
test_data.sort(key=int)
test_gen = utils.graph_generator(TEST_PATH,
test_data,
dim,
mb_size=1,
train=False)
test_mb_size = np.ceil(len(test_data) / 1)
out_file = open("../prediction.txt", "w")
for graphs, _ in tqdm.tqdm(test_gen, total=test_mb_size):
assert len(graphs) == 1
pred_label, _ = gnn.prediction(graphs[0])
out_file.write(str(pred_label))
out_file.write("\n")
out_file.close()
win_start=0,
win_stop=0.5):
'''Chaining above function above to get useful dataframe'''
targets = get_targets(df)
peaks = get_peaks(co,
dt,
min_heights,
min_distance=cfg['min_peak_spacing'])
peak_df, _ = get_latencies(targets,
peaks,
win_start,
win_stop,
trial_len=trial_len)
return peak_df
if __name__ == '__main__':
trial_type = 'all'
lfads_filename = "/home/pmalonis/lfads_analysis/data/model_output/rockstar_2OLS24_%s.h5" % trial_type
data_filename = "/home/pmalonis/lfads_analysis/data/intermediate/rockstar.p"
inputInfo_filename = "/home/pmalonis/lfads_analysis/data/model_output/rockstar_inputInfo.mat"
df = pd.read_pickle(data_filename)
input_info = io.loadmat(inputInfo_filename)
with h5py.File(lfads_filename) as h5file:
co = h5file['controller_outputs'][:]
used_inds = utils.get_indices(input_info, trial_type)
def main(argv):
play_chord(utils.get_indices(lut.lut[utils.get_canonical_name(argv[1])]),
60, 1)
time.sleep(1)
