def fold_era(era_name, eras_test, features, original_x: pd.DataFrame,
predict_x: pd.DataFrame):
era_train_x = original_x[original_x.era == era_name].copy()
era_train_y = era_train_x.target.copy()
era_train_x.drop('data_type', axis=1, inplace=True)
era_train_x.drop('target', axis=1, inplace=True)
era_train_x.drop('id', axis=1, inplace=True)
era_train_x.drop('era', axis=1, inplace=True)
era_test_x = predict_x[features].copy()
era_valid_x = original_x[original_x.era.isin(eras_test)].copy()
era_valid_y = era_valid_x.target.copy()
era_valid_x = era_valid_x[features]
blend_x_train = era_train_and_score(era_name, era_train_x, era_train_y,
era_valid_x, era_valid_y)
blend_x_test = era_predict(era_name, era_test_x)
era_pred_x = predict_x[predict_x.data_type == 'validation'].copy()
era_pred_y = era_pred_x.target.copy()
era_pred_x = era_pred_x[features]
for index, clf in enumerate(clfs[era_name]):
clf_name = clf.__class__.__name__
if hasattr(clf, 'predict_proba'):
prtmp = clf.predict_proba(era_pred_x)[:, 1]
else:
prtmp = pd.DataFrame(clf.predict(era_pred_x)).values.ravel()
score('V-%s-%s' % (era_name, clf_name), prtmp, era_pred_y)
return blend_x_train, blend_x_test
def main():
args = parse_args()
C = importlib.import_module(args.config).TrainConfig
print("MODEL ID: {}".format(C.model_id))
summary_writer = SummaryWriter(C.log_dpath)
train_iter, val_iter, test_iter, vocab = build_loaders(C)
model = build_model(C, vocab)
optimizer = torch.optim.Adam(model.parameters(),
lr=C.lr,
weight_decay=C.weight_decay,
amsgrad=True)
lr_scheduler = ReduceLROnPlateau(optimizer,
mode='min',
factor=C.lr_decay_gamma,
patience=C.lr_decay_patience,
verbose=True)
best_val_scores = {'CIDEr': 0.}
best_epoch = 0
best_ckpt_fpath = None
for e in range(1, C.epochs + 1):
print("\n\n\nEpoch {:d}".format(e))
ckpt_fpath = C.ckpt_fpath_tpl.format(e)
""" Train """
print("\n[TRAIN]")
train_loss = train(e, model, optimizer, train_iter, vocab,
C.decoder.rnn_teacher_forcing_ratio, C.reg_lambda,
C.recon_lambda, C.gradient_clip)
log_train(C, summary_writer, e, train_loss, get_lr(optimizer))
""" Validation """
print("\n[VAL]")
val_loss = evaluate(model, val_iter, vocab, C.reg_lambda,
C.recon_lambda)
val_scores, _, _ = score(model, val_iter, vocab)
log_val(C, summary_writer, e, val_loss, val_scores)
if e % C.save_every == 0:
print("Saving checkpoint at epoch={} to {}".format(e, ckpt_fpath))
save_checkpoint(e, model, ckpt_fpath, C)
if e >= C.lr_decay_start_from:
lr_scheduler.step(val_loss)
if e == 1 or val_scores['CIDEr'] > best_val_scores['CIDEr']:
best_epoch = e
best_val_scores = val_scores
best_ckpt_fpath = ckpt_fpath
""" Test with Best Model """
print("\n\n\n[BEST]")
best_model = load_checkpoint(model, best_ckpt_fpath)
test_scores, _, _ = score(best_model, test_iter, vocab)
log_test(C, summary_writer, best_epoch, test_scores)
save_checkpoint(best_epoch, best_model, C.ckpt_fpath_tpl.format("best"), C)
def can_form_stacking_pair_at(self, i, j):
if j <= i + 2:
return False
else:
S = self.seq
score = self.score_func
if score(S[i], S[j]) * score(S[i + 1], S[j - 1]) != 0:
return True
else:
return False
def model_linear(train , test , flag):
train_x , train_y = train[0] , train[1]
test_x , test_y = test[0] , test[1]
regr = linear_model.Lasso(alpha = 0.3 , max_iter=5000)
#regr = linear_model.LassoCV()
print "start training"
regr.fit(train_x , train_y['gap'].values)
if flag == 'offline':
submit = regr.predict(test_x)
print Util.score(test_y.values , submit)
def main():
with open('4.txt', 'r') as f:
ip = f.read().split()
keys = [max(range(256), key=lambda k: score(decode(i, k))) for i in ip]
print(
max([{
"ciphertext": ip[i],
"key": keys[i],
"decoded": decode(ip[i], keys[i])
} for i in range(len(keys))],
key=lambda k: score(k["decoded"])))
def model_xgb(train , test , flag):
train_x , train_y = train[0] , train[1]
test_x , test_y = test[0] , test[1]
print train_x.shape
print train_y[2].shape
print fea_names
dtrain = xgb.DMatrix(train_x , label = train_y[2].values)
dtest = xgb.DMatrix(test_x)
def mapeobj(preds , dtrain):
gaps = dtrain.get_label()
delta = (preds - gaps) / gaps
k = 6
e = np.exp(k * delta)
grad = (e - 1 / e) / (e + 1 / e)
for i , t in enumerate(delta):
if t > 1:
grad[i] = 1
elif t < -1:
grad[i] = -1
grad = grad / gaps
hess = (4 * k * gaps**2) / (e + 1 / e)**2
for i , t in enumerate(delta):
if abs(t) > 1:
hess[i] = 0
for i , g in enumerate(gaps):
if g == 0:
grad[i] = 0
hess[i] = 0
return grad , hess
def evalmape(preds , dtrain):
gaps = dtrain.get_label()
errs = abs(gaps - preds) / gaps
for i , g in enumerate(gaps):
if g == 0:
errs[i] = 0
err = np.mean(errs)
return 'error' , err
if flag == 'online':
watchlist = [(dtrain , 'train')]
m_xgb = xgb.train(Params.xgb_reg_params , dtrain , 800 , watchlist , mapeobj , evalmape)
prd = m_xgb.predict(dtest)
Util.submit(test_y.values , prd)
elif flag == 'offline':
dtest.set_label(test_y[2].values)
watchlist = [(dtrain , 'train') , (dtest , 'eval')]
#m_xgb = xgb.train(Params.xgb_reg_params , dtrain , 100 , watchlist)
m_xgb = xgb.train(Params.xgb_reg_params , dtrain , 800 , watchlist , mapeobj , evalmape)
prd = m_xgb.predict(dtest)
#prd = postprocess(train , test_y.values , prd)
print Util.score(test_y.values , prd)
imp = m_xgb.get_fscore()
print sorted(imp.items() , key = lambda d:d[1] , reverse = True)
def main():
set_random_seed(C.seed)
summary_writer = SummaryWriter(C.log_dpath)
train_iter, val_iter, test_iter, vocab = build_loaders(C)
model = build_model(C, vocab)
print("#params: ", count_parameters(model))
model = model.cuda()
optimizer = torch.optim.Adamax(model.parameters(),
lr=C.lr,
weight_decay=1e-5)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
C.epochs,
eta_min=0,
last_epoch=-1)
best_val_scores = {'CIDEr': -1.}
for e in range(1, C.epochs + 1):
print()
ckpt_fpath = C.ckpt_fpath_tpl.format(e)
""" Train """
teacher_forcing_ratio = get_teacher_forcing_ratio(
C.decoder.max_teacher_forcing_ratio,
C.decoder.min_teacher_forcing_ratio, e, C.epochs)
train_loss = train(e, model, optimizer, train_iter, vocab,
teacher_forcing_ratio, C.CA_lambda, C.gradient_clip)
log_train(C, summary_writer, e, train_loss, get_lr(optimizer),
teacher_forcing_ratio)
lr_scheduler.step()
""" Validation """
val_loss = evaluate(model, val_iter, vocab, C.CA_lambda)
val_scores, _, _, _ = score(model, val_iter, vocab)
log_val(C, summary_writer, e, val_loss, val_scores)
if val_scores['CIDEr'] > best_val_scores['CIDEr']:
best_val_scores = val_scores
best_epoch = e
best_model = model
print("Saving checkpoint at epoch={} to {}".format(e, ckpt_fpath))
save_checkpoint(ckpt_fpath, e, model, optimizer)
""" Test """
test_scores, _, _, _ = score(best_model, test_iter, vocab)
for metric in C.metrics:
summary_writer.add_scalar("BEST SCORE/{}".format(metric),
test_scores[metric], best_epoch)
best_ckpt_fpath = C.ckpt_fpath_tpl.format("best")
save_checkpoint(best_ckpt_fpath, best_epoch, best_model, optimizer)
def randomized_motif_search(dna, k, t, x):
random.seed(x)
motifs = ['CCA', 'CCT', 'CTT', 'TTG']
# for dnastr in dna:
# randnum = random.randint(0, len(dnastr)-k)
# motifs.append(dnastr[randnum:randnum+k])
bestmotifs = motifs
while True:
profile = profile_list(motifs)
motifs = motifs_list(profile, dna, k)
if score(motifs) < score(bestmotifs):
bestmotifs = motifs
else:
return bestmotifs, score(bestmotifs)
def main():
args= argparser()
fs_ = 500
thr_ = 0.075
only_test_data = True
patient_ecg,patient_ref,R_ref,HR_ref = load_patient_data(args.datapath,args.refpath,fs_)
if args.algorithm == 3:
windowed_data = patient_ecg[1936:,:]
windowed_beats = patient_ref[1936:]
scaler = StandardScaler()
scaled_data = scaler.fit_transform(np.asarray(windowed_data).transpose()).transpose()
BATCH_SIZE = 16
patient_ecg_t = torch.from_numpy(scaled_data).float()
patient_ecg_t = patient_ecg_t.view((patient_ecg_t.shape[0],1,patient_ecg_t.shape[1]))
patient_ecg_tl = TensorDataset(patient_ecg_t)
trainloader = DataLoader(patient_ecg_tl, batch_size=BATCH_SIZE)
SAVED_MODEL_PATH = args.model_path
y_pred = load_model_CNN(SAVED_MODEL_PATH,trainloader,args.device)
y_pred_1 = []
for batch in range(len(y_pred)):
for record in range(len(y_pred[batch])):
y_pred_1.append(y_pred[batch][record].cpu().numpy())
y_pred_array = np.asarray(y_pred_1)
y_pred_array_1 = np.asarray(y_pred_1)
peak_locs = []
for i in range(y_pred_array.shape[0]):
peak_locs.append(scipy.signal.find_peaks(-y_pred_array[i,:],distance = 45,height = -0.2,prominence = 0.035)[0])
### Getting the amplitude values at valley location.
y_roll_valleys = []
y = []
for j in range(len(peak_locs)):
y = [y_pred_array[j,i] for i in peak_locs[j]]
y_roll_valleys.append(y)
### Calling the scoring Function
else:
peak_locs,results = call_detector(args,patient_ecg,patient_ref,R_ref,fs_)
r_ans,hr_ans = compute_HR(peak_locs,fs_)
if(only_test_data):
r_ref,hr_ref = R_ref[1936:],HR_ref[1936:]
rec_acc,hr_acc = score(r_ref, hr_ref, r_ans, hr_ans, fs_, thr_)
else:
rec_acc,hr_acc = score(r_ref, hr_ref, r_ans, hr_ans, fs_, thr_)
def era_train_and_score(era_name, x, y, x_valid, y_valid):
blend_x = np.zeros((x_valid.shape[0], len(clfs[era_name])))
for index, clf in enumerate(clfs[era_name]):
clf.fit(x, y)
clf_name = clf.__class__.__name__
if hasattr(clf, 'predict_proba'):
prtmp = clf.predict_proba(x_valid)[:, 1]
else:
prtmp = pd.DataFrame(clf.predict(x_valid)).values.ravel()
blend_x[:, index] = prtmp
score('%s-%s' % (era_name, clf_name), prtmp, y_valid)
return blend_x
def evaluate(model, test_dataset, args, epoch):
test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=True)
criterion = InpaintingLoss(VGG16FeatureExtractor()).to(device)
model.eval()
val_loss = 0.
val_mse_score = 0.
val_ssim_score = 0.
for step, batch in enumerate(test_dataloader):
img, mask, gt = [t.type(torch.float).to(device) for t in batch]
mask = mask.permute(0,3,1,2)
output, _ = model(img, mask)
loss_dict = criterion(img, mask, output, gt)
loss = 0.0
for key, coef in args.LAMBDA_DICT.items():
loss += coef * loss_dict[key].item()
mse, ssim = score(output.detach().cpu().numpy(), gt.detach().cpu().numpy())
val_mse_score += mse
val_ssim_score += ssim
val_loss += loss
del loss_dict
del loss
torch.cuda.empty_cache()
return val_loss, val_mse_score, val_ssim_score
def evaluate(model, test_pairs, batch_size=128):
src_sentences, trg_sentences = zip(*test_pairs)
prd_sentences, _, _ = model.predict(src_sentences, batch_size=batch_size)
assert len(prd_sentences) == len(src_sentences) == len(trg_sentences)
total_score = 0
for i, (src, trg, prd) in enumerate(
tqdm(
zip(src_sentences, trg_sentences, prd_sentences),
desc="scoring",
total=len(src_sentences),
)):
pred_score = score(trg, prd)
total_score += pred_score
if i < 10:
print(f"\n\n\n---- Example {i} ----")
print(f"src = {src}")
print(f"trg = {trg}")
print(f"prd = {prd}")
print(f"score = {pred_score}")
final_score = total_score / len(prd_sentences)
print(f"{total_score}/{len(prd_sentences)} = {final_score:.4f}")
return final_score
def alpha_beta(board, depth, player, alpha, beta, maxim):
if depth == 0 or utils.lastMove(board, player):
return utils.score(board, player), -1
valid = utils.get_moves(board, player)
if maxim:
v = -99999
for move in valid:
(tmp_board, tot) = utils.move(move % 8, int(move / 8), player,
copy.deepcopy(board))
v = max(
alpha_beta(tmp_board, depth - 1, utils.opponent(player), alpha,
beta, False)[0], v)
alpha = max(alpha, v)
if beta <= alpha:
break
return v, move
else: # minimizingPlayer
v = 99999
for move in valid:
(tmp_board, tot) = utils.move(move % 8, int(move / 8), player,
copy.deepcopy(board))
v = min(
alpha_beta(tmp_board, depth - 1, utils.opponent(player), alpha,
beta, True)[0], v)
beta = min(beta, v)
if beta <= alpha:
break
return v, move
def fill_matrix(self):
seq = self.seq
score = self.score_func
n = len(seq)
V = [[0 for j in range(n)] for i in range(n)]
for offset in range(3, n):
for i in range(n - offset):
j = i + offset
# no stacking pair at position j
v0 = V[i][j - 1]
# no stacking pair at position i
v1 = V[i + 1][j]
m = floor((j - i + 1) / 2.0) - 1
# form a length 'h' stacking pair at (k, j)
v2 = max([V[i+h+1][j-h-1] + \
h*reduce(
lambda a,b:a*b,
[score(seq[i+t], seq[j-t]) for t in range(h+1)])
for h in range(1, m+1)] + [0])
# form a stacking pairs at (i, k) and (k, j)
v3 = max([V[i][k] + V[k + 1][j]
for k in range(i + 3, j)]) if j > i + 3 else 0
V[i][j] = max([v0, v1, v2, v3])
self.V_matrix = V
def greedy(libraries, n_days, scores):
day = 0
solution = [-1 for i in range(n_days)]
scanned_books_set = set()
scanned_books_dict = dict()
all_libraries = copy.deepcopy(libraries)
while day < n_days and len(all_libraries) > 0:
lib_id, books = choose_best_score(
n_days - day, all_libraries, scores,
scanned_books_set) # gets the best library to sign up
if lib_id == -1:
break
lib = libraries[lib_id]
scanned_books_dict[lib_id] = books
scanned_books_set.update(books)
for _ in range(
lib.signup_days): # stores in the solution the chosen library
solution[day] = lib_id
day += 1
all_libraries.remove(
lib) # removes chosen library from the list of available libraries
while len(solution) < n_days:
solution.append(-1)
return Solution(solution, score(scanned_books_set, scores),
scanned_books_dict)
def create_evaluation_distinctiveness(config, Kind):
model_fname = config.model_fname % Kind.__name__
try:
model = LdaModel.load(model_fname)
logger.info('Opened previously created model at file %s' % model_fname)
except:
error('Cannot evalutate LDA models not built yet!')
scores = utils.score(model, utils.kullback_leibler_divergence)
total = sum([x[1] for x in scores])
logger.info("%s model KL: %f" % (model_fname, total))
with open(config.path + 'evaluate-results.csv', 'a') as f:
w = csv.writer(f)
w.writerow([model_fname, total])
etas = list()
for topic in model.state.get_lambda():
topic_eta = list()
for p_w in topic:
topic_eta.append(p_w * numpy.log2(p_w))
etas.append(-sum(topic_eta))
entropy = sum(etas) / len(etas)
logger.info("%s model entropy mean: %f" % (model_fname, entropy))
with open(config.path + 'evaluate-entropy-results.csv', 'a') as f:
w = csv.writer(f)
w.writerow([model_fname, entropy])
def train_kfold_all(save_history=False):
'''
This function performs kfold models training
on all targets together and saves results.
Returns data frame of predicted validation data.
:param save_history: bool, save history.json or not.
:return: oof_df
'''
oof_df = pd.DataFrame(data=np.zeros(train_data[TARGETS].shape), columns=TARGETS)
print('Training on all targets starts.')
for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data, train_data)):
print(f'Fold {fold_+1}')
tr_data = train_data.loc[trn_idx]
val_data = train_data.loc[val_idx]
tr_dataset = MRIDataset(MRI_TR, tr_data, 'train', (TARGETS))
val_dataset = MRIDataset(MRI_TR, val_data, 'val', TARGETS)
model = create_nn(N_CLASSES)
opt = torch.optim.SGD(model.parameters(), lr=LR,
momentum=0.9, weight_decay=7e-4)
criterion = nn.L1Loss()
with torch.autograd.set_detect_anomaly(True):
history = train(tr_dataset, val_dataset, model, epochs=N_EPOCHS,
batch_size=BS, opt=opt, criterion=criterion)
if save_history:
loss, acc, val_loss, val_acc = zip(*history)
history_dict = {
'train_loss': loss,
'val_loss': val_loss,
'train_acc': acc,
'val_acc': val_acc
}
with open(f'{HISTORY_DIR}/all_fold{fold_}_history.json', 'w') as f:
json.dump(history_dict, f, indent=2, ensure_ascii=False)
weights = model.state_dict()
torch.save(weights,
f'{WEIGHTS_DIR}/model_all_fold{fold_}_epochs{N_EPOCHS}_bs{BS}.pth')
val_loader = DataLoader(dataset=val_dataset, shuffle=False,
batch_size=BS, num_workers=8)
val_preds = make_val_preds(model, val_loader)
oof_df.loc[val_idx] = val_preds['predicted']
print('==================================')
print(f'All val score on fold {fold_+1}: ')
print(score(val_preds['labels'], val_preds['predicted']))
print(f'All val MAE on fold {fold_+1}: ')
print(mean_absolute_error(val_preds['labels'], val_preds['predicted']))
print('==================================')
return oof_df
def model_rf(train , test , flag):
train_x , train_y = train[0] , train[1]
test_x , test_y = test[0] , test[1]
rf = RandomForestRegressor()
rf.set_params(**Params.rf_reg_params)
print "start training"
rf.fit(train_x , train_y['gap'].values)
if flag == 'online':
prd = rf.predict(test_x)
prd = postprocess(train , test_y.values , prd)
Util.submit(test_y.values , prd)
elif flag == 'offline':
prd = rf.predict(test_x)
prd = postprocess(train , test_y.values , prd)
print 'test : ' , Util.score(test_y.values , prd)
prd = rf.predict(train_x)
print 'train : ' , Util.score(train_y.values , prd)
def run(ckpt_fpath):
checkpoint = torch.load(ckpt_fpath)
""" Load Config """
config = dict_to_cls(checkpoint['config'])
""" Build Data Loader """
if config.corpus == "MSVD":
corpus = MSVD(config)
elif config.corpus == "MSR-VTT":
corpus = MSRVTT(config)
train_iter, val_iter, test_iter, vocab = \
corpus.train_data_loader, corpus.val_data_loader, corpus.test_data_loader, corpus.vocab
print('#vocabs: {} ({}), #words: {} ({}). Trim words which appear less than {} times.'.format(
vocab.n_vocabs, vocab.n_vocabs_untrimmed, vocab.n_words, vocab.n_words_untrimmed, config.loader.min_count))
""" Build Models """
decoder = Decoder(
rnn_type=config.decoder.rnn_type,
num_layers=config.decoder.rnn_num_layers,
num_directions=config.decoder.rnn_num_directions,
feat_size=config.feat.size,
feat_len=config.loader.frame_sample_len,
embedding_size=config.vocab.embedding_size,
hidden_size=config.decoder.rnn_hidden_size,
attn_size=config.decoder.rnn_attn_size,
output_size=vocab.n_vocabs,
rnn_dropout=config.decoder.rnn_dropout)
decoder.load_state_dict(checkpoint['decoder'])
model = CaptionGenerator(decoder, config.loader.max_caption_len, vocab)
model = model.cuda()
""" Train Set """
train_scores, train_refs, train_hypos = score(model, train_iter, beam_width=5, beam_alpha=0.)
save_result(train_refs, train_hypos, C.result_dpath, config.corpus, 'train')
print("[TRAIN] {}".format(train_scores))
""" Validation Set """
val_scores, val_refs, val_hypos = score(model, val_iter, beam_width=5, beam_alpha=0.)
save_result(val_refs, val_hypos, C.result_dpath, config.corpus, 'val')
print("[VAL] scores: {}".format(val_scores))
""" Test Set """
test_scores, test_refs, test_hypos = score(model, test_iter, beam_width=5, beam_alpha=0.)
save_result(test_refs, test_hypos, C.result_dpath, config.corpus, 'test')
print("[TEST] {}".format(test_scores))
def gibbsSampler(dna, k, t, N, x):
# initialize bestmotifs as motifs
random.seed(x)
motifs = []
for dnastr in dna:
randnum = random.randint(0, len(dnastr) - k)
motifs.append(dnastr[randnum:randnum + k])
bestmotifs = motifs
# for N iterations
for i in range(1, N):
i = random.randint(0, t - 1)
motifs.pop(i)
# profile_list
profile = profile_list(motifs)
motifi = prof_random_kmer(dna[i], k, profile)[0]
motifs.insert(i, motifi)
if score(motifs) < score(bestmotifs):
bestmotifs = motifs
return bestmotifs, score(bestmotifs)
def model_rf_cv(train , test):
train_x , train_y = train[0] , train[1]
cv = cross_validation.KFold(len(train_x) , n_folds = 5)
results = []
rf = RandomForestRegressor()
rf.set_params(**Params.rf_reg_params)
for traincv , testcv in cv:
print traincv , testcv
probas = rf.fit(train_x[traincv] , train_y['gap'][traincv].values).predict(train_x[testcv])
results.append(Util.score(train_y.loc[testcv , y_fea_names].values , probas))
print results
print np.mean(results)
def fill_matrix(self):
seq = self.seq
score = self.score_func
n = len(seq)
V = [[0 for j in range(n)] for i in range(n)]
for m in range(n - 1):
for i in range(n - m - 1):
j = i + m + 1
v0 = V[i + 1][j - 1] + score(seq[i], seq[j])
v1 = max([V[i][k] + V[k + 1][j] for k in range(i, j)])
V[i][j] = max(v0, v1)
self.V_matrix = V
def auxiliar_neighbour(solution, current_lib, libraries, free_slots, n_days,
scores):
day = 0
books2lib = dict()
scanned_books = set()
new_list = []
while day < solution.libraries_list.index(current_lib):
lib = solution.libraries_list[day]
books2lib[lib] = solution.books2lib[lib]
scanned_books.update(books2lib[lib])
for _ in range(libraries[lib].signup_days):
new_list.append(lib)
day += 1
remaining_days = libraries[current_lib].signup_days + free_slots
all_libraries = [
lib for lib in libraries if lib.id not in solution.libraries_list
and lib.signup_days <= remaining_days
] # gets all the libraries that can sign up and are not in the solution
lib_id, books = choose_best_score(
n_days - day, all_libraries, scores,
scanned_books) # finds the best library to sign up (compares scores)
new_day = day
if lib_id != -1:
books2lib[lib_id] = books
scanned_books.update(books)
for _ in range(libraries[lib_id].signup_days):
new_list.append(lib_id)
new_day += 1
day += libraries[current_lib].signup_days
while day < n_days:
lib = solution.libraries_list[day]
if lib == -1:
break
books2lib[lib] = libraries[lib].get_books(n_days - new_day,
scanned_books)
scanned_books.update(books2lib[lib])
for _ in range(libraries[lib].signup_days):
new_list.append(lib)
new_day += 1
day += 1
while len(new_list) < n_days:
new_list.append(-1)
new_score = score(scanned_books, scores)
return new_list, new_score, books2lib
def train_model(data_path):
X_train, X_valid, X_test, y_train, y_valid, y_test = load_data(data_path)
X_train, y_train = preprocess(X_train, y_train)
X_valid, y_valid = preprocess(X_valid, y_valid)
X_test, _ = preprocess(X_test, y_test)
#y_pred = mlp_model(X_train, y_train, X_valid, y_valid, X_test)
y_pred = cnn_model(X_train, y_train, X_valid, y_valid, X_test)
#y_pred = dummy_model(X_train, y_train, X_valid, y_valid, X_test)
return score(y_pred, y_test)
def train(self, train_loader, dev_loader, epochs=2, verbose=1):
batch_size = train_loader.batch_size
iterations = len(train_loader)
for epoch in range(epochs):
running_loss = 0.0
running_acc = 0.0
epoch_init_time = time.time()
for i, batch in enumerate(train_loader):
self.net = self.net.to(self.device).train()
self.optimizer.zero_grad()
X, Y, lengths = batch
X, Y, lengths = X.to(self.device), Y.to(
self.device), lengths.float().to(self.device)
Y_pred = self.net(X, lengths)
loss = self.criterion(Y_pred, Y)
loss.backward()
self.optimizer.step()
running_acc += score(Y_pred, Y)
running_loss += loss.item()
if verbose == 1:
self.__report_iteration(iterations, i, epoch_init_time,
epoch, running_loss, running_acc,
verbose)
self.__log_detailed_history(epoch, i, running_loss,
running_acc)
if dev_loader:
dev_loss, dev_acc = self.evaluate(dev_loader)
else:
dev_loss, dev_acc = None, None
self.__report_epoch(epoch, epochs, epoch_init_time, iterations,
running_loss, running_acc, dev_loss, dev_acc,
verbose)
self.__log_history(epoch, iterations, running_loss, running_acc,
dev_loss, dev_acc)
if self.patience and self.epochs_not_improving >= self.patience:
best_dev_loss = self.best_dev_loss['value']
best_dev_acc = self.best_dev_acc['value']
final_msg = 'Enough of not improving...' +\
f'best_dev_loss:{best_dev_loss:02.4f}, ' +\
f'best_dev_acc:{best_dev_acc*100:02.2f}%'
print(final_msg)
break
return self.history_output
def evaluate(self, data_loader):
with torch.no_grad():
running_loss = 0
running_acc = 0
for batch in data_loader:
self.net = self.net.to(self.device).eval()
X, Y, lengths = batch
X, Y, lengths = X.to(self.device), Y.to(
self.device), lengths.float().to(self.device)
Y_pred = self.net(X, lengths)
running_loss += self.criterion(Y_pred, Y).item()
running_acc += score(Y_pred, Y)
loss = running_loss / len(data_loader)
acc = running_acc / len(data_loader)
return loss, acc
def cross_validate(self, x, y):
scores = []
raws = []
n = math.ceil(len(x)/5)
x_chunks = [x[k:k+n] for k in range(0, len(x), n)]
y_chunks = [y[k:k+n] for k in range(0, len(y), n)]
for i in range(5):
xtrain = x_chunks[(i+1) % 5]
ytrain = y_chunks[(i+1) % 5]
for j in range(3):
x_train = np.concatenate((x_chunks[(i+j+2) % 5], xtrain))
y_train = np.concatenate((y_chunks[(i+j+2) % 5], ytrain))
self.train(x_train, y_train)
p = self.predict(x_chunks[i])
scores.append(utils.score(y_chunks[i], p))
return scores
def calculate_solution_score(sol, libraries, scores):
books = set()
day = 0
n_days = len(sol)
# iterates over the list, in which each position has the id of the library being signup up in the day corresponding to that index
while day < n_days:
if sol[day] == -1:
# already signed up all libraries
break
lib = libraries[sol[day]]
# get all books that the current library can send, given the remaining days and already scanned books
books.update(lib.get_books(n_days - day, books))
# skip to next library's index
day += lib.signup_days
# return total score of the scanned books
return score(books, scores)
def evaluate_metrics(self,
data_loader,
metrics=['accuracy', 'precision', 'recall', 'f1']):
out_metrics = []
Ys, Ypreds = [], []
with torch.no_grad():
for batch in data_loader:
self.net = self.net.to(self.device).eval()
X, Y, lengths = batch
X, Y, lengths = X.to(self.device), Y.to(
self.device), lengths.float().to(self.device)
Y_pred = self.net(X, lengths)
Ys.append(Y)
Ypreds.append(Y_pred)
Y = torch.cat(Ys)
Y_pred = torch.cat(Ypreds)
for metric in metrics:
out_metrics.append(score(Y_pred, Y, metric))
return out_metrics
def run(corpus, ckpt_fpath):
C.corpus = corpus
if corpus == 'MSVD':
C.loader = MSVDLoaderConfig
elif corpus == 'MSR-VTT':
C.loader = MSRVTTLoaderConfig
else:
raise NotImplementedError('Unknown corpus: {}'.format(corpus))
checkpoint = torch.load(ckpt_fpath)
train_iter, val_iter, test_iter, vocab = build_loaders(C)
model = build_model(C, vocab)
model.load_state_dict(torch.load(ckpt_fpath))
model.cuda()
model.eval()
scores, _, _, _ = score(model, test_iter, vocab)
print(scores)
def min_max(board, depth, player, maxim):
if depth == 0 or utils.lastMove(board, player):
return utils.score(board, player)
valid = utils.get_moves(board, player)
if maxim:
bestValue = -99999
for move in valid:
(tmp_board, tot) = utils.move(move, player, copy.deepcopy(board))
v = min_max(tmp_board, depth - 1, utils.opponent(player), False)
bestValue = max(bestValue, v)
# print(np.array(tmp_board).reshape((8,8)))
return bestValue
else: # minimizingPlayer
bestValue = 99999
for move in valid:
(tmp_board, tot) = utils.move(move, player, copy.deepcopy(board))
v = min_max(tmp_board, depth - 1, utils.opponent(player), True)
bestValue = min(bestValue, v)
# print(np.array(tmp_board).reshape((8,8)))
return bestValue
def free_energy_of_stacking_pairs_with_length(self, i, j, h, penalty_func=logistic):
seq = self.seq
matchs = []
for t in range(h+1):
matchs.append(score(seq[i+t], seq[j-t]))
all_match = reduce(lambda a,b:a*b, matchs)
if all_match:
energys = []
for t in range(h):
e = stacking_pair_free_energy(seq[i+t], seq[j-t], seq[i+t+1], seq[j-t-1])
if penalty_func:
d = float(j - i - 2*t)
r = penalty_func(d, len(seq))
e = e * r
energys.append(e)
eS = -sum(energys)
return eS
else:
return 0
def read_output(file, libraries, scores, n_days):
with open(file, 'r', encoding='utf-8') as ifp:
lines = ifp.readlines()
n_libraries = int(lines[0])
libraries_list = []
books2lib = dict()
books = set()
for i in range(n_libraries):
line = lines[i * 2 + 1].split()
lib_id = int(line[0])
for _ in range(libraries[lib_id].signup_days):
libraries_list.append(lib_id)
books2lib[lib_id] = [int(x) for x in lines[i * 2 + 2].split()]
books.update(books2lib[lib_id])
while len(libraries_list) < n_days:
libraries_list.append(-1)
return Solution(libraries_list, score(books, scores), books2lib)
def main(args):
patient_ecg, windowed_beats = obtain_data(args)
BATCH_SIZE = 64
patient_ecg_t = torch.from_numpy(patient_ecg).float()
patient_ecg_t = patient_ecg_t.view(
(patient_ecg_t.shape[0], 1, patient_ecg_t.shape[1]))
patient_ecg_tl = TensorDataset(patient_ecg_t)
trainloader = DataLoader(patient_ecg_tl, batch_size=BATCH_SIZE)
SAVED_MODEL_PATH = args.model_path
y_pred = load_model_CNN(SAVED_MODEL_PATH, trainloader, args.device)
y_pred_1 = []
for batch in range(len(y_pred)):
for record in range(len(y_pred[batch])):
y_pred_1.append(y_pred[batch][record].cpu().numpy())
y_pred_array = np.asarray(y_pred_1)
y_pred_array_1 = np.asarray(y_pred_1)
resampled_dt = []
for record in range(y_pred_array.shape[0]):
resampled_dt.append(scipy.signal.resample(y_pred_array_1[record],
3600))
y_pred_array = np.asarray(resampled_dt)
peak_locs = []
for i in range(y_pred_array.shape[0]):
peak_locs.append(
scipy.signal.find_peaks(-y_pred_array[i, :],
distance=45,
height=-0.2,
prominence=0.035)[0])
### Getting the amplitude values at valley location.
y_roll_valleys = []
y = []
for j in range(len(peak_locs)):
y = [y_pred_array[j, i] for i in peak_locs[j]]
y_roll_valleys.append(y)
### Calling the scoring Function
FS = 360
THR = 0.075
rec_acc, all_FP, all_FN, all_TP = score(windowed_beats, peak_locs, FS, THR)
def simulated_annealing(solution, tags, n_iter=1000000):
temperature = 100
best_score, best_solution = utils.score(file, solution), solution
current_score = best_score
with tqdm.trange(n_iter) as progress:
for i in progress:
u = np.random.randint(len(solution))
v = np.random.randint(len(solution))
score_delta = _score_swap(solution, tags, u, v)
prob = 1 if score_delta > 0 else np.exp(score_delta / temperature)
if np.random.random() < prob:
solution[u], solution[v] = solution[v], solution[u]
current_score += score_delta
if (i + 1) % 100 == 0:
temperature *= 0.95
if current_score > best_score:
best_score = current_score
best_solution = copy.deepcopy(solution)
progress.set_postfix(best_score=best_score,
current_score=current_score)
return best_solution
def test_clf(clf, docs_test, y_test):
global best_score, best_clf
print "######################################################################"
print "# Showing for %s" % clf
print_best_params(clf)
show_most_informative_features(clf)
y_predict = clf.predict(docs_test)
current_score = u.score(y_test, y_predict)
print 30 * "="
print "# Total Score: %s " % current_score
print 30 * "="
print clf_report(y_test, y_predict)
print clf_confusion_matrix(y_test, y_predict)
if current_score >= best_score:
best_clf = clf
best_score = current_score
print "######################################################################"
from utils import score names = sorted( eval( open( "txt/p022" ).read() ) ) print sum( ( i+1 ) * score( name ) for i, name in enumerate( names ) )
from utils import isTriangle, score words = eval( open( "txt/p042" ).read() ) print sum( 1 for word in words if isTriangle( score( word ) ) )
# load the source space
source_sp = Space.build(source_file, source_words.union(set(lexicon)))
source_sp.normalize()
print "Reading: %s" % target_file
target_sp = Space.build(target_file)
target_sp.normalize()
print "Translating" # translates all the elements loaded in the source space
mapped_source_sp = apply_tm(source_sp, tm)
print "Retrieving translations"
test_data = get_valid_data(source_sp, target_sp, test_data)
# turn test data into a dictionary (a word can have mutiple translation)
gold = collections.defaultdict(set)
for k, v in test_data:
gold[k].add(v)
score(mapped_source_sp, target_sp, gold, additional)
print "Printing mapped vectors: %s" % out_file
np.savetxt("%s.vecs.txt" % out_file, mapped_source_sp.mat)
np.savetxt("%s.wds.txt" % out_file, mapped_source_sp.id2row, fmt="%s")
if __name__ == '__main__':
main(sys.argv)
def score(self, selection_strength, games):
"""
Calculates player's fitness score based on payoffs
"""
return utils.score(self.payoffs, selection_strength, games)
def score(self, selection_strength, games):
return utils.score(self.payoffs, selection_strength, games)
def get_score(query, m): seq = query[m.query_ix:m.query_ix + m.length] gen = genome[m.genome_ix:m.genome_ix + m.length] return utils.score(gen, probs[m.genome_ix:m.genome_ix+m.length], seq)