def pipe(event, context):
print(event)
raw = extract(event)
print(raw)
transformed = transform(raw.values.tolist())
print(transformed)
load(transformed)
def __init__(self, args):
self.args = args
# check parameters
if self.args.adversarial:
assert 0 <= self.args.dis_dropout < 1
assert 0 <= self.args.dis_input_dropout < 1
assert 0 <= self.args.dis_smooth < 0.5
assert self.args.dis_lambda > 0 and self.args.dis_steps > 0
assert 0 < self.args.lr_shrink <= 1
assert self.args.model_path is not None
self.dataset = None
# build model / trainer / evaluator
if not self.args.pred and not self.args.cal_sent_sim:
self.logger = initialize_exp(self.args)
if self.args.adversarial or self.args.cal_sent_sim:
assert os.path.isfile(self.args.input_file)
self.dataset, unique_id_to_feature, self.features = load(self.args.vocab_file,
self.args.input_file, batch_size=self.args.batch_size,
do_lower_case=self.args.do_lower_case, max_seq_length=self.args.max_seq_length,
local_rank=self.args.local_rank, vocab_file1=self.args.vocab_file1)
self.bert_model, self.mapping, self.discriminator, self.bert_model1 = build_model(self.args, True)
if self.args.adversarial or self.args.pred:
self.trainer = BertTrainer(self.bert_model, self.dataset, self.mapping, self.discriminator,
self.args, bert_model1=self.bert_model1)
if self.args.adversarial or self.args.cal_sent_sim:
self.evaluator = BertEvaluator(self.bert_model, self.dataset, self.mapping, self.discriminator,
self.args, self.features, bert_model1=self.bert_model1)
if self.args.local_rank == -1 or self.args.no_cuda:
self.device = torch.device("cuda" if torch.cuda.is_available() and not self.args.no_cuda else "cpu")
else:
self.device = torch.device("cuda", self.args.local_rank)
def display():
for test in load.load():
if test.name: print(test.name + " :")
print(" entrée : {}".format(test.inp))
print(" code : {}".format(str(test.code)))
print(" sortie : {}".format(test.outpout))
print("")
def etl_accident():
for file in INPUT_DATA_GOUV_URL:
year = file.get("year")
print(year)
dict = {}
for input_url in file.get("urls"):
url = input_url.get("url")
type = input_url.get("type")
data = extract(url, type)
data = transform(data, type)
dict[type] = data
load(dict['vehicles'], dict['users'], dict['places'],
dict['characteristics'])
def start():
result = {"ok": 0, "fail": 0, "crash": 0}
tests = load.load()
param = load.param()
if param["compile"]:
print("\n== COMPILATION ==\n")
rt = subprocess.run(["make", "re"])
if rt.returncode != 0:
print(message["crash"], "make failed")
sys.exit(0)
print("\n== TESTS ==\n")
for test in tests:
print(test.name, end=" : ")
try:
rt = subprocess.run(["./" + param["bin_name"]] +
shlex.split(test.inp),
timeout=param["timeout"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outpout = rt.stdout.decode("utf-8")
except:
print(message["crash"], "\nTimeout\n")
result["crash"] += 1
continue
if rt.returncode < 0:
print(
message["crash"],
"\nLe programme à reçu le signal {} {}".format(
str(abs(rt.returncode)),
"(Segmentation Fault)" if rt.returncode == -11 else ""))
result["crash"] += 1
elif rt.returncode != test.code:
print(
message["fail"], "\nCode attendu : {}\nCode reçu : {}".format(
str(test.code), str(rt.returncode)))
result["fail"] += 1
elif outpout != test.outpout and test.outpout:
print(message["fail"],
"\nAttendu : {}\nReçu : {}".format(test.outpout, outpout))
result["fail"] += 1
else:
print(message["succes"])
result["ok"] += 1
print("")
print(message["succes"], result["ok"], message["fail"], result["fail"],
message["crash"], result["crash"])
def add(argv):
tests = load.load()
if len(argv) <= 2:
inp = input_inp()
else:
inp = int(argv[2])
if len(argv) <= 3:
code = input_code()
else:
code = int(argv[3])
if len(argv) <= 4:
outpout = input_outp()
else:
outpout = argv[4]
if len(argv) > 5:
name = argv[5 ]
else:
name = input_name()
if name:
tests.append(Tests(inp,code,outpout,name))
else:
tests.append(Tests(inp,code,outpout))
load.save(tests)
Program: Python unittest
Version: 20210604
@author: Pranab Das (Twitter: @pranab_das)
Run : python3 test.py
or : python3 -m unittest test.py
"""
import unittest
import numpy as np
from src.load import load
from src.calc_area import calc_area
from src.fit_gauss import fit_gauss
from src.fit_lorentz import fit_lorentz
from src.lock_peak import lock_peak
from src.norm_bg import norm_bg
data = load("dataset/sample_data.txt", 12)
s1 = load("dataset/sample_XAS.txt", 1)
s2 = load("dataset/sample_XAS.txt", 2)
class Test(unittest.TestCase):
def test_load(self):
self.assertEqual(data.shape, (6501, 11))
self.assertAlmostEqual(data[1200, 0], 27.0)
self.assertAlmostEqual(data[1500, 6], 26.0)
def test_calc_area(self):
self.assertAlmostEqual(calc_area(data[:, 6], data[:, 0], 20, 120), 270053.1, places=1)
def test_fit_gauss(self):
x_fit, y_fit = fit_gauss(data[:, 0], data[:, 6], xmin=93, xmax=95)
x0 = x_fit[np.where(y_fit==max(y_fit))[0][0]]
def train(self):
"""
"""
if self.args.load_pred_bert:
assert self.args.bert_file0 is not None
assert self.args.bert_file1 is not None
assert self.args.vocab_file is not None
assert self.args.vocab_file1 is not None
self.dataset, unique_id_to_feature, self.features = load_from_bert(
self.args.vocab_file,
self.args.bert_file0,
self.args.bert_file1,
do_lower_case=self.args.do_lower_case,
max_seq_length=self.args.max_seq_length,
n_max_sent=self.args.n_max_sent,
vocab_file1=self.args.vocab_file1,
align_file=self.args.align_file,
align_punc=self.args.align_punc,
policy=self.args.align_policy)
else:
self.dataset, unique_id_to_feature, self.features = load(
self.args.vocab_file,
self.args.input_file,
batch_size=self.args.batch_size,
do_lower_case=self.args.do_lower_case,
max_seq_length=self.args.max_seq_length,
local_rank=self.args.local_rank,
vocab_file1=self.args.vocab_file1,
align_file=self.args.align_file)
self.trainer = SupervisedBertTrainer(
self.bert_model,
self.mapping,
self.args,
bert_model1=self.bert_model1,
trans_types=self.transformer_types)
sampler = RandomSampler(self.dataset)
train_loader = DataLoader(self.dataset,
sampler=sampler,
batch_size=self.args.batch_size)
n_without_improvement = 0
min_loss = 1e6
path4loss = self.args.model_path + '/model4loss'
if not os.path.exists(path4loss):
os.makedirs(path4loss)
if self.args.save_all:
model_log = open(path4loss + '/model.log', 'w')
# training loop
for n_epoch in range(self.args.n_epochs):
#self.logger.info('Starting epoch %i...' % n_epoch)
if (n_epoch + 1) % self.args.decay_step == 0:
self.trainer.decay_map_lr()
n_inst = 0
n_batch = 0
to_log = {"avg_cosine_similarity": 0, "loss": 0}
if self.args.load_pred_bert:
for input_embs_a, input_mask_a, input_embs_b, input_mask_b, align_ids_a, align_ids_b, align_mask, example_indices in train_loader:
n_batch += 1
with torch.no_grad():
input_embs_a = input_embs_a.to(self.device)
input_mask_a = input_mask_a.to(self.device)
input_embs_b = input_embs_b.to(self.device)
align_ids_a = align_ids_a.to(self.device)
align_ids_b = align_ids_b.to(self.device)
align_mask = align_mask.to(self.device)
#print (align_ids_a, align_ids_b, align_mask)
src_bert = self.trainer.get_indexed_mapped_bert_from_bert(
input_embs_a,
input_mask_a,
align_ids_a,
align_mask,
bert_layer=self.args.bert_layer)
tgt_bert = self.trainer.get_indexed_bert_from_bert(
input_embs_b,
align_ids_b,
align_mask,
bert_layer=self.args.bert_layer)
avg_cos_sim, loss = self.trainer.supervised_mapping_step(
src_bert, tgt_bert)
n_inst += src_bert.size()[0]
cos_sim = avg_cos_sim.cpu().detach().numpy()
loss_ = loss.cpu().detach().numpy()
to_log["avg_cosine_similarity"] += cos_sim
to_log["loss"] += loss_
else:
for input_ids_a, input_mask_a, input_ids_b, input_mask_b, align_ids_a, align_ids_b, align_mask, example_indices in train_loader:
n_batch += 1
input_ids_a = input_ids_a.to(self.device)
input_mask_a = input_mask_a.to(self.device)
input_ids_b = input_ids_b.to(self.device)
input_mask_b = input_mask_b.to(self.device)
align_ids_a = align_ids_a.to(self.device)
align_ids_b = align_ids_b.to(self.device)
align_mask = align_mask.to(self.device)
#print (align_ids_a, align_ids_b, align_mask)
src_bert = self.trainer.get_indexed_mapped_bert(
input_ids_a,
input_mask_a,
align_ids_a,
align_mask,
bert_layer=self.args.bert_layer,
model_id=0)
tgt_bert = self.trainer.get_indexed_bert(
input_ids_b,
input_mask_b,
align_ids_b,
align_mask,
bert_layer=self.args.bert_layer,
model_id=1)
avg_cos_sim, loss = self.trainer.supervised_mapping_step(
src_bert, tgt_bert)
n_inst += src_bert.size()[0]
cos_sim = avg_cos_sim.cpu().detach().numpy()
loss_ = loss.cpu().detach().numpy()
to_log["avg_cosine_similarity"] += cos_sim
to_log["loss"] += loss_
to_log["avg_cosine_similarity"] /= n_batch
to_log["loss"] /= n_batch
self.logger.info(
"Epoch:{}, avg cos sim:{:.6f}, avg loss:{:.6f}, instances:{}".
format(n_epoch, to_log["avg_cosine_similarity"],
to_log["loss"], n_inst))
if to_log[
"avg_cosine_similarity"] <= self.trainer.best_valid_metric and to_log[
"loss"] >= min_loss:
n_without_improvement += 1
else:
n_without_improvement = 0
if to_log["loss"] < min_loss:
self.logger.info(" Minimum loss : {:.6f}".format(
to_log["loss"]))
if self.args.save_all:
save_path = path4loss + '/epoch-' + str(n_epoch)
model_log.write(
"Epoch:{}, avg cos sim:{:.6f}, avg loss:{:.6f}\n".
format(n_epoch, to_log["avg_cosine_similarity"],
to_log["loss"]))
else:
save_path = path4loss
self.trainer.save_model(save_path + '/best_mapping.pkl')
min_loss = to_log["loss"]
if self.args.save_sim:
self.trainer.save_best(to_log, "avg_cosine_similarity")
else:
if to_log[
"avg_cosine_similarity"] > self.trainer.best_valid_metric:
self.trainer.best_valid_metric = to_log[
"avg_cosine_similarity"]
self.logger.info(
"Max avg cos sim:{:.6f}, Min avg loss:{:.6f}".format(
self.trainer.best_valid_metric, min_loss))
#self.logger.info('End of epoch %i.\n\n' % n_epoch)
if n_without_improvement >= self.args.quit_after_n_epochs_without_improvement:
self.logger.info(
'After {} epochs without improvement, quiting!'.format(
n_without_improvement))
break