def add_to_cart(item_id):
cart_id = session.get('current_user_cart')['id']
query = f"""
INSERT INTO Service_cart_rel (Service_id, Cart_id) VALUES ({item_id}, {cart_id})
"""
db.db_save(query)
return redirect('/')
def save_secret(text: str = Form("secret_form"),
ttl: int = Form("secret_form")):
if 31 < ttl < 0:
raise HTTPException(status_code=400, detail="wrong ttl value")
uid = uuid.uuid4()
expires = datetime.datetime.now() + datetime.timedelta(days=ttl)
try:
db.db_save(uid, text, expires)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
return "https://secret.exo.icu/get/{}".format(uid)
def load(lingo: duolingo.Duolingo, learning_lang: str, mother_lang: str):
vocab = lingo.get_vocabulary()
old_words = db_load("words") or []
words = vocab["vocab_overview"]
print(
f"Loaded vocabulary. You have studied {len(words)} words ({len(words) - len(old_words)} new since last time)."
)
print(f"Loading translations to {mother_lang}...")
translations = get_translations(lingo, words, learning_lang, mother_lang)
words = [{
**word, "translations": translations[word["word_string"]]
} for word in words]
print(f"Loaded translations.")
updated_words = init_practice_counts(update_word_dict(old_words, words))
db_save("words", updated_words)
print("Updated")
def register():
form = RegisterForm(request.form)
if request.method == 'POST' and form.validate():
user = db.db_get(
f"""select * from Buser where login='******';""")
if user:
return render_template(
'register.html',
form=form,
alert='Юзер с таким логином уже существует'), 400
query = f"insert into Buser (login, password, Role_id) values ('{form.login.data}', '{form.password.data}', 2);"
db.db_save(query)
user = db.db_get(
f"""select * from Buser where login='******';""")
session['current_user'] = user
db.db_save(
f"""insert into cart (Buser_id, Status_id) values ((SELECT MAX(id) FROM buser), 6);"""
)
session['current_user_cart'] = db.db_get(
f"""select * from cart where Buser_id=(SELECT MAX(id) FROM buser);"""
)
return redirect('/')
return render_template('register.html', form=form)
def manage_orders(status_id):
if not session.get('current_user'):
return redirect('/login')
elif not session.get('current_user')['role_id'] == 1:
return render_template('401.html')
form = StatusForm(request.form)
if request.method == 'POST':
db.db_save(
f'update border set Status_id={form.status.data} where id={request.args.get("ord_id")}'
)
query = f"""
select distinct o.id, o.buser_id, o.payment_type, o.time , u.name, u.last_name, u.surname, u.phone , s.sname, o.price from border o
full outer join buser u on o.buser_id = u.id inner join cart c on u.id = c.buser_id inner join Order_cart_rel ok
on o.id = ok.border_id and c.id=ok.cart_id inner join Service_cart_rel r on r.cart_id = c.id inner join
service s on r.service_id = s.id and r.cart_id = c.id where o.status_id={status_id} and c.status_id=7 order by id;
"""
db_data = db.db_get(query, cur_type='all')
data = prepare_data(db_data)
return render_template('orders.html',
data=data,
form=form,
status_id=status_id)
def create_order():
if not session.get('current_user'):
return redirect('/login')
form = OrderForm(request.form)
price = request.args.get("price")
user_id = session.get('current_user')['id']
cart_id = session.get('current_user_cart')['id']
if request.method == 'POST' and request.args.get("reject"):
query = f"""update cart set status_id=7 where id={cart_id};
insert into cart (Buser_id, Status_id) values ({user_id}, 6);
insert into border (Buser_id,Status_id,payment_type) values ({user_id}, 3, '{form.payment_type.data}');
insert into Order_cart_rel (Cart_id, Border_id) values ({cart_id}, (SELECT MAX(id) FROM border));"""
db.db_save(query)
session['current_user_cart'] = db.db_get(
f'select * from cart where buser_id={user_id} and status_id=6;')
return redirect('/rejected')
elif request.method == 'POST' and price and form.validate():
query = f"""UPDATE Buser set name='{form.name.data}', last_name='{form.last_name.data}', surname='{form.surname.data}', phone='{form.phone.data}' where id={user_id};
update cart set status_id=7 where id={cart_id};
insert into cart (Buser_id, Status_id) values ({user_id}, 6);
insert into border (Buser_id,Status_id,payment_type, time, price) values ({user_id}, 1, '{form.payment_type.data}', to_timestamp('{form.time.data}', 'yyyy-mm-dd hh24:mi:ss'), {price});
insert into Order_cart_rel (Cart_id, Border_id) values ({cart_id}, (SELECT MAX(id) FROM border));
"""
db.db_save(query)
session['current_user_cart'] = db.db_get(
f'select * from cart where buser_id={user_id} and status_id=6;')
return redirect('/success')
data = db.db_get(
f"""select * from service where id in (select service_id from Service_cart_rel where cart_id={session.get('current_user_cart')['id']});""",
cur_type='all')
price = 0
if data:
for item in data:
price += item['price']
context = {'data': data, 'price': price}
return render_template('order.html', data=context, form=form)
const=True,
help='Loads the newest vocabulary data for your user')
parser.add_argument('--play',
action='store_const',
const=True,
help='Start practicing')
args = parser.parse_args()
password = db_load('password')
username = db_load('username')
missing_pass = password is None and args.password is None
missing_username = username is None and args.username is None
if missing_pass or missing_username:
print("Please log in with username and password")
exit(1)
if args.password is not None:
db_save('password', args.password)
password = args.password
if args.username is not None:
db_save('username', args.username)
username = args.username
learning_lang = db_load('learning_lang')
mother_lang = db_load('mother_lang')
if learning_lang is None and args.learning_lang is None:
print("Please specify a learning language")
exit(1)
if args.learning_lang is not None:
db_save('learning_lang', args.learning_lang)
print(f"Set your learning language to {args.learning_lang}")
learning_lang = args.learning_lang
if mother_lang is None and args.mother_lang is None:
def inc_practice_count(word: dict):
words = db_load("words")
old_word = find_word(word, words)
old_word["practice_count"] += 1
db_save("words", words)
def train(args, db_conn):
model = None
usecase = args.usecase
source = args.source
print("Building model for usecase: %s , Source: %s" % (usecase, source))
output_dir = "{}/usecases/{}/Model2".format(MODEL_PATH, usecase)
output_dir = Path(output_dir)
if output_dir.exists():
model = output_dir
training_data = []
labels = []
if source == 'csv':
trainingset = pd.read_csv(DATA_PATH + '/usecases_trainingset.csv')[[
'sentence', 'entities'
]].values.tolist()
else:
trainingset = db_conn.query(
'select sentence, entities from usecases_trainingset where use_case_id=%s'
% usecase)
for data in trainingset:
training_data.append((data[0], {
"entities": ast.literal_eval(data[1])
}))
for label in ast.literal_eval(data[1]):
# if label[2] not in labels and label[2] != 'EndDate':
labels.append(label[2])
# CHECK THE 'training_data' Format
# training_data = [('what is the price of polo?', {'entities': [(21, 25, 'PrdName')]}), ('what is the price of ball?', {'entities': [(21, 25, 'PrdName')]})]
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
# nlp = spacy.load('en_core_web_sm') # create blank Language class
nlp = spacy.load('en_core_web_lg')
# nlp.vocab.vectors.name = 'spacy_pretrained_vectors'
print("Created blank 'en' model")
# Add entity recognizer to model if it's not in the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'ner' not in nlp.pipe_names:
ner = nlp.create_pipe('ner')
nlp.add_pipe(ner)
# otherwise, get it, so we can add labels to it
else:
ner = nlp.get_pipe('ner')
for label in labels:
ner.add_label(label)
# if model is None:
# optimizer = nlp.begin_training()
# else:
# # Note that 'begin_training' initializes the models, so it'll zero out
# # existing entity types.
# optimizer = nlp.entity.create_optimizer()
n_iter = 1
Loops_Lowest_loss = 10000
Loops_Lowest_alpha = 10000
Loops_Lowest_LR = 10000
Loops_Lowest_Drop = 10000
Best_Iteration = 0
Loops_Lowest_evaluation_score = {}
Loops_Lowest_beam_score = {}
for inum in range(n_iter):
print("ITERATION >>> ", inum)
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
Lowest_loss = 10000
Lowest_i = 10000
Lowest_opt = 10000
Lowest_alpha = 10000
Lowest_LR = 10000
Lowest_Drop = 10000
is_break = False
Lowest_evaluation_score = {}
Lowest_beam_score = {}
all_Anotations = []
all_Anotations.clear()
# optimizer = nlp.begin_training()
# optimizer = nlp.entity.create_optimizer()
for iLearnRate in numpy.arange(0.001, 0.002, 0.001):
# print("IN-2",iLearnRate)
# optimizer.learn_rate = iLearnRate
for ialpha in numpy.arange(0.001, 0.1, 0.01):
# print("IN",ialpha)
# optimizer.alpha = ialpha
for i in numpy.arange(0.1, 0.9, 0.05):
# print("IN-3",i)
# ============ <=alpha=> 0.001 <=LearnRate=> 0.09099999999999998 <=drop=> 0.1 ============
print("\n\n============", "ITERATION >>> ", inum,
"<=LearnRate=>", iLearnRate, "<=alpha=>", ialpha,
"<=drop=>", i, "============")
# random.shuffle(training_data)
losses = {}
cfg = {}
# if model is None:
# optimizer = nlp.begin_training()
# else:
# optimizer = nlp.entity.create_optimizer()
optimizer = nlp.begin_training()
optimizer.beta1 = 0.9
optimizer.learn_rate = iLearnRate
optimizer.alpha = ialpha
# batches = minibatch(training_data, size=compounding(4., 32., 1.001))
batches = minibatch(training_data)
with nlp.disable_pipes(*other_pipes):
for batch in batches:
# print(batch)
texts, annotations = zip(*batch)
nlp.update(texts,
annotations,
sgd=optimizer,
drop=i,
losses=losses)
print('losses => ', losses['ner'])
print('\n\n', "Lowest Iteration Loose >> ",
Loops_Lowest_loss, '\t Lowest_loss', Lowest_loss,
'<==>', 'Losses', losses['ner'], '\n\n')
if Lowest_loss > losses['ner']:
print('\t\t evaluate.score')
evaluation_score = Evaluate.score(
nlp, training_data, labels)
print('\n', evaluation_score)
print('\t\t evaluate.BeamScore')
beam_score = Evaluate.beam_score(nlp, training_data)
print('\n', beam_score)
Lowest_loss = losses['ner']
Lowest_evaluation_score = evaluation_score
Lowest_beam_score = beam_score
Lowest_alpha = ialpha
Lowest_LR = iLearnRate
Lowest_Drop = i
print(
">>>>>>>>>>>>>> inside Lowest Value <<<<<<<<<<<<<<<"
)
print(Lowest_loss, Loops_Lowest_loss)
if Lowest_loss < Loops_Lowest_loss:
print(
">>>>>>>>>>>>>> inside Lowest Loop Value <<<<<<<<<<<<<<<"
)
Loops_Lowest_loss = Lowest_loss
Best_Iteration = inum
Loops_Lowest_alpha = Lowest_alpha
Loops_Lowest_LR = Lowest_LR
Loops_Lowest_Drop = Lowest_Drop
Loops_Lowest_evaluation_score = Lowest_evaluation_score
Loops_Lowest_beam_score = Lowest_beam_score
if output_dir is not None:
if not output_dir.exists():
output_dir.mkdir()
nlp.meta['name'] = 'usecase_{}.model'.format(
usecase)
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# END of drop Loop
# END of alpha Loop
# END of learn_rate Loop
print(
"\t=================================================================================================="
)
print(
"\t=================================================================================================="
)
print("\t<=alpha=>", Lowest_alpha, "<=LearnRate=>", Lowest_LR,
"<=drop=>", Lowest_Drop)
print('\tLowest_loss : ', Lowest_loss)
print(Lowest_evaluation_score)
print(Lowest_beam_score)
# END of ITERATION Loop
print(
"===================================ITERATION==============================================================="
)
print(
"=================================================================================================="
)
print("Best Iretation index : ", Best_Iteration, "<=alpha=>",
Loops_Lowest_alpha, "<=LearnRate=>", Loops_Lowest_LR, "<=drop=>",
Loops_Lowest_Drop)
print('Lowest_loss in all LOOP : ', Loops_Lowest_loss)
print(Loops_Lowest_evaluation_score)
print(Loops_Lowest_beam_score)
if source == 'csv':
entity_obj = pd.read_csv(DATA_PATH + '/usecases_entity.csv')[[
'name'
]].values.tolist()
else:
entity_obj = db_conn.query(
'select name from usecases_entity where use_case_id=%s' % usecase)
entity_label_list = []
for e in entity_obj:
entity_label_list.append(e[0])
# -------- Calling Functions for evaluating ----------
beam_score = Evaluate.beam_score(nlp, training_data)
eval_score = Evaluate.score(nlp, training_data, entity_label_list)
model_name = "Model2"
# Add/Update to DB model/create
if source == 'db':
db_save(model_name, usecase, beam_score, eval_score, output_dir)
print("Saved The Model Details!")