def train(Train_images, Train_labels, Test_images, Test_label):
model = create_model()
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
print("model created successfully...")
print(model.summary())
es = [
EarlyStopping(monitor='val_loss', patience=50),
ModelCheckpoint(filepath='best_model.h5',
monitor='val_loss',
save_best_only=True)
]
print("Training...")
telegram_bot_sendtext("Training Start")
history = model.fit(
Train_images,
Train_labels,
batch_size=batch_size,
epochs=training_epoch,
callbacks=es,
validation_data=(Test_images, Test_label),
# validation_split=validation_ratio,
shuffle=True)
telegram_bot_sendtext("Training Completed")
return history, model
import telegram_bot
from datetime import date
today = date.today()
PINCODES = ['516001', '516002', '516003', '516004']
DATE = '03-05-2021'
config = {'accept': 'application/json', 'Accept-Language': 'hi_IN'}
for PINCODE in PINCODES:
url = 'https://cdn-api.co-vin.in/api/v2/appointment/sessions/public/calendarByPin?pincode=' + PINCODE + '&date=' + DATE
result = requests.get(url, headers=config)
json.loads(result.text)
data = json.loads(result.text)
for center in data['centers']:
info = {}
info['name'] = center['name']
info['district_name'] = center['district_name']
info['fee_type'] = center['fee_type']
s = {}
i = 1
for session in center['sessions']:
s['date'] = session['date']
s['available_capacity'] = session['available_capacity']
s['min_age_limit'] = session['min_age_limit']
s['vaccine'] = session['vaccine']
s['slots'] = session['slots']
info['session_' + str(i)] = s
i = i + 1
obj = json.dumps(info, indent=1)
ans = telegram_bot.telegram_bot_sendtext(obj)
def scrap_projects(query=None):
# options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('headless')
chrome_options.add_argument("--incognito")
# read csv
try:
data = pd.read_csv(f'{abs_path}workana_jobs.csv')
data = data[data['query'] == query]
except FileNotFoundError:
data = pd.DataFrame()
# web driver
driver = webdriver.Chrome(ChromeDriverManager().install(),
options=chrome_options)
# request
query_string = f'query={query}&' if query else ''
driver.get(
f"https://www.workana.com/jobs?{query_string}category=it-programming&language=es%2Cen"
)
# content
projects = driver.find_elements_by_xpath(
"//div[@class='project-item js-project']")
skills = driver.find_elements_by_xpath("//div[@class='skills']")
proj = {
'query': [],
'date': [],
'link': [],
'title': [],
'budget': [],
'skills': []
}
for project in zip(projects, skills):
# date
src_code = project[0].get_attribute("outerHTML")
i = src_code.find('h5 title="') + len('h5 title="')
j = src_code.find('"', i)
date = src_code[i:j]
dt = to_datetime(date)
if not data.empty:
if datetime.strptime(data.iloc[0]['date'],
'%Y-%m-%d %H:%M:%S') <= dt:
break
proj['date'].append(dt)
# query
proj['query'].append(query)
# i += 20
# link
i = src_code.find('a href', j) + 8
j = src_code.find('"', i)
link = f'https://www.workana.com{src_code[i:j]}/'
proj['link'].append(link)
# title
i = src_code.find('title', j) + 7
j = src_code.find('"', i + 1)
title = src_code[i:j]
proj['title'].append(title)
# budget
i = src_code.find('USD')
j = src_code.find('<', i)
budget = src_code[i:j]
proj['budget'].append(budget)
# skills
skills = project[1].text
proj['skills'].append(skills)
# send notification
send_notification = f'[New Project]({link})\n{date}\n{budget}'
telegram_bot_sendtext(send_notification)
data_proj = pd.DataFrame.from_dict(proj)
# writing
if data.empty:
data_proj.to_csv(f'{abs_path}workana_jobs.csv', index=False)
else:
frames = [data_proj, data]
new_data = pd.concat(frames)
new_data.to_csv(f'{abs_path}workana_jobs.csv', index=False)
# making sure of closing chrome (?)
driver.close()
driver.quit()
def retrain(model, training_loader, loss_fn, optimizer):
tr_loss = 0
n_correct = 0
nb_tr_steps = 0
nb_tr_examples = 0
model.train()
bot.telegram_bot_sendtext("re-training started for : " +
config.generic_path)
for _, data in enumerate(tqdm(training_loader, 0)):
ids = data['ids'].to(device, dtype=torch.long)
mask = data['mask'].to(device, dtype=torch.long)
targets = data['targets'].to(device, dtype=torch.long)
# print("len(targets): ".format(len(targets)))
# print("len(ids): ".format(len(targets)))
# print("len(ids[0]): ".format(len(ids[0])))
# print("*"*120)
# print("ids: {}".format(ids))
# print("mask: {}".format(mask))
# print("targets: {}".format(targets))
# print("*"*120)
# Calling the created model
# outputs, probability = model(ids, mask)
outputs = model(ids, mask)
# print("MODEL OUTPUTS: {}".format(outputs))
# print("MODEL probability: {}".format(probability))
loss = loss_fn(outputs, targets)
# print("loss: {}".format(loss))
tr_loss += loss.item()
# print("loss.item(): {}".format(loss.item()))
# print("outputs.data: {}".format(outputs.data))
# print("torch.max(outputs.data, dim=1): {}".format(torch.max(outputs.data, dim=1)))
big_val, big_idx = torch.max(outputs.data, dim=1)
# print("big_idx: {}".format(big_idx))
n_correct += utility.calculate_accuracy(big_idx, targets)
# print("+"*120)
nb_tr_steps += 1
nb_tr_examples += targets.size(0)
if _ % 5000 == 0:
loss_step = tr_loss / nb_tr_steps
accu_step = (n_correct * 100) / nb_tr_examples
bot.telegram_bot_sendtext(config.generic_path)
bot.telegram_bot_sendtext("Training Loss per 5000 steps: " +
str(loss_step))
bot.telegram_bot_sendtext("Training Accuracy per 5000 steps: " +
str(accu_step))
print(f"Training Loss per 5000 steps: {loss_step}")
print(f"Training Accuracy per 5000 steps: {accu_step}")
print(strftime("%Y-%m-%d %H:%M:%S", gmtime()))
optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
# # When using GPU
optimizer.step()
print(
f'The Total Accuracy for Epoch {epoch}: {(n_correct * 100) / nb_tr_examples}'
)
epoch_loss = tr_loss / nb_tr_steps
epoch_accu = (n_correct * 100) / nb_tr_examples
bot.telegram_bot_sendtext(config.generic_path)
bot.telegram_bot_sendtext("Final Training Loss Epoch " + str(epoch_loss))
bot.telegram_bot_sendtext("Final Training Accuracy Epoch: " +
str(epoch_accu))
bot.telegram_bot_sendtext("EPOCH completed {Re-training}")
print(f"Training Loss Epoch: {epoch_loss}")
print(f"Training Accuracy Epoch: {epoch_accu}")
return epoch_loss, epoch_accu
classification_report_df.to_csv(config.generic_path +
"validation_classification_report.csv")
graph["train_epoch_loss_list"].append(train_epoch_loss)
graph['train_epoch_accu_list'].append(train_epoch_accu)
graph['valid_epoch_loss_list'].append(valid_epoch_loss)
graph['valid_epoch_accu_list'].append(valid_epoch_accu)
validation_f1_score_macro = f1_score(y_valid_actual,
y_valid_predicted,
average="macro")
print("validation_f1_score_macro: {}".format(validation_f1_score_macro))
graph['validation_f1_score_macro_list'].append(validation_f1_score_macro)
if valid_epoch_accu > best_validation_accuracy:
# Creating check point
utility.save_model(EPOCH=epoch,
model=model,
optimizer=optimizer,
LOSS=train_epoch_loss,
ACCURACY=train_epoch_accu,
PATH=config.checkpoint_path)
best_validation_accuracy = valid_epoch_accu
graph["best_validation_accuracy"] = best_validation_accuracy
print("graph: {}".format(graph))
utility.save_graph(graph_data=graph, path=config.generic_path)
bot.telegram_bot_sendtext(graph)