def hanEvaluate(dataset_name, model_path, MAX_FEATURES=200000, MAX_SENTENCE_NUM=15, MAX_WORD_NUM=25):
"""
Test set evaluating for Han model. This function print accuracy and a scikit learn classification.
report with f1-score. It uses gpu.
:param dataset_name: string of dataset name.
:param model_path: path of saved pytorch model fine tuned bert network (or checkpoint with isCheckpoint as True).
:return: None
"""
if (os.path.isfile('datasets/' + dataset_name + '_cleaned.txt')):
with open('datasets/' + dataset_name + '_cleaned.txt', 'rb') as f:
data_cleaned = pickle.load(f)
x_test = data_cleaned[4]
y_test = data_cleaned[5]
embedding_matrix = data_cleaned[6]
word_index = data_cleaned[7]
n_classes = data_cleaned[8]
else:
print("Please, use preprocessing function to save dataset first.")
return None
model = load_model(model_path, custom_objects={'AttentionLayer': AttentionLayer})
print("Evaluating network on Test Set...")
BATCH_SIZE = 64
total_t0 = time.time()
predictions = model.predict(x_test, batch_size=BATCH_SIZE)
print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - total_t0)))
print("")
print(classification_report(y_test.argmax(axis=1), predictions.argmax(axis=1), digits=4))
class Search:
Keys = ("artist", "title", "duration", "rating")
_searchText = ""
_searchResults = []
@UserAttrib(type=Traits.EditableText, searchLook=True)
def searchText(self, updateText=None):
if updateText is not None and self._searchText != updateText:
self._searchText = updateText
self._searchResults = songdb.search(updateText)
self.searchResults_updateEvent.push()
return self._searchText
@UserAttrib(type=Traits.Table(
keys=Keys,
format_duration=lambda d: formatTime(d) if d > 0 else "",
format_rating=lambda r: "★" * int(round(r * 5))))
@property
def searchResults(self):
return list(self._searchResults)
@searchResults.setUpdateEvent
@initBy
def searchResults_updateEvent(self):
return Event()
def petHunger(currentHunger = 0):
if currentHunger == 0:
movey(970, 335)
currentHunger = float(readImage(takeImage(540, 555, 595, 575)))
hungerSecond = 100 / (12 * 60 * 60)
to75 = max(currentHunger - 75, 0)
to50 = max(currentHunger - 50, 0)
to10 = max(currentHunger - 10, 0)
for x in [to10, to50, to75]:
if x > 0:
print(formatTime(x / hungerSecond))
tim = x
soundOffIn(tim / hungerSecond)
return ''
def userLongString(self): import utils s = self.userString duration = getattr(self, "duration", -1) if duration >= 0: s += ", " + utils.formatTime(duration) try: import os size = os.stat(self.url).st_size except Exception: size = None s += ", " + self.fileext if size and duration > 0: s += ", %.3g kbit/s" % (size * 8 / 1024. / duration) if size: s += ", " + utils.formatFilesize(size) return s
def userLongString(self): import utils s = self.userString duration = getattr(self, "duration", -1) if duration >= 0: s += ", " + utils.formatTime(duration) try: import os size = os.stat(self.url).st_size except Exception: size = None s += ", " + self.fileext if size and duration > 0: s += ", %.3g kbit/s" % (size * 8 / 1024. / duration) if size: s += ", " + utils.formatFilesize(size) return s
def softTargetsEvaluate(dataset_name, n_classes):
if (os.path.isfile('datasets/' + dataset_name + '_kd_cleaned.txt')):
with open('datasets/' + dataset_name + '_kd_cleaned.txt', 'rb') as f:
data_cleaned = pickle.load(f)
else:
print('Please, run kdPreprocessing first.')
return
training_set = data_cleaned[0]
train_params = {'batch_size': 128,
'shuffle': False,
'num_workers': 0
}
training_loader = DataLoader(training_set, **train_params)
print('Soft targets evaluating...')
t0 = time.time()
fin_targets = []
fin_outputs = []
with torch.no_grad():
for step, batch in enumerate(training_loader):
targets = batch['targets']
soft_targets = batch['soft_targets']
if step % 100 == 0 and not step == 0:
elapsed = formatTime(time.time() - t0)
print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(training_loader), elapsed))
fin_targets.extend(targets.detach().numpy().tolist())
fin_outputs.extend(soft_targets.detach().numpy().tolist())
fin_outputs = np.array(fin_outputs)
fin_targets = np.array(fin_targets)
accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1))
print(" Train Accuracy: {0:.4f}".format(accuracy))
print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))
def __str__(self):
logWord = ""
if self.log_type == LogTypes.BAN.value:
logWord = "Banned"
elif self.log_type == LogTypes.NOTE.value:
logWord = "Note"
elif self.log_type == LogTypes.KICK.value:
logWord = "Kicked"
elif self.log_type == LogTypes.UNBAN.value:
logWord = "Unbanned"
else: # LogTypes.WARN
logWord = "Warning #{}".format(self.log_type)
return "[{date}] **{name}** - {word} by {staff} - {message}\n".format(
date = formatTime(self.timestamp),
name = self.name,
word = logWord,
staff = self.staff,
message = self.log_message
)
class Search:
Keys = ("artist", "title", "duration", "rating")
def __init__(self):
self._searchText = ""
self._searchResults = []
import threading
self._lock = threading.RLock()
def _startSearch(self, txt):
def search():
with self._lock:
if self._searchText != txt: return
res = songdb.search(txt)
with self._lock:
if self._searchText == txt:
self._searchResults = res
self.__class__.searchResults.updateEvent(self).push()
with self._lock:
self._searchText = txt
TaskSystem.daemonThreadCall(search, name="Song DB search")
@UserAttrib(type=Traits.EditableText, searchLook=True)
def searchText(self, updateText=None):
with self._lock:
if updateText is not None and self._searchText != updateText:
self._startSearch(updateText)
return self._searchText
@UserAttrib(type=Traits.Table(
keys=Keys,
format_duration=lambda d: formatTime(d) if d > 0 else "",
format_rating=lambda r: "★" * int(round(r * 5))),
variableHeight=True,
addUpdateEvent=True)
@property
def searchResults(self):
with self._lock:
return list(self._searchResults)
def bertTrain(dataset_name,
n_classes,
TRAIN_BATCH_SIZE=16,
EPOCHS=3,
LEARNING_RATE=1e-05,
validation=True,
from_checkpoint=False,
model_path=None):
device = 'cuda' if cuda.is_available() else 'cpu'
if (os.path.isfile('datasets/' + dataset_name + '_bert_cleaned.txt')):
with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f:
data_cleaned = pickle.load(f)
else:
print('Please, run bertPreprocessing first.')
return
training_set = data_cleaned[0]
validation_set = data_cleaned[1]
test_set = data_cleaned[2]
MAX_LEN = data_cleaned[3]
VALID_BATCH_SIZE = 8
start_epoch = 0
train_params = {
'batch_size': TRAIN_BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
test_params = {
'batch_size': VALID_BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
training_loader = DataLoader(training_set, **train_params)
validation_loader = DataLoader(validation_set, **test_params)
testing_loader = DataLoader(test_set, **test_params)
model = BertModel(n_classes=n_classes, dropout=0.3)
optimizer = torch.optim.Adam(params=model.parameters(), lr=LEARNING_RATE)
criterion = torch.nn.CrossEntropyLoss()
if from_checkpoint == True:
print('Restoring model from checkpoint...')
torch.cuda.empty_cache()
checkpoint = torch.load(model_path, map_location=device)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.cuda()
start_epoch = checkpoint['epoch'] + 1
del checkpoint
print('Start epoch: {:}'.format(start_epoch + 1))
model.to(device)
# Stats with Tensorboard
log_dir = "logs/" + dataset_name + "_bert/" + datetime.datetime.now(
).strftime("%Y%m%d-%H%M%S")
shutil.rmtree(log_dir, ignore_errors=True)
writer = SummaryWriter(log_dir=log_dir)
total_t0 = time.time()
for epoch in range(start_epoch, EPOCHS):
print("")
print(
'============================== Epoch {:} / {:} =============================='
.format(epoch + 1, EPOCHS))
print('Training...')
t0 = time.time()
model.train()
for step, batch in enumerate(training_loader):
ids = batch['ids'].to(device, dtype=torch.long)
mask = batch['mask'].to(device, dtype=torch.long)
token_type_ids = batch['token_type_ids'].to(device,
dtype=torch.long)
targets = batch['targets'].to(device, dtype=torch.long)
model.zero_grad()
outputs = model(ids, mask, token_type_ids)
outputs = torch.softmax(outputs, dim=1)
loss = criterion(outputs, torch.max(targets, 1)[1])
if step % 100 == 0 and not step == 0:
# Calculate elapsed time in minutes.
elapsed = formatTime(time.time() - t0)
# Report progress.
print(
' Batch {:>5,} of {:>5,}. Loss: {:>19,} Elapsed: {:}.'
.format(step, len(training_loader), loss, elapsed))
writer.add_scalar('batch_loss', loss,
step + (epoch * len(training_loader)))
optimizer.zero_grad()
loss.backward()
optimizer.step()
training_time = formatTime(time.time() - t0)
print(" Training epoch took: {:}".format(training_time))
print(" Saving checkpoint...")
os.makedirs(
os.path.dirname('models/model_' + dataset_name + '_bert/ckp_' +
str(epoch) + 'epochs_' +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S")),
exist_ok=True)
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'loss': loss
}, 'models/model_' + dataset_name + '_bert/ckp_' + str(epoch) +
'epochs_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
print("")
if validation == True:
print("Running Validation...")
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
fin_targets = []
fin_outputs = []
with torch.no_grad():
for batch in validation_loader:
ids = batch['ids'].to(device, dtype=torch.long)
mask = batch['mask'].to(device, dtype=torch.long)
token_type_ids = batch['token_type_ids'].to(
device, dtype=torch.long)
targets = batch['targets'].to(device, dtype=torch.long)
outputs = model(ids, mask, token_type_ids)
total_eval_loss += criterion(torch.softmax(outputs, dim=1),
torch.max(targets, 1)[1])
fin_targets.extend(targets.cpu().detach().numpy().tolist())
fin_outputs.extend(
torch.softmax(outputs,
dim=1).cpu().detach().numpy().tolist())
valid_loss = total_eval_loss / len(validation_loader)
fin_outputs = np.array(fin_outputs)
fin_targets = np.array(fin_targets)
accuracy = accuracy_score(fin_targets.argmax(axis=1),
fin_outputs.argmax(axis=1))
print(" Validation Accuracy: {0:.4f}".format(accuracy))
print(" Validation Loss: {0:.4f}".format(valid_loss))
writer.add_scalar('epoch_loss', valid_loss, epoch)
writer.add_scalar('epoch_accuracy', accuracy, epoch)
print("")
print("Training complete!")
print("Saving model...")
os.makedirs(
os.path.dirname('models/model_' + dataset_name + '_bert/' +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S")),
exist_ok=True)
torch.save(
model.state_dict(), 'models/model_' + dataset_name + '_bert/' +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
print("Total training took {:} (h:mm:ss)".format(
formatTime(time.time() - total_t0)))
async def logUser(m, state):
# Attempt to generate user object
userid = ul.parse_mention(m)
if userid == None:
if state == LogTypes.NOTE:
await m.channel.send("I wasn't able to understand that message: `$note USER`")
else:
await m.channel.send("I wasn't able to understand that message: `$log USER`")
return
# Calculate value for 'num' category in database
# For warns, it's the newest number of warns, otherwise it's a special value
if state == LogTypes.WARN:
count = db.get_warn_count(userid)
else:
count = state.value
currentTime = datetime.datetime.utcnow()
# Attempt to fetch the username for the user
username = ul.fetch_username(m.guild, userid)
if username == None:
username = "******" + str(userid)
await m.channel.send("I wasn't able to find a username for that user, but whatever, I'll do it anyway.")
# Generate log message, adding URLs of any attachments
content = utils.combineMessage(m)
mes = utils.parseMessage(content, username)
# If they didn't give a message, abort
if mes == "":
await m.channel.send("Please give a reason for why you want to log them.")
return
# Update records for graphing
import visualize
if state == LogTypes.BAN:
visualize.updateCache(m.author.name, (1, 0), utils.formatTime(currentTime))
elif state == LogTypes.WARN:
visualize.updateCache(m.author.name, (0, 1), utils.formatTime(currentTime))
elif state == LogTypes.UNBAN:
await m.channel.send("Removing all old logs for unbanning")
db.clear_user_logs(userid)
# Generate message for log channel
globalcount = db.get_dbid()
new_log = db.UserLogEntry(globalcount + 1, userid, username, count, currentTime, mes, m.author.name, None)
logMessage = str(new_log)
await m.channel.send(logMessage)
# Send ban recommendation, if needed
if (state == LogTypes.WARN and count >= config.WARN_THRESHOLD):
await m.channel.send("This user has received {} warnings or more. It is recommended that they be banned.".format(config.WARN_THRESHOLD))
logMesID = 0
# If we aren't noting, need to also write to log channel
if state != LogTypes.NOTE:
# Post to channel, keep track of message ID
try:
chan = discord.utils.get(m.guild.channels, id=config.LOG_CHAN)
logMes = await chan.send(logMessage)
logMesID = logMes.id
except discord.errors.InvalidArgument:
await m.channel.send("The logging channel has not been set up in `config.json`. In order to have a visual record, please specify a channel ID.")
try:
# Send a DM to the user
u = ul.fetch_user(m.guild, userid)
if u != None:
DMchan = u.dm_channel
# If first time DMing, need to create channel
if DMchan == None:
DMchan = await u.create_dm()
# Only send DM when specified in configs
if state == LogTypes.BAN and config.DM_BAN:
await DMchan.send("Hi there! You've been banned from the Stardew Valley Discord for violating the rules: `{}`. If you have any questions, you can send a message to the moderators via the sidebar at <https://www.reddit.com/r/StardewValley>, and they'll forward it to us.".format(mes))
elif state == LogTypes.WARN and config.DM_WARN:
await DMchan.send("Hi there! You received warning #{} in the Stardew Valley Discord for violating the rules: `{}`. Please review <#445729591533764620> and <#445729663885639680> for more info. If you have any questions, you can reply directly to this message to contact the staff.".format(count, mes))
elif state == LogTypes.KICK and config.DM_BAN:
await DMchan.send("Hi there! You've been kicked from the Stardew Valley Discord for violating the following reason: `{}`. If you have any questions, you can send a message to the moderators via the sidebar at <https://www.reddit.com/r/StardewValley>, and they'll forward it to us.".format(mes))
# Exception handling
except discord.errors.HTTPException as e:
await m.channel.send("ERROR: While attempting to DM, there was an unexpected error. Tell aquova this: {}".format(e))
except Exception as e:
await m.channel.send( "ERROR: An unexpected error has occurred. Tell aquova this: {}".format(e))
# Update database
new_log.message_url = logMesID
db.add_log(new_log)
async def removeError(m, edit):
userid = ul.parse_mention(m)
if userid == None:
if edit:
await m.channel.send("I wasn't able to understand that message: `$remove USER [num] new_message`")
else:
await m.channel.send("I wasn't able to understand that message: `$remove USER [num]`")
return
username = ul.fetch_username(m.guild, userid)
if username == None:
username = str(userid)
# If editing, and no message specified, abort.
mes = utils.parseMessage(m.content, username)
if mes == "":
if edit:
await m.channel.send("You need to specify an edit message")
return
else:
mes = "0"
try:
index = int(mes.split()[0]) - 1
mes = utils.strip(mes)
except (IndexError, ValueError):
index = -1
# Find most recent entry in database for specified user
search_results = db.search(userid)
# If no results in database found, can't modify
if search_results == []:
await m.channel.send("I couldn't find that user in the database")
# If invalid index given, yell
elif (index > len(search_results) - 1) or index < -1:
await m.channel.send("I can't modify item number {}, there aren't that many for this user".format(index + 1))
else:
item = search_results[index]
import visualize
if edit:
if item.log_type == LogTypes.NOTE.value:
currentTime = datetime.datetime.utcnow()
item.timestamp = currentTime
item.log_message = mes
item.staff = m.author.name
db.add_log(item)
out = "The log now reads as follows:\n{}\n".format(str(item))
await m.channel.send(out)
return
else:
await m.channel.send("You can only edit notes for now")
return
# Everything after here is deletion
db.remove_log(item.dbid)
out = "The following log was deleted:\n"
out += str(item)
if item.log_type == LogTypes.BAN:
visualize.updateCache(item.staff, (-1, 0), utils.formatTime(item.timestamp))
elif item.log_type == LogTypes.WARN:
visualize.updateCache(item.staff, (0, -1), utils.formatTime(item.timestamp))
await m.channel.send(out)
# Search logging channel for matching post, and remove it
try:
if item.message_url != 0:
chan = discord.utils.get(m.guild.channels, id=config.LOG_CHAN)
m = await chan.fetch_message(item.message_url)
await m.delete()
# Print message if unable to find message to delete, but don't stop
except discord.errors.HTTPException as e:
print("Unable to find message to delete: {}", str(e))
choice = choices[choice](*args)
else:
print("Invalid selection. Choose:",
', '.join([k for k in choices.keys()]))
choice = ''
return
if __name__ == "__main__":
fileName = 'Error Logs/Error Log - ' + datetime.datetime.strftime(
datetime.datetime.now(), '%I_%M_%S_%p') + '.log'
logging.basicConfig(filename=fileName, level=logging.DEBUG, filemode='w')
try:
if False:
import gui
gui.MainWindow().run()
else:
start = time.time()
ctypes.windll.kernel32.SetConsoleTitleW("Idle Gods Controller X")
args = [''] * len(sys.argv)
if len(sys.argv) > 1:
for x in range(1, len(sys.argv)):
args[x - 1] = sys.argv[x]
askChoice(*args)
except SystemExit:
pass
except BaseException:
logging.getLogger(__name__).exception("Program terminated")
logging.debug("Ran for: " + utils.formatTime(int(time.time() - start)))
raise
logging.shutdown()
def kdPreprocessing(dataset_name,
n_classes,
data_df,
teacher_path,
MAX_LEN=128,
save_all=True,
isCheckpoint=False):
"""
Dataset preparation for Bert Model and KD models. It is splitted (0.8 train, 0.1 valid and 0.1 test) and sets are
returned. Every set is a CustomDatasetWithSoftTargets class (see utils.py) that return data in Bert format.
:param dataset_name: string of dataset name.
:param data_df: dataset in dataframe pandas format.
:param MAX_LEN: it represents total words represented in bert encoding (other words will be ignored).
:param save_all: boolean that specifies if save all data for time saving before training or network evaluating.
:return: training_set, validation_set, test_set in CustomDataset format.
"""
device = 'cuda' if cuda.is_available() else 'cpu'
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
data_df['soft_targets'] = 0
train_size = 0.8
train_dataset = data_df.sample(frac=train_size,
random_state=200).reset_index(drop=True)
tmp_dataset = data_df.drop(train_dataset.index).reset_index(drop=True)
test_dataset = tmp_dataset.sample(frac=0.5,
random_state=200).reset_index(drop=True)
val_dataset = tmp_dataset.drop(test_dataset.index).reset_index(drop=True)
print("FULL Dataset: {}".format(data_df.shape))
print("TRAIN Dataset: {}".format(train_dataset.shape))
print("TEST Dataset: {}".format(test_dataset.shape))
print("VALID Dataset: {}".format(val_dataset.shape))
training_set = CustomDatasetWithSoftTargets(train_dataset, tokenizer,
MAX_LEN)
validation_set = CustomDatasetWithSoftTargets(val_dataset, tokenizer,
MAX_LEN)
test_set = CustomDatasetWithSoftTargets(test_dataset, tokenizer, MAX_LEN)
train_params = {'batch_size': 32, 'shuffle': False, 'num_workers': 0}
training_loader = DataLoader(training_set, **train_params)
teacher_model = BertModel(n_classes=n_classes, dropout=0.3)
if isCheckpoint:
teacher_model.load_state_dict(
torch.load(teacher_path)['model_state_dict'])
else:
teacher_model.load_state_dict(torch.load(teacher_path))
print(teacher_model)
total_params = sum(p.numel() for p in teacher_model.parameters())
print('Teacher total parameters: {:}'.format(total_params))
teacher_model.to(device)
teacher_model.eval()
soft_targets = []
t0 = time.time()
print('Creating soft targets...')
with torch.no_grad():
for step, batch in enumerate(training_loader):
ids = batch['ids'].to(device, dtype=torch.long)
mask = batch['mask'].to(device, dtype=torch.long)
token_type_ids = batch['token_type_ids'].to(device,
dtype=torch.long)
if step % 100 == 0 and not step == 0:
# Calculate elapsed time in minutes.
elapsed = formatTime(time.time() - t0)
# Report progress.
print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(
step, len(training_loader), elapsed))
soft_target = torch.softmax(teacher_model(ids, mask,
token_type_ids),
dim=1)
soft_targets.extend(soft_target.cpu().detach().numpy().tolist())
del teacher_model
training_set.setSoftTargets(soft_targets)
if save_all is True:
os.makedirs(os.path.dirname('datasets/' + dataset_name +
'_kd_cleaned.txt'),
exist_ok=True)
with open('datasets/' + dataset_name + '_kd_cleaned.txt', 'wb') as f:
pickle.dump([training_set, validation_set, test_set, MAX_LEN], f)
return training_set, validation_set, test_set
def lstmEvaluate(dataset_name, n_classes, model_path, isCheckpoint=False):
device = 'cuda' if cuda.is_available() else 'cpu'
with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f:
data_cleaned = pickle.load(f)
test_set = data_cleaned[2]
MAX_LEN = data_cleaned[3]
TEST_BATCH_SIZE = 64
EMBEDDING_DIM = 50
HIDDEN_DIM = 256
test_params = {'batch_size': TEST_BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
testing_loader = DataLoader(test_set, **test_params)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = LSTMBase(embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, vocab_size=tokenizer.vocab_size,
n_classes=n_classes)
print(model)
total_params = sum(p.numel() for p in model.parameters())
total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
print('Total parameters: {:}'.format(total_params))
print('Total trainable parameters: {:}'.format(total_trainable_params))
if isCheckpoint:
model.load_state_dict(torch.load(model_path)['model_state_dict'])
else:
model.load_state_dict(torch.load(model_path))
model.to(device)
model.eval()
criterion = torch.nn.CrossEntropyLoss()
# evaluate the network
print("Evaluating network on Test Set")
t0 = time.time()
total_eval_loss = 0
fin_targets = []
fin_outputs = []
with torch.no_grad():
for step, batch in enumerate(testing_loader):
ids = batch['ids'].to(device, dtype=torch.long)
targets = batch['targets'].to(device, dtype=torch.long)
outputs = model(ids)
total_eval_loss += criterion(torch.softmax(outputs, dim=1), torch.max(targets, 1)[1])
if step % 100 == 0 and not step == 0:
elapsed = formatTime(time.time() - t0)
print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(testing_loader), elapsed))
fin_targets.extend(targets.cpu().detach().numpy().tolist())
fin_outputs.extend(torch.softmax(outputs, dim=1).cpu().detach().numpy().tolist())
valid_loss = total_eval_loss / len(testing_loader)
fin_outputs = np.array(fin_outputs)
fin_targets = np.array(fin_targets)
accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1))
print("")
print(" Test Accuracy: {0:.2f}".format(accuracy))
print(" Test Loss: {0:.2f}".format(valid_loss))
print("")
print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - t0)))
print("")
print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))
def bertEvaluate(dataset_name, n_classes, model_path, isCheckpoint=False):
"""
Test set evaluating for a fine tuned Bert model. This function print accuracy and a scikit learn classification
report with f1-score. It uses gpu.
:param dataset_name: string of dataset name.
:param n_classes: int of number of dataset classes.
:param model_path: path of saved pytorch model fine tuned bert network (or checkpoint with isCheckpoint as True).
:param isCheckpoint: boolean that specifies neither model_path is a model checkpoint.
:return: None
"""
device = 'cuda' if cuda.is_available() else 'cpu'
with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f:
data_cleaned = pickle.load(f)
test_set = data_cleaned[2]
MAX_LEN = data_cleaned[3]
TEST_BATCH_SIZE = 16
test_params = {'batch_size': TEST_BATCH_SIZE,
'shuffle': True,
'num_workers': 0
}
testing_loader = DataLoader(test_set, **test_params)
model = BertModel(n_classes=n_classes, dropout=0.3)
if isCheckpoint:
model.load_state_dict(torch.load(model_path)['model_state_dict'])
else:
model.load_state_dict(torch.load(model_path))
model.to(device)
model.eval()
criterion = torch.nn.CrossEntropyLoss()
# evaluate the network
print("Evaluating network on Test Set")
t0 = time.time()
total_eval_loss = 0
fin_targets = []
fin_outputs = []
with torch.no_grad():
for step, batch in enumerate(testing_loader):
ids = batch['ids'].to(device, dtype=torch.long)
mask = batch['mask'].to(device, dtype=torch.long)
token_type_ids = batch['token_type_ids'].to(device, dtype=torch.long)
targets = batch['targets'].to(device, dtype=torch.long)
outputs = model(ids, mask, token_type_ids)
total_eval_loss += criterion(outputs, torch.max(targets, 1)[1])
if step % 100 == 0 and not step == 0:
elapsed = formatTime(time.time() - t0)
print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(testing_loader), elapsed))
fin_targets.extend(targets.cpu().detach().numpy().tolist())
fin_outputs.extend(torch.softmax(outputs, dim=1).cpu().detach().numpy().tolist())
valid_loss = total_eval_loss / len(testing_loader)
fin_outputs = np.array(fin_outputs)
fin_targets = np.array(fin_targets)
accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1))
print("")
print(" Test Accuracy: {0:.2f}".format(accuracy))
print(" Test Loss: {0:.2f}".format(valid_loss))
print("")
print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - t0)))
print("")
print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))
