def hanEvaluate(dataset_name, model_path, MAX_FEATURES=200000, MAX_SENTENCE_NUM=15, MAX_WORD_NUM=25): """ Test set evaluating for Han model. This function print accuracy and a scikit learn classification. report with f1-score. It uses gpu. :param dataset_name: string of dataset name. :param model_path: path of saved pytorch model fine tuned bert network (or checkpoint with isCheckpoint as True). :return: None """ if (os.path.isfile('datasets/' + dataset_name + '_cleaned.txt')): with open('datasets/' + dataset_name + '_cleaned.txt', 'rb') as f: data_cleaned = pickle.load(f) x_test = data_cleaned[4] y_test = data_cleaned[5] embedding_matrix = data_cleaned[6] word_index = data_cleaned[7] n_classes = data_cleaned[8] else: print("Please, use preprocessing function to save dataset first.") return None model = load_model(model_path, custom_objects={'AttentionLayer': AttentionLayer}) print("Evaluating network on Test Set...") BATCH_SIZE = 64 total_t0 = time.time() predictions = model.predict(x_test, batch_size=BATCH_SIZE) print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - total_t0))) print("") print(classification_report(y_test.argmax(axis=1), predictions.argmax(axis=1), digits=4))
class Search: Keys = ("artist", "title", "duration", "rating") _searchText = "" _searchResults = [] @UserAttrib(type=Traits.EditableText, searchLook=True) def searchText(self, updateText=None): if updateText is not None and self._searchText != updateText: self._searchText = updateText self._searchResults = songdb.search(updateText) self.searchResults_updateEvent.push() return self._searchText @UserAttrib(type=Traits.Table( keys=Keys, format_duration=lambda d: formatTime(d) if d > 0 else "", format_rating=lambda r: "★" * int(round(r * 5)))) @property def searchResults(self): return list(self._searchResults) @searchResults.setUpdateEvent @initBy def searchResults_updateEvent(self): return Event()
def petHunger(currentHunger = 0): if currentHunger == 0: movey(970, 335) currentHunger = float(readImage(takeImage(540, 555, 595, 575))) hungerSecond = 100 / (12 * 60 * 60) to75 = max(currentHunger - 75, 0) to50 = max(currentHunger - 50, 0) to10 = max(currentHunger - 10, 0) for x in [to10, to50, to75]: if x > 0: print(formatTime(x / hungerSecond)) tim = x soundOffIn(tim / hungerSecond) return ''
def userLongString(self): import utils s = self.userString duration = getattr(self, "duration", -1) if duration >= 0: s += ", " + utils.formatTime(duration) try: import os size = os.stat(self.url).st_size except Exception: size = None s += ", " + self.fileext if size and duration > 0: s += ", %.3g kbit/s" % (size * 8 / 1024. / duration) if size: s += ", " + utils.formatFilesize(size) return s
def softTargetsEvaluate(dataset_name, n_classes): if (os.path.isfile('datasets/' + dataset_name + '_kd_cleaned.txt')): with open('datasets/' + dataset_name + '_kd_cleaned.txt', 'rb') as f: data_cleaned = pickle.load(f) else: print('Please, run kdPreprocessing first.') return training_set = data_cleaned[0] train_params = {'batch_size': 128, 'shuffle': False, 'num_workers': 0 } training_loader = DataLoader(training_set, **train_params) print('Soft targets evaluating...') t0 = time.time() fin_targets = [] fin_outputs = [] with torch.no_grad(): for step, batch in enumerate(training_loader): targets = batch['targets'] soft_targets = batch['soft_targets'] if step % 100 == 0 and not step == 0: elapsed = formatTime(time.time() - t0) print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(training_loader), elapsed)) fin_targets.extend(targets.detach().numpy().tolist()) fin_outputs.extend(soft_targets.detach().numpy().tolist()) fin_outputs = np.array(fin_outputs) fin_targets = np.array(fin_targets) accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1)) print(" Train Accuracy: {0:.4f}".format(accuracy)) print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))
def __str__(self): logWord = "" if self.log_type == LogTypes.BAN.value: logWord = "Banned" elif self.log_type == LogTypes.NOTE.value: logWord = "Note" elif self.log_type == LogTypes.KICK.value: logWord = "Kicked" elif self.log_type == LogTypes.UNBAN.value: logWord = "Unbanned" else: # LogTypes.WARN logWord = "Warning #{}".format(self.log_type) return "[{date}] **{name}** - {word} by {staff} - {message}\n".format( date = formatTime(self.timestamp), name = self.name, word = logWord, staff = self.staff, message = self.log_message )
class Search: Keys = ("artist", "title", "duration", "rating") def __init__(self): self._searchText = "" self._searchResults = [] import threading self._lock = threading.RLock() def _startSearch(self, txt): def search(): with self._lock: if self._searchText != txt: return res = songdb.search(txt) with self._lock: if self._searchText == txt: self._searchResults = res self.__class__.searchResults.updateEvent(self).push() with self._lock: self._searchText = txt TaskSystem.daemonThreadCall(search, name="Song DB search") @UserAttrib(type=Traits.EditableText, searchLook=True) def searchText(self, updateText=None): with self._lock: if updateText is not None and self._searchText != updateText: self._startSearch(updateText) return self._searchText @UserAttrib(type=Traits.Table( keys=Keys, format_duration=lambda d: formatTime(d) if d > 0 else "", format_rating=lambda r: "★" * int(round(r * 5))), variableHeight=True, addUpdateEvent=True) @property def searchResults(self): with self._lock: return list(self._searchResults)
def bertTrain(dataset_name, n_classes, TRAIN_BATCH_SIZE=16, EPOCHS=3, LEARNING_RATE=1e-05, validation=True, from_checkpoint=False, model_path=None): device = 'cuda' if cuda.is_available() else 'cpu' if (os.path.isfile('datasets/' + dataset_name + '_bert_cleaned.txt')): with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f: data_cleaned = pickle.load(f) else: print('Please, run bertPreprocessing first.') return training_set = data_cleaned[0] validation_set = data_cleaned[1] test_set = data_cleaned[2] MAX_LEN = data_cleaned[3] VALID_BATCH_SIZE = 8 start_epoch = 0 train_params = { 'batch_size': TRAIN_BATCH_SIZE, 'shuffle': True, 'num_workers': 0 } test_params = { 'batch_size': VALID_BATCH_SIZE, 'shuffle': True, 'num_workers': 0 } training_loader = DataLoader(training_set, **train_params) validation_loader = DataLoader(validation_set, **test_params) testing_loader = DataLoader(test_set, **test_params) model = BertModel(n_classes=n_classes, dropout=0.3) optimizer = torch.optim.Adam(params=model.parameters(), lr=LEARNING_RATE) criterion = torch.nn.CrossEntropyLoss() if from_checkpoint == True: print('Restoring model from checkpoint...') torch.cuda.empty_cache() checkpoint = torch.load(model_path, map_location=device) model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) for state in optimizer.state.values(): for k, v in state.items(): if isinstance(v, torch.Tensor): state[k] = v.cuda() start_epoch = checkpoint['epoch'] + 1 del checkpoint print('Start epoch: {:}'.format(start_epoch + 1)) model.to(device) # Stats with Tensorboard log_dir = "logs/" + dataset_name + "_bert/" + datetime.datetime.now( ).strftime("%Y%m%d-%H%M%S") shutil.rmtree(log_dir, ignore_errors=True) writer = SummaryWriter(log_dir=log_dir) total_t0 = time.time() for epoch in range(start_epoch, EPOCHS): print("") print( '============================== Epoch {:} / {:} ==============================' .format(epoch + 1, EPOCHS)) print('Training...') t0 = time.time() model.train() for step, batch in enumerate(training_loader): ids = batch['ids'].to(device, dtype=torch.long) mask = batch['mask'].to(device, dtype=torch.long) token_type_ids = batch['token_type_ids'].to(device, dtype=torch.long) targets = batch['targets'].to(device, dtype=torch.long) model.zero_grad() outputs = model(ids, mask, token_type_ids) outputs = torch.softmax(outputs, dim=1) loss = criterion(outputs, torch.max(targets, 1)[1]) if step % 100 == 0 and not step == 0: # Calculate elapsed time in minutes. elapsed = formatTime(time.time() - t0) # Report progress. print( ' Batch {:>5,} of {:>5,}. Loss: {:>19,} Elapsed: {:}.' .format(step, len(training_loader), loss, elapsed)) writer.add_scalar('batch_loss', loss, step + (epoch * len(training_loader))) optimizer.zero_grad() loss.backward() optimizer.step() training_time = formatTime(time.time() - t0) print(" Training epoch took: {:}".format(training_time)) print(" Saving checkpoint...") os.makedirs( os.path.dirname('models/model_' + dataset_name + '_bert/ckp_' + str(epoch) + 'epochs_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")), exist_ok=True) torch.save( { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'loss': loss }, 'models/model_' + dataset_name + '_bert/ckp_' + str(epoch) + 'epochs_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) print("") if validation == True: print("Running Validation...") model.eval() total_eval_accuracy = 0 total_eval_loss = 0 fin_targets = [] fin_outputs = [] with torch.no_grad(): for batch in validation_loader: ids = batch['ids'].to(device, dtype=torch.long) mask = batch['mask'].to(device, dtype=torch.long) token_type_ids = batch['token_type_ids'].to( device, dtype=torch.long) targets = batch['targets'].to(device, dtype=torch.long) outputs = model(ids, mask, token_type_ids) total_eval_loss += criterion(torch.softmax(outputs, dim=1), torch.max(targets, 1)[1]) fin_targets.extend(targets.cpu().detach().numpy().tolist()) fin_outputs.extend( torch.softmax(outputs, dim=1).cpu().detach().numpy().tolist()) valid_loss = total_eval_loss / len(validation_loader) fin_outputs = np.array(fin_outputs) fin_targets = np.array(fin_targets) accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1)) print(" Validation Accuracy: {0:.4f}".format(accuracy)) print(" Validation Loss: {0:.4f}".format(valid_loss)) writer.add_scalar('epoch_loss', valid_loss, epoch) writer.add_scalar('epoch_accuracy', accuracy, epoch) print("") print("Training complete!") print("Saving model...") os.makedirs( os.path.dirname('models/model_' + dataset_name + '_bert/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")), exist_ok=True) torch.save( model.state_dict(), 'models/model_' + dataset_name + '_bert/' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) print("Total training took {:} (h:mm:ss)".format( formatTime(time.time() - total_t0)))
async def logUser(m, state): # Attempt to generate user object userid = ul.parse_mention(m) if userid == None: if state == LogTypes.NOTE: await m.channel.send("I wasn't able to understand that message: `$note USER`") else: await m.channel.send("I wasn't able to understand that message: `$log USER`") return # Calculate value for 'num' category in database # For warns, it's the newest number of warns, otherwise it's a special value if state == LogTypes.WARN: count = db.get_warn_count(userid) else: count = state.value currentTime = datetime.datetime.utcnow() # Attempt to fetch the username for the user username = ul.fetch_username(m.guild, userid) if username == None: username = "******" + str(userid) await m.channel.send("I wasn't able to find a username for that user, but whatever, I'll do it anyway.") # Generate log message, adding URLs of any attachments content = utils.combineMessage(m) mes = utils.parseMessage(content, username) # If they didn't give a message, abort if mes == "": await m.channel.send("Please give a reason for why you want to log them.") return # Update records for graphing import visualize if state == LogTypes.BAN: visualize.updateCache(m.author.name, (1, 0), utils.formatTime(currentTime)) elif state == LogTypes.WARN: visualize.updateCache(m.author.name, (0, 1), utils.formatTime(currentTime)) elif state == LogTypes.UNBAN: await m.channel.send("Removing all old logs for unbanning") db.clear_user_logs(userid) # Generate message for log channel globalcount = db.get_dbid() new_log = db.UserLogEntry(globalcount + 1, userid, username, count, currentTime, mes, m.author.name, None) logMessage = str(new_log) await m.channel.send(logMessage) # Send ban recommendation, if needed if (state == LogTypes.WARN and count >= config.WARN_THRESHOLD): await m.channel.send("This user has received {} warnings or more. It is recommended that they be banned.".format(config.WARN_THRESHOLD)) logMesID = 0 # If we aren't noting, need to also write to log channel if state != LogTypes.NOTE: # Post to channel, keep track of message ID try: chan = discord.utils.get(m.guild.channels, id=config.LOG_CHAN) logMes = await chan.send(logMessage) logMesID = logMes.id except discord.errors.InvalidArgument: await m.channel.send("The logging channel has not been set up in `config.json`. In order to have a visual record, please specify a channel ID.") try: # Send a DM to the user u = ul.fetch_user(m.guild, userid) if u != None: DMchan = u.dm_channel # If first time DMing, need to create channel if DMchan == None: DMchan = await u.create_dm() # Only send DM when specified in configs if state == LogTypes.BAN and config.DM_BAN: await DMchan.send("Hi there! You've been banned from the Stardew Valley Discord for violating the rules: `{}`. If you have any questions, you can send a message to the moderators via the sidebar at <https://www.reddit.com/r/StardewValley>, and they'll forward it to us.".format(mes)) elif state == LogTypes.WARN and config.DM_WARN: await DMchan.send("Hi there! You received warning #{} in the Stardew Valley Discord for violating the rules: `{}`. Please review <#445729591533764620> and <#445729663885639680> for more info. If you have any questions, you can reply directly to this message to contact the staff.".format(count, mes)) elif state == LogTypes.KICK and config.DM_BAN: await DMchan.send("Hi there! You've been kicked from the Stardew Valley Discord for violating the following reason: `{}`. If you have any questions, you can send a message to the moderators via the sidebar at <https://www.reddit.com/r/StardewValley>, and they'll forward it to us.".format(mes)) # Exception handling except discord.errors.HTTPException as e: await m.channel.send("ERROR: While attempting to DM, there was an unexpected error. Tell aquova this: {}".format(e)) except Exception as e: await m.channel.send( "ERROR: An unexpected error has occurred. Tell aquova this: {}".format(e)) # Update database new_log.message_url = logMesID db.add_log(new_log)
async def removeError(m, edit): userid = ul.parse_mention(m) if userid == None: if edit: await m.channel.send("I wasn't able to understand that message: `$remove USER [num] new_message`") else: await m.channel.send("I wasn't able to understand that message: `$remove USER [num]`") return username = ul.fetch_username(m.guild, userid) if username == None: username = str(userid) # If editing, and no message specified, abort. mes = utils.parseMessage(m.content, username) if mes == "": if edit: await m.channel.send("You need to specify an edit message") return else: mes = "0" try: index = int(mes.split()[0]) - 1 mes = utils.strip(mes) except (IndexError, ValueError): index = -1 # Find most recent entry in database for specified user search_results = db.search(userid) # If no results in database found, can't modify if search_results == []: await m.channel.send("I couldn't find that user in the database") # If invalid index given, yell elif (index > len(search_results) - 1) or index < -1: await m.channel.send("I can't modify item number {}, there aren't that many for this user".format(index + 1)) else: item = search_results[index] import visualize if edit: if item.log_type == LogTypes.NOTE.value: currentTime = datetime.datetime.utcnow() item.timestamp = currentTime item.log_message = mes item.staff = m.author.name db.add_log(item) out = "The log now reads as follows:\n{}\n".format(str(item)) await m.channel.send(out) return else: await m.channel.send("You can only edit notes for now") return # Everything after here is deletion db.remove_log(item.dbid) out = "The following log was deleted:\n" out += str(item) if item.log_type == LogTypes.BAN: visualize.updateCache(item.staff, (-1, 0), utils.formatTime(item.timestamp)) elif item.log_type == LogTypes.WARN: visualize.updateCache(item.staff, (0, -1), utils.formatTime(item.timestamp)) await m.channel.send(out) # Search logging channel for matching post, and remove it try: if item.message_url != 0: chan = discord.utils.get(m.guild.channels, id=config.LOG_CHAN) m = await chan.fetch_message(item.message_url) await m.delete() # Print message if unable to find message to delete, but don't stop except discord.errors.HTTPException as e: print("Unable to find message to delete: {}", str(e))
choice = choices[choice](*args) else: print("Invalid selection. Choose:", ', '.join([k for k in choices.keys()])) choice = '' return if __name__ == "__main__": fileName = 'Error Logs/Error Log - ' + datetime.datetime.strftime( datetime.datetime.now(), '%I_%M_%S_%p') + '.log' logging.basicConfig(filename=fileName, level=logging.DEBUG, filemode='w') try: if False: import gui gui.MainWindow().run() else: start = time.time() ctypes.windll.kernel32.SetConsoleTitleW("Idle Gods Controller X") args = [''] * len(sys.argv) if len(sys.argv) > 1: for x in range(1, len(sys.argv)): args[x - 1] = sys.argv[x] askChoice(*args) except SystemExit: pass except BaseException: logging.getLogger(__name__).exception("Program terminated") logging.debug("Ran for: " + utils.formatTime(int(time.time() - start))) raise logging.shutdown()
def kdPreprocessing(dataset_name, n_classes, data_df, teacher_path, MAX_LEN=128, save_all=True, isCheckpoint=False): """ Dataset preparation for Bert Model and KD models. It is splitted (0.8 train, 0.1 valid and 0.1 test) and sets are returned. Every set is a CustomDatasetWithSoftTargets class (see utils.py) that return data in Bert format. :param dataset_name: string of dataset name. :param data_df: dataset in dataframe pandas format. :param MAX_LEN: it represents total words represented in bert encoding (other words will be ignored). :param save_all: boolean that specifies if save all data for time saving before training or network evaluating. :return: training_set, validation_set, test_set in CustomDataset format. """ device = 'cuda' if cuda.is_available() else 'cpu' tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') data_df['soft_targets'] = 0 train_size = 0.8 train_dataset = data_df.sample(frac=train_size, random_state=200).reset_index(drop=True) tmp_dataset = data_df.drop(train_dataset.index).reset_index(drop=True) test_dataset = tmp_dataset.sample(frac=0.5, random_state=200).reset_index(drop=True) val_dataset = tmp_dataset.drop(test_dataset.index).reset_index(drop=True) print("FULL Dataset: {}".format(data_df.shape)) print("TRAIN Dataset: {}".format(train_dataset.shape)) print("TEST Dataset: {}".format(test_dataset.shape)) print("VALID Dataset: {}".format(val_dataset.shape)) training_set = CustomDatasetWithSoftTargets(train_dataset, tokenizer, MAX_LEN) validation_set = CustomDatasetWithSoftTargets(val_dataset, tokenizer, MAX_LEN) test_set = CustomDatasetWithSoftTargets(test_dataset, tokenizer, MAX_LEN) train_params = {'batch_size': 32, 'shuffle': False, 'num_workers': 0} training_loader = DataLoader(training_set, **train_params) teacher_model = BertModel(n_classes=n_classes, dropout=0.3) if isCheckpoint: teacher_model.load_state_dict( torch.load(teacher_path)['model_state_dict']) else: teacher_model.load_state_dict(torch.load(teacher_path)) print(teacher_model) total_params = sum(p.numel() for p in teacher_model.parameters()) print('Teacher total parameters: {:}'.format(total_params)) teacher_model.to(device) teacher_model.eval() soft_targets = [] t0 = time.time() print('Creating soft targets...') with torch.no_grad(): for step, batch in enumerate(training_loader): ids = batch['ids'].to(device, dtype=torch.long) mask = batch['mask'].to(device, dtype=torch.long) token_type_ids = batch['token_type_ids'].to(device, dtype=torch.long) if step % 100 == 0 and not step == 0: # Calculate elapsed time in minutes. elapsed = formatTime(time.time() - t0) # Report progress. print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format( step, len(training_loader), elapsed)) soft_target = torch.softmax(teacher_model(ids, mask, token_type_ids), dim=1) soft_targets.extend(soft_target.cpu().detach().numpy().tolist()) del teacher_model training_set.setSoftTargets(soft_targets) if save_all is True: os.makedirs(os.path.dirname('datasets/' + dataset_name + '_kd_cleaned.txt'), exist_ok=True) with open('datasets/' + dataset_name + '_kd_cleaned.txt', 'wb') as f: pickle.dump([training_set, validation_set, test_set, MAX_LEN], f) return training_set, validation_set, test_set
def lstmEvaluate(dataset_name, n_classes, model_path, isCheckpoint=False): device = 'cuda' if cuda.is_available() else 'cpu' with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f: data_cleaned = pickle.load(f) test_set = data_cleaned[2] MAX_LEN = data_cleaned[3] TEST_BATCH_SIZE = 64 EMBEDDING_DIM = 50 HIDDEN_DIM = 256 test_params = {'batch_size': TEST_BATCH_SIZE, 'shuffle': True, 'num_workers': 0 } testing_loader = DataLoader(test_set, **test_params) tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = LSTMBase(embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, vocab_size=tokenizer.vocab_size, n_classes=n_classes) print(model) total_params = sum(p.numel() for p in model.parameters()) total_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print('Total parameters: {:}'.format(total_params)) print('Total trainable parameters: {:}'.format(total_trainable_params)) if isCheckpoint: model.load_state_dict(torch.load(model_path)['model_state_dict']) else: model.load_state_dict(torch.load(model_path)) model.to(device) model.eval() criterion = torch.nn.CrossEntropyLoss() # evaluate the network print("Evaluating network on Test Set") t0 = time.time() total_eval_loss = 0 fin_targets = [] fin_outputs = [] with torch.no_grad(): for step, batch in enumerate(testing_loader): ids = batch['ids'].to(device, dtype=torch.long) targets = batch['targets'].to(device, dtype=torch.long) outputs = model(ids) total_eval_loss += criterion(torch.softmax(outputs, dim=1), torch.max(targets, 1)[1]) if step % 100 == 0 and not step == 0: elapsed = formatTime(time.time() - t0) print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(testing_loader), elapsed)) fin_targets.extend(targets.cpu().detach().numpy().tolist()) fin_outputs.extend(torch.softmax(outputs, dim=1).cpu().detach().numpy().tolist()) valid_loss = total_eval_loss / len(testing_loader) fin_outputs = np.array(fin_outputs) fin_targets = np.array(fin_targets) accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1)) print("") print(" Test Accuracy: {0:.2f}".format(accuracy)) print(" Test Loss: {0:.2f}".format(valid_loss)) print("") print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - t0))) print("") print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))
def bertEvaluate(dataset_name, n_classes, model_path, isCheckpoint=False): """ Test set evaluating for a fine tuned Bert model. This function print accuracy and a scikit learn classification report with f1-score. It uses gpu. :param dataset_name: string of dataset name. :param n_classes: int of number of dataset classes. :param model_path: path of saved pytorch model fine tuned bert network (or checkpoint with isCheckpoint as True). :param isCheckpoint: boolean that specifies neither model_path is a model checkpoint. :return: None """ device = 'cuda' if cuda.is_available() else 'cpu' with open('datasets/' + dataset_name + '_bert_cleaned.txt', 'rb') as f: data_cleaned = pickle.load(f) test_set = data_cleaned[2] MAX_LEN = data_cleaned[3] TEST_BATCH_SIZE = 16 test_params = {'batch_size': TEST_BATCH_SIZE, 'shuffle': True, 'num_workers': 0 } testing_loader = DataLoader(test_set, **test_params) model = BertModel(n_classes=n_classes, dropout=0.3) if isCheckpoint: model.load_state_dict(torch.load(model_path)['model_state_dict']) else: model.load_state_dict(torch.load(model_path)) model.to(device) model.eval() criterion = torch.nn.CrossEntropyLoss() # evaluate the network print("Evaluating network on Test Set") t0 = time.time() total_eval_loss = 0 fin_targets = [] fin_outputs = [] with torch.no_grad(): for step, batch in enumerate(testing_loader): ids = batch['ids'].to(device, dtype=torch.long) mask = batch['mask'].to(device, dtype=torch.long) token_type_ids = batch['token_type_ids'].to(device, dtype=torch.long) targets = batch['targets'].to(device, dtype=torch.long) outputs = model(ids, mask, token_type_ids) total_eval_loss += criterion(outputs, torch.max(targets, 1)[1]) if step % 100 == 0 and not step == 0: elapsed = formatTime(time.time() - t0) print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(testing_loader), elapsed)) fin_targets.extend(targets.cpu().detach().numpy().tolist()) fin_outputs.extend(torch.softmax(outputs, dim=1).cpu().detach().numpy().tolist()) valid_loss = total_eval_loss / len(testing_loader) fin_outputs = np.array(fin_outputs) fin_targets = np.array(fin_targets) accuracy = accuracy_score(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1)) print("") print(" Test Accuracy: {0:.2f}".format(accuracy)) print(" Test Loss: {0:.2f}".format(valid_loss)) print("") print("Total evaluating took {:} (h:mm:ss)".format(formatTime(time.time() - t0))) print("") print(classification_report(fin_targets.argmax(axis=1), fin_outputs.argmax(axis=1), digits=4))