def fit(self, train_loader, valid_loader, start_epoch=0, max_epochs=200):
best_acc = 0.
bar = IncrementalBar(max=max_epochs - start_epoch)
for e in range(start_epoch, max_epochs):
bar.message = '{:>5.2f}%%'.format(bar.percent)
bar.suffix = '{}/{} [{}<{}\t{:.2f}it/s]'.format(
bar.index, bar.max, bar.elapsed_td, bar.eta_td, bar.avg)
bar.next()
if e == self.milestones[0]:
schedule_lr(self.optimizer) # update learning rate once
if e == self.milestones[1]:
schedule_lr(self.optimizer)
if e == self.milestones[2]:
schedule_lr(self.optimizer)
self.train(train_loader, self.model, self.criterion,
self.optimizer, e)
accuracy, best_threshold, roc_curve_tensor = self.evaluate(
self.conf, *valid_loader['agedb_30'])
self.board_val('agedb_30', accuracy, best_threshold,
roc_curve_tensor, e)
if accuracy > best_acc:
best_acc = accuracy
save_checkpoint(self.model, self.optimizer, self.conf,
best_acc, e)
bar.finish()
def generate_submission(model: Type[BaseModel],
model_config: ModelConfig = None) -> None:
if model_config is None:
model_config = {}
print('Generating data/submission.csv with {}'.format(model.__name__))
model_instance = model()
model_instance.learn(LoadedData.training_games, LoadedData.training_decks,
model_config)
n = len(bot_list) * len(LoadedData.test_decks)
try:
bar = IncrementalBar('',
max=n,
suffix='%(index)d/%(max)d ETA: %(eta)ds')
with open('data/submission.csv', 'w') as file:
for bot in bot_list:
bar.message = bot
for deck in LoadedData.test_decks:
print('{};{};{}'.format(bot, deck['deckName'],
model_instance.predict(bot, deck)),
file=file)
bar.next()
bar.finish()
except KeyboardInterrupt:
print(SHOW_CURSOR)
raise KeyboardInterrupt
def crossval(folds: int, games: List[Game], decks: List[Deck],
model: Type[BaseModel], model_config: ModelConfig) -> float:
random.shuffle(decks)
fold_decks = len(decks) // folds
wins = defaultdict(lambda: (0, 0))
for game in games:
player0, player1 = (game['bot0'], game['deck0']), (game['bot1'],
game['deck1'])
wins0, wins1 = wins[player0], wins[player1]
if game['winner'] == 0:
wins0 = (wins0[0] + 1, wins0[1] + 1)
wins1 = (wins1[0], wins1[1] + 1)
else:
wins0 = (wins0[0], wins0[1] + 1)
wins1 = (wins1[0] + 1, wins1[1] + 1)
wins[player0], wins[player1] = wins0, wins1
winrates = {}
for player, wr in wins.items():
winrates[player] = 100 * wr[0] / wr[1]
avg_rmse = 0
for i in range(folds):
test_decks = decks[i * fold_decks:(i + 1) * fold_decks]
training_decks = decks[:i * fold_decks] + decks[(i + 1) * fold_decks:]
training_deck_names = set([d['deckName'] for d in training_decks])
training_games = list(
filter(
lambda g: g['deck0'] in training_deck_names and g['deck1'] in
training_deck_names, games))
model_instance = model()
model_instance.learn(training_games, training_decks, model_config)
rmse = 0.0
ctr = 0
n = len(test_decks) * len(bot_list)
bar = IncrementalBar('',
max=n,
suffix='%(index)d/%(max)d ETA: %(eta)ds')
for deck in test_decks:
for bot in bot_list:
ctr += 1
rmse += (model_instance.predict(bot, deck) -
winrates[(bot, deck['deckName'])])**2
bar.message = '{}/{} | {:.5f}'.format(i + 1, folds,
math.sqrt(rmse / ctr))
bar.next()
bar.finish()
rmse = math.sqrt(rmse / n)
avg_rmse += rmse
avg_rmse /= folds
return avg_rmse
def processTerms(targetFile):
"""
A function that opens a csv file and producdes .mp3 files of each term.abs
Takes a local file .csv
"""
with open(targetFile) as csv_file:
terms = csv.reader(csv_file, delimiter=',')
length_of_terms = sum(1 for row in terms)
csv_file.seek(0)
prog_bar = IncrementalBar('Generating speech...',
max=length_of_terms,
suffix='Remaining time: %(eta)ds')
for term in terms:
prog_bar.message = 'Processing ID: ' + term[1]
gTTS(text=term[0], lang='en', slow=False).save(term[1] + '.mp3')
prog_bar.next()
prog_bar.finish()
state = 'FINISHED'
sys.stdout.write('\b' * 11)
sys.stdout.flush()
sys.stdout.write('\r ')
print("Finished")
state = 'Running'
spinner = MoonSpinner('Running ')
counter = 0
while state != 'FINISHED':
# Do some work
time.sleep(.1)
spinner.next()
counter += 1
if counter > 20:
state = 'FINISHED'
sys.stdout.write('\b' * 11)
sys.stdout.flush()
sys.stdout.write('\r ')
print("Finished")
iCount = 0
bar = IncrementalBar('Mirgrating process e_gdbgra',
max=10,
suffix='%(percent).1f%% - %(eta)ds')
for i in range(10):
time.sleep(round(random.random(), 1))
bar.next()
iCount += 1
if iCount > 5:
bar.message = "Migrating process p_gdbgra"
bar.finish()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'path', help='Path to directory containing measurement data'
)
args = parser.parse_args()
# Use given path as working directory for script so glob works from there
os.chdir(args.path)
total_lines = get_total_lines(glob.iglob('**/*.CSV', recursive=True))
total_size = int(total_lines / BUFFER_SIZE)
bar = IncrementalBar(
max=total_size, suffix='%(percent)d%% [ETA: %(eta_td)s]'
)
bar.start()
csv_glob = peekable(glob.iglob('**/*.CSV', recursive=True))
line_reader = buffered_line_reader(
line_reader(csv_glob), buffer_size=BUFFER_SIZE
)
with requests.Session() as session:
# Iterate through the progress bar to auto update the bar
for lines in bar.iter(line_reader):
bar.message = csv_glob.peek(bar.message)
response = session.post(URL, data=lines)
assert response.status_code == 204, response.text