return (page, full_smape, val_smape)
def wrapper(pages):
base_log_info = '[Process:{0}] '.format(mp.current_process().name)
val_results = []
lg.info(base_log_info + 'starting its pages loop')
for page in tqdm(pages):
val_results.append(process_page(page))
lg.info(base_log_info + 'finished its pages loop')
return val_results
total_proc = mp.cpu_count()
# NOTE: shuffle the cols to that any pages that still need models built get distributied evenly
# NOTE: shuffling the index directly switches all the pages from their corresponding series... BAD
cols = pagedf.columns.values.copy()
np.random.shuffle(cols)
col_split = np.array_split(cols, total_proc)
mp_pool = mp.Pool(total_proc)
with utils.clock():
val_results = mp_pool.map(wrapper, col_split)
lg.info('Finished pool map')
lg.info('Val results recieved - processes ended')
val_results = [item for sublist in val_results for item in sublist]
val_results = pd.DataFrame(
val_results,
columns=['page_index', VERSION[:-1] + '_full', VERSION[:-1] + '_val'])
val_results.to_feather(PROPHET_PATH + RESULTS_PATH + VERSION[:-1] + 'df.f')
lg.info('Script complete')
batch_size=batch_size, shuffle=False
)
# In[9]:
loss_func = nn.L1Loss()
model = rnn.RNN(loss_func=loss_func).cuda()
# In[60]:
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
with clock():
model.fit(trainloader, valloader=None, optimizer=optimizer, num_epochs=17)
# In[ ]:
save_best_path = base_dir+'rnn_stage2_FINAL_v2_lr1.mdl'
torch.save(model.state_dict(), save_best_path)
# In[ ]:
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
with clock():
