DEBUG = False
if DEBUG:
BATCH_SIZE = 3
NUM_WORKERS = 0
PRINT_EVERY = 10
SAVE_EVERY = 1000
else:
BATCH_SIZE = 512
NUM_WORKERS = 6
PRINT_EVERY = 100
SAVE_EVERY = 1000
NUM_EPOCHS = 250
t_acq, t_idx_to_seq, t_seq, t_macros, t_ym2idx = load_data(
TRAIN_PATH, verbose=True, oneChunkOnly=True)
v_acq, v_idx_to_seq, v_seq, v_macros, v_ym2idx = load_data(
VALID_PATH, verbose=True, oneChunkOnly=True)
train_ds = FNMCCARDataset(t_acq, t_idx_to_seq, t_seq, t_macros, t_ym2idx,
12, 1)
valid_ds = FNMCCARDataset(v_acq, v_idx_to_seq, v_seq, v_macros, v_ym2idx,
12, 1)
print("Number of train acq: {:,}".format(len(train_ds)))
print("Number of valid acq: {:,}".format(len(valid_ds)))
trainDL = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle = True, \
collate_fn=paddingCollator, num_workers=NUM_WORKERS, pin_memory=True)
validDL = DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle = True, \
collate_fn=paddingCollator, num_workers=NUM_WORKERS, pin_memory=True)
DEBUG = True
if DEBUG:
BATCH_SIZE = 3
NUM_WORKERS = 0
PRINT_EVERY = 10
SAVE_EVERY = 1000
else:
BATCH_SIZE = 256
NUM_WORKERS = 6
PRINT_EVERY = 100
SAVE_EVERY = 1000
NUM_EPOCHS = 5
t_acq, t_idx_to_seq, t_seq, t_macros, t_ym2idx = load_data(
TRAIN_PATH, True, True)
v_acq, v_idx_to_seq, v_seq, v_macros, v_ym2idx = load_data(
VALID_PATH, True, True)
train_ds = FNMCCARDataset(t_acq, t_idx_to_seq, t_seq, t_macros, t_ym2idx,
12, 0)
valid_ds = FNMCCARDataset(v_acq, v_idx_to_seq, v_seq, v_macros, v_ym2idx,
12, 0)
print("Number of train acq: {:,}".format(len(train_ds)))
print("Number of valid acq: {:,}".format(len(valid_ds)))
trainDL = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle = True, \
collate_fn=paddingCollator, num_workers=NUM_WORKERS, pin_memory=True)
validDL = DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle = True, \
collate_fn=paddingCollator, num_workers=NUM_WORKERS, pin_memory=True)