def test_dimension_methods(): movieLensDataHandler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) assert 8936 == movieLensDataHandler.get_input_dim() assert 8936 == movieLensDataHandler.get_output_dim()
def test_droplast(): movieLensDataHandler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) train_dataloader = movieLensDataHandler.get_train_dataloader( batch_size=200, drop_last=False) count = 0 for batch in train_dataloader: assert 8936 == len(batch[0][0]) assert 8936 == len(batch[1][0]) count += 1 assert 51 == count
def test_batchsize(): movieLensDataHandler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) # test the number of batches train_dataloader = movieLensDataHandler.get_train_dataloader( batch_size=200) count = 0 for batch in train_dataloader: assert 200 == len(batch[0]) assert 200 == len(batch[1]) assert 8936 == len(batch[0][0]) assert 8936 == len(batch[1][0]) count += 1 assert 50 == count
def test_length(): movieLensDataHandler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) assert 10001 == movieLensDataHandler.get_traindata_len() assert 2000 == movieLensDataHandler.get_testdata_len() assert 2000 == movieLensDataHandler.get_validationdata_len() assert 10001 == movieLensDataHandler.get_traindata_len() assert 2000 == movieLensDataHandler.get_testdata_len() assert 2000 == movieLensDataHandler.get_validationdata_len()
def test_shuffle(): movieLensDataHandler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) # test the number of batches train_dataloader = movieLensDataHandler.get_train_dataloader(shuffle=False) first = True first_batch = None for batch in train_dataloader: if first: first_batch = batch first = False first = True for batch in train_dataloader: if first: comparison = batch[0] == first_batch[0] assert comparison.all() break
logger.setLevel(logging.INFO) # set cuda if available device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_data_path = os.path.join(dir_path, 'movielens_small_training.npy') validation_input_data_path = os.path.join( dir_path, 'movielens_small_validation_input.npy') validation_output_data_path = os.path.join( dir_path, 'movielens_small_validation_test.npy') test_input_data_path = os.path.join(dir_path, 'movielens_small_test_input.npy') test_output_data_path = os.path.join(dir_path, 'movielens_small_test_test.npy') products_data_path = os.path.join(dir_path, 'movielens_products_data.npy') data_handler = AEDataHandler('MovieLensSmall', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) input_dim = data_handler.get_input_dim() output_dim = data_handler.get_output_dim() products_data_np = np.load(products_data_path) products_data_torch = torch.tensor(products_data_np, dtype=torch.float32).to(device) # create model model = MultiVAE(params='yaml_files/params_multi_VAE_training.yaml') correctness_loss = VAELoss() revenue_loss = VAELoss(weighted_vector=products_data_torch) losses = [correctness_loss, revenue_loss]
dir_path, 'movielens_small_validation_test.npy') test_input_data_path = os.path.join(dir_path, 'movielens_small_test_input.npy') test_output_data_path = os.path.join(dir_path, 'movielens_small_test_test.npy') products_data_path = os.path.join(dir_path, 'movielens_products_data.npy') np.save(train_data_path, np.random.rand(10000, 8936).astype('float32')) np.save(validation_input_data_path, np.random.rand(2000, 8936).astype('float32')) np.save(validation_output_data_path, np.random.rand(2000, 8936).astype('float32')) np.save(test_input_data_path, np.random.rand(2000, 8936).astype('float32')) np.save(test_output_data_path, np.random.rand(2000, 8936).astype('float32')) np.save(products_data_path, np.random.rand(8936)) dataHandler = AEDataHandler('Testing trainer random dataset', train_data_path, validation_input_data_path, validation_output_data_path, test_input_data_path, test_output_data_path) input_dim = dataHandler.get_input_dim() output_dim = dataHandler.get_output_dim() products_data_np = np.load(products_data_path) products_data_torch = torch.tensor(products_data_np, dtype=torch.float32).to(device) # create model model = MultiVAE(params='yaml_files/params_multi_VAE.yaml') correctness_loss = VAELoss() revenue_loss = VAELoss(weighted_vector=products_data_torch) losses = [correctness_loss, revenue_loss]