def test_sum(idx):
output = load_train_df(
dataset_dir='/store/tellus/train',
output='/store/tellus/train.pqt'
)
df = pd.read_parquet(output)
dataset = TellusDataset(
df=df,
has_y=True,
)
writer = SummaryWriter(f'{config["TENSORBORAD_LOG_DIR"]}/sum')
writer.add_image(
f"palser/{idx}/{dataset[idx]['label']}",
vutils.make_grid(
pipe(range(1),
map(lambda x: dataset[idx]),
map(lambda x: [
x['palser_after'] + x['palser_before'],
]),
concat,
list)
),
)
writer.add_image(
f"landsat/{idx}/{dataset[idx]['label']}",
vutils.make_grid(
pipe(range(1),
map(lambda x: dataset[idx]),
map(lambda x: [
x['landsat_after'] + x['landsat_before']
]),
concat,
list)
),
)
def test_esampler():
output = load_train_df(
dataset_dir='/store/tellus/train',
output='/store/tmp/train.pqt'
)
df = pd.read_parquet(output)
dataset = TellusDataset(
df=df,
has_y=True,
)
subset = Subset(
dataset,
list(range(1500, 1600))
)
epoch_size = 10
s = ChunkSampler(
epoch_size=epoch_size,
len_indices=len(subset),
shuffle=True,
)
batch_size = 2
train_loader = DataLoader(
subset,
sampler=s,
batch_size=batch_size,
pin_memory=True,
)
for i in range(11):
samples = pipe(
train_loader,
map(lambda x: x['id']),
filter(lambda x: len(x) == batch_size),
list
)
assert len(samples) == epoch_size//batch_size