def test_ignores_misnamed_files():
dataroot, _, _, _ = fake_geo_data(1)
open(os.path.join(dataroot, 'garbage'), 'w').close()
open(os.path.join(dataroot, 'garbage_norm.dat'), 'w').close()
p = GeoPickler(dataroot)
p.initialise()
assert (len(p.get_folder_by_id(0)) == 1)
def test_searches_subfolders():
dataroot, _, _, _ = fake_geo_data(1)
subfolder, _, _, _ = fake_geo_data(2)
shutil.move(subfolder, dataroot)
p = GeoPickler(dataroot)
p.collect_all()
p.group_by_series()
assert (list(p.folders.keys()) == ['', os.path.basename(subfolder)])
assert (len(p.get_folder_by_id(0)) == 1)
assert (len(p.get_folder_by_id(1)) == 2)
def test_matches_other_file_pattern():
dataroot, _, _, _ = fake_geo_data(1)
shutil.move(os.path.join(dataroot, 'serie100000_Vx.dat'),
os.path.join(dataroot, 'serie1_0_Vx.dat'))
shutil.move(os.path.join(dataroot, 'serie100000_Vy.dat'),
os.path.join(dataroot, 'serie1_0_Vy.dat'))
assert ('serie1_0_Vx.dat' in os.listdir(dataroot))
assert ('serie1_0_Vy.dat' in os.listdir(dataroot))
p = GeoPickler(dataroot)
p.collect_all()
p.group_by_series()
assert (len(p.get_folder_by_id(0)) == 1)
def test_groups_by_series():
dataroot, _, _, _ = fake_geo_data(3)
p = GeoPickler(dataroot)
p.collect_all()
p.group_by_series()
folder = p.get_folder_by_id(0)
assert (len(folder) == 3)
assert (p.get_series_in_folder(0) == [0, 1, 2])
assert (folder[0] == [
'serie100000_DIV.dat', 'serie100000_Vx.dat', 'serie100000_Vy.dat'
])
assert (folder[1] == [
'serie100001_DIV.dat', 'serie100001_Vx.dat', 'serie100001_Vy.dat'
])
assert (folder[2] == [
'serie100002_DIV.dat', 'serie100002_Vx.dat', 'serie100002_Vy.dat'
])
def test_pickling_preserves_folder_structure(mocker):
dataroot, DIV_base, Vx_base, Vy_base = fake_geo_data(1)
subfolder1, DIV_sub1, Vx_sub1, Vy_sub1 = fake_geo_data(2)
subfolder2, DIV_sub2, Vx_sub2, Vy_sub2 = fake_geo_data(2)
for data_group in [
'DIV_base', 'Vx_base', 'Vy_base', 'DIV_sub1', 'Vx_sub1', 'Vy_sub1',
'DIV_sub2', 'Vx_sub2', 'Vy_sub2'
]:
for i, data in enumerate(eval(data_group)):
eval(data_group)[i] = np.interp(
data, (np.min(data.ravel()), np.max(data.ravel())), [-1, 1])
shutil.move(subfolder1, dataroot)
shutil.move(subfolder2, dataroot)
p = GeoPickler(dataroot, 'out_dir')
p.collect_all()
p.group_by_series()
# Doesn't always load in order, but it's not a big deal
assert (sorted(p.folders.keys()) == sorted(
['', os.path.basename(subfolder1),
os.path.basename(subfolder2)]))
assert (len(p.get_folder_by_id(0)) == 1)
assert (len(p.get_folder_by_id(1)) == 2)
assert (len(p.get_folder_by_id(2)) == 2)
dirs = list(p.folders.keys())
mocker.patch('torch.save')
# Just set mask threshold to zero, otherwise it randomly fails if not enough
# pixels in randomly generated test data
p.pickle_all(1000, 4, 0)
assert (len(torch.save.call_args_list) == 5)
for args, kwargs in torch.save.call_args_list:
series_number = args[0]['series_number']
# It doesn't matter which order they're written in
if args[1] == os.path.join('out_dir',
'{:05}.pkl'.format(series_number)):
assert ((args[0]['A_DIV'] == DIV_base[0]).all())
assert ((args[0]['A_Vx'] == Vx_base[0]).all())
assert ((args[0]['A_Vy'] == Vy_base[0]).all())
elif args[1] == os.path.join('out_dir', os.path.basename(subfolder1),
'{:05}.pkl'.format(series_number)):
assert (series_number == 0 or series_number == 1)
assert ((args[0]['A_DIV'] == DIV_sub1[series_number]).all())
assert ((args[0]['A_Vx'] == Vx_sub1[series_number]).all())
assert ((args[0]['A_Vy'] == Vy_sub1[series_number]).all())
elif args[1] == os.path.join('out_dir', os.path.basename(subfolder2),
'{:05}.pkl'.format(series_number)):
assert (series_number == 0 or series_number == 1)
assert ((args[0]['A_DIV'] == DIV_sub2[series_number]).all())
assert ((args[0]['A_Vx'] == Vx_sub2[series_number]).all())
assert ((args[0]['A_Vy'] == Vy_sub2[series_number]).all())
else:
assert False, args[1]