def test_notreadable(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
m['test'] = 'abc'
m = mmapdict(f, picklers=[])
with self.assertRaises(ValueError):
m['test'] == 'abc'
def test_readonly(self):
with tempfile.NamedTemporaryFile() as f:
m = mmapdict(f.name, picklers=[GenericPickler])
m['test'] = 'abc'
self.assertEqual(m['test'], 'abc')
m2 = mmapdict(f.name, readonly=True, picklers=[GenericPickler])
self.assertEqual(m2['test'], 'abc')
def _test_bad_file(self, d, exc=None):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(d)
f.flush()
f.close()
if exc is not None:
with self.assertRaises(exc):
m = mmapdict(f.name, picklers=[GenericPickler])
else:
m = mmapdict(f.name, picklers=[GenericPickler])
os.unlink(f.name)
def test_nonexistent(self):
import os
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
os.unlink(f.name)
with self.assertRaises(FileNotFoundError):
m = mmapdict(f.name, True, picklers=[GenericPickler])
# This works and re-create the file
m = mmapdict(f.name, False, picklers=[GenericPickler])
del m
os.unlink(f.name)
def test_readonly(self):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
m = mmapdict(f.name, picklers=[GenericPickler])
m['test'] = 'abc'
self.assertEqual(m['test'], 'abc')
del m
m2 = mmapdict(f.name, readonly=True, picklers=[GenericPickler])
self.assertEqual(m2['test'], 'abc')
del m2
os.unlink(f.name)
def test_readonly(self):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
m = mmapdict(f.name, picklers=[ArrayPickler])
m['test'] = numpy.zeros((1, ), dtype=numpy.float32)
m2 = mmapdict(f.name, True, picklers=[ArrayPickler])
with self.assertRaises(ValueError):
m2['test'][0] = 2
del m
del m2
os.unlink(f.name)
def test_valid_pickle(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
f.seek(0)
d = pickle.load(f)
self.assertDictEqual(d, {})
def test_nonexistentkey(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
with self.assertRaises(KeyError):
del m['test']
with self.assertRaises(KeyError):
m['test']
def test_readonly(self):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
m = mmapdict(f.name, picklers=[GenericPickler])
m['test1'] = 234
self.assertTrue(m.writable)
m2 = mmapdict(f.name, True, picklers=[GenericPickler])
self.assertFalse(m2.writable)
with self.assertRaises(io.UnsupportedOperation):
m2['test1'] = 123
with self.assertRaises(io.UnsupportedOperation):
m2['test2'] = 123
with self.assertRaises(io.UnsupportedOperation):
del m2['test1']
def test_store_empty(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[ArrayPickler])
m['test'] = EmptyNDArray((10, 9), dtype=numpy.float32)
self.assertIsInstance(m['test'], numpy.memmap)
self.assertEqual(m['test'].shape, (10, 9))
def test_convert_not_possible(self):
with tempfile.TemporaryFile() as f:
d = 'abc'
pickle.dump(d, f)
with self.assertRaises(ValueError):
m = mmapdict(f, picklers=[GenericPickler])
def test_destructor(self):
import weakref
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers = [GenericPickler])
m_ref = weakref.ref(m)
del m
self.assertIsNone(m_ref(), "Reference to object is still valid, something is wrong (using object instance instead of weakref.ref?)")
def test_store_simple(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers = [GenericPickler])
m['test'] = 'abc'
self.assertEqual(m['test'], 'abc')
d = pickle.load(f)
self.assertDictEqual(d, {'test': 'abc',})
def test_convert(self):
with tempfile.NamedTemporaryFile() as f:
v = {'abc': 123}
pickle.dump(v, f)
f.flush()
m = mmapdict(f, picklers=[GenericPickler])
assert dict(m) == v
def test_delitem(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
m['test'] = 'abc'
self.assertEqual(m['test'], 'abc')
del m['test']
with self.assertRaises(KeyError):
print(m['test'])
#reset
m = mmapdict(f)
with self.assertRaises(KeyError):
print(m['test'])
f.seek(0, io.SEEK_SET)
d = pickle.load(f)
self.assertDictEqual(d, {})
def test_store_dims(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[ArrayPickler])
for i in range(1, 9):
m['test{}'.format(i)] = numpy.zeros(tuple([1] * i),
dtype=numpy.float32)
for i in range(1, 9):
self.assertEqual(m['test{}'.format(i)].ndim, i)
def test_broken(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f)
m['a'] = 1
# self.assertTrue(m.fsck())
m['b'] = (1, 2, 3)
m['c'] = 'test'
f.seek(0, io.SEEK_END)
original_size = f.tell()
original_dict = dict(m)
# self.assertTrue(m.fsck())
f.seek(0, io.SEEK_END)
self.assertDictEqual(original_dict, dict(m))
self.assertEqual(original_size, f.tell())
for i in range(1, 13):
del m
f.truncate(original_size - i)
m = mmapdict(f)
m.fsck()
f.seek(0, io.SEEK_END)
self.assertDictEqual(original_dict, dict(m))
self.assertEqual(original_size, f.tell())
# We loose one key...
del original_dict['c']
for i in range(14, 20):
del m
f.truncate(original_size - i)
m = mmapdict(f)
m.fsck()
f.seek(0, io.SEEK_END)
self.assertDictEqual(original_dict, dict(m))
def test_store_simple(self):
with tempfile.TemporaryFile() as f:
data = numpy.array([[1, 2, 3], [4, 5, 6]])
m = mmapdict(f, picklers=[ArrayPickler, GenericPickler])
m['test'] = data
self.assertIsInstance(m['test'], numpy.memmap)
numpy.testing.assert_array_equal(m['test'], data)
f.seek(0)
d = pickle.load(f)
numpy.testing.assert_array_equal(d['test'], data)
def test_masked_empty(self):
with tempfile.TemporaryFile() as f:
data = numpy.ma.zeros([2, 3], dtype=numpy.int64)
m = mmapdict(f, picklers = [MaskedArrayPickler, ArrayPickler, GenericPickler])
m['test'] = data
self.assertIsInstance(m['test'], numpy.ma.masked_array)
self.assertIsInstance(m['test'].data, numpy.memmap)
self.assertIsInstance(m['test'].mask, numpy.memmap)
numpy.testing.assert_array_equal(m['test'], data)
d = pickle.load(f)
numpy.testing.assert_array_equal(d['test'], data)
def test_convert(self):
with tempfile.TemporaryFile() as f:
d = {
'a': 1,
'b': (1, 2, 3),
'c': 'test',
}
pickle.dump(d, f)
m = mmapdict(f, picklers=[GenericPickler])
self.assertDictEqual(dict(m), d)
def test_store_masked(self):
with tempfile.TemporaryFile() as f:
data = numpy.ma.MaskedArray([[1, 2, 3], [4, 5, 6]], [[False, True, False], [True, False, True]])
m = mmapdict(f, picklers = [MaskedArrayPickler])
m['test'] = data
self.assertIsInstance(m['test'].data, numpy.memmap)
self.assertIsInstance(m['test'].mask, numpy.memmap)
numpy.testing.assert_array_equal(m['test'], data)
f.seek(0)
d = pickle.load(f)
numpy.testing.assert_array_equal(d['test'], data)
def test_masked_bug(self):
with tempfile.TemporaryFile() as f:
data = numpy.random.rand(100)
data = numpy.ma.masked_invalid(data)
m = mmapdict(f, picklers=[MaskedArrayPickler])
m['test'] = data
self.assertIsInstance(m['test'].data, numpy.memmap)
self.assertIsInstance(m['test'].mask, numpy.memmap)
numpy.testing.assert_array_equal(m['test'], data)
f.seek(0)
d = pickle.load(f)
numpy.testing.assert_array_equal(d['test'], data)
def test_concurrent_mmapdict_pickle(self):
# This is not a real test, but it fixes the converage computation since the previous test in not counted
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
m = mmapdict(f.name)
m2 = pickle.loads(pickle.dumps(m))
del m
del m2
os.unlink(f.name)
def test_commit_number(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
self.assertEqual(m._header.commit_number, 0)
self.assertTrue(m._header.is_valid())
m._header.commit_number = 32
self.assertEqual(m._header.commit_number, 32)
self.assertTrue(m._header.is_valid())
m._header.commit_number = 465468
self.assertEqual(m._header.commit_number, 465468)
self.assertTrue(m._header.is_valid())
def test_put_opcodes(self):
import string
import itertools
d_vals = {}
for s in itertools.product(string.ascii_letters, string.ascii_letters):
s = ''.join(s)
d_vals[s] = s
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
for i in range(5):
m['test-{}'.format(i)] = d_vals
for i in range(5):
self.assertDictEqual(m['test-{}'.format(i)], d_vals)
def migrate_base_data(args, module, override_input=None, override_output=None):
import os, subprocess
from mmappickle import mmapdict
if override_input is not None:
input_data = mmapdict(override_input, True)
else:
input_data = mmapdict(args.input, True)
if override_output is not None:
output_data = mmapdict(override_output)
else:
output_data = mmapdict(args.output)
output_data['description'] = input_data['description']
output_data['wavelengths'] = input_data['wavelengths']
if 'processing_steps' in input_data:
processing_steps = input_data['processing_steps']
else:
processing_steps = []
try:
git_revision = subprocess.check_output(
[
'git', 'describe', '--match=NeVeRmAtChRandoMJuNK', '--always',
'--abbrev=40', '--dirty'
],
cwd=os.path.dirname(__file__)).strip()
except:
git_revision = None
this_step_params = {}
for k, v in args._get_kwargs():
this_step_params[k] = v
processing_steps.append((module, git_revision, this_step_params))
output_data['processing_steps'] = processing_steps
return input_data, output_data
def test_vacuum(self):
with tempfile.TemporaryFile() as f:
m = mmapdict(f, picklers=[GenericPickler])
m['a'] = 1
m['b'] = 2
m['c'] = 3
m['d'] = ' ' * 2 * 1024 * 1024
m['e'] = 5
m['f'] = 6
m['g'] = 7
m['h'] = 8
f.seek(0, io.SEEK_END)
fsizebefore = f.tell()
del m['b']
del m['d']
del m['f']
del m['g']
f.seek(0, io.SEEK_SET)
valid_dict = pickle.load(f)
self.assertDictEqual(dict(m), valid_dict)
f.seek(0, io.SEEK_END)
fsizeafterdel = f.tell()
self.assertEqual(fsizebefore, fsizeafterdel)
old_commit_number = m.commit_number
m.vacuum()
f.seek(0, io.SEEK_END)
fsizeaftervacuum = f.tell()
self.assertNotEqual(fsizebefore, fsizeaftervacuum)
self.assertNotEqual(old_commit_number, m.commit_number)
self.assertDictEqual(dict(m), valid_dict)
m.commit_number = 143
m.vacuum()
self.assertEqual(143, m.commit_number)
# We have to do something otherwise the vacuum won't do anything
del m['a']
m.commit_number = 0
m.vacuum()
self.assertNotEqual(0, m.commit_number)
def test_store_ref(self):
with tempfile.TemporaryFile() as f:
obj = "1234"
obj2 = "machin"
dict_a = {
obj: obj,
'3': obj,
}
dict_b = {
obj: obj2,
obj2: obj2,
2: 4,
4: obj,
}
m = mmapdict(f)
m['dict_a'] = dict_a
m['dict_b'] = dict_b
self.assertEqual(m['dict_a'], dict_a)
self.assertEqual(m['dict_b'], dict_b)
d = pickle.load(f)
self.assertDictEqual(d, {
'dict_a': dict_a,
'dict_b': dict_b,
})
import collections
od = collections.OrderedDict()
od['obj'] = obj
od['obj2'] = obj2
od[obj] = 3
m['od'] = od
#This should not fail, but will have no effect
m['od']['machin'] = 'abc'
self.assertEqual(m['od'], od)
f.seek(0, io.SEEK_SET)
d = pickle.load(f)
m['od'] = 'abc'
self.assertEqual(m['od'], 'abc')
f.seek(0, io.SEEK_SET)
d = pickle.load(f)
self.assertEqual(d['od'], 'abc')
def create_dataset(d, n, output):
prefixes = [
d + '/' + p.split('.')[0] for p in os.listdir(d)
if '_' not in p and 'tif' in p
]
prefixes = np.random.choice(prefixes, n, replace=False)
out_path = output + '/out'
if os.path.isfile(out_path):
os.remove(out_path)
m = mmappickle.mmapdict(out_path)
for prefix in prefixes:
crop, cells = image_for_prefix(prefix)
d = {'crop': crop, 'cell': cells}
key = os.path.basename(prefix)
m[key] = d
def test_concurrent_1(self):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
m = mmapdict(f.name)
m['value'] = numpy.zeros((4, ), numpy.float)
import multiprocessing
import itertools
with multiprocessing.Pool(4) as p:
p.map(_tc_increment, itertools.product([m], range(4)))
self.assertEqual(m['value'][0], 1)
self.assertEqual(m['value'][1], 1)
self.assertEqual(m['value'][2], 1)
self.assertEqual(m['value'][3], 1)
del m
os.unlink(f.name)
