def test_read_demo(self):
print("MDF read tests")
ret = True
for enable in (True, False):
for mdf in os.listdir('tmpdir_demo'):
for memory in MEMORY:
with MDF(os.path.join('tmpdir_demo', mdf),
memory=memory) as input_file:
if input_file.version == '2.00':
continue
for name in set(
input_file.channels_db) - {'time', 't'}:
signal = input_file.get(name)
original_samples = CHANNELS_DEMO[name]
if signal.samples.dtype.kind == 'f':
signal = signal.astype(np.float32)
res = np.array_equal(signal.samples,
original_samples)
if not res:
ret = False
self.assertTrue(ret)
cleanup_files()
def test_convert_demo(self):
print("MDF convert tests")
for out in SUPPORTED_VERSIONS:
for mdfname in os.listdir('tmpdir_demo'):
for memory in MEMORY:
input_file = os.path.join('tmpdir_demo', mdfname)
if MDF(input_file).version == '2.00':
continue
print(input_file, memory, out)
with MDF(input_file, memory=memory) as mdf:
outfile = mdf.convert(out, memory=memory).save(
'tmp', overwrite=True)
equal = True
with MDF(input_file, memory=memory) as mdf, \
MDF(outfile, memory=memory) as mdf2:
for name in set(mdf.channels_db) - {'t', 'time'}:
original = mdf.get(name)
converted = mdf2.get(name)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut_demo(self):
print("MDF cut absolute tests")
cntr = 0
for mdfname in os.listdir("tmpdir_demo"):
for memory in MEMORY:
input_file = os.path.join("tmpdir_demo", mdfname)
if "2.00" in input_file:
continue
for whence in (0, 1):
print(input_file, memory, whence)
outfile1 = (MDF(input_file, memory=memory).cut(
stop=2, whence=whence,
include_ends=False).save("tmp1", overwrite=True))
outfile2 = (MDF(input_file, memory=memory).cut(
start=2, stop=6, whence=whence,
include_ends=False).save("tmp2", overwrite=True))
outfile3 = (MDF(input_file, memory=memory).cut(
start=6, whence=whence,
include_ends=False).save("tmp3", overwrite=True))
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
vedrsion=MDF(input_file, memory="minimum").version,
memory=memory,
).save("tmp", overwrite=True)
print("OUT", outfile)
equal = True
with MDF(input_file,
memory=memory) as mdf, MDF(outfile,
memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group["channels"][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut_demo(self):
print("MDF cut absolute tests")
cntr = 0
for mdfname in os.listdir('tmpdir_demo'):
for memory in MEMORY:
input_file = os.path.join('tmpdir_demo', mdfname)
if '2.00' in input_file:
continue
for whence in (0, 1):
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory).cut(
stop=2, whence=whence).save('tmp1', overwrite=True)
outfile2 = MDF(input_file, memory=memory).cut(
start=2, stop=6, whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory).cut(
start=6, whence=whence).save('tmp3', overwrite=True)
outfile = MDF.merge(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp', overwrite=True)
print('OUT', outfile)
equal = True
with MDF(input_file, memory=memory) as mdf, \
MDF(outfile, memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group['channels'][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_get_channel_units(self):
print("MDF get channel units tests")
ret = True
for mdf in os.listdir('tmpdir_demo'):
for memory in MEMORY:
with MDF(os.path.join('tmpdir_demo', mdf),
memory=memory) as input_file:
if input_file.version == '2.00':
continue
print(mdf, memory)
for channel_name, original_unit in UNITS.items():
comment = input_file.get_channel_unit(channel_name)
if comment != original_unit:
ret = False
self.assertTrue(ret)
cleanup_files()
def test_filter(self):
print("MDF filter tests")
for mdfname in os.listdir("tmpdir_demo"):
for memory in MEMORY:
input_file = os.path.join("tmpdir_demo", mdfname)
if MDF(input_file, memory=memory).version <= "2.00":
# if MDF(input_file, memory=memory).version < '4.00':
continue
channels_nr = np.random.randint(1, len(CHANNELS_DEMO) + 1)
channel_list = random.sample(list(CHANNELS_DEMO), channels_nr)
filtered_mdf = MDF(input_file,
memory=memory).filter(channel_list,
memory=memory)
target = set(k for k in filtered_mdf.channels_db
if not k.endswith("[0]")
and not k.startswith("DI") and "\\" not in k)
self.assertTrue((target - {"t", "time"}) == set(channel_list))
equal = True
with MDF(input_file, memory=memory) as mdf:
print(input_file, memory)
for name in channel_list:
original = mdf.get(name)
filtered = filtered_mdf.get(name)
if not np.array_equal(original.samples,
filtered.samples):
equal = False
if not np.array_equal(original.timestamps,
filtered.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_select(self):
print("MDF select tests")
for mdfname in os.listdir("tmpdir_demo"):
for memory in MEMORY:
input_file = os.path.join("tmpdir_demo", mdfname)
if MDF(input_file).version == "2.00":
continue
print(input_file, memory)
channels_nr = np.random.randint(1, len(CHANNELS_DEMO) + 1)
channel_list = random.sample(list(CHANNELS_DEMO), channels_nr)
selected_signals = MDF(input_file,
memory=memory).select(channel_list)
self.assertTrue(len(selected_signals) == len(channel_list))
self.assertTrue(
all(ch.name == name
for ch, name in zip(selected_signals, channel_list)))
equal = True
with MDF(input_file, memory=memory) as mdf:
for selected in selected_signals:
original = mdf.get(selected.name)
if not np.array_equal(original.samples,
selected.samples):
equal = False
if not np.array_equal(original.timestamps,
selected.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_filter(self):
print("MDF filter tests")
for mdfname in os.listdir('tmpdir_demo'):
for memory in MEMORY:
input_file = os.path.join('tmpdir_demo', mdfname)
if MDF(input_file, memory=memory).version <= '2.00':
# if MDF(input_file, memory=memory).version < '4.00':
continue
channels_nr = np.random.randint(1, len(CHANNELS_DEMO) + 1)
channel_list = random.sample(list(CHANNELS_DEMO), channels_nr)
filtered_mdf = MDF(input_file,
memory=memory).filter(channel_list,
memory=memory)
self.assertTrue((set(filtered_mdf.channels_db) -
{'t', 'time'}) == set(channel_list))
equal = True
with MDF(input_file, memory=memory) as mdf:
print(input_file, memory)
for name in channel_list:
original = mdf.get(name)
filtered = filtered_mdf.get(name)
if not np.array_equal(original.samples,
filtered.samples):
equal = False
if not np.array_equal(original.timestamps,
filtered.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_get_channel_comment_v4(self):
print("MDF get channel comment tests")
ret = True
for mdf in os.listdir('tmpdir_demo'):
for memory in MEMORY:
with MDF(os.path.join('tmpdir_demo', mdf),
memory=memory) as input_file:
if input_file.version < '4.00':
continue
print(mdf, memory)
for channel_name, original_comment in COMMENTS.items():
comment = input_file.get_channel_comment(channel_name)
if comment != original_comment:
print(mdf, channel_name, original_comment, comment)
1 / 0
ret = False
self.assertTrue(ret)
cleanup_files()
def test_read_demo(self):
print("MDF read tests")
ret = True
for enable in (True, False):
for mdf in os.listdir("tmpdir_demo"):
for memory in MEMORY:
with MDF(os.path.join("tmpdir_demo", mdf),
memory=memory) as input_file:
if input_file.version == "2.00":
continue
print(mdf, memory)
for name in set(
input_file.channels_db) - {"time", "t"}:
if (name.endswith("[0]") or name.startswith("DI")
or "\\" in name):
continue
signal = input_file.get(name)
try:
original_samples = CHANNELS_DEMO[name.split(
"\\")[0]]
except:
print(name)
raise
if signal.samples.dtype.kind == "f":
signal = signal.astype(np.float32)
res = np.array_equal(signal.samples,
original_samples)
if not res:
ret = False
self.assertTrue(ret)
cleanup_files()
def test_read_arrays(self):
print("MDF read big array files")
for mdfname in os.listdir("tmpdir_array"):
for memory in MEMORY:
input_file = os.path.join("tmpdir_array", mdfname)
print(input_file, memory)
equal = True
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
("Channel_{}".format(j), "(2, 3)<u8"),
("channel_{}_axis_1".format(j),
"(2, )<u8"),
("channel_{}_axis_2".format(j),
"(3, )<u8"),
]
types = np.dtype(types)
vals = mdf.get("Channel_{}".format(j),
group=i,
samples_only=True)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 1:
samples = np.ones((cycles, 2, 3), dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [("Channel_{}".format(j), "(2, 3)<u8")]
types = np.dtype(types)
vals = mdf.get("Channel_{}".format(j),
group=i,
samples_only=True)[0]
target = [samples * j]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
("struct_{}_channel_0".format(j),
np.uint8),
("struct_{}_channel_1".format(j),
np.uint16),
("struct_{}_channel_2".format(j),
np.uint32),
("struct_{}_channel_3".format(j),
np.uint64),
("struct_{}_channel_4".format(j), np.int8),
("struct_{}_channel_5".format(j),
np.int16),
("struct_{}_channel_6".format(j),
np.int32),
("struct_{}_channel_7".format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get("Channel_{}".format(j),
group=i,
samples_only=True)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
print(target)
print(vals)
1 / 0
self.assertTrue(equal)
cleanup_files()
def test_convert(self):
print("MDF convert big files tests")
t = np.arange(cycles, dtype=np.float64)
for out in ('3.30', '4.10'):
for mdfname in os.listdir('tmpdir'):
for memory in MEMORY:
input_file = os.path.join('tmpdir', mdfname)
print(input_file, memory, out)
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
outfile = mdf.convert(out, memory=memory).save(
'tmp_convert_{}_{}'.format(out, memory),
overwrite=True,
)
equal = True
with MDF(outfile) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
if not np.array_equal(vals, v * (j - 1)):
equal = False
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
if not np.array_equal(
vals,
v * (j - 1) - 0.5):
equal = False
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = '{} * sin(v)'
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
f = form.format(j - 1)
if not np.array_equal(
vals, numexpr.evaluate(f)):
equal = False
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = '({} * v -0.5) / 1'
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)
if not np.array_equal(
vals, numexpr.evaluate(f)):
equal = False
elif i == 4:
for j in range(1, 20):
target = np.array([
'Channel {} sample {}'.format(
j, k).encode('ascii')
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype('(8,)u1'))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
self.assertTrue(equal)
cleanup_files()
def tearDownClass(cls):
shutil.rmtree('tmpdir_demo', True)
shutil.rmtree('tmpdir_array', True)
shutil.rmtree('tmpdir', True)
os.remove('test.zip')
cleanup_files()
def test_read(self):
print("MDF read big files")
for mdfname in os.listdir("tmpdir"):
for memory in MEMORY:
input_file = os.path.join("tmpdir", mdfname)
print(input_file, memory)
equal = True
with MDF(input_file, memory=memory) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(vals, v * (j - 1)):
equal = False
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(vals, v * (j - 1) - 0.5):
equal = False
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = "{} * sin(v)"
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
if not np.array_equal(vals,
numexpr.evaluate(f)):
equal = False
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = "({} * v -0.5) / 1"
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(vals,
numexpr.evaluate(f)):
equal = False
elif i == 4:
for j in range(1, 20):
target = np.array([
"Channel {} sample {}".format(
j, k).encode("ascii")
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
self.assertTrue(equal)
cleanup_files()
def test_cut_arrays(self):
print("MDF cut big array files")
for mdfname in os.listdir("tmpdir_array"):
for memory in MEMORY:
for whence in (0, 1):
input_file = os.path.join("tmpdir_array", mdfname)
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105.5,
whence=whence,
include_ends=False).save(
"tmp1", overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.5,
stop=201.5,
whence=whence,
include_ends=False).save(
"tmp2", overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.5,
whence=whence,
include_ends=False).save(
"tmp3", overwrite=True)
outfile = MDF.concatenate(
[outfile1, outfile2, outfile3],
MDF(input_file, memory="minimum").version,
memory=memory,
).save("tmp_cut", overwrite=True)
equal = True
with MDF(outfile, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
("Channel_{}".format(j), "(2, 3)<u8"),
("channel_{}_axis_1".format(j),
"(2, )<u8"),
("channel_{}_axis_2".format(j),
"(3, )<u8"),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
print(
i,
j,
len(target),
len(vals),
vals,
target,
sep="\n\n",
)
1 / 0
elif i == 1:
samples = np.ones((cycles, 2, 3),
dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [("Channel_{}".format(j),
"(2, 3)<u8")]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [samples * j]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
("struct_{}_channel_0".format(j),
np.uint8),
("struct_{}_channel_1".format(j),
np.uint16),
("struct_{}_channel_2".format(j),
np.uint32),
("struct_{}_channel_3".format(j),
np.uint64),
("struct_{}_channel_4".format(j),
np.int8),
("struct_{}_channel_5".format(j),
np.int16),
("struct_{}_channel_6".format(j),
np.int32),
("struct_{}_channel_7".format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get(
"Channel_{}".format(j),
group=i,
samples_only=True,
)[0]
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target, dtype=types)
if not np.array_equal(vals, target):
equal = False
1 / 0
self.assertTrue(equal)
cleanup_files()
def etearDownClass(cls):
shutil.rmtree("tmpdir_demo", True)
shutil.rmtree("tmpdir_array", True)
shutil.rmtree("tmpdir", True)
os.remove("test.zip")
cleanup_files()
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for mdfname in os.listdir("tmpdir"):
for memory in MEMORY:
input_file = os.path.join("tmpdir", mdfname)
for whence in (0, 1):
print(input_file, memory)
outfile0 = MDF(input_file, memory=memory)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence,
include_ends=False).save(
"tmp0", overwrite=True)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence,
include_ends=False).save(
"tmp1", overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence,
include_ends=False).save(
"tmp2", overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.1,
whence=whence,
include_ends=False).save(
"tmp3", overwrite=True)
outfile4 = MDF(input_file, memory=memory)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence,
include_ends=False).save(
"tmp4", overwrite=True)
outfile = MDF.concatenate(
[outfile0, outfile1, outfile2, outfile3, outfile4],
version=MDF(input_file, memory="minimum").version,
memory=memory,
).save("tmp_cut", overwrite=True)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = "{} * sin(v)"
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = "({} * v -0.5) / 1"
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
cond = np.array_equal(
vals, numexpr.evaluate(f))
target = numexpr.evaluate(f)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
"Channel {} sample {}".format(
j, k).encode("ascii")
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
cleanup_files()
def test_convert(self):
print("MDF convert big files tests")
t = np.arange(cycles, dtype=np.float64)
for out in ("3.30", ):
for mdfname in os.listdir("tmpdir"):
for memory in MEMORY:
input_file = os.path.join("tmpdir", mdfname)
print(input_file, memory, out)
if "4.00" in input_file or "4.10" in input_file:
continue
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
outfile = mdf.convert(out, memory=memory).save(
"tmp_convert_{}_{}".format(out, memory),
overwrite=True)
equal = True
with MDF(outfile) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(vals, v * (j - 1)):
equal = False
print(vals, len(vals))
print(v * (j - 1), len(v))
1 / 0
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(
vals,
v * (j - 1) - 0.5):
equal = False
1 / 0
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = "{} * sin(v)"
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
f = form.format(j - 1)
if not np.array_equal(
vals, numexpr.evaluate(f)):
equal = False
1 / 0
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = "({} * v -0.5) / 1"
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)[0]
if not np.array_equal(
vals, numexpr.evaluate(f)):
equal = False
target = numexpr.evaluate(f)
print(i, j, vals, target, len(vals),
len(target))
1 / 0
elif i == 4:
for j in range(1, 20):
target = np.array([
"Channel {} sample {}".format(
j, k).encode("ascii")
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
1 / 0
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype("(8,)u1"))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
1 / 0
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)[0]
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
1 / 0
self.assertTrue(cond)
self.assertTrue(equal)
cleanup_files()
def test_cut_arrays(self):
print("MDF cut big array files")
for mdfname in os.listdir('tmpdir_array'):
for memory in MEMORY:
for whence in (0, 1):
input_file = os.path.join('tmpdir_array', mdfname)
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105, stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201,
whence=whence).save('tmp3',
overwrite=True)
outfile = MDF.merge(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
equal = True
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
('channel_{}_axis_1'.format(j),
'(2, )<u8'),
('channel_{}_axis_2'.format(j),
'(3, )<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 1:
samples = np.ones((cycles, 2, 3),
dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [
samples * j,
]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
('struct_{}_channel_0'.format(j),
np.uint8),
('struct_{}_channel_1'.format(j),
np.uint16),
('struct_{}_channel_2'.format(j),
np.uint32),
('struct_{}_channel_3'.format(j),
np.uint64),
('struct_{}_channel_4'.format(j),
np.int8),
('struct_{}_channel_5'.format(j),
np.int16),
('struct_{}_channel_6'.format(j),
np.int32),
('struct_{}_channel_7'.format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
self.assertTrue(equal)
cleanup_files()
