def test_merge(self):
print("MDF merge tests")
for out in SUPPORTED_VERSIONS:
for mdfname in os.listdir('tmpdir'):
for memory in MEMORY:
input_file = os.path.join('tmpdir', mdfname)
files = [
input_file,
] * 4
MDF.merge(files, out, memory).save('tmp', overwrite=True)
equal = True
with MDF(input_file, memory=memory) as mdf, \
MDF('tmp', memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, channel in enumerate(
group['channels'][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(
np.tile(original.samples, 4),
converted.samples):
equal = False
self.assertTrue(equal)
def merge_v4(output, fmt, memory):
files = [r'test.mf4', ] * 3
with Timer('Merge 3 files',
'asammdf {} {} v4'.format(asammdf_version, memory),
fmt) as timer:
MDF.merge(files, memory=memory, outversion='4.10')
output.send([timer.output, timer.error])
def merge_v4(path, output, fmt):
files = [r'test.mf4', ] * 2
os.chdir(path)
with Timer('Merge files',
'asammdf {} v4'.format(asammdf_version),
fmt) as timer:
MDF.merge(files)
output.send([timer.output, timer.error])
def merge_v3(path, output, fmt, memory):
os.chdir(path)
files = [r'test.mdf', ] * 2
with Timer('Merge files',
'asammdf {} {} v3'.format(asammdf_version, memory),
fmt) as timer:
MDF.merge(files, memory=memory, outversion='3.30')
output.send([timer.output, timer.error])
def merge_v4():
files = [
r'test.mf4',
] * 2
os.chdir(path)
with Timer('asammdf {} v4'.format(asammdf_version)):
y = MDF.merge(files)
def merge_v3():
os.chdir(path)
files = [
r'test.mdf',
] * 2
with Timer('asammdf {} v3'.format(asammdf_version)):
y = MDF.merge(files)
def merge_v4_nodata():
files = [
r'test.mf4',
] * 2
os.chdir(path)
with Timer('asammdf {} v4 nodata'.format(asammdf_version)):
y = MDF.merge(files, load_measured_data=False)
y.close()
def merge_v4_nodata(path, output, fmt):
files = [r'test.mf4', ] * 2
os.chdir(path)
with Timer('Merge files',
'asammdf {} v4 nodata'.format(asammdf_version),
fmt) as timer:
y = MDF.merge(files, load_measured_data=False)
y.close()
output.send([timer.output, timer.error])
def test_cut_demo(self):
print("MDF cut absolute tests")
cntr = 0
for mdfname in os.listdir('tmpdir_demo'):
for memory in MEMORY:
input_file = os.path.join('tmpdir_demo', mdfname)
if '2.00' in input_file:
continue
for whence in (0, 1):
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory).cut(
stop=2, whence=whence).save('tmp1', overwrite=True)
outfile2 = MDF(input_file, memory=memory).cut(
start=2, stop=6, whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory).cut(
start=6, whence=whence).save('tmp3', overwrite=True)
outfile = MDF.merge(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp', overwrite=True)
print('OUT', outfile)
equal = True
with MDF(input_file, memory=memory) as mdf, \
MDF(outfile, memory=memory) as mdf2:
for i, group in enumerate(mdf.groups):
for j, _ in enumerate(group['channels'][1:], 1):
original = mdf.get(group=i, index=j)
converted = mdf2.get(group=i, index=j)
if not np.array_equal(original.samples,
converted.samples):
equal = False
if not np.array_equal(original.timestamps,
converted.timestamps):
equal = False
self.assertTrue(equal)
cleanup_files()
def generate_test_files():
print('Generating test files:')
for version in ('3.30', '4.10'):
print("-> generating file for version", version)
mdf = MDF(version=version, memory='minimum')
if version == '3.30':
cycles = 500
channels_count = 8000
filename = 'test.mdf'
else:
cycles = 500
channels_count = 20000
filename = 'test.mf4'
if os.path.exists(filename):
continue
t = np.arange(cycles, dtype=np.float64)
# no conversion
sigs = []
for i in range(channels_count):
sig = Signal(
np.ones(cycles, dtype=np.uint16),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=None,
comment='Unsinged int 16bit channel {}'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# linear
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_LINEAR,
'a': float(i),
'b': -0.5,
}
sig = Signal(
np.ones(cycles, dtype=np.int16),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Signed 16bit channel {} with linear conversion'.
format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# algebraic
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_ALGEBRAIC,
'formula': '{} * sin(X)'.format(i),
}
sig = Signal(
np.arange(cycles, dtype=np.int32) / 100,
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Sinus channel {} with algebraic conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
# rational
sigs = []
for i in range(channels_count):
conversion = {
'type': SignalConversions.CONVERSION_RATIONAL,
'P1': 0,
'P2': i,
'P3': -0.5,
'P4': 0,
'P5': 0,
'P6': 1,
}
sig = Signal(
np.ones(cycles, dtype=np.int64),
t,
name='Channel_{}'.format(i),
unit='unit_{}'.format(i),
conversion=conversion,
comment='Channel {} with rational conversion'.format(i),
raw=True,
)
sigs.append(sig)
mdf.append(sigs)
mdf.save(filename, overwrite=True)
del mdf
MDF.merge([
filename,
] * 10, version, memory='minimum').save(filename, overwrite=True)
def test_cut_arrays(self):
print("MDF cut big array files")
for mdfname in os.listdir('tmpdir_array'):
for memory in MEMORY:
for whence in (0, 1):
input_file = os.path.join('tmpdir_array', mdfname)
print(input_file, memory, whence)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105, stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201,
whence=whence).save('tmp3',
overwrite=True)
outfile = MDF.merge(
[outfile1, outfile2, outfile3],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
equal = True
with MDF(input_file, memory=memory) as mdf:
mdf.configure(read_fragment_size=8000)
for i, group in enumerate(mdf.groups):
if i == 0:
samples = [
np.ones((cycles, 2, 3), dtype=np.uint64),
np.ones((cycles, 2), dtype=np.uint64),
np.ones((cycles, 3), dtype=np.uint64),
]
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
('channel_{}_axis_1'.format(j),
'(2, )<u8'),
('channel_{}_axis_2'.format(j),
'(3, )<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 1:
samples = np.ones((cycles, 2, 3),
dtype=np.uint64)
axis_0 = np.ones((cycles, 2), dtype=np.uint64)
axis_1 = np.ones((cycles, 3), dtype=np.uint64)
for j in range(1, 20):
types = [
('Channel_{}'.format(j), '(2, 3)<u8'),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [
samples * j,
]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
elif i == 2:
samples = [
np.ones(cycles, dtype=np.uint8),
np.ones(cycles, dtype=np.uint16),
np.ones(cycles, dtype=np.uint32),
np.ones(cycles, dtype=np.uint64),
np.ones(cycles, dtype=np.int8),
np.ones(cycles, dtype=np.int16),
np.ones(cycles, dtype=np.int32),
np.ones(cycles, dtype=np.int64),
]
for j in range(1, 20):
types = [
('struct_{}_channel_0'.format(j),
np.uint8),
('struct_{}_channel_1'.format(j),
np.uint16),
('struct_{}_channel_2'.format(j),
np.uint32),
('struct_{}_channel_3'.format(j),
np.uint64),
('struct_{}_channel_4'.format(j),
np.int8),
('struct_{}_channel_5'.format(j),
np.int16),
('struct_{}_channel_6'.format(j),
np.int32),
('struct_{}_channel_7'.format(j),
np.int64),
]
types = np.dtype(types)
vals = mdf.get('Channel_{}'.format(j),
group=i,
samples_only=True)
target = [arr * j for arr in samples]
target = np.core.records.fromarrays(
target,
dtype=types,
)
if not np.array_equal(vals, target):
equal = False
self.assertTrue(equal)
cleanup_files()
def test_cut(self):
print("MDF cut big files tests")
t = np.arange(cycles, dtype=np.float64)
for mdfname in os.listdir('tmpdir'):
for memory in MEMORY:
input_file = os.path.join('tmpdir', mdfname)
for whence in (0, 1):
print(input_file, memory)
outfile0 = MDF(input_file, memory=memory)
outfile0.configure(read_fragment_size=8000)
outfile0 = outfile0.cut(stop=-1,
whence=whence).save('tmp0',
overwrite=True)
outfile1 = MDF(input_file, memory=memory)
outfile1.configure(read_fragment_size=8000)
outfile1 = outfile1.cut(stop=105,
whence=whence).save('tmp1',
overwrite=True)
outfile2 = MDF(input_file, memory=memory)
outfile2.configure(read_fragment_size=8000)
outfile2 = outfile2.cut(start=105.1,
stop=201,
whence=whence).save('tmp2',
overwrite=True)
outfile3 = MDF(input_file, memory=memory)
outfile3.configure(read_fragment_size=8000)
outfile3 = outfile3.cut(start=201.1,
whence=whence).save('tmp3',
overwrite=True)
outfile4 = MDF(input_file, memory=memory)
outfile4.configure(read_fragment_size=8000)
outfile4 = outfile4.cut(start=7000,
whence=whence).save('tmp4',
overwrite=True)
outfile = MDF.merge(
[outfile0, outfile1, outfile2, outfile3, outfile4],
MDF(input_file, memory='minimum').version,
memory=memory,
).save('tmp_cut', overwrite=True)
with MDF(outfile) as mdf:
for i, group in enumerate(mdf.groups):
if i == 0:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(vals, v * (j - 1))
if not cond:
print(i, j, vals, v * (j - 1),
len(vals), len(v))
self.assertTrue(cond)
elif i == 1:
v = np.ones(cycles, dtype=np.int64)
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(
vals,
v * (j - 1) - 0.5)
if not cond:
print(vals,
v * (j - 1) - 0.5, len(vals),
len(v))
self.assertTrue(cond)
elif i == 2:
v = np.arange(cycles, dtype=np.int64) / 100.0
form = '{} * sin(v)'
for j in range(1, 20):
vals = mdf.get(group=i,
index=j,
samples_only=True)
f = form.format(j - 1)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 3:
v = np.ones(cycles, dtype=np.int64)
form = '({} * v -0.5) / 1'
for j in range(1, 20):
f = form.format(j - 1)
vals = mdf.get(group=i,
index=j,
samples_only=True)
cond = np.array_equal(
vals, numexpr.evaluate(f))
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 4:
for j in range(1, 20):
target = np.array([
'Channel {} sample {}'.format(
j, k).encode('ascii')
for k in range(cycles)
])
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 5:
v = np.ones(cycles, dtype=np.dtype('(8,)u1'))
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
elif i == 6:
v = np.ones(cycles, dtype=np.uint64)
for j in range(1, 20):
target = v * j
vals = mdf.get(group=i,
index=j + 1,
samples_only=True)
cond = np.array_equal(vals, target)
if not cond:
print(i, j, vals, target, len(vals),
len(target))
self.assertTrue(cond)
cleanup_files()