def test_table_random_write_versions():
"""
table write w/versions
1. write data set 1
2. write data set 2
3. scan & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file, need_sort=True))
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True, snapshot=0, is_async=True)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file, need_sort=True))
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False)
nose.tools.assert_true(common.compare_files(dump_file2, scan_file, need_sort=True))
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False, snapshot=0, is_async=True)
nose.tools.assert_true(common.compare_files(dump_file2, scan_file, need_sort=True))
def test_table_random_write(self):
common.create_singleversion_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)],
op='w',
table_name=table_name,
cf='cf0:q,cf1:q',
random='random',
key_seed=1,
value_seed=10,
value_size=100,
num=10000,
key_size=20)
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=False)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=False,
snapshot=0,
is_async=True)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
'''
def table_snapshot_relaunch(self):
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
common.cluster_op('kill')
common.cluster_op('launch')
time.sleep(2)
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
'''
def test_table_random_write():
"""
table write simple
1. write data set 1
2. scan & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)],
op='w',
table_name=table_name,
cf='cf0:q,cf1:q',
random='random',
key_seed=1,
value_seed=10,
value_size=100,
num=10000,
key_size=20)
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=False)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=False,
snapshot=0,
is_async=True)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
def test_table_write_multiversion_snapshot():
"""
table write w/version w/snapshot
1. write data set 1, 2
2. create snapshot
3. write data set 3, 4
4. scan w/snapshot, scan w/o snapshot & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file2, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
def test_table_write_delete_version():
"""
table write and deletion w/versions
1. write data set 1, 2, 3, 4
2. scan
3. delete data set 3
4. scan & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False), (dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=12, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=13, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=0)
common.run_tera_mark([], op='d', table_name=table_name, cf='cf0:q,cf1:q', random='random', key_seed=1,
value_seed=12, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
def test_kv_snapshot_relaunch():
"""
kv cluster relaunch
1. write data set 1
2. create snapshot
3. write data set 2
4. scan w/snapshot, scan w/o snapshot & compare
5. kill & launch cluster
6. repeat 4
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
common.cluster_op('kill')
common.cluster_op('launch')
time.sleep(2)
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=False, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=False, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
def table_snapshot_relaunch(self):
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
common.cluster_op('kill')
common.cluster_op('launch')
time.sleep(2)
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
'''
def test_table_write_delete_version():
"""
table write and deletion w/versions
1. write data set 1, 2, 3, 4
2. scan
3. delete data set 3
4. scan & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False), (dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=12, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, True)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=13, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=True, snapshot=0)
common.run_tera_mark([], op='d', table_name=table_name, cf='cf0:q,cf1:q', random='random', key_seed=1,
value_seed=12, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=True, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
def test_table_random_write(self):
common.create_singleversion_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False, snapshot=0, is_async=True)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
'''
def test_table_write_snapshot():
"""
table write w/snapshot
1. write data set 1
2. create snapshot
3. write data set 2
4. scan w/snapshot, scan w/o snapshot & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)],
op='w',
table_name=table_name,
cf='cf0:q,cf1:q',
random='random',
key_seed=1,
value_seed=10,
value_size=100,
num=10000,
key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)],
op='w',
table_name=table_name,
cf='cf0:q,cf1:q',
random='random',
key_seed=1,
value_seed=11,
value_size=100,
num=10000,
key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name,
file_path=scan_file1,
allversion=False,
snapshot=snapshot)
common.scan_table(table_name=table_name,
file_path=scan_file2,
allversion=False,
snapshot=0)
nose.tools.assert_true(
common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(
common.compare_files(dump_file2, scan_file2, need_sort=True))
def test_dnxhd(self):
frames = 3
for profile_name in [
'dnx_1080p_36_23.97', 'dnx_720p_90x_25', 'dnx_1080i_120_25',
'dnx_1080p_175x_23.97'
]:
new_file = os.path.join(common.sandbox(),
'%s_import_essence.aaf' % profile_name)
sample = common.generate_dnxhd(profile_name,
'%s-import.dnxhd' % profile_name,
frames=frames)
with aaf2.open(new_file, 'w') as f:
profile = video.dnx_profiles.get(profile_name)
mob = f.create.MasterMob(profile_name)
f.content.mobs.append(mob)
mob.import_dnxhd_essence(sample, profile['frame_rate'])
with aaf2.open(new_file, 'r') as f:
mob = next(f.content.sourcemobs())
stream = mob.essence.open('r')
dump_path = os.path.join(common.sample_dir(),
'%s-import-dump.dnxhd' % profile_name)
with io.open(dump_path, 'wb') as out:
out.write(stream.read())
assert common.compare_files(dump_path, sample)
assert mob.slots[0].segment.length == frames
assert mob.descriptor.length == frames
def test_wav(self):
# name, sample_rate = 48000, duration = 2, sample_fmt='s16le', format='wav'):
# profile_name = 'pcm_48000_s24le'
frame_rate = 30
timecode_fps = 30
start_time = int(timecode_fps * 60 * 60) * 2
duration = 1.5
for profile_name in sorted(audio.pcm_profiles):
sample_format = audio.pcm_profiles[profile_name]['sample_format']
sample_rate = audio.pcm_profiles[profile_name]['sample_rate']
sample = common.generate_pcm_audio_mono(
profile_name,
sample_format=sample_format,
sample_rate=sample_rate,
duration=duration)
new_file = os.path.join(common.sandbox(),
'%s_import_essence.aaf' % profile_name)
with aaf2.open(new_file, 'w') as f:
tape_mob = f.create.SourceMob()
f.content.mobs.append(tape_mob)
tape_mob.create_tape_slots(profile_name,
frame_rate,
timecode_fps,
media_kind='sound')
tape = tape_mob.create_source_clip(1, start_time)
mob = f.create.MasterMob(profile_name)
f.content.mobs.append(mob)
mob.import_audio_essence(sample, frame_rate, tape)
with aaf2.open(new_file, 'r') as f:
source_mobs = []
tape_mobs = []
for mob in f.content.sourcemobs():
if isinstance(mob.descriptor, aaf2.essence.TapeDescriptor):
tape_mobs.append(mob)
else:
source_mobs.append(mob)
mob = source_mobs[0]
tape_mob = tape_mobs[0]
stream = mob.essence.open('r')
dump_path = os.path.join(common.sample_dir(),
'%s-import-dump.wav' % profile_name)
mob.export_audio(dump_path)
assert common.compare_files(dump_path, sample)
audio_samples = mob.descriptor['Length'].value
assert audio_samples == sample_rate * duration
edit_length = duration * frame_rate
assert mob.slots[0].segment.length == edit_length
assert mob.slots[0].segment.start == start_time
assert mob.slots[0].segment.mob_id == tape_mob.mob_id
def test_dnxhr(self):
frame_rate = '23.97'
uhd2160 = (960, 540)
frames = 3
timecode_fps = 30
start_time = int(timecode_fps * 60 * 60) # 1 hour
for profile_name in ['dnxhr_lb', 'dnxhr_sq', 'dnxhr_hq']:
new_file = os.path.join(common.sandbox(),
'%s_import_essence.aaf' % profile_name)
with aaf2.open(new_file, 'w') as f:
profile = video.dnx_profiles.get(profile_name)
sample = common.generate_dnxhd(profile_name,
"%s-import.dnxhd" %
profile_name,
frames=frames,
size=uhd2160,
frame_rate=frame_rate)
# create a tape
tape_mob = f.create.SourceMob()
tape_mob.create_tape_slots(profile_name, frame_rate,
timecode_fps)
f.content.mobs.append(tape_mob)
tape = tape_mob.create_source_clip(1, start=start_time)
mob = f.create.MasterMob(profile_name)
f.content.mobs.append(mob)
timecode = f.create.Timecode(timecode_fps)
mob.import_dnxhd_essence(sample, frame_rate, tape=tape)
with aaf2.open(new_file, 'r') as f:
source_mobs = []
tape_mobs = []
for mob in f.content.sourcemobs():
if isinstance(mob.descriptor, aaf2.essence.TapeDescriptor):
tape_mobs.append(mob)
else:
source_mobs.append(mob)
mob = source_mobs[0]
stream = mob.essence.open('r')
dump_path = os.path.join(common.sample_dir(),
'%s-import-dump.dnxhd' % profile_name)
with io.open(dump_path, 'wb') as out:
out.write(stream.read())
assert common.compare_files(dump_path, sample)
assert mob.slots[0].segment.length == frames
assert mob.slots[0].segment.start == start_time
assert mob.descriptor.length == frames
def test_table_random_write_versions():
"""
table write w/versions
1. write data set 1
2. write data set 2
3. scan & compare
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q',
random='random', key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.run_tera_mark([(dump_file1, True), (dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file, need_sort=True))
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False)
nose.tools.assert_true(common.compare_files(dump_file2, scan_file, need_sort=True))
def test_table_write_snapshot(self):
common.create_singleversion_table()
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=False, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=False, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
'''
def test_table_write_snapshot(self):
common.create_singleversion_table()
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=False, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=False, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(common.compare_files(dump_file2, scan_file2, need_sort=True))
'''
def test_kv_random_write(self):
common.create_kv_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name='test', random='random',
value_size=100, num=5000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
'''
def test_rowreader_lowlevelscan(self):
common.create_singleversion_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name=table_name,
cf='cf0:q00,cf0:q01,cf0:q02,cf1:q00,cf1:q01,cf1:q02', random='seq',
key_seed=1, value_seed=10, value_size=64, num=10, key_size=20)
common.rowread_table(table_name=table_name, file_path=scan_file)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=False))
'''
def test_showsize_table():
time.sleep(10)
show_ret = common.parse_showinfo()
table_name = "test"
nose.tools.assert_equal(show_ret[table_name]["size"], "4K")
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name='test', cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=500000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
nose.tools.assert_not_equal(show_ret[table_name]["size"], "4K")
def test_kv_random_write():
"""
kv table write
1. write data set 1
2. scan & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name='test', random='random',
value_size=100, num=5000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=True)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
def test_table_random_write():
"""
table write simple
1. write data set 1
2. scan & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
common.scan_table(table_name=table_name, file_path=scan_file, allversion=False)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=True))
def test_rowreader_lowlevelscan():
"""
table rowreader_lowlevelscan
1. write data set 1
2. scan & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name=table_name,
cf='cf0:q00,cf0:q01,cf0:q02,cf1:q00,cf1:q01,cf1:q02', random='seq',
key_seed=1, value_seed=10, value_size=64, num=10, key_size=20)
common.rowread_table(table_name=table_name, file_path=scan_file)
nose.tools.assert_true(common.compare_files(dump_file, scan_file, need_sort=False))
def test_kv_random_write(self):
common.create_kv_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)],
op='w',
table_name='test',
random='random',
value_size=100,
num=5000,
key_size=20)
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=True)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
'''
def test_rowreader_lowlevelscan(self):
common.create_singleversion_table()
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark(
[(dump_file, False)],
op='w',
table_name=table_name,
cf='cf0:q00,cf0:q01,cf0:q02,cf1:q00,cf1:q01,cf1:q02',
random='seq',
key_seed=1,
value_seed=10,
value_size=64,
num=10,
key_size=20)
common.rowread_table(table_name=table_name, file_path=scan_file)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=False))
'''
def test_showsize_kv_table():
time.sleep(10)
show_ret = common.parse_showinfo()
table_name = "test"
nose.tools.assert_equal(show_ret[table_name]["size"], "4K")
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)],
op='w',
table_name='test',
random='random',
key_seed=1,
value_seed=10,
value_size=100,
num=5000,
key_size=20)
common.scan_table(table_name=table_name,
file_path=scan_file,
allversion=True)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=True))
nose.tools.assert_not_equal(show_ret[table_name]["size"], "4K")
def test_rowreader_lowlevelscan():
"""
table rowreader_lowlevelscan
1. write data set 1
2. scan & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file = 'scan.out'
common.run_tera_mark([(dump_file, False)],
op='w',
table_name=table_name,
cf='cf0:q00,cf0:q01,cf0:q02,cf1:q00,cf1:q01,cf1:q02',
random='seq',
key_seed=1,
value_seed=10,
value_size=64,
num=10,
key_size=20)
common.rowread_table(table_name=table_name, file_path=scan_file)
nose.tools.assert_true(
common.compare_files(dump_file, scan_file, need_sort=False))
def test_table_write_del_snapshot():
"""
table write deletion w/snapshot
1. write data set 1
2. take snapshot
3. delete data set 1
4. scan w/snapshot, scan w/o snapshot & compare
:return: None
"""
table_name = 'test'
dump_file = 'dump.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file, False)], op='w', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=10, value_size=100, num=10000, key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([], op='d', table_name=table_name, cf='cf0:q,cf1:q', random='random',
key_seed=1, value_seed=11, value_size=100, num=10000, key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name, file_path=scan_file1, allversion=False, snapshot=snapshot)
common.scan_table(table_name=table_name, file_path=scan_file2, allversion=False, snapshot=0)
nose.tools.assert_true(common.compare_files(dump_file, scan_file1, need_sort=True))
nose.tools.assert_true(common.file_is_empty(scan_file2))
def test_kv_snapshot_relaunch():
"""
kv cluster relaunch
1. write data set 1
2. create snapshot
3. write data set 2
4. scan w/snapshot, scan w/o snapshot & compare
5. kill & launch cluster
6. repeat 4
:return: None
"""
table_name = 'test'
dump_file1 = 'dump1.out'
dump_file2 = 'dump2.out'
scan_file1 = 'scan1.out'
scan_file2 = 'scan2.out'
common.run_tera_mark([(dump_file1, False)],
op='w',
table_name=table_name,
random='random',
key_seed=1,
value_seed=10,
value_size=100,
num=10000,
key_size=20)
snapshot = common.snapshot_op(table_name)
common.run_tera_mark([(dump_file2, False)],
op='w',
table_name=table_name,
random='random',
key_seed=1,
value_seed=11,
value_size=100,
num=10000,
key_size=20)
common.compact_tablets(common.get_tablet_list(table_name))
common.scan_table(table_name=table_name,
file_path=scan_file1,
allversion=True,
snapshot=snapshot)
common.scan_table(table_name=table_name,
file_path=scan_file2,
allversion=True,
snapshot=0)
nose.tools.assert_true(
common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(
common.compare_files(dump_file2, scan_file2, need_sort=True))
common.cluster_op('kill')
common.cluster_op('launch')
time.sleep(2)
common.scan_table(table_name=table_name,
file_path=scan_file1,
allversion=False,
snapshot=snapshot)
common.scan_table(table_name=table_name,
file_path=scan_file2,
allversion=False,
snapshot=0)
nose.tools.assert_true(
common.compare_files(dump_file1, scan_file1, need_sort=True))
nose.tools.assert_true(
common.compare_files(dump_file2, scan_file2, need_sort=True))
def test_multi(self):
frame_rate = 30.0
frames = 4
audio_duration = frames / frame_rate
uhd2160 = (960, 540)
new_file = os.path.join(common.sandbox(), 'multi_import_essence.aaf')
audio_profile_name = 'pcm_48000_s24le'
sample_format = audio.pcm_profiles[audio_profile_name]['sample_format']
sample_rate = audio.pcm_profiles[audio_profile_name]['sample_rate']
audio_sample = common.generate_pcm_audio_mono(
audio_profile_name,
sample_format=sample_format,
sample_rate=sample_rate,
duration=audio_duration)
samples = {}
mastermobs = {}
timecode_fps = 30
start_time = int(timecode_fps * 60 * 60)
tape_id = None
with aaf2.open(new_file, 'w') as f:
# create a tape
tape_mob = f.create.SourceMob()
tape_mob.create_tape_slots("mulit-import", frame_rate,
timecode_fps)
tape_id = tape_mob.mob_id
f.content.mobs.append(tape_mob)
for i, profile_name in enumerate(
['dnxhr_lb', 'dnxhr_sq', 'dnxhr_hq'], 1):
profile = video.dnx_profiles.get(profile_name)
sample = common.generate_dnxhd(profile_name,
"%s-mulit-import.dnxhd" %
profile_name,
frames=frames,
size=uhd2160,
frame_rate=frame_rate)
mob = f.create.MasterMob(profile_name)
f.content.mobs.append(mob)
tape = tape_mob.create_source_clip(1, start=start_time)
vs_slot = mob.import_dnxhd_essence(sample,
frame_rate,
tape=tape)
as_slot = mob.import_audio_essence(audio_sample, frame_rate)
vs_mob = vs_slot.segment.mob
as_mob = as_slot.segment.mob
mob.comments['profile'] = profile_name
mob.comments['integer'] = 100
mob.comments['integer'] = i
mastermobs[mob.mob_id] = (profile_name, i)
v_dump_path = os.path.join(
common.sample_dir(),
'%s-multi-import-dump.wav' % profile_name)
a_dump_path = os.path.join(
common.sample_dir(),
'%s-multi-import-dump.dnxhd' % profile_name)
samples[vs_mob.mob_id] = (sample, v_dump_path, start_time)
samples[as_mob.mob_id] = (audio_sample, a_dump_path,
start_time)
start_time += 100
with aaf2.open(new_file, 'r') as f:
tape_mob = f.content.mobs.get(tape_id)
for mob_id, (profile_name, v) in mastermobs.items():
mob = f.content.mobs.get(mob_id)
assert mob.comments['profile'] == profile_name
assert mob.comments['integer'] == v
for mob_id, (original_path, dump_path,
start_time) in samples.items():
mob = f.content.mobs.get(mob_id)
if isinstance(mob.descriptor, aaf2.essence.PCMDescriptor):
mob.export_audio(dump_path)
assert common.compare_files(dump_path, original_path)
assert mob.slots[0].segment.length == frames
if isinstance(mob.descriptor, aaf2.essence.CDCIDescriptor):
stream = mob.essence.open('r')
with io.open(dump_path, 'wb') as out:
out.write(stream.read())
assert common.compare_files(dump_path, original_path)
assert mob.slots[0].segment.length == frames
assert mob.slots[0].segment.start == start_time
assert mob.slots[0].segment.mob_id == tape_id