def _test_values_file_speed(self, use_sqlite):
values = []
for i in range(20000):
value = SimpleEntity("http://aql.org/download",
name="target_url%s" % i)
values.append(value)
with Tempfile() as tmp:
print("Opening a database '%s' ..." % tmp)
timer = Chrono()
with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf:
with timer:
keys = vf.add_entities(values)
print("add values time: %s" % (timer, ))
with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf:
with timer:
keys = vf.add_entities(values)
print("re-add values time: %s" % (timer, ))
with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf:
with timer:
vf.find_entities_by_key(keys)
print("get values time: %s" % timer)
with timer:
with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf:
pass
print("reopen values file time: %s" % timer)
def test_node_speed(self):
try:
tmp_files = []
source_files = _generate_files(tmp_files, 4000, 50 * 1024)
# idep_files = _copy_files(tmp_files, source_files, 'h')
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
builder = TestSpeedBuilder("TestSpeedBuilder", "tmp", "h")
for source in source_files:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
for tmp_file in node.get_targets():
tmp_files.append(tmp_file)
t = lambda: _test_no_build_speed(vfile,
builder,
source_files)
t = timeit.timeit(t, number=1)
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_speed(self):
try:
tmp_files = []
source_files = _generate_files(tmp_files, 4000, 50 * 1024)
# idep_files = _copy_files(tmp_files, source_files, 'h')
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
builder = _SpeedBuilder("SpeedBuilder", "tmp", "h")
for source in source_files:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
for tmp_file in node.get_targets():
tmp_files.append(tmp_file)
t = lambda: _test_no_build_speed(vfile, builder,
source_files)
t = timeit.timeit(t, number=1)
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_batch(self):
with Tempdir() as tmp_dir:
vfile_name = Tempfile(root_dir=tmp_dir)
vfile_name.close()
with EntitiesFile(vfile_name) as vfile:
src_files = self.generate_source_files(tmp_dir, 5, 100)
self._rebuild_batch_node(vfile, src_files, len(src_files))
self._rebuild_batch_node(vfile, src_files, 0)
self._rebuild_batch_node(vfile, src_files[:-2], 0)
self._rebuild_batch_node(vfile, src_files[0:1], 0)
# -----------------------------------------------------------
write_bin_file(src_files[1], b"src_file1")
write_bin_file(src_files[2], b"src_file1")
self._rebuild_batch_node(vfile, src_files, 2)
def __getitem__(self, builder):
builder_name = builder.name
try:
vfilename = self.names[builder_name]
except KeyError:
vfilename = os.path.join(builder.get_build_dir(), '.aql.db')
self.names[builder_name] = vfilename
try:
return self.handles[vfilename]
except KeyError:
vfile = EntitiesFile(vfilename,
use_sqlite=self.use_sqlite,
force=self.force_lock)
self.handles[vfilename] = vfile
return vfile
def test_node_file(self):
try:
tmp_files = []
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
with Tempfile(suffix='.1') as tmp1:
with Tempfile(suffix='.2') as tmp2:
value1 = FileChecksumEntity(tmp1)
value2 = FileChecksumEntity(tmp2)
options = builtin_options()
builder = CopyBuilder(options, "tmp", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "d")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
tmp1.write(b'123')
tmp1.flush()
value1 = FileChecksumEntity(tmp1)
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
with Tempfile(suffix='.3') as tmp3:
value3 = FileChecksumEntity(tmp3)
node3 = self._rebuild_node(
vfile, builder, [value3], [], tmp_files)
node = self._rebuild_node(vfile, builder,
[value1, node3], [],
tmp_files)
builder3 = CopyBuilder(options, "xxx", "3")
node3 = self._rebuild_node(vfile, builder3,
[value3], [],
tmp_files)
# node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx,
# ,tmp3.3
node = self._rebuild_node(vfile, builder,
[value1, node3], [],
tmp_files)
# node: CopyBuilder, tmp, i, tmp1, tmp3.xxx ->
# tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
dep = SimpleEntity("1", name="dep1")
node = self._rebuild_node(vfile, builder,
[value1, node3],
[dep], tmp_files)
dep = SimpleEntity("11", name="dep1")
node = self._rebuild_node(vfile, builder,
[value1, node3],
[dep], tmp_files)
node3 = self._rebuild_node(vfile, builder3,
[value1], [],
tmp_files)
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
node3 = self._rebuild_node(vfile, builder3,
[value2], [],
tmp_files)
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
node_tname = node.get_target_entities()[0].name
with open(node_tname, 'wb') as f:
f.write(b'333')
f.flush()
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
side_effects = node.get_side_effect_entities()
with open(side_effects[0].name, 'wb') as f:
f.write(b'abc')
f.flush()
node = Node(builder, [value1])
node.depends([node3])
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile,
False))
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_value(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
value1 = SimpleEntity(
"http://aql.org/download1", name="target_url1")
value2 = SimpleEntity(
"http://aql.org/download2", name="target_url2")
value3 = SimpleEntity(
"http://aql.org/download3", name="target_url3")
dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value")
dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value")
options = builtin_options()
builder = ChecksumBuilder(options)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
finally:
vfile.close()
def test_values_file_2(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
self.assertItemsEqual(vfile.find_entities([]), [])
value1 = SimpleEntity(
"http://aql.org/download", name="target_url1")
value2 = SimpleEntity(
"http://aql.org/download2", name="target_url2")
value3 = SimpleEntity(
"http://aql.org/download3", name="target_url3")
value4 = SimpleEntity(
"http://aql.org/download4", name="target_url4")
value5 = SimpleEntity(
"http://aql.org/download5", name="target_url5")
value6 = SimpleEntity(
"http://aql.org/download6", name="target_url6")
values = [value1, value2, value3]
dep_values_1 = values
dep_values_2 = dep_values_1 + [value4]
dep_values_3 = dep_values_2 + [value5]
dep_values_4 = dep_values_3 + [value6]
all_values = dep_values_4
all_keys = vfile.add_entities(all_values)
vfile.self_test()
self.assertItemsEqual(
vfile.find_entities(all_values), all_values)
self.assertItemsEqual(
vfile.find_entities_by_key(all_keys), all_values)
dep_keys_1 = vfile.add_entities(dep_values_1)
vfile.self_test()
dep_keys_2 = vfile.add_entities(dep_values_2)
vfile.self_test()
dep_keys_3 = vfile.add_entities(dep_values_3)
vfile.self_test()
dep_keys_4 = vfile.add_entities(dep_values_4)
vfile.self_test()
self.assertTrue(set(dep_keys_1).issubset(dep_keys_2))
self.assertTrue(set(dep_keys_2).issubset(dep_keys_3))
self.assertTrue(set(dep_keys_3).issubset(dep_keys_4))
self.assertItemsEqual(all_keys, dep_keys_4)
vfile.close()
vfile.open(tmp)
vfile.self_test()
self.assertItemsEqual(
vfile.find_entities(all_values), all_values)
vfile.self_test()
self.assertItemsEqual(
vfile.find_entities_by_key(dep_keys_1), dep_values_1)
self.assertItemsEqual(
vfile.find_entities_by_key(dep_keys_2), dep_values_2)
self.assertItemsEqual(
vfile.find_entities_by_key(dep_keys_3), dep_values_3)
self.assertItemsEqual(
vfile.find_entities_by_key(dep_keys_4), dep_values_4)
value4 = SimpleEntity(
"http://aql.org/download3/0", name=value4.name)
vfile.add_entities([value4])
vfile.self_test()
self.assertItemsEqual(
vfile.find_entities_by_key(dep_keys_1), dep_values_1)
self.assertIsNone(vfile.find_entities_by_key(dep_keys_2))
self.assertIsNone(vfile.find_entities_by_key(dep_keys_3))
self.assertIsNone(vfile.find_entities_by_key(dep_keys_4))
finally:
vfile.close()
def test_values_file_same_name(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
value1 = SimpleEntity("test", name="test1")
value2 = SignatureEntity(b"1234354545", name=value1.name)
vfile.add_entities([value1, value2])
vfile.self_test()
values = [SimpleEntity(name=value1.name),
SignatureEntity(name=value2.name)]
values = vfile.find_entities(values)
self.assertItemsEqual(values, [value1, value2])
vfile.close()
vfile.open(tmp)
vfile.self_test()
finally:
vfile.close()
def test_values_file(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
value1 = SimpleEntity("http://aql.org/download")
value2 = SimpleEntity("http://aql.org/download2")
value3 = SimpleEntity("http://aql.org/download3")
values = [value1, value2, value3]
value_keys = vfile.add_entities(values)
vfile.self_test()
other_value_keys = vfile.add_entities(values)
vfile.self_test()
self.assertItemsEqual(value_keys, other_value_keys)
values = sorted(values, key=operator.attrgetter('id'))
s_values = vfile.find_entities(values)
s_values = sorted(s_values, key=operator.attrgetter('id'))
self.assertItemsEqual(values, s_values)
vfile.clear()
vfile.self_test()
# -----------------------------------------------------------
value_keys = vfile.add_entities(values)
vfile.self_test()
s_values = vfile.find_entities(values)
vfile.self_test()
dep_values = vfile.find_entities_by_key(value_keys)
vfile.self_test()
self.assertItemsEqual(s_values, dep_values)
# -----------------------------------------------------------
value1_key = vfile.add_entities([value1])[0]
vfile.self_test()
s_dep_value = vfile.find_entities_by_key([value1_key])[0]
self.assertEqual(value1, s_dep_value)
value1 = SimpleEntity("abc", name=value1.name)
vfile.add_entities([value1])
vfile.self_test()
s_dep_value = vfile.find_entities_by_key(value_keys)
vfile.self_test()
self.assertIsNone(s_dep_value)
finally:
vfile.close()
def test_node_file(self):
try:
tmp_files = []
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
with Tempfile(suffix='.1') as tmp1:
with Tempfile(suffix='.2') as tmp2:
value1 = FileChecksumEntity(tmp1)
value2 = FileChecksumEntity(tmp2)
options = builtin_options()
builder = CopyBuilder(options, "tmp", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "d")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
tmp1.write(b'123')
tmp1.flush()
value1 = FileChecksumEntity(tmp1)
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
with Tempfile(suffix='.3') as tmp3:
value3 = FileChecksumEntity(tmp3)
node3 = self._rebuild_node(
vfile, builder, [value3], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1, node3], [],
tmp_files)
builder3 = CopyBuilder(options, "xxx", "3")
node3 = self._rebuild_node(
vfile, builder3, [value3], [], tmp_files)
# node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx,
# ,tmp3.3
node = self._rebuild_node(
vfile, builder, [value1, node3], [],
tmp_files)
# node: CopyBuilder, tmp, i, tmp1, tmp3.xxx ->
# tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
dep = SimpleEntity("1", name="dep1")
node = self._rebuild_node(
vfile, builder, [value1, node3], [dep],
tmp_files)
dep = SimpleEntity("11", name="dep1")
node = self._rebuild_node(
vfile, builder, [value1, node3], [dep],
tmp_files)
node3 = self._rebuild_node(
vfile, builder3, [value1], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
node3 = self._rebuild_node(
vfile, builder3, [value2], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
node_tname = node.get_target_entities()[0].name
with open(node_tname, 'wb') as f:
f.write(b'333')
f.flush()
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
side_effects = node.get_side_effect_entities()
with open(side_effects[0].name, 'wb') as f:
f.write(b'abc')
f.flush()
node = Node(builder, [value1])
node.depends([node3])
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(
vfile, False))
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_value(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
value1 = SimpleEntity("http://aql.org/download1",
name="target_url1")
value2 = SimpleEntity("http://aql.org/download2",
name="target_url2")
value3 = SimpleEntity("http://aql.org/download3",
name="target_url3")
dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value")
dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value")
options = builtin_options()
builder = ChecksumBuilder(options)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
finally:
vfile.close()
def test_values_file_2(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
self.assertItemsEqual(vfile.find_entities([]), [])
value1 = SimpleEntity("http://aql.org/download",
name="target_url1")
value2 = SimpleEntity("http://aql.org/download2",
name="target_url2")
value3 = SimpleEntity("http://aql.org/download3",
name="target_url3")
value4 = SimpleEntity("http://aql.org/download4",
name="target_url4")
value5 = SimpleEntity("http://aql.org/download5",
name="target_url5")
value6 = SimpleEntity("http://aql.org/download6",
name="target_url6")
values = [value1, value2, value3]
dep_values_1 = values
dep_values_2 = dep_values_1 + [value4]
dep_values_3 = dep_values_2 + [value5]
dep_values_4 = dep_values_3 + [value6]
all_values = dep_values_4
all_keys = vfile.add_entities(all_values)
vfile.self_test()
self.assertItemsEqual(vfile.find_entities(all_values),
all_values)
self.assertItemsEqual(vfile.find_entities_by_key(all_keys),
all_values)
dep_keys_1 = vfile.add_entities(dep_values_1)
vfile.self_test()
dep_keys_2 = vfile.add_entities(dep_values_2)
vfile.self_test()
dep_keys_3 = vfile.add_entities(dep_values_3)
vfile.self_test()
dep_keys_4 = vfile.add_entities(dep_values_4)
vfile.self_test()
self.assertTrue(set(dep_keys_1).issubset(dep_keys_2))
self.assertTrue(set(dep_keys_2).issubset(dep_keys_3))
self.assertTrue(set(dep_keys_3).issubset(dep_keys_4))
self.assertItemsEqual(all_keys, dep_keys_4)
vfile.close()
vfile.open(tmp)
vfile.self_test()
self.assertItemsEqual(vfile.find_entities(all_values),
all_values)
vfile.self_test()
self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_1),
dep_values_1)
self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_2),
dep_values_2)
self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_3),
dep_values_3)
self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_4),
dep_values_4)
value4 = SimpleEntity("http://aql.org/download3/0",
name=value4.name)
vfile.add_entities([value4])
vfile.self_test()
self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_1),
dep_values_1)
self.assertIsNone(vfile.find_entities_by_key(dep_keys_2))
self.assertIsNone(vfile.find_entities_by_key(dep_keys_3))
self.assertIsNone(vfile.find_entities_by_key(dep_keys_4))
finally:
vfile.close()
def test_values_file_same_name(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
value1 = SimpleEntity("test", name="test1")
value2 = SignatureEntity(b"1234354545", name=value1.name)
vfile.add_entities([value1, value2])
vfile.self_test()
values = [
SimpleEntity(name=value1.name),
SignatureEntity(name=value2.name)
]
values = vfile.find_entities(values)
self.assertItemsEqual(values, [value1, value2])
vfile.close()
vfile.open(tmp)
vfile.self_test()
finally:
vfile.close()
def test_values_file(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
vfile.self_test()
value1 = SimpleEntity("http://aql.org/download")
value2 = SimpleEntity("http://aql.org/download2")
value3 = SimpleEntity("http://aql.org/download3")
values = [value1, value2, value3]
value_keys = vfile.add_entities(values)
vfile.self_test()
other_value_keys = vfile.add_entities(values)
vfile.self_test()
self.assertItemsEqual(value_keys, other_value_keys)
values = sorted(values, key=operator.attrgetter('id'))
s_values = vfile.find_entities(values)
s_values = sorted(s_values, key=operator.attrgetter('id'))
self.assertItemsEqual(values, s_values)
vfile.clear()
vfile.self_test()
# -----------------------------------------------------------
value_keys = vfile.add_entities(values)
vfile.self_test()
s_values = vfile.find_entities(values)
vfile.self_test()
dep_values = vfile.find_entities_by_key(value_keys)
vfile.self_test()
self.assertItemsEqual(s_values, dep_values)
# -----------------------------------------------------------
value1_key = vfile.add_entities([value1])[0]
vfile.self_test()
s_dep_value = vfile.find_entities_by_key([value1_key])[0]
self.assertEqual(value1, s_dep_value)
value1 = SimpleEntity("abc", name=value1.name)
vfile.add_entities([value1])
vfile.self_test()
s_dep_value = vfile.find_entities_by_key(value_keys)
vfile.self_test()
self.assertIsNone(s_dep_value)
finally:
vfile.close()