def test_values_file_same_name(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: vfile.self_test() value1 = SimpleEntity("test", name="test1") value2 = SignatureEntity(b"1234354545", name=value1.name) vfile.add_entities([value1, value2]) vfile.self_test() values = [ SimpleEntity(name=value1.name), SignatureEntity(name=value2.name) ] values = vfile.find_entities(values) self.assertItemsEqual(values, [value1, value2]) vfile.close() vfile.open(tmp) vfile.self_test() finally: vfile.close()
def test_value_pickler(self): with Tempfile() as tmp: tmp_name = str(tmp) vpick = EntityPickler() value = FileChecksumEntity(tmp) vl = vpick.dumps(value) vl = vpick.dumps(value) vl = vpick.dumps(value) v = vpick.loads(vl) v = vpick.loads(vl) v = vpick.loads(vl) self.assertEqual(value, v) value = FileTimestampEntity(tmp) v = vpick.loads(vpick.dumps(value)) self.assertEqual(value, v) value = SimpleEntity('123-345', name=tmp_name) v = vpick.loads(vpick.dumps(value)) self.assertEqual(value, v) value = SimpleEntity('123-345', name=tmp_name) v = vpick.loads(vpick.dumps(value)) self.assertEqual(value, v) value = SimpleEntity(name=tmp_name) v = vpick.loads(vpick.dumps(value)) self.assertEqual(value.name, v.name) self.assertIsNone(v.signature)
def test_values_file(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: vfile.self_test() value1 = SimpleEntity("http://aql.org/download") value2 = SimpleEntity("http://aql.org/download2") value3 = SimpleEntity("http://aql.org/download3") values = [value1, value2, value3] value_keys = vfile.add_entities(values) vfile.self_test() other_value_keys = vfile.add_entities(values) vfile.self_test() self.assertItemsEqual(value_keys, other_value_keys) values = sorted(values, key=operator.attrgetter('id')) s_values = vfile.find_entities(values) s_values = sorted(s_values, key=operator.attrgetter('id')) self.assertItemsEqual(values, s_values) vfile.clear() vfile.self_test() # ----------------------------------------------------------- value_keys = vfile.add_entities(values) vfile.self_test() s_values = vfile.find_entities(values) vfile.self_test() dep_values = vfile.find_entities_by_key(value_keys) vfile.self_test() self.assertItemsEqual(s_values, dep_values) # ----------------------------------------------------------- value1_key = vfile.add_entities([value1])[0] vfile.self_test() s_dep_value = vfile.find_entities_by_key([value1_key])[0] self.assertEqual(value1, s_dep_value) value1 = SimpleEntity("abc", name=value1.name) vfile.add_entities([value1]) vfile.self_test() s_dep_value = vfile.find_entities_by_key(value_keys) vfile.self_test() self.assertIsNone(s_dep_value) finally: vfile.close()
def test_str_empty_value_save_load(self): value1 = SimpleEntity(name='results_link') value2 = SimpleEntity(name=value1.name) self.assertEqual(value1, value2) self.assertFalse(value1.is_actual()) self.assertFalse(value2.is_actual()) self._test_save_load(value1) self._test_save_load(value2)
def test_prj_implicit_value_args(self): with Tempdir() as tmp_dir: cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir]) prj = Project(cfg) tool = prj.tools.add_tool(_NullTool) tool.noop(v1="a", v2="b", v3="c") prj.build() self.assertEqual(self.built_nodes, 1) # ----------------------------------------------------------- self.built_nodes = 0 tool.noop(v1="aa", v2="bb", v3="cc") prj.build() self.assertEqual(self.built_nodes, 0) # ----------------------------------------------------------- self.built_nodes = 0 v1 = SimpleEntity("a", name="value1") tool.noop(v1=v1, v2="b", v3="c") prj.build() self.assertEqual(self.built_nodes, 1) # ----------------------------------------------------------- self.built_nodes = 0 v1 = SimpleEntity("ab", name="value1") tool.noop(v1=v1, v2="b", v3="c") prj.build() self.assertEqual(self.built_nodes, 1) # ----------------------------------------------------------- self.built_nodes = 0 v1 = SimpleEntity("ab", name="value1") tool.noop(v1=v1, v2="b", v3="c") prj.build() self.assertEqual(self.built_nodes, 0)
def _test_values_file_speed(self, use_sqlite): values = [] for i in range(20000): value = SimpleEntity("http://aql.org/download", name="target_url%s" % i) values.append(value) with Tempfile() as tmp: print("Opening a database '%s' ..." % tmp) timer = Chrono() with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf: with timer: keys = vf.add_entities(values) print("add values time: %s" % (timer, )) with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf: with timer: keys = vf.add_entities(values) print("re-add values time: %s" % (timer, )) with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf: with timer: vf.find_entities_by_key(keys) print("get values time: %s" % timer) with timer: with EntitiesFile(tmp, use_sqlite=use_sqlite) as vf: pass print("reopen values file time: %s" % timer)
def test_str_value(self): value1 = SimpleEntity('http://buildsrv.com/results.out') value2 = SimpleEntity('http://buildsrv.com/results.out') self._test_save_load(value1) self.assertEqual(value1, value1) self.assertEqual(value1, value2) self.assertTrue(value1.is_actual()) self._test_save_load(value2) value2 = SimpleEntity('http://buildsrv.com/results2.out') self.assertNotEqual(value1, value2)
def _rebuild_batch_node(self, vfile, src_files, built_count): options = builtin_options() options.batch_build = True options.batch_groups = 2 builder = CopyBuilder(options, "tmp", "i") node = Node(builder, src_files) dep = SimpleEntity("11", name="dep1") node.depends(dep) node.initiate() split_nodes = node.build_split(vfile, False) if built_count == 0: self.assertFalse(split_nodes) self.assertTrue(node.check_actual(vfile, False)) else: for split_node in split_nodes: self.assertFalse(split_node.check_actual(vfile, False)) split_node.build() split_node.save(vfile) self.assertEqual(len(split_nodes), 2) self.assertTrue(node.check_actual(vfile, False))
def build(self, source_entities, targets): target_entities = [] for source_value in source_entities: copy_value = SimpleEntity(source_value.get(), name=source_value.name + '_copy') target_entities.append(copy_value) targets.add_targets(target_entities)
def test_bm_deps_speed(self): bm = BuildManager() value = SimpleEntity("http://aql.org/download", name="target_url1") builder = CopyValueBuilder() node = Node(builder, value) bm.add([node]) _generate_node_tree(bm, builder, node, 5000)
def build_batch(self, source_entities, targets): idep = SimpleEntity(b'1234') for src_value in source_entities: src = src_value.get() new_name = src + '.' + self.ext new_iname = src + '.' + self.iext shutil.copy(src, new_name) shutil.copy(src, new_iname) src_targets = targets[src_value] src_targets.add_targets(new_name) src_targets.add_side_effects(new_iname) src_targets.add_implicit_deps(idep)
def test_bm_node_build_fail(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 builder = FailedBuilder(options) nodes = [ Node(builder, SimpleEntity("123-%s" % (i, ))) for i in range(4) ] bm.add(nodes) self.assertRaises(Exception, _build, bm) self.assertEqual(self.built_nodes, 0)
def build(self, source_entities, targets): target_values = [] itarget_values = [] idep = SimpleEntity(b'1234') for src in source_entities: src = src.get() new_name = src + '.' + self.ext new_iname = src + '.' + self.iext shutil.copy(src, new_name) shutil.copy(src, new_iname) target_values.append(new_name) itarget_values.append(new_iname) targets.add_targets(target_values) targets.add_side_effects(itarget_values) targets.add_implicit_deps(idep)
def test_bm_sync_nodes(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 nodes = [ Node(SyncValueBuilder(options, name="%s" % i, number=n), SimpleEntity("123-%s" % i)) for i, n in zip(range(4), [3, 5, 7, 11]) ] bm.add(nodes) bm.sync(nodes) _build(bm, jobs=4)
def test_values_file_2(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: vfile.self_test() self.assertItemsEqual(vfile.find_entities([]), []) value1 = SimpleEntity("http://aql.org/download", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") value4 = SimpleEntity("http://aql.org/download4", name="target_url4") value5 = SimpleEntity("http://aql.org/download5", name="target_url5") value6 = SimpleEntity("http://aql.org/download6", name="target_url6") values = [value1, value2, value3] dep_values_1 = values dep_values_2 = dep_values_1 + [value4] dep_values_3 = dep_values_2 + [value5] dep_values_4 = dep_values_3 + [value6] all_values = dep_values_4 all_keys = vfile.add_entities(all_values) vfile.self_test() self.assertItemsEqual(vfile.find_entities(all_values), all_values) self.assertItemsEqual(vfile.find_entities_by_key(all_keys), all_values) dep_keys_1 = vfile.add_entities(dep_values_1) vfile.self_test() dep_keys_2 = vfile.add_entities(dep_values_2) vfile.self_test() dep_keys_3 = vfile.add_entities(dep_values_3) vfile.self_test() dep_keys_4 = vfile.add_entities(dep_values_4) vfile.self_test() self.assertTrue(set(dep_keys_1).issubset(dep_keys_2)) self.assertTrue(set(dep_keys_2).issubset(dep_keys_3)) self.assertTrue(set(dep_keys_3).issubset(dep_keys_4)) self.assertItemsEqual(all_keys, dep_keys_4) vfile.close() vfile.open(tmp) vfile.self_test() self.assertItemsEqual(vfile.find_entities(all_values), all_values) vfile.self_test() self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_1), dep_values_1) self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_2), dep_values_2) self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_3), dep_values_3) self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_4), dep_values_4) value4 = SimpleEntity("http://aql.org/download3/0", name=value4.name) vfile.add_entities([value4]) vfile.self_test() self.assertItemsEqual(vfile.find_entities_by_key(dep_keys_1), dep_values_1) self.assertIsNone(vfile.find_entities_by_key(dep_keys_2)) self.assertIsNone(vfile.find_entities_by_key(dep_keys_3)) self.assertIsNone(vfile.find_entities_by_key(dep_keys_4)) finally: vfile.close()
def test_node_file(self): try: tmp_files = [] with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: with Tempfile(suffix='.1') as tmp1: with Tempfile(suffix='.2') as tmp2: value1 = FileChecksumEntity(tmp1) value2 = FileChecksumEntity(tmp2) options = builtin_options() builder = CopyBuilder(options, "tmp", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "d") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) tmp1.write(b'123') tmp1.flush() value1 = FileChecksumEntity(tmp1) node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) with Tempfile(suffix='.3') as tmp3: value3 = FileChecksumEntity(tmp3) node3 = self._rebuild_node( vfile, builder, [value3], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1, node3], [], tmp_files) builder3 = CopyBuilder(options, "xxx", "3") node3 = self._rebuild_node( vfile, builder3, [value3], [], tmp_files) # node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx, # ,tmp3.3 node = self._rebuild_node( vfile, builder, [value1, node3], [], tmp_files) # node: CopyBuilder, tmp, i, tmp1, tmp3.xxx -> # tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) dep = SimpleEntity("1", name="dep1") node = self._rebuild_node( vfile, builder, [value1, node3], [dep], tmp_files) dep = SimpleEntity("11", name="dep1") node = self._rebuild_node( vfile, builder, [value1, node3], [dep], tmp_files) node3 = self._rebuild_node( vfile, builder3, [value1], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) node3 = self._rebuild_node( vfile, builder3, [value2], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) node_tname = node.get_target_entities()[0].name with open(node_tname, 'wb') as f: f.write(b'333') f.flush() node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) side_effects = node.get_side_effect_entities() with open(side_effects[0].name, 'wb') as f: f.write(b'abc') f.flush() node = Node(builder, [value1]) node.depends([node3]) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual( vfile, False)) finally: vfile.close() finally: for tmp_file in tmp_files: try: os.remove(tmp_file) except OSError: pass
def test_node_value(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value") dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value") options = builtin_options() builder = ChecksumBuilder(options) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) finally: vfile.close()
def test_bm_deps(self): bm = BuildManager() value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") options = builtin_options() builder = CopyValueBuilder(options) node0 = Node(builder, value1) node1 = Node(builder, node0) node2 = Node(builder, node1) node3 = Node(builder, value2) node4 = Node(builder, value3) node5 = Node(builder, node4) node6 = Node(builder, node5) node6.depends([node0, node1]) bm.add([node0]) bm.self_test() self.assertEqual(len(bm), 1) bm.add([node1]) bm.self_test() self.assertEqual(len(bm), 2) bm.add([node2]) bm.self_test() self.assertEqual(len(bm), 3) bm.add([node3]) bm.self_test() self.assertEqual(len(bm), 4) bm.add([node4]) bm.self_test() self.assertEqual(len(bm), 5) bm.add([node5]) bm.self_test() self.assertEqual(len(bm), 6) bm.add([node6]) bm.self_test() self.assertEqual(len(bm), 7) node0.depends(node3) bm.depends(node0, [node3]) bm.self_test() node1.depends(node3) bm.depends(node1, [node3]) bm.self_test() node2.depends(node3) bm.depends(node2, [node3]) bm.self_test() node3.depends(node4) bm.depends(node3, [node4]) bm.self_test() node0.depends(node5) bm.depends(node0, [node5]) bm.self_test() node5.depends(node3) bm.depends(node5, [node3]) bm.self_test() def _cyclic_deps(src_node, dep_node): src_node.depends(dep_node) bm.depends(src_node, [dep_node]) self.assertRaises(ErrorNodeDependencyCyclic, _cyclic_deps, node4, node3)
def test_bm_require_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node( SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node( SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31, )) node22 = Node( SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33, )) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33, )) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) # print( "node30: %s" % node30 ) # print( "node31: %s" % node31 ) # print( "node32: %s" % node32 ) # print( "node33: %s" % node33 ) # # print( "node20: %s" % node20 ) # print( "node21: %s" % node21 ) # print( "node22: %s" % node22 ) # print( "node23: %s" % node23 ) # print( "node24: %s" % node24 ) # # print( "node10: %s" % node10 ) # print( "node11: %s" % node11 ) bm.add((node10, node11)) bm.module_depends(node10, [node11]) _build(bm, jobs=4)
def test_bm_sync_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node( SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node( SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31, )) node22 = Node( SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33, )) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33, )) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) bm.add((node10, node11)) bm.sync((node10, node11), deep=True) _build(bm, jobs=4)
def make_entity(self, data, name=None): return SimpleEntity(data=data, name=name)
def make_entity(self, value, tags=None): if isinstance(value, FilePath): return self.make_file_entity(name=value, tags=tags) return SimpleEntity(value, tags=tags)
def test_bm_nodes(self): def _make_nodes(builder): node1 = Node(builder, value1) copy_node1 = Node(builder, node1) copy2_node1 = Node(builder, copy_node1) node2 = Node(builder, value2) node3 = Node(builder, value3) copy_node3 = Node(builder, node3) copy2_node3 = Node(builder, copy_node3) copy2_node3.depends([node1, copy_node1]) return node1, node2, node3, copy_node1,\ copy_node3, copy2_node1, copy2_node3 with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") builder = CopyValueBuilder(options) bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 7) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 0) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.removed_nodes = 0 bm.clear() bm.close() self.assertEqual(self.removed_nodes, 7) # ---------------------------------------------------------- nodes = _make_nodes(builder) copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[copy_node3]) bm.close() self.assertEqual(self.built_nodes, 2) # ---------------------------------------------------------- nodes = _make_nodes(builder) node2 = nodes[1] copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3]) bm.close() self.assertEqual(self.built_nodes, 1)
def get_entities(self): entities = super(NodeBaseNameFilter, self).get_entities() return tuple(SimpleEntity(os.path.basename(entity.get())) for entity in entities)
def make_simple_entity(self, value, tags=None): return SimpleEntity(value, tags=tags)