def test_bm_node_names(self): with Tempdir() as tmp_dir: src_files = self.generate_source_files(tmp_dir, 3, 201) options = builtin_options() options.build_dir = tmp_dir builder = ChecksumBuilder(options, 0, 256, replace_ext=False) bm = BuildManager() try: src_entities = [] for s in src_files: src_entities.append(FileChecksumEntity(s)) node0 = Node(builder, None) node1 = Node(builder, src_entities) node2 = Node(builder, node1) node3 = Node(builder, node2) node4 = Node(builder, node3) bm.add([node0, node1, node2, node3, node4]) bm.build(1, False) print("node2: %s" % str(node4)) print("node2: %s" % str(node3)) print("node2: %s" % str(node2)) print("node1: %s" % str(node1)) print("node0: %s" % str(node0)) finally: bm.close()
def _rebuild_batch_node(self, vfile, src_files, built_count): options = builtin_options() options.batch_build = True options.batch_groups = 2 builder = CopyBuilder(options, "tmp", "i") node = Node(builder, src_files) dep = SimpleEntity("11", name="dep1") node.depends(dep) node.initiate() split_nodes = node.build_split(vfile, False) if built_count == 0: self.assertFalse(split_nodes) self.assertTrue(node.check_actual(vfile, False)) else: for split_node in split_nodes: self.assertFalse(split_node.check_actual(vfile, False)) split_node.build() split_node.save(vfile) self.assertEqual(len(split_nodes), 2) self.assertTrue(node.check_actual(vfile, False))
def test_bm_node_index(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 2 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) bm = BuildManager() self.built_nodes = 0 builder = ChecksumSingleBuilder(options, 0, 256) node = Node(builder, src_files) nodes = [Node(builder, node[i * 2]) for i in range(num_src_files + 1)] node2 = Node(builder, node[1:2][:]) # bm.add([node2]) bm.add(nodes) _build(bm) self.assertEqual( self.built_nodes, num_src_files + num_src_files + 1 + 1)
def test_bm_batch(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir options.batch_build = True src_files = self.generate_source_files(tmp_dir, 3, 201) builder = ChecksumBuilder(options, 0, 256, replace_ext=True) self.building_nodes = self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.building_nodes, 2) self.assertEqual(self.building_nodes, self.built_nodes) self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.built_nodes, 0) bm = _add_nodes_to_bm(builder, src_files) try: bm.clear() bm.self_test() finally: bm.close()
def test_bm_no_conflicts(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) bm = BuildManager() self.built_nodes = 0 builder1 = ChecksumSingleBuilder(options, 0, 256) builder2 = ChecksumSingleBuilder(options, 0, 256) node1 = Node(builder1, src_files) node2 = Node(builder2, src_files) node1 = Node(builder1, node1) node2 = Node(builder2, node2) bm.add([node1, node2]) _build(bm) self.assertEqual( self.built_nodes, num_src_files + num_src_files * 2)
def test_bm_node_index(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 2 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) bm = BuildManager() self.built_nodes = 0 builder = ChecksumSingleBuilder(options, 0, 256) node = Node(builder, src_files) nodes = [ Node(builder, node[i * 2]) for i in range(num_src_files + 1) ] node2 = Node(builder, node[1:2][:]) # bm.add([node2]) bm.add(nodes) _build(bm) self.assertEqual(self.built_nodes, num_src_files + num_src_files + 1 + 1)
def test_bm_no_conflicts(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) bm = BuildManager() self.built_nodes = 0 builder1 = ChecksumSingleBuilder(options, 0, 256) builder2 = ChecksumSingleBuilder(options, 0, 256) node1 = Node(builder1, src_files) node2 = Node(builder2, src_files) node1 = Node(builder1, node1) node2 = Node(builder2, node2) bm.add([node1, node2]) _build(bm) self.assertEqual(self.built_nodes, num_src_files + num_src_files * 2)
def test_bm_sync_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node(SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node(SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31,)) node22 = Node(SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33,)) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33,)) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) bm.add((node10, node11)) bm.sync((node10, node11), deep=True) _build(bm, jobs=4)
def test_options_build_dir_name(self): options = builtin_options() self.assertEqual(options.build_dir_name, 'debug') options.target_os = "windows" self.assertEqual(options.build_dir_name.get(), 'windows_debug') options.target_arch = "x86-32" self.assertEqual(options.build_dir_name, 'windows_x86-32_debug')
def test_builtin_options(self): options = builtin_options() self.assertEqual(options.optimization, 'off') self.assertEqual(options.build_variant, 'debug') options.build_variant = 'release' self.assertEqual(options.optimization, 'speed') options.build_variant = 'release_size' self.assertEqual(options.optimization, 'size') options.build_variant = 'final' self.assertEqual(options.optimization, 'speed') self.assertEqual(options.build_variant.option_type().group, "Build")
def test_bm_skip_nodes_by_value(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 node = Node(ValueBuilder(options), [1, 2, 3, 4]) bm.add([node]) bm.build_if(False, node) _build(bm, jobs=4) self.assertEqual(self.built_nodes, 0)
def test_bm_rebuild(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 10 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) def _build_nodes(num_dups, uptodate): bm = BuildManager() self.building_nodes = self.built_nodes = 0 builder = ChecksumSingleBuilder(options, 0, 256) src_entities = [] for s in src_files: src_entities.append(FileChecksumEntity(s)) for i in range(num_dups): num_built_nodes = 1 node = Node(builder, src_entities) node = Node(builder, node) num_built_nodes += 2 node = Node(builder, node) num_built_nodes += 2**2 node = Node(builder, node) num_built_nodes += 2**3 bm.add([node]) _build(bm, jobs=10, explain=False) if uptodate: num_built_nodes = 0 else: num_built_nodes *= num_src_files self.assertEqual(self.building_nodes, num_built_nodes) _build_nodes(3, False) _build_nodes(3, True)
def test_bm_dup_names(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir options.batch_build = True # options.batch_groups = 4 options.batch_size = 3 src_files = self.generate_source_files(tmp_dir, 10, 201) builder = ChecksumBadBuilder(options, 0, 256, replace_ext=True) self.building_nodes = self.built_nodes = 0 with self.assertRaises(ErrorNodeDuplicateNames): _build_checksums(builder, src_files, jobs=4)
def test_bm_rebuild(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 10 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) def _build_nodes(num_dups, uptodate): bm = BuildManager() self.building_nodes = self.built_nodes = 0 builder = ChecksumSingleBuilder(options, 0, 256) src_entities = tuple(map(FileChecksumEntity, src_files)) num_built_nodes = 0 for i in range(num_dups): num_built_nodes = 1 node = Node(builder, src_entities) node = Node(builder, node) num_built_nodes += 2 node = Node(builder, node) num_built_nodes += 2**2 node = Node(builder, node) num_built_nodes += 2**3 bm.add([node]) _build(bm, jobs=10, explain=False) if uptodate: num_built_nodes = 0 else: num_built_nodes *= num_src_files self.assertEqual(self.building_nodes, num_built_nodes) _build_nodes(3, False) _build_nodes(3, True)
def test_bm_tags_batch(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir options.batch_build = True num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) builder = ChecksumBuilder(options, 0, 256) options.batch_build = False single_builder = ChecksumSingleBuilder(options, 0, 256) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(single_builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, num_src_files + 1) # ----------------------------------------------------------- self.touch_cpp_file(src_files[0]) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(single_builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, 2)
def test_bm_tags_batch(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir options.batch_build = True num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) builder = ChecksumBuilder(options, 0, 256) options.batch_build = False single_builder = ChecksumSingleBuilder(options, 0, 256) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(single_builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, num_src_files + 1) # ----------------------------------------------------------- self.regenerate_file(src_files[0], 201) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(single_builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, 2)
def test_options_join(self): built_options = builtin_options() p_options = built_options.override() options = p_options.override() options2 = options.override() options2.target_os = "windows" options2.target_arch = "x86-32" self.assertEqual(options2.build_dir_name.get(), 'windows_x86-32_debug') self.assertEqual(options.build_dir_name, 'debug') options2.join() self.assertEqual(options.build_dir_name.get(), 'windows_x86-32_debug')
def test_bm_sync_nodes(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 nodes = [Node(SyncValueBuilder(options, name="%s" % i, number=n), SimpleEntity("123-%s" % i)) for i, n in zip(range(4), [3, 5, 7, 11])] bm.add(nodes) bm.sync(nodes) _build(bm, jobs=4)
def test_bm_node_build_fail(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 builder = FailedBuilder(options) nodes = [Node(builder, SimpleEntity("123-%s" % (i,))) for i in range(4)] bm.add(nodes) self.assertRaises(Exception, _build, bm) self.assertEqual(self.built_nodes, 0)
def test_bm_node_build_fail(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 builder = FailedBuilder(options) nodes = [ Node(builder, SimpleEntity("123-%s" % (i, ))) for i in range(4) ] bm.add(nodes) self.assertRaises(Exception, _build, bm) self.assertEqual(self.built_nodes, 0)
def test_bm_expensive(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 event = threading.Event() heavy = ExpensiveValueBuilder(options, event, do_expensive=True) light = ExpensiveValueBuilder(options, event, do_expensive=False) node1 = Node(heavy, [1, 2, 3, 4, 5, 7]) node2 = Node(light, list(range(10, 100, 10))) bm.add([node2, node1]) bm.expensive(node1) _build(bm, jobs=16)
def test_bm_tags(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) builder = ChecksumSingleBuilder(options, 0, 256) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, num_src_files * 2) # ----------------------------------------------------------- self.regenerate_file(src_files[0], ) bm = BuildManager() self.built_nodes = 0 node = Node(builder, src_files) node_md5 = Node(builder, node.at('md5')) bm.add([node_md5]) _build(bm) self.assertEqual(self.built_nodes, 2)
def test_bm_sync_nodes(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 nodes = [ Node(SyncValueBuilder(options, name="%s" % i, number=n), SimpleEntity("123-%s" % i)) for i, n in zip(range(4), [3, 5, 7, 11]) ] bm.add(nodes) bm.sync(nodes) _build(bm, jobs=4)
def test_bm_skip_nodes_by_option(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 cond_node = Node(CondBuilder(options), False) options.has_openmp = BoolOptionType(default=True) options.has_openmp = cond_node node = Node(ValueBuilder(options), None) bm.add([node]) bm.build_if(options.has_openmp, node) bm.depends(node, [cond_node]) _build(bm, jobs=4) self.assertEqual(self.built_nodes, 1)
def test_exec(self): with Tempdir() as tmp_dir: build_dir = os.path.join(tmp_dir, 'build') options = builtin_options() cmd = [sys.executable, '-c', 'print("TEST EXEC")'] options.build_dir = build_dir exec_cmd = ExecuteCommandBuilder(options) bm = BuildManager() try: result = Node(exec_cmd, cmd) bm.add([result]) self._build(bm, jobs=1, keep_going=False) self.assertEqual(self.building_started, 1) self.assertEqual(self.building_started, self.built_nodes) bm.close() result = Node(exec_cmd, cmd) bm = BuildManager() bm.add([result]) self.building_started = 0 self._build(bm, jobs=1, keep_going=False) self.assertEqual(self.building_started, 0) finally: bm.close()
def test_bm_build(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir src_files = self.generate_source_files(tmp_dir, 5, 201) builder = ChecksumBuilder(options, 0, 256) self.building_nodes = self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.building_nodes, 2) self.assertEqual(self.building_nodes, self.built_nodes) # ----------------------------------------------------------- self.building_nodes = self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.building_nodes, 0) self.assertEqual(self.building_nodes, self.built_nodes) # ----------------------------------------------------------- builder = ChecksumBuilder(options, 32, 1024) self.building_nodes = self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.building_nodes, 2) self.assertEqual(self.building_nodes, self.built_nodes) # ----------------------------------------------------------- self.building_nodes = self.built_nodes = 0 _build_checksums(builder, src_files) self.assertEqual(self.building_nodes, 0) self.assertEqual(self.building_nodes, self.building_nodes)
def test_bm_conflicts(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir num_src_files = 3 src_files = self.generate_source_files(tmp_dir, num_src_files, 201) bm = BuildManager() self.built_nodes = 0 builder1 = ChecksumSingleBuilder(options, 0, 256) builder2 = ChecksumSingleBuilder(options, 0, 1024) node1 = Node(builder1, src_files) node2 = Node(builder2, src_files) # node1 = Node( builder1, node1 ) # node2 = Node( builder2, node2 ) bm.add([node1, node2]) self.assertRaises(ErrorNodeSignatureDifferent, _build, bm)
def test_bm_skip_nodes_by_node(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 cond_node1 = Node(CondBuilder(options), False) node1 = Node(ValueBuilder(options), [1, 2]) cond_node2 = Node(CondBuilder(options), True) node2 = Node(ValueBuilder(options), [3, 4]) main = Node(ValueBuilder(options), [7, 8, node1, node2]) bm.add([main]) bm.build_if(cond_node1, node1) bm.build_if(cond_node2, node2) _build(bm, jobs=4) self.assertEqual(main.get(), "7-8-3-4") self.assertEqual(self.built_nodes, 4)
def test_node_value(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value") dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value") options = builtin_options() builder = ChecksumBuilder(options) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) finally: vfile.close()
def test_bm_deps(self): bm = BuildManager() value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") options = builtin_options() builder = CopyValueBuilder(options) node0 = Node(builder, value1) node1 = Node(builder, node0) node2 = Node(builder, node1) node3 = Node(builder, value2) node4 = Node(builder, value3) node5 = Node(builder, node4) node6 = Node(builder, node5) node6.depends([node0, node1]) bm.add([node0]) bm.self_test() self.assertEqual(len(bm), 1) bm.add([node1]) bm.self_test() self.assertEqual(len(bm), 2) bm.add([node2]) bm.self_test() self.assertEqual(len(bm), 3) bm.add([node3]) bm.self_test() self.assertEqual(len(bm), 4) bm.add([node4]) bm.self_test() self.assertEqual(len(bm), 5) bm.add([node5]) bm.self_test() self.assertEqual(len(bm), 6) bm.add([node6]) bm.self_test() self.assertEqual(len(bm), 7) node0.depends(node3) bm.depends(node0, [node3]) bm.self_test() node1.depends(node3) bm.depends(node1, [node3]) bm.self_test() node2.depends(node3) bm.depends(node2, [node3]) bm.self_test() node3.depends(node4) bm.depends(node3, [node4]) bm.self_test() node0.depends(node5) bm.depends(node0, [node5]) bm.self_test() node5.depends(node3) bm.depends(node5, [node3]) bm.self_test() def _cyclic_deps(src_node, dep_node): src_node.depends(dep_node) bm.depends(src_node, [dep_node]) self.assertRaises(ErrorNodeDependencyCyclic, _cyclic_deps, node4, node3)
def test_node_file(self): try: tmp_files = [] with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: with Tempfile(suffix='.1') as tmp1: with Tempfile(suffix='.2') as tmp2: value1 = FileChecksumEntity(tmp1) value2 = FileChecksumEntity(tmp2) options = builtin_options() builder = CopyBuilder(options, "tmp", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "d") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) tmp1.write(b'123') tmp1.flush() value1 = FileChecksumEntity(tmp1) node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) with Tempfile(suffix='.3') as tmp3: value3 = FileChecksumEntity(tmp3) node3 = self._rebuild_node( vfile, builder, [value3], [], tmp_files) node = self._rebuild_node(vfile, builder, [value1, node3], [], tmp_files) builder3 = CopyBuilder(options, "xxx", "3") node3 = self._rebuild_node(vfile, builder3, [value3], [], tmp_files) # node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx, # ,tmp3.3 node = self._rebuild_node(vfile, builder, [value1, node3], [], tmp_files) # node: CopyBuilder, tmp, i, tmp1, tmp3.xxx -> # tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i node = self._rebuild_node(vfile, builder, [value1], [node3], tmp_files) dep = SimpleEntity("1", name="dep1") node = self._rebuild_node(vfile, builder, [value1, node3], [dep], tmp_files) dep = SimpleEntity("11", name="dep1") node = self._rebuild_node(vfile, builder, [value1, node3], [dep], tmp_files) node3 = self._rebuild_node(vfile, builder3, [value1], [], tmp_files) node = self._rebuild_node(vfile, builder, [value1], [node3], tmp_files) node3 = self._rebuild_node(vfile, builder3, [value2], [], tmp_files) node = self._rebuild_node(vfile, builder, [value1], [node3], tmp_files) node_tname = node.get_target_entities()[0].name with open(node_tname, 'wb') as f: f.write(b'333') f.flush() node = self._rebuild_node(vfile, builder, [value1], [node3], tmp_files) side_effects = node.get_side_effect_entities() with open(side_effects[0].name, 'wb') as f: f.write(b'abc') f.flush() node = Node(builder, [value1]) node.depends([node3]) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) finally: vfile.close() finally: for tmp_file in tmp_files: try: os.remove(tmp_file) except OSError: pass
def test_node_value(self): with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: value1 = SimpleEntity( "http://aql.org/download1", name="target_url1") value2 = SimpleEntity( "http://aql.org/download2", name="target_url2") value3 = SimpleEntity( "http://aql.org/download3", name="target_url3") dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value") dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value") options = builtin_options() builder = ChecksumBuilder(options) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value1) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual(vfile, False)) # ----------------------------------------------------------- node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) node.build() node.save(vfile) node = Node(builder, [value1, value2, value3]) node.depends(dep_value2) node.depends(NullEntity()) node.initiate() node.build_split(vfile, False) self.assertFalse(node.check_actual(vfile, False)) finally: vfile.close()
def test_bm_require_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node( SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node( SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31, )) node22 = Node( SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33, )) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33, )) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) # print( "node30: %s" % node30 ) # print( "node31: %s" % node31 ) # print( "node32: %s" % node32 ) # print( "node33: %s" % node33 ) # # print( "node20: %s" % node20 ) # print( "node21: %s" % node21 ) # print( "node22: %s" % node22 ) # print( "node23: %s" % node23 ) # print( "node24: %s" % node24 ) # # print( "node10: %s" % node10 ) # print( "node11: %s" % node11 ) bm.add((node10, node11)) bm.module_depends(node10, [node11]) _build(bm, jobs=4)
def test_bm_nodes(self): def _make_nodes(builder): node1 = Node(builder, value1) copy_node1 = Node(builder, node1) copy2_node1 = Node(builder, copy_node1) node2 = Node(builder, value2) node3 = Node(builder, value3) copy_node3 = Node(builder, node3) copy2_node3 = Node(builder, copy_node3) copy2_node3.depends([node1, copy_node1]) return node1, node2, node3, copy_node1,\ copy_node3, copy2_node1, copy2_node3 with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() value1 = SimpleEntity("http://aql.org/download1", name="target_url1") value2 = SimpleEntity("http://aql.org/download2", name="target_url2") value3 = SimpleEntity("http://aql.org/download3", name="target_url3") builder = CopyValueBuilder(options) bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 7) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 0) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.removed_nodes = 0 bm.clear() bm.close() self.assertEqual(self.removed_nodes, 7) # ---------------------------------------------------------- nodes = _make_nodes(builder) copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[copy_node3]) bm.close() self.assertEqual(self.built_nodes, 2) # ---------------------------------------------------------- nodes = _make_nodes(builder) node2 = nodes[1] copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3]) bm.close() self.assertEqual(self.built_nodes, 1)
def __init__(self, args=None): paths_type = value_list_type(UniqueList, AbsFilePath) strings_type = value_list_type(UniqueList, str) cli_options = ( CLIOption("-C", "--directory", "directory", AbsFilePath, '', "Change directory before reading the make files.", 'FILE PATH', cli_only=True), CLIOption("-f", "--makefile", "makefile", FilePath, 'make.aql', "Path to a make file.", 'FILE PATH', cli_only=True), CLIOption("-l", "--list-options", "list_options", bool, False, "List current options and exit."), CLIOption("-L", "--list-tool-options", "list_tool_options", strings_type, [], "List tool options and exit.", "TOOL_NAME", cli_only=True), CLIOption("-t", "--list-targets", "list_targets", bool, False, "List all available targets and exit.", cli_only=True), CLIOption("-c", "--config", "config", AbsFilePath, None, "The configuration file used to read CLI arguments.", cli_only=True), CLIOption("-R", "--clean", "clean", bool, False, "Cleans targets.", cli_only=True), CLIOption("-u", "--up", "search_up", bool, False, "Search up directory tree for a make file.", cli_only=True), CLIOption("-e", "--no-tool-errors", "no_tool_errors", bool, False, "Stop on any error during initialization of tools."), CLIOption("-I", "--tools-path", "tools_path", paths_type, [], "Path to tools and setup scripts.", 'FILE PATH, ...'), CLIOption("-k", "--keep-going", "keep_going", bool, False, "Keep going when some targets can't be built."), CLIOption("-j", "--jobs", "jobs", int, None, "Number of parallel jobs to process targets.", 'NUMBER'), CLIOption("-v", "--verbose", "verbose", bool, False, "Verbose mode."), CLIOption("-s", "--silent", "silent", bool, False, "Don't print any messages except warnings and errors."), CLIOption(None, "--no-output", "no_output", bool, False, "Don't print builder's output messages."), CLIOption(None, "--debug-memory", "debug_memory", bool, False, "Display memory usage."), CLIOption( "-P", "--debug-profile", "debug_profile", AbsFilePath, None, "Run under profiler and save the results " "in the specified file.", 'FILE PATH'), CLIOption( "-T", "--debug-profile-top", "debug_profile_top", int, 30, "Show the specified number of top functions " "from profiler report.", 'FILE PATH'), CLIOption(None, "--debug-explain", "debug_explain", bool, False, "Show the reasons why targets are being rebuilt"), CLIOption(None, "--debug-exec", "debug_exec", bool, False, "Full trace of all executed commands."), CLIOption("--bt", "--debug-backtrace", "debug_backtrace", bool, False, "Show call stack back traces for errors."), CLIOption(None, "--force-lock", "force_lock", bool, False, "Forces to lock AQL DB file.", cli_only=True), CLIOption(None, "--use-sqlite", "use_sqlite", bool, False, "Use SQLite DB."), CLIOption("-V", "--version", "version", bool, False, "Show version and exit.", cli_only=True), ) cli_config = CLIConfig(cli_options, args) options = builtin_options() # ----------------------------------------------------------- # Add tools path # Read a config file from user's home user_config = os.path.join(_get_user_config_dir(), 'default.cfg') if os.path.isfile(user_config): _read_config(user_config, cli_config, options) # ----------------------------------------------------------- # Read a config file specified from CLI config = cli_config.config if config: _read_config(config, cli_config, options) # ----------------------------------------------------------- # Apply non-cli arguments to options arguments = {} ignore_options = set(ProjectConfig.__slots__) ignore_options.add('config') for name, value in cli_config.items(): if (name not in ignore_options) and (value is not None): arguments[name] = value options.update(arguments) # ----------------------------------------------------------- self.options = options self.arguments = arguments self.directory = os.path.abspath(cli_config.directory) makefile = cli_config.makefile if makefile.find(os.path.sep) != -1: makefile = os.path.abspath(makefile) self.makefile = makefile self.search_up = cli_config.search_up self.tools_path = cli_config.tools_path self.default_tools_path = _get_default_tools_path() self.no_tool_errors = cli_config.no_tool_errors self.targets = cli_config.targets self.verbose = cli_config.verbose self.silent = cli_config.silent self.show_version = cli_config.version self.no_output = cli_config.no_output self.keep_going = cli_config.keep_going self.clean = cli_config.clean self.list_options = cli_config.list_options self.list_tool_options = cli_config.list_tool_options self.list_targets = cli_config.list_targets self.jobs = cli_config.jobs self.force_lock = cli_config.force_lock self.use_sqlite = cli_config.use_sqlite self.debug_profile = cli_config.debug_profile self.debug_profile_top = cli_config.debug_profile_top self.debug_memory = cli_config.debug_memory self.debug_explain = cli_config.debug_explain self.debug_backtrace = cli_config.debug_backtrace self.debug_exec = cli_config.debug_exec
def test_bm_nodes(self): def _make_nodes(builder): node1 = Node(builder, value1) copy_node1 = Node(builder, node1) copy2_node1 = Node(builder, copy_node1) node2 = Node(builder, value2) node3 = Node(builder, value3) copy_node3 = Node(builder, node3) copy2_node3 = Node(builder, copy_node3) copy2_node3.depends([node1, copy_node1]) return node1, node2, node3, copy_node1,\ copy_node3, copy2_node1, copy2_node3 with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() value1 = SimpleEntity( "http://aql.org/download1", name="target_url1") value2 = SimpleEntity( "http://aql.org/download2", name="target_url2") value3 = SimpleEntity( "http://aql.org/download3", name="target_url3") builder = CopyValueBuilder(options) bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 7) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.built_nodes = 0 bm.build(jobs=1, keep_going=False) bm.close() self.assertEqual(self.built_nodes, 0) # ---------------------------------------------------------- bm.add(_make_nodes(builder)) self.removed_nodes = 0 bm.clear() bm.close() self.assertEqual(self.removed_nodes, 7) # ---------------------------------------------------------- nodes = _make_nodes(builder) copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[copy_node3]) bm.close() self.assertEqual(self.built_nodes, 2) # ---------------------------------------------------------- nodes = _make_nodes(builder) node2 = nodes[1] copy_node3 = nodes[4] bm.add(nodes) self.built_nodes = 0 bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3]) bm.close() self.assertEqual(self.built_nodes, 1)
def test_bm_sync_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node( SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node( SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31, )) node22 = Node( SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33, )) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33, )) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) bm.add((node10, node11)) bm.sync((node10, node11), deep=True) _build(bm, jobs=4)
def __init__(self, args=None): paths_type = value_list_type(UniqueList, AbsFilePath) strings_type = value_list_type(UniqueList, str) cli_options = ( CLIOption("-C", "--directory", "directory", AbsFilePath, '', "Change directory before reading the make files.", 'FILE PATH', cli_only=True), CLIOption("-f", "--makefile", "makefile", FilePath, 'make.aql', "Path to a make file.", 'FILE PATH', cli_only=True), CLIOption("-l", "--list-options", "list_options", bool, False, "List current options and exit."), CLIOption("-L", "--list-tool-options", "list_tool_options", strings_type, [], "List tool options and exit.", "TOOL_NAME", cli_only=True), CLIOption("-t", "--list-targets", "list_targets", bool, False, "List all available targets and exit.", cli_only=True), CLIOption("-c", "--config", "config", AbsFilePath, None, "The configuration file used to read CLI arguments.", cli_only=True), CLIOption("-R", "--clean", "clean", bool, False, "Cleans targets.", cli_only=True), CLIOption("-u", "--up", "search_up", bool, False, "Search up directory tree for a make file.", cli_only=True), CLIOption("-e", "--no-tool-errors", "no_tool_errors", bool, False, "Stop on any error during initialization of tools."), CLIOption("-I", "--tools-path", "tools_path", paths_type, [], "Path to tools and setup scripts.", 'FILE PATH, ...'), CLIOption("-k", "--keep-going", "keep_going", bool, False, "Keep going when some targets can't be built."), CLIOption("-j", "--jobs", "jobs", int, None, "Number of parallel jobs to process targets.", 'NUMBER'), CLIOption("-v", "--verbose", "verbose", bool, False, "Verbose mode."), CLIOption("-s", "--silent", "silent", bool, False, "Don't print any messages except warnings and errors."), CLIOption(None, "--no-output", "no_output", bool, False, "Don't print builder's output messages."), CLIOption(None, "--debug-memory", "debug_memory", bool, False, "Display memory usage."), CLIOption("-P", "--debug-profile", "debug_profile", AbsFilePath, None, "Run under profiler and save the results " "in the specified file.", 'FILE PATH'), CLIOption("-T", "--debug-profile-top", "debug_profile_top", int, 30, "Show the specified number of top functions " "from profiler report.", 'FILE PATH'), CLIOption(None, "--debug-explain", "debug_explain", bool, False, "Show the reasons why targets are being rebuilt"), CLIOption(None, "--debug-exec", "debug_exec", bool, False, "Full trace of all executed commands."), CLIOption("--bt", "--debug-backtrace", "debug_backtrace", bool, False, "Show call stack back traces for errors."), CLIOption(None, "--force-lock", "force_lock", bool, False, "Forces to lock AQL DB file.", cli_only=True), CLIOption(None, "--use-sqlite", "use_sqlite", bool, False, "Use SQLite DB."), CLIOption("-V", "--version", "version", bool, False, "Show version and exit.", cli_only=True), ) cli_config = CLIConfig(cli_options, args) options = builtin_options() # ----------------------------------------------------------- # Add tools path # Read a config file from user's home user_config = os.path.join(_get_user_config_dir(), 'default.cfg') if os.path.isfile(user_config): _read_config(user_config, cli_config, options) # ----------------------------------------------------------- # Read a config file specified from CLI config = cli_config.config if config: _read_config(config, cli_config, options) # ----------------------------------------------------------- # Apply non-cli arguments to options arguments = {} ignore_options = set(ProjectConfig.__slots__) ignore_options.add('config') for name, value in cli_config.items(): if (name not in ignore_options) and (value is not None): arguments[name] = value options.update(arguments) # ----------------------------------------------------------- self.options = options self.arguments = arguments self.directory = os.path.abspath(cli_config.directory) makefile = cli_config.makefile if makefile.find(os.path.sep) != -1: makefile = os.path.abspath(makefile) self.makefile = makefile self.search_up = cli_config.search_up self.tools_path = cli_config.tools_path self.default_tools_path = _get_default_tools_path() self.no_tool_errors = cli_config.no_tool_errors self.targets = cli_config.targets self.verbose = cli_config.verbose self.silent = cli_config.silent self.show_version = cli_config.version self.no_output = cli_config.no_output self.keep_going = cli_config.keep_going self.clean = cli_config.clean self.list_options = cli_config.list_options self.list_tool_options = cli_config.list_tool_options self.list_targets = cli_config.list_targets self.jobs = cli_config.jobs self.force_lock = cli_config.force_lock self.use_sqlite = cli_config.use_sqlite self.debug_profile = cli_config.debug_profile self.debug_profile_top = cli_config.debug_profile_top self.debug_memory = cli_config.debug_memory self.debug_explain = cli_config.debug_explain self.debug_backtrace = cli_config.debug_backtrace self.debug_exec = cli_config.debug_exec
def test_node_file(self): try: tmp_files = [] with Tempfile() as tmp: vfile = EntitiesFile(tmp) try: with Tempfile(suffix='.1') as tmp1: with Tempfile(suffix='.2') as tmp2: value1 = FileChecksumEntity(tmp1) value2 = FileChecksumEntity(tmp2) options = builtin_options() builder = CopyBuilder(options, "tmp", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "i") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) builder = CopyBuilder(options, "ttt", "d") node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) tmp1.write(b'123') tmp1.flush() value1 = FileChecksumEntity(tmp1) node = self._rebuild_node(vfile, builder, [value1, value2], [], tmp_files) with Tempfile(suffix='.3') as tmp3: value3 = FileChecksumEntity(tmp3) node3 = self._rebuild_node( vfile, builder, [value3], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1, node3], [], tmp_files) builder3 = CopyBuilder(options, "xxx", "3") node3 = self._rebuild_node( vfile, builder3, [value3], [], tmp_files) # node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx, # ,tmp3.3 node = self._rebuild_node( vfile, builder, [value1, node3], [], tmp_files) # node: CopyBuilder, tmp, i, tmp1, tmp3.xxx -> # tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) dep = SimpleEntity("1", name="dep1") node = self._rebuild_node( vfile, builder, [value1, node3], [dep], tmp_files) dep = SimpleEntity("11", name="dep1") node = self._rebuild_node( vfile, builder, [value1, node3], [dep], tmp_files) node3 = self._rebuild_node( vfile, builder3, [value1], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) node3 = self._rebuild_node( vfile, builder3, [value2], [], tmp_files) node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) node_tname = node.get_target_entities()[0].name with open(node_tname, 'wb') as f: f.write(b'333') f.flush() node = self._rebuild_node( vfile, builder, [value1], [node3], tmp_files) side_effects = node.get_side_effect_entities() with open(side_effects[0].name, 'wb') as f: f.write(b'abc') f.flush() node = Node(builder, [value1]) node.depends([node3]) node.initiate() node.build_split(vfile, False) self.assertTrue(node.check_actual( vfile, False)) finally: vfile.close() finally: for tmp_file in tmp_files: try: os.remove(tmp_file) except OSError: pass
def test_bm_require_modules(self): with Tempdir() as tmp_dir: options = builtin_options() options.build_dir = tmp_dir bm = BuildManager() self.built_nodes = 0 """ 10 11__ / | \ / \ \ 20 21 22 23 24 / \ | / \ \ | 30 31 32 33 """ node30 = Node(SyncValueBuilder(options, name="30", number=7), SimpleEntity("30")) node31 = Node(SyncValueBuilder(options, name="31", number=0, sleep_interval=0), SimpleEntity("31")) node32 = Node(SyncValueBuilder(options, name="32", number=0, sleep_interval=0), SimpleEntity("32")) node33 = Node(SyncValueBuilder(options, name="33", number=17), SimpleEntity("33")) node20 = Node(SyncValueBuilder(options, name="20", number=7), (node30, node31)) node21 = Node(SyncValueBuilder(options, name="21", number=7), (node31,)) node22 = Node(SyncValueBuilder(options, name="22", number=0, sleep_interval=5), (node31, node32)) node23 = Node(SyncValueBuilder(options, name="23", number=17), (node33,)) node24 = Node(SyncValueBuilder(options, name="24", number=17), (node33,)) node10 = Node(SyncValueBuilder(options, name="10", number=7), (node20, node21, node22)) node11 = Node(SyncValueBuilder(options, name="11", number=17), (node22, node23, node24)) # print( "node30: %s" % node30 ) # print( "node31: %s" % node31 ) # print( "node32: %s" % node32 ) # print( "node33: %s" % node33 ) # # print( "node20: %s" % node20 ) # print( "node21: %s" % node21 ) # print( "node22: %s" % node22 ) # print( "node23: %s" % node23 ) # print( "node24: %s" % node24 ) # # print( "node10: %s" % node10 ) # print( "node11: %s" % node11 ) bm.add((node10, node11)) bm.module_depends(node10, [node11]) _build(bm, jobs=4)