def main(): """ The main function Runs the selection pipeline. """ args = parse_arguments() config = parse_config(args) set_environment(config['environment']) logging.basicConfig(format='%(levelname)s %(asctime)s %(message)s', filename=args.log_file, filemode='w', level=args.level) check_paths(config) try: cores = int(config['system']['cores_avaliable']) except: logging.error("Could not read cores_avaliable from the config file") # TODO - add real error messages sys.exit(1) logging.info("Creating Job Queue") job_queue = JobQueue(cores) logging.info("Success") ngadnap_graph = CreateNGaDNAPGraph(args=args, config=config, job_queue=job_queue) job_queue.set_command_graph(ngadnap_graph.command_graph) ngadnap_graph.populate() ngadnap_graph.run() print("NGaDNAP ran successfully")
def test_run_multicore_job(self): one_hundred_jobs = [] for i in range(10): one_hundred_jobs.append(CommandNode("sleep 1", str(i))) com = self._create_simple_job() q = JobQueue(10) for job in one_hundred_jobs: q.add_job(job) q.join()
def test_run_save_stdout(self): os.mkdir("test_dir") c = CommandNode("echo hello", "1", stdout="test.txt", working_dir="test_dir") q = JobQueue(1) q.add_job(c) q.join() f = open("test_dir/test.txt").read().strip() assert f == "hello" os.remove("test_dir/test.txt") os.rmdir("test_dir")
def test_basic_depends_on(self): q = JobQueue(1) g = CommandGraph(q) q.set_command_graph(g) c1 = CommandNode("sleep 1", "1") c2 = CommandNode("echo hello", "2", stdout="test.txt") g.add_node(command_node=c1, depends_on=[c2]) assert set(g.nodes()) == set(['1', '2']) g.start() g.finish_block() os.remove('test.txt')
def test_basic_depends_on(self): q = JobQueue(1) g = CommandGraph(q) q.set_command_graph(g) c1 = CommandNode("sleep 1", "1") c2 = CommandNode("echo hello", "2", stdout="test.txt") g.add_node(command_node=c1, depends_on=[c2]) assert set(g.nodes())== set(['1', '2']) g.start() g.finish_block() os.remove('test.txt')
def test_two_depends_on(self): q = JobQueue(1) g = CommandGraph(q) q.set_command_graph(g) c1 = CommandNode("sleep 1", "1") c2 = CommandNode("echo hello", "2", stdout="test.txt") c3 = CommandNode("echo hello", "3", stdout="test2.txt") g.add_node(command_node=c1, depends_on=[c3, c2]) g.add_node(command_node=c3, depends_on=[c2]) g.start() g.finish_block() os.remove("test.txt") os.remove("test2.txt")
def test_two_depends_on(self): q = JobQueue(1) g = CommandGraph(q) q.set_command_graph(g) c1 = CommandNode("sleep 1", "1") c2 = CommandNode("echo hello", "2", stdout="test.txt") c3 = CommandNode("echo hello", "3", stdout="test2.txt") g.add_node(command_node=c1, depends_on=[c3,c2]) g.add_node(command_node=c3, depends_on=[c2]) g.start() g.finish_block() os.remove("test.txt") os.remove("test2.txt")
def test_create_graph(self): q = JobQueue(1) g = CommandGraph(q) g.add("n1", "n2") g.add("n1", "n4") g.add("n2", "n3") assert (set(g._graph.keys()) == set(["n1", "n2"])) g.remove("n1") assert (set(g._graph.keys()) == set(["n2"])) g.add("n1", "n4") g.remove("n4") assert (g._graph['n1'] == set([])) assert (g.get_adjacent('n1') == set([])) assert (g.get_adjacent('n2') == set(["n3"]))
def test_run_simple_job(self): com = self._create_simple_job() q = JobQueue(1) q.add_job(com) q.join()