def main():
""" The main function
Runs the selection pipeline.
"""
args = parse_arguments()
config = parse_config(args)
set_environment(config['environment'])
logging.basicConfig(format='%(levelname)s %(asctime)s %(message)s',
filename=args.log_file,
filemode='w',
level=args.level)
check_paths(config)
try:
cores = int(config['system']['cores_avaliable'])
except:
logging.error("Could not read cores_avaliable from the config file")
# TODO - add real error messages
sys.exit(1)
logging.info("Creating Job Queue")
job_queue = JobQueue(cores)
logging.info("Success")
ngadnap_graph = CreateNGaDNAPGraph(args=args,
config=config,
job_queue=job_queue)
job_queue.set_command_graph(ngadnap_graph.command_graph)
ngadnap_graph.populate()
ngadnap_graph.run()
print("NGaDNAP ran successfully")
def main():
""" The main function
Runs the selection pipeline.
"""
args = parse_arguments()
config = parse_config(args)
set_environment(config['environment'])
logging.basicConfig(format='%(levelname)s %(asctime)s %(message)s',
filename=args.log_file, filemode='w',
level=args.level)
check_paths(config)
try:
cores = int(config['system']['cores_avaliable'])
except:
logging.error("Could not read cores_avaliable from the config file")
# TODO - add real error messages
sys.exit(1)
logging.info("Creating Job Queue")
job_queue = JobQueue(cores)
logging.info("Success")
ngadnap_graph = CreateNGaDNAPGraph(args=args, config=config, job_queue=job_queue)
job_queue.set_command_graph(ngadnap_graph.command_graph)
ngadnap_graph.populate()
ngadnap_graph.run()
print("NGaDNAP ran successfully")
def test_run_multicore_job(self):
one_hundred_jobs = []
for i in range(10):
one_hundred_jobs.append(CommandNode("sleep 1", str(i)))
com = self._create_simple_job()
q = JobQueue(10)
for job in one_hundred_jobs:
q.add_job(job)
q.join()
def test_run_save_stdout(self):
os.mkdir("test_dir")
c = CommandNode("echo hello", "1", stdout="test.txt", working_dir="test_dir")
q = JobQueue(1)
q.add_job(c)
q.join()
f = open("test_dir/test.txt").read().strip()
assert f == "hello"
os.remove("test_dir/test.txt")
os.rmdir("test_dir")
def test_basic_depends_on(self):
q = JobQueue(1)
g = CommandGraph(q)
q.set_command_graph(g)
c1 = CommandNode("sleep 1", "1")
c2 = CommandNode("echo hello", "2", stdout="test.txt")
g.add_node(command_node=c1, depends_on=[c2])
assert set(g.nodes()) == set(['1', '2'])
g.start()
g.finish_block()
os.remove('test.txt')
def test_basic_depends_on(self):
q = JobQueue(1)
g = CommandGraph(q)
q.set_command_graph(g)
c1 = CommandNode("sleep 1", "1")
c2 = CommandNode("echo hello", "2", stdout="test.txt")
g.add_node(command_node=c1, depends_on=[c2])
assert set(g.nodes())== set(['1', '2'])
g.start()
g.finish_block()
os.remove('test.txt')
def test_two_depends_on(self):
q = JobQueue(1)
g = CommandGraph(q)
q.set_command_graph(g)
c1 = CommandNode("sleep 1", "1")
c2 = CommandNode("echo hello", "2", stdout="test.txt")
c3 = CommandNode("echo hello", "3", stdout="test2.txt")
g.add_node(command_node=c1, depends_on=[c3, c2])
g.add_node(command_node=c3, depends_on=[c2])
g.start()
g.finish_block()
os.remove("test.txt")
os.remove("test2.txt")
def test_two_depends_on(self):
q = JobQueue(1)
g = CommandGraph(q)
q.set_command_graph(g)
c1 = CommandNode("sleep 1", "1")
c2 = CommandNode("echo hello", "2", stdout="test.txt")
c3 = CommandNode("echo hello", "3", stdout="test2.txt")
g.add_node(command_node=c1, depends_on=[c3,c2])
g.add_node(command_node=c3, depends_on=[c2])
g.start()
g.finish_block()
os.remove("test.txt")
os.remove("test2.txt")
def test_create_graph(self):
q = JobQueue(1)
g = CommandGraph(q)
g.add("n1", "n2")
g.add("n1", "n4")
g.add("n2", "n3")
assert (set(g._graph.keys()) == set(["n1", "n2"]))
g.remove("n1")
assert (set(g._graph.keys()) == set(["n2"]))
g.add("n1", "n4")
g.remove("n4")
assert (g._graph['n1'] == set([]))
assert (g.get_adjacent('n1') == set([]))
assert (g.get_adjacent('n2') == set(["n3"]))
def test_run_simple_job(self):
com = self._create_simple_job()
q = JobQueue(1)
q.add_job(com)
q.join()
