def run_exp(args): # command expansion must be done in two phases: # first, inputs are expanded and indentified # then, the experimental hash is computed and substituted # if arguments are partially specified, fill them in from previous # runs args = fill_last_args(args) # now fill in any missing arguments that have defaults args = fill_defaults(args) # now make sure we have all the necessary arguments check_args(args) # find out the hash of experimental commit hsh = util.exec_output(['git', 'rev-parse', args.commit]).strip() # parse parameters from command line params = parse_params(args.params) job = dag.dag_node(args.description, params, hsh, args.command, rerun = args.rerun, subdir_only = args.subdir_only) jobs = dag.dag([job,]) lb = local_backend.local_backend() jobs.backend = lb jobs.mainloop()
def aggregate_dag(self): assert(False) res = dag() for (n, l) in group_vars([v.term for v in self.vertices.values()]): nv = self.vtype(n) nv.pos = list(l) res.add_vertex(nv) for ((s, e), ev) in self.edges.items(): ne = edge(var_key_noarray(s), var_key_noarray(e)) ne.raps = [ev.rap] if ne.key() not in res.edges: res.add_edge(ne) else: res.edges[ne.key()].raps += ne.raps return res
def run(): toplevel_nodes = [] for nodeGroup in nodes: for node in nodes[nodeGroup]: if not node.parents: toplevel_nodes += [node] mydag = dag.dag(toplevel_nodes) mydag.backend = local_backend.local_backend() status = mydag.mainloop() if status == dag.RUN_STATE_SUCCESS: print "Task completed successfully." elif status == dag.RUN_STATE_FAIL: print "Task failed!" else: print "Unrecognized exit status"
def connected_components(self): """ Return a list of connected components ordered by split links""" comps = super(inference_dag, self).connected_components() # Build a dag representing the relationship between the connected components. comp_dag = dag() for d in range(0, len(comps)): comp_dag.add_vertex(self.vtype(d)) isplit = [set(var_key_noarray(v).replace("_inref__split_", "") for v in comps[d].roots() if "_split_" in var_key_noarray(v)) for d in range(0, len(comps))] osplit = [set(var_key_noarray(v).replace("_outref__split_", "") for v in comps[d].leaves() if "_split_" in var_key_noarray(v)) for d in range(0, len(comps))] for d1 in range(0, len(comps)): for d2 in range(d1 + 1, len(comps)): if len(osplit[d1] & isplit[d2]) > 0: comp_dag.add_edge(edge(d1, d2)) if len(osplit[d2] & isplit[d1]) > 0: comp_dag.add_edge(edge(d2, d1)) # Order the components based on a topological sort of the component dag. return [comps[d] for d in comp_dag.topo_sort()]
def __init__(self, value): self.value = value isGit = os.environ["TEST_SEARCHER"] == "git" print isGit try: ancestryFilename = os.environ["TEST_ANCESTRY"] testDir = os.environ["TEST_DIR"] loc = int(os.environ["TEST_LOC"]) f = file(ancestryFilename, "r") (identifiers, parents) = dagRead.read(f) f.close() N = len(parents) thisDag = dag.dag(parents, N) t = testDetector.detect(N, 1.0, thisDag, False, loc) numbers = {} for i in range(len(identifiers)): numbers[identifiers[i]] = i if isGit: os.system("git rev-parse HEAD >$TEST_DIR/__head__") where = open(testDir + "/__head__", "r").readline().strip() where = numbers[where] else: where = sys.argv[1] where = numbers[where] os.system("echo 'foo' >>$TEST_DIR/tries")
#!/usr/bin/env python import dag, util, local_backend import time # this file is just for testing; if a user actually wanted to run a # job they'd use exp.py. hsh = util.exec_output(['git', 'rev-parse', 'HEAD']).strip() test_node = dag.dag_node(desc = "testscript", commit = hsh, command = "./test.sh") test_node2=dag.dag_node(desc = "testscript2", commit = hsh, command = "./test2.sh {testscript}/log") test_node2.add_parents(set([test_node,])); test_dag = dag.dag([test_node,]) lb = local_backend.local_backend() test_dag.backend = lb test_dag.mainloop()
def __init__(self, value): self.value = value isGit = os.environ['TEST_SEARCHER']=='git' print isGit try: ancestryFilename=os.environ['TEST_ANCESTRY'] testDir = os.environ["TEST_DIR"] loc=int(os.environ['TEST_LOC']) f=file(ancestryFilename,"r") (identifiers,parents)=dagRead.read(f) f.close() N=len(parents) thisDag=dag.dag(parents,N) t=testDetector.detect(N,1.0,thisDag,False,loc) numbers={} for i in range(len(identifiers)): numbers[identifiers[i]]=i if isGit: os.system("git rev-parse HEAD >$TEST_DIR/__head__") where=open(testDir+"/__head__","r").readline().strip() where=numbers[where] else: where = sys.argv[1] where=numbers[where] os.system("echo 'foo' >>$TEST_DIR/tries")