def check_parameters(parameters_searched, parameters_to_search, desc, commit, command, dependencies):
this_parameters_to_search = copy.deepcopy(parameters_to_search)
if len(parameters_to_search) != 0:
#cur_params_searched = parameters_searched.copy()
#cur_params_to_search = parameters_to_search.copy()
#for key in parameters_to_search:
(key,value) = this_parameters_to_search.popitem()
if isinstance(value, list):
#Need to create at least one node for each element in list. May have to create more if other
#parameters are also lists.
#print("value")
#print(value)
for l in value:
#print("l = " + str(l))
this_parameters_searched = copy.deepcopy(parameters_searched)
this_parameters_searched[key] = l
#print cur_params_searched
#print cur_params_to_search
check_parameters(this_parameters_searched, this_parameters_to_search, desc,commit,command,dependencies)
else:
this_parameters_searched = copy.deepcopy(parameters_searched)
this_parameters_searched[key] = value
check_parameters(this_parameters_searched, this_parameters_to_search, desc,commit,command,dependencies)
else:
# print("ready to make node, parameters = ")
# print(parameters_searched)
#if command starts with @, it is a macro
if(command[0]=='@'):
code=command[1:]
newNode = dag.dag_node(desc,parameters_searched,commit,None, code, dependencies)
else:
newNode = dag.dag_node(desc,parameters_searched,commit,command, None, dependencies)
# print("newNode = " )
# print newNode
# print("parameters = ")
# print(newNode.params)
if desc in nodes:
nodes[desc].add(newNode)
else:
nodes[desc] = Set()
nodes[desc].add(newNode)
#Add to all nodes in dependencies that they have a child nodes[desc]
#print("dependencies = ")
#print dependencies
newNodeSet = Set()
newNodeSet.add(newNode)
if dependencies != None:
for d in dependencies:
d.add_children(newNodeSet)
def run_exp(args):
# command expansion must be done in two phases:
# first, inputs are expanded and indentified
# then, the experimental hash is computed and substituted
# if arguments are partially specified, fill them in from previous
# runs
args = fill_last_args(args)
# now fill in any missing arguments that have defaults
args = fill_defaults(args)
# now make sure we have all the necessary arguments
check_args(args)
# find out the hash of experimental commit
hsh = util.exec_output(['git', 'rev-parse', args.commit]).strip()
# parse parameters from command line
params = parse_params(args.params)
job = dag.dag_node(args.description, params, hsh, args.command, rerun = args.rerun, subdir_only = args.subdir_only)
jobs = dag.dag([job,])
lb = local_backend.local_backend()
jobs.backend = lb
jobs.mainloop()
def read_descrs(keep_unreadable=False, keep_unfinished=False, keep_failed=False,
keep_broken_deps=False):
resultsdir = os.path.join(util.abs_root_path(), RESULTS_DIR)
try:
exp_dirs = os.listdir(resultsdir)
except OSError:
exp_dirs = []
exps = []
for exp_dir in exp_dirs:
exp = dag.dag_node(hsh = exp_dir)
if (exp.success() or
(exp.failure() and keep_failed) or
keep_unfinished):
if keep_broken_deps or not exp.broken_deps():
exps.append(exp)
sys.stderr.write('Finished reading descriptions...\n')
return exps
#!/usr/bin/env python
import dag, util, local_backend
import time
# this file is just for testing; if a user actually wanted to run a
# job they'd use exp.py.
hsh = util.exec_output(['git', 'rev-parse', 'HEAD']).strip()
test_node = dag.dag_node(desc = "testscript", commit = hsh, command = "./test.sh")
test_node2=dag.dag_node(desc = "testscript2", commit = hsh, command = "./test2.sh {testscript}/log")
test_node2.add_parents(set([test_node,]));
test_dag = dag.dag([test_node,])
lb = local_backend.local_backend()
test_dag.backend = lb
test_dag.mainloop()