def main():
"""
runs experiment in different settings
"""
#data = ["cancer", "toy", "landmine", "mnist"]
#compare_solvers(500, "splicing")
#compare_solvers("cancer", 100)
#compare_solvers("landmine", 30000)
#compare_solvers("mnist", 30000)
#compare_solvers("toy", 10000)
#args = [["mnist", 30000],["toy", 10000]]
ar = []
#ar.append({"data_name": "mnist", "min_interval": 1000})
ar.append({"data_name": "toy", "min_interval": 1000})
#ar.append({"data_name": "landmine", "min_interval": 300})
#ar.append({"data_name": "cancer", "min_interval": 100})
local = True
max_num_threads = 1
pythongrid.pg_map(compare_solvers,
ar, {},
local,
max_num_threads,
mem="18G")
def runExample():
"""
execute map example
"""
args = [1, 2, 4, 8, 16]
local = True
max_num_threads = 3
param = {"h_vmem": "1G"}
intermediate_results = pg_map(computeFactorial, args, param, local, max_num_threads)
print "reducing result"
for i, ret in enumerate(intermediate_results):
print "f(%i) = %i" % (args[i], ret)
def train(tax, tax_name, num_train):
"""
hierarchical training procedure
"""
datadir = "../../data"
num_vald = num_train*3
params_c = "[1 10 100 250]"
params_b = "[0 0.25 0.5 0.75 1.0]"
# enqueue root node
grey_nodes = [tax]
#####################################################
# top-down processing of taxonomy #
#####################################################
leaf_cmds = []
while len(grey_nodes)>0:
node = grey_nodes.pop(0) # pop first item
# enqueue children
if node.children != None:
grey_nodes.extend(node.children)
# get data below current node
data_keys = node.get_data_keys()
# create org list in matlab format
org_list = str(data_keys).replace("[", "{").replace("]", "}")
# set up presvm
if node.is_root():
# no parent at root node
parent_model = "none"
else:
# regularize against parent predictor
parent_model = "../../out/%s/%s" % (tax_name, node.parent.name)
# determine current output directory
current_out = "../out/%s/%s" % (tax_name, node.name)
# create dirs if not present
if not os.path.exists(current_out):
#Super-mkdir; create a leaf directory and all intermediate ones.
print "creating dir %s" % (current_out)
os.makedirs(current_out)
# construct commands
matlab_call_train = "matlab -nojvm -nosplash -nodesktop -r \"train_orgs('%s',%s,'%s',%i,%i,%s,%s,'%s/evaluation.mat'), exit;\" " % (datadir, org_list, current_out, num_train, num_vald, params_c, params_b, parent_model)
matlab_call_eval = "matlab -nojvm -nosplash -nodesktop -r \"evaluate_result('%s'), exit;\" " % ("../" + current_out)
# example call:
# matlab -nojvm -nosplash -nodesktop -r train_orgs('../../data',{'Bacillus_anthracis_Ames_uid57909', 'Bacillus_subtilis_168_uid57675'},'../out/tax1/bacillus',10,50,[0.1 1 10],[0 0.1 0.5 0.8 0.9],'../../out/All_Tasks/evaluation.mat'), exit;
cmd_tuple = matlab_call_train, matlab_call_eval
# execute processing at inner nodes, enqueue for leaves
if node.is_leaf():
leaf_cmds.append(cmd_tuple)
else:
system_call.call2(cmd_tuple)
# execute leaf processing in parallel
result = pg.pg_map(system_call.call2, leaf_cmds, local=True, maxNumThreads=4, mem="16G")