def launch_many(run_id):
'''
Generate script paramaters and launch a bunch of bsub jobs.
Designed to be run on the cluster via an interactive shell.
Note: If this is not run on cluster, since it does not look
up a remote url for files, it won't be able to find expression
data.
'''
print 'Launching all jobs!'
#MAKE INPUTS
expr_filenames = ['soheil/expression_c4d_n4_tt_{0}.mat'.format(ttnum)
for ttnum in range(70)] + ['soheil/expression_c4d_n4_intercluster.mat']
urls = [ cfg.dataURL(f) for f in expr_filenames ]
remote_exprnames =[ cfg.dataPath(url) for url in urls ]
inp_dicts = [dict(out_iter_num = out_iter_num,
in_iter_num = in_iter_num,
k = k,
beta = beta,
f_mix = f_mix,
f_sim = f_sim,
f_en_in = f_en_in,
f_en_out = f_en_out,
th_cor = th_cor,
trunc_value = trunc_value,
degree_bound = degree_bound,
filename = filename)
for out_iter_num in array([25],double)
for in_iter_num in array([100],double)
for k in array([6],double)
for beta in array([4],double)
for f_mix in array([2],double)
for f_sim in array([.8],double)
for f_en_in in array([1.],double)
for f_en_out in array([1.],double)
for th_cor in array([.6],double)
for trunc_value in array([3],double)
for degree_bound in array([3],double)
for filename in remote_exprnames ]
#MAKE EYEBALL
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run_single',
name = 'mcmc_',
mem = 3)
#LAUNCH EYEBALL JOBS
eyeball.launch()
#RETURN A LIST OF LAUNCHED JOBS
return dict(cmds=eyeball.cmds,
inputs = inp_dicts)
def runmany(run_id):
print 'TESTING WITH A LIMITED RANGE OF FAMILIES'
inp_dicts = [dict([('ofs',r)]) for r in range(0,1493)]
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run',
name = 'ra2_runs_',
mem = 3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def runmany(run_id):
print 'TESTING WITH A LIMITED RANGE OF FAMILIES'
inp_dicts = [dict([('ofs', r)]) for r in range(0, 1493)]
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='run',
name='ra2_runs_',
mem=3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_riboswitches(run_id):
inp_dicts = []
for k,v in rutils.switch_dicts().iteritems():
rfid = 'RF{0:05}'.format(v)
inp_dicts.append(dict(family = rfid,
inp_run_id = 'RS_{0}'.format(rfid),
run_id = 'RS_tree_{0}'.format(rfid)))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run_treebuilder',
mem = 3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_all(run_id):
inp_dicts = []
for r in range(0,1493):
rfid = 'RF{0:05}'.format(r)
inp_dicts.append(dict(family = rfid,
inp_run_id = 'FA_{0}'.format(rfid),
run_id = 'FA_tree_{0}'.format(rfid)))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run_treebuilder',
mem = 3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_riboswitches(run_id):
inp_dicts = []
for k, v in rutils.switch_dicts().iteritems():
rfid = 'RF{0:05}'.format(v)
savename = 'Riboswitch_list_{1}_{0}'.format(rfid, k)
inp_dicts.append(
dict(family=rfid, run_id='RS_{0}'.format(rfid), savename=savename))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='run_structmaker',
mem=3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_all(run_id):
inp_dicts = []
for r in range(0, 1493):
rfid = 'RF{0:05}'.format(r)
savename = 'Family_list_{0}'.format(rfid)
inp_dicts.append(
dict(family=rfid, run_id='FA_{0}'.format(rfid), savename=savename))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='run_structmaker',
mem=3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_riboswitches(run_id):
inp_dicts = []
for k, v in rutils.switch_dicts().iteritems():
rfid = 'RF{0:05}'.format(v)
inp_dicts.append(
dict(family=rfid,
inp_run_id='RS_{0}'.format(rfid),
run_id='RS_tree_{0}'.format(rfid)))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='run_treebuilder',
mem=3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_all(run_id):
inp_dicts = []
for r in range(0, 1493):
rfid = 'RF{0:05}'.format(r)
inp_dicts.append(
dict(family=rfid,
inp_run_id='FA_{0}'.format(rfid),
run_id='FA_tree_{0}'.format(rfid)))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='run_treebuilder',
mem=3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_riboswitches(run_id):
inp_dicts = []
for k,v in rutils.switch_dicts().iteritems():
rfid = 'RF{0:05}'.format(v)
savename = 'Riboswitch_list_{1}_{0}'.format(rfid,k)
inp_dicts.append(dict(family = rfid,
run_id = 'RS_{0}'.format(rfid),
savename = savename))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run_structmaker',
mem = 3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def bsub_all(run_id):
inp_dicts = []
for r in range(0,1493):
rfid = 'RF{0:05}'.format(r)
savename = 'Family_list_{0}'.format(rfid)
inp_dicts.append(dict(family = rfid,
run_id = 'FA_{0}'.format(rfid),
savename = savename))
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func = 'run_structmaker',
mem = 3)
eyeball.launch()
return dict(cmds=eyeball.cmds)
def remote_make_tests(run_id):
'''
the idea is that this function will queue up the batch jobs
and submit them with bsub. Using eyeball, it will then wait
until all jobs are done and when they are, export output back to
gliese.
inputs:
run_id
output:
the datapath (same for local and remote) of data output from
threads.
'''
test = False
if test:
mirnaf = os.path.join(os.path.dirname(inspect.stack()[0][1]),
'miRNA.mat')
mirna = sio.loadmat(mirnaf)
expr = mirna['expression']
e_norms = sum(expr**2, 1)
cluster_dists = e_norms[:,newaxis] + e_norms[newaxis,:] \
- 2 * dot(expr, expr.T)
sims = -cluster_dists
inp_dicts = []
percentiles = logspace(-2, 1.99, 3)
for p in percentiles:
inp_dicts.append(
dict(similarities=sims,
self_similarity=percentile(sims.flatten(), p)))
else:
inp_dicts = butils.load_data(run_id, 'input')
eyeball = bsub.eyeball(run_id,
os.path.abspath(inspect.stack()[0][1]),
inp_dicts,
func='test_bsubfun',
name=run_id + '_test_',
mem=2)
eyeball.launch()
eyeball. await ()
eyeball.package()
eyeball.complete()
def remote_make_tests(run_id):
"""
the idea is that this function will queue up the batch jobs
and submit them with bsub. Using eyeball, it will then wait
until all jobs are done and when they are, export output back to
gliese.
inputs:
run_id
output:
the datapath (same for local and remote) of data output from
threads.
"""
test = False
if test:
mirnaf = os.path.join(os.path.dirname(inspect.stack()[0][1]), "miRNA.mat")
mirna = sio.loadmat(mirnaf)
expr = mirna["expression"]
e_norms = sum(expr ** 2, 1)
cluster_dists = e_norms[:, newaxis] + e_norms[newaxis, :] - 2 * dot(expr, expr.T)
sims = -cluster_dists
inp_dicts = []
percentiles = logspace(-2, 1.99, 3)
for p in percentiles:
inp_dicts.append(dict(similarities=sims, self_similarity=percentile(sims.flatten(), p)))
else:
inp_dicts = butils.load_data(run_id, "input")
eyeball = bsub.eyeball(
run_id, os.path.abspath(inspect.stack()[0][1]), inp_dicts, func="test_bsubfun", name=run_id + "_test_", mem=2
)
eyeball.launch()
eyeball.await()
eyeball.package()
eyeball.complete()
