from osmosis.parallel import sge
import osmosis.parallel.kfold_xval_precision_template as mb_template
reload(mb_template)
template = sge.getsourcelines(mb_template)[0]
ssh = sge.SSH(hostname='proclus.stanford.edu', username='******', port=22)
batch_sge = []
for i in np.arange(65):
params_dict = dict(i=i)
code = sge.add_params(template, params_dict)
name = 'kfold_xval_sph_cc_m%s' % (i)
cmd_file = '/home/klchan13/pycmd/%s.py' % name
print("Generating: %s" % cmd_file)
sge.py_cmd(ssh,
code,
file_name=cmd_file,
python='/home/klchan13/anaconda/bin/python')
cmd_file = '/home/klchan13/pycmd/%s.py' % name
batch_sge.append(
sge.qsub_cmd('/home/klchan13/bashcmd.sh %s' % cmd_file,
name,
email='*****@*****.**'))
# Add some header stuff:
batch_sge = ['#!/bin/bash'] + batch_sge
sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
shorthand_im))
other_file_names.append("sfm_cod_%s_%s"%(fODF,
shorthand_im))
# Model Parameters
for i in np.arange(others_file_num):
import osmosis.parallel.model_params_template as template
qsub_cmd_gen(template, 'sfm_mp_%s'%sid, i, sid, fODF, im,
data_path, mem=25)
if (sid_idx == 0) & (i == 0):
other_file_names.append("model_params_%s_%s"%(fODF,
shorthand_im))
# Add some header stuff:
batch_sge = ['#!/bin/bash'] + batch_sge
sge.write_file_ssh(ssh, batch_sge, batch_sge_file)
# Read batch_sge.sh and get the individual qsub commands.
batch_sge = sp.check_output(["cat", "batch_sge.sh"])
cmd_line_split = batch_sge.split('\n')
# Check to see if the output files from each command exists already
# and eliminate them from the cmd_line
red_cmd_line = []
for cmd_idx in np.arange(1, len(cmd_line_split)-1):
pycmd = cmd_line_split[cmd_idx].split(' ')[2].split('_')
if pycmd[0] == "im":
sid_idx = len(pycmd) - 2
else:
sid_idx = len(pycmd) - 3
from osmosis.parallel import sge
import osmosis.parallel.mb_predict_template_smm as mb_template
reload(mb_template)
template = sge.getsourcelines(mb_template)[0]
ssh = sge.SSH(hostname='proclus.stanford.edu',username='******', port=22)
batch_sge = []
for i in range(65):
params_dict = dict(i=i)
code = sge.add_params(template,params_dict)
name = 'multi_bi_exp_rs%s'%(i)
cmd_file = '/home/klchan13/pycmd/%s.py'%name
print("Generating: %s"%cmd_file)
sge.py_cmd(ssh,
code,
file_name=cmd_file,
python='/home/klchan13/anaconda/bin/python')
cmd_file = '/home/klchan13/pycmd/%s.py'%name
batch_sge.append(sge.qsub_cmd(
'/home/klchan13/bashcmd.sh %s'%cmd_file,name))
# Add some header stuff:
batch_sge = ['#!/bin/bash'] + batch_sge
sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
params_file=params_file)
code = sge.add_params(template, params_dict)
name = 'ssd_%s_b%s_data%s_l1ratio%s_alpha%s_%03d' % (
subject, b, data_i + 1, l1_ratio, alpha, i)
cmd_file = '/home/arokem/pycmd/%s.py' % name
print("Generating: %s" % cmd_file)
sge.py_cmd(ssh,
code,
file_name=cmd_file,
python='/home/arokem/anaconda/bin/python')
cmd_file = '/home/arokem/pycmd/%s.py' % name
batch_sge.append(
sge.qsub_cmd(
'/home/arokem/bashcmd.sh %s' % cmd_file, name))
# Add some header stuff:
#batch_sge = ['export PATH=$PATH:/hsgs/software/oge2011.11p1/bin/linux-x64/'] + batch_sge
#batch_sge = ['export SGE_ROOT=/hsgs/software/oge2011.11p1'] + batch_sge
batch_sge = ['#!/bin/bash'] + batch_sge
sge.write_file_ssh(ssh, batch_sge, '/home/arokem/batch_sge.sh')
#stat = os.system('scp -c blowfish -C tmp/* %s:~/pycmd/.'%ssh.hostname)
#if stat != 0:
# print "what what!"
#ssh.exec_command('./batch_sge.sh')
#ssh.disconnect()
params_file = params_file)
code = sge.add_params(template,params_dict)
name = 'ssd_%s_b%s_data%s_l1ratio%s_alpha%s_%03d'%(
subject, b, data_i+1, l1_ratio, alpha, i)
cmd_file = '/home/arokem/pycmd/%s.py'%name
print("Generating: %s"%cmd_file)
sge.py_cmd(ssh,
code,
file_name=cmd_file,
python='/home/arokem/anaconda/bin/python')
cmd_file = '/home/arokem/pycmd/%s.py'%name
batch_sge.append(sge.qsub_cmd(
'/home/arokem/bashcmd.sh %s'%cmd_file,name))
# Add some header stuff:
#batch_sge = ['export PATH=$PATH:/hsgs/software/oge2011.11p1/bin/linux-x64/'] + batch_sge
#batch_sge = ['export SGE_ROOT=/hsgs/software/oge2011.11p1'] + batch_sge
batch_sge = ['#!/bin/bash'] + batch_sge
sge.write_file_ssh(ssh, batch_sge, '/home/arokem/batch_sge.sh')
#stat = os.system('scp -c blowfish -C tmp/* %s:~/pycmd/.'%ssh.hostname)
#if stat != 0:
# print "what what!"
#ssh.exec_command('./batch_sge.sh')
#ssh.disconnect()