def qsub_cmd_gen(template, job_name, i, sid, fODF, im, data_path, cmd_file_path=cmd_file_path, python_path=python_path, bashcmd=bashcmd, mem=25): reload(template) template = sge.getsourcelines(template)[0] # Name the job and generate the parameters for each job if job_name[0:2] != "im": params_dict = dict(i=i, sid=sid, fODF=fODF, im=im, data_path=data_path) name = '%s_%s_%s%s'%(job_name,fODF,shorthand_im,i) else: params_dict = dict(i=i, sid=sid, im=im, data_path=data_path) name = '%s_%s%s'%(job_name,shorthand_im,i) code = sge.add_params(template,params_dict) cmd_file = os.path.join(cmd_file_path, '%s.py'%name) print("Generating: %s"%cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python=python_path) cmd_file = os.path.join(cmd_file_path, '%s.py'%name) batch_sge.append(sge.qsub_cmd( '%s %s'%(bashcmd, cmd_file), name, mem_usage=mem))
from osmosis.parallel import sge import osmosis.parallel.kfold_xval_precision_template as mb_template reload(mb_template) template = sge.getsourcelines(mb_template)[0] ssh = sge.SSH(hostname='proclus.stanford.edu', username='******', port=22) batch_sge = [] for i in np.arange(65): params_dict = dict(i=i) code = sge.add_params(template, params_dict) name = 'kfold_xval_sph_cc_m%s' % (i) cmd_file = '/home/klchan13/pycmd/%s.py' % name print("Generating: %s" % cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/klchan13/anaconda/bin/python') cmd_file = '/home/klchan13/pycmd/%s.py' % name batch_sge.append( sge.qsub_cmd('/home/klchan13/bashcmd.sh %s' % cmd_file, name, email='*****@*****.**')) # Add some header stuff: batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
params_file=params_file) code = sge.add_params(template, params_dict) name = 'ssd_%s_b%s_data%s_l1ratio%s_alpha%s_%03d' % ( subject, b, data_i + 1, l1_ratio, alpha, i) cmd_file = '/home/arokem/pycmd/%s.py' % name print("Generating: %s" % cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/arokem/anaconda/bin/python') cmd_file = '/home/arokem/pycmd/%s.py' % name batch_sge.append( sge.qsub_cmd( '/home/arokem/bashcmd.sh %s' % cmd_file, name)) # Add some header stuff: #batch_sge = ['export PATH=$PATH:/hsgs/software/oge2011.11p1/bin/linux-x64/'] + batch_sge #batch_sge = ['export SGE_ROOT=/hsgs/software/oge2011.11p1'] + batch_sge batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/arokem/batch_sge.sh') #stat = os.system('scp -c blowfish -C tmp/* %s:~/pycmd/.'%ssh.hostname) #if stat != 0: # print "what what!" #ssh.exec_command('./batch_sge.sh') #ssh.disconnect()
from osmosis.parallel import sge import osmosis.parallel.mb_predict_template_smm as mb_template reload(mb_template) template = sge.getsourcelines(mb_template)[0] ssh = sge.SSH(hostname='proclus.stanford.edu',username='******', port=22) batch_sge = [] for i in range(65): params_dict = dict(i=i) code = sge.add_params(template,params_dict) name = 'multi_bi_exp_rs%s'%(i) cmd_file = '/home/klchan13/pycmd/%s.py'%name print("Generating: %s"%cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/klchan13/anaconda/bin/python') cmd_file = '/home/klchan13/pycmd/%s.py'%name batch_sge.append(sge.qsub_cmd( '/home/klchan13/bashcmd.sh %s'%cmd_file,name)) # Add some header stuff: batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
wm_file=wm_file, params_file = params_file) code = sge.add_params(template,params_dict) name = 'ssd_%s_b%s_data%s_l1ratio%s_alpha%s_%03d'%( subject, b, data_i+1, l1_ratio, alpha, i) cmd_file = '/home/arokem/pycmd/%s.py'%name print("Generating: %s"%cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/arokem/anaconda/bin/python') cmd_file = '/home/arokem/pycmd/%s.py'%name batch_sge.append(sge.qsub_cmd( '/home/arokem/bashcmd.sh %s'%cmd_file,name)) # Add some header stuff: #batch_sge = ['export PATH=$PATH:/hsgs/software/oge2011.11p1/bin/linux-x64/'] + batch_sge #batch_sge = ['export SGE_ROOT=/hsgs/software/oge2011.11p1'] + batch_sge batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/arokem/batch_sge.sh') #stat = os.system('scp -c blowfish -C tmp/* %s:~/pycmd/.'%ssh.hostname) #if stat != 0: # print "what what!" #ssh.exec_command('./batch_sge.sh') #ssh.disconnect()
from osmosis.parallel import sge import osmosis.parallel.multib_template as mb_template reload(mb_template) template = sge.getsourcelines(mb_template)[0] ssh = sge.SSH(hostname='proclus.stanford.edu',username='******', port=22) batch_sge = [] #for i in range(1): #params_dict = dict(i=i) #code = sge.add_params(template,params_dict) name = 'mb_model_params' cmd_file = '/home/klchan13/pycmd/%s.py'%name print("Generating: %s"%cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/klchan13/anaconda/bin/python') cmd_file = '/home/klchan13/pycmd/%s.py'%name batch_sge.append(sge.qsub_cmd( '/home/klchan13/bashcmd.sh %s'%cmd_file,name)) # Add some header stuff: batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
import numpy as np import osmosis.io as oio from osmosis.parallel import sge import osmosis.parallel.mb_multi_model_params_template as mb_template reload(mb_template) template = sge.getsourcelines(mb_template)[0] ssh = sge.SSH(hostname='proclus.stanford.edu', username='******', port=22) batch_sge = [] for i in range(65): params_dict = dict(i=i) code = sge.add_params(template, params_dict) name = 'mp_multi_se%s' % (i) cmd_file = '/home/klchan13/pycmd/%s.py' % name print("Generating: %s" % cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/klchan13/anaconda/bin/python') cmd_file = '/home/klchan13/pycmd/%s.py' % name batch_sge.append( sge.qsub_cmd('/home/klchan13/bashcmd.sh %s' % cmd_file, name)) # Add some header stuff: batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')
import osmosis.parallel.kfold_xval_precision_template as mb_template reload(mb_template) template = sge.getsourcelines(mb_template)[0] ssh = sge.SSH(hostname='proclus.stanford.edu',username='******', port=22) batch_sge = [] for i in np.arange(65): params_dict = dict(i=i) code = sge.add_params(template,params_dict) name = 'kfold_xval_sph_cc_m%s'%(i) cmd_file = '/home/klchan13/pycmd/%s.py'%name print("Generating: %s"%cmd_file) sge.py_cmd(ssh, code, file_name=cmd_file, python='/home/klchan13/anaconda/bin/python') cmd_file = '/home/klchan13/pycmd/%s.py'%name batch_sge.append(sge.qsub_cmd( '/home/klchan13/bashcmd.sh %s'%cmd_file,name, email = '*****@*****.**')) # Add some header stuff: batch_sge = ['#!/bin/bash'] + batch_sge sge.write_file_ssh(ssh, batch_sge, '/home/klchan13/batch_sge.sh')