class MakeGerald(CommandOp):
def __init__(self, logging):
#full path
self.logging = logging
self.transRsync = Transfer('rsync')
def set_params(self, run_folder, gerald_conf, dist_conf, dest, wall):
self.logging.debug("setting params")
if dest == 'OSC':
self.local = 0
self.gerald_pl = "/nfs/03/osu5422/CASAVA/CASAVA_v1.7.0-build/bin/GERALD.pl"
elif dest == 'P01':
self.local = 1
self.gerald_pl = 'GERALD.pl'
self.config_file = gerald_conf
self.runid = run_folder.split('_')[0]
self.ran_pipeline = 0
self.pbs_path = '/nfs/03/osu5422/pbs_scripts/'
self.options = "--EXPT_DIR"
self.root_dir = "/slxdata/solexadata/"
self.clust_server = 'glenn.osc.edu'
self.clust_uname = 'osu5422'
self.clust_passwd = 'A706412O'
if not self.local:
self.all_runs = "/nfs/proj06/PAS0447/"
else:
self.all_runs = "/slxdata/solexadata/"
if not self.local:
self.expt_dir = self.all_runs+'illumina_runs/'+run_folder+\
"/Data/Intensities/BaseCalls/"
else:
self.expt_dir = self.all_runs+run_folder+\
"/Data/Intensities/BaseCalls/"
self.run_folder = run_folder
self.compress_conf = dist_conf
self.conf_obj = config.Parameters(self.compress_conf, 1)
#############################################################
#Take these from a config???
#############################################################
self.ssh_obj = run_ssh.SSH_cmd(self.clust_server,
self.clust_uname,
self.clust_passwd)
self.transRsync.setServerParams(self.clust_server,
self.clust_uname,
self.clust_passwd)
self.pbs_string = "#PBS -N gerald-job-"+self.runid+"-"+\
str(self.conf_obj.numOfNodes)+"n"+\
str(self.conf_obj.numOfCores)+"p-"+\
"test30tiles\n#PBS -l walltime="+\
str(self.conf_obj.wallTime)+":00:00\n"+\
"#PBS -l nodes="+\
str(self.conf_obj.numOfNodes)+":ppn="+\
str(self.conf_obj.numOfCores)+\
"\n#PBS -j oe\n"+\
"#PBS -S /bin/ksh\n#PBS -m b\n"+\
"#PBS -m e\n\n#tdate=$(date +%T)\n\n"+\
"set -x\n\ncd "
self.backup_dir = self.all_runs+"illumina_runs/\nbackup"
self.comp_conf = "[main]\nrunid="+self.runid+"\n"+\
"[compress]\nroot="+self.all_runs+\
"illumina_runs/\nbackup="+\
self.all_runs+"illumina_runs/backup/\nGerald="
#self.InitiateRsync()
def clean_content(self, in_l):
in_l.reverse()
ret_l = []
for i in in_l:
if i.strip().endswith('directory'):
#error?
raw_input("should not be here")
break
if i.startswith('ls'):
break
ret_l.append(i.split('/')[-1][:-7])
return ret_l
def rsync_back(self):
#if not self.ran_pipeline:
#print "error!!!"
#throw error
#return
self.runid_dir = self.backup_dir+"/"+self.runid
self.runid_dir = "".join(self.runid_dir.split("\n"))
for lane, name_folder in self.conf_obj.ftp_user_dict.\
iteritems():
temp_dir = self.runid_dir
temp_dir = os.path.join(temp_dir, name_folder[0]) + '/'
out_file = open("temp", 'w')
if not test:
self.ssh_obj.run("ls "+temp_dir+"*s_"+lane+"*",
out_file)
else:
print "ls "+temp_dir+"*s_"+lane+"*"
out_file.close()
out_file = open("temp", 'r')
file_cont = out_file.readlines()
file_cont.pop()
out_file.close()
file_cont = self.clean_content(file_cont)
self.remove("temp")
file_list = file_cont
file_list.append(self.runid+"_summary.tar.gz")
file_list.append(self.runid+"_Summary.htm")
for file in file_list:
self.transRsync.copy(temp_dir+file, '/cccstor/'+\
name_folder[1]+'/', 0)
self.logging.debug("transferred "+temp_dir+file+" to /cccstor/"+name_folder[1]+"/")
#tell quest ???
pass
#rsync back to p01
if len(name_folder) == 4:
sid = name_folder[2]
eid = name_folder[3]
self.conf_obj.server.UpdateQuestSampleResults(sid,\
eid, file_list, self.runid, \
"IlluminaGAII", "heWex45--")
#"terry", "chess")
self.logging.info("Finished run "+self.runid)
def write_comppbs(self):
pbs_file_n = self.runid+'compress.pbs'
self.pbs_file_h = open(pbs_file_n, 'w')
self.comp_pbs_string = self.pbs_string+"py_source\n"+\
"PYTHONPATH=$PYTHONPATH:/nfs/03/"+\
"osu5422/Solid-programs/"+\
"Solid_Corona_Lite_4.2.2/lib/python\n"+\
"export PYTHONPATH\n"+\
"/usr/bin/python cat_gzip.py "+\
self.rem_comp_conf+\
"\n/usr/bin/python osc_quest.py "+\
self.rem_comp_conf+" "+self.run_folder+" 0"
self.pbs_file_h.write(self.comp_pbs_string)
self.pbs_file_h.close()
self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
self.remove(pbs_file_n)
def write_makepbs(self, path):
pbs_file_n = self.runid+'.pbs'
self.pbs_file_h = open(pbs_file_n, 'w')
self.make_pbs_string = self.pbs_string+path+\
"\ntime make -j "+\
str(self.conf_obj.numOfCores)+\
"\ncd $HOME/py_source/\n"+\
"\n/usr/bin/python osc_quest.py "+\
self.all_runs+'compression_configs/conf_'+\
pbs_file_n.strip('.pbs')+\
".txt "+self.run_folder+" 1"
self.pbs_file_h.write(self.make_pbs_string)
self.pbs_file_h.close()
self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
self.remove(pbs_file_n)
def loadConfigObj(self, runFolder):
temp = os.getcwd()
os.chdir(self.codeBasePath)
os.chdir("compression_configs")
localPath = os.path.basename(self.rem_comp_conf)
print localPath
self.conf_obj = config.Parameters(localPath, 0)
self.conf_obj.popAll()
os.chdir(temp)
def InitiateRsync(self):
self.logging.debug("Starting Rsync to osc")
if not os.access(self.root_dir+self.run_folder, os.F_OK):
raise FatalError("Cannot access "+\
self.root_dir+self.run_folder)
os.chdir(self.root_dir)
if not self.local:
self.transRsync.copy(os.path.join(os.getcwd(),self.run_folder),
self.all_runs+'illumina_runs/', 1)
self.config_file = self.write_config(self.config_file,
'gerald')
self.logging.info("Finished writing Gerald config")
temp = self.gerald_pl+' '+self.config_file+' '+\
self.options+' '+self.expt_dir+' --make'
self.logging.debug("Running command "+temp)
reqd_path = self.gen_make(temp)
self.logging.info("Path for gerald "+reqd_path)
self.comp_conf += reqd_path+"\nBustard="
reqd_path = '/'.join(reqd_path.rstrip('/').split('/')[:-1])
self.comp_conf += reqd_path+"\n"
self.compress_conf = self.write_config(self.comp_conf+\
self.compress_conf,
'compression')
self.conf_obj = config.Parameters(self.compress_conf, 0)
self.conf_obj.popAll()
#try:
#self.conf_obj.server.\
#UpdateRunStatusJobId(self.run_folder, self.job_id,'')
#except:
#self.logging.info("Quest error")
self.logging.info("Finished writing compression config")
#temp = self.gerald_pl+' '+self.config_file+' '
if not self.local:
self.write_comppbs()
self.logging.info("Finished writing compression pbs")
def write_config(self, conf, name):
file_n = "conf_"+self.runid+".txt"
gerald_h = open(file_n, 'w')
gerald_h.write(conf)
gerald_h.close()
self.transRsync.copy(file_n, self.all_runs+\
name+"_configs", 1)
if name == "compression":
self.move(file_n, os.path.join(self.codeBasePath, name+"_configs"))
self.rem_comp_conf = self.all_runs+name+\
"_configs/"+file_n
return name+"_configs/"+file_n
else:
self.remove(file_n)
return self.all_runs+name+'_configs/'+file_n
def gen_compress(self):
out_file = open("temp", 'w')
if not test:
self.ssh_obj.run("qsub pbs_scripts/"+self.runid+\
"compress.pbs", out_file)
print "qsub pbs_scripts/"+self.runid+"compress.pbs"
self.logging.info("Submitting compression job")
self.logging.debug("qsub pbs_scripts/"+self.runid+\
"compress.pbs")
out_file.close()
out_file = open("temp", 'r')
file_cont = out_file.readlines()
out_file.close()
self.remove("temp")
main_line = file_cont[-2]
self.job_id = main_line.split('.')[0]
if test:
job_id = '123456'
#self.conf_obj.server.UpdateRunStatusCompressingResults(\
# self.run_folder, self.job_id,'')
def gen_make(self, cmd):
if not self.local:
out_file = open("temp", 'w')
if not test:
self.ssh_obj.run(cmd, out_file)
else:
print cmd
out_file.close()
elif self.local:
if not test:
os.system(cmd+" > temp")
else:
os.system("touch temp")
print cmd+" > temp"
self.logging.info("Finished generating gerald make file")
out_file = open("temp", 'r')
file_cont = out_file.readlines()
out_file.close()
# -4 because there is a logout line at the end
main_line = file_cont[-4].strip()
reqd_path = ""
if main_line.endswith("completed with no problems"):
self.remove("temp")
reqd_path = main_line.split()[4]
#out_file = open("temp", 'w')
if not self.local:
self.write_makepbs(reqd_path)
#self.ssh_obj.run("python py_source/cluster_side.py "+\
#"pbs_scripts/"+self.runid+".pbs"+" "+\
#reqd_path+" "+self.runid, out_file)
#out_file.close()
#os.system("rm temp")
out_file = open("temp", 'w')
if not test:
#TODO:raghu. uncomment later
self.ssh_obj.run("qsub pbs_scripts/"+self.runid+\
".pbs", out_file)
print "qsub pbs_scripts/"+self.runid+".pbs"
out_file.close()
out_file = open("temp", 'r')
file_cont = out_file.readlines()
out_file.close()
main_line = file_cont[-2]
self.remove("temp")
self.job_id = main_line.split('.')[0]
if test:
self.job_id = '123456'
self.logging.info("Submitted gerald job. Jobid = "+self.job_id)
elif not self.local: #case of compression pbs
pass
self.ran_pipeline = 1
#call quest method to give job_id
#out_file = open("temp", 'w')
#self.ssh_obj.run("qstat "+job_id, out_file)
#out_file.close()
else:
self.logging.info("Error!!! in making gerald. "+\
"Cannot submit job for make")
raise myerror
return reqd_path
#############################################################
#raise error if failure and tell quest???
#############################################################
def safetyCheck(self):
pass
class CatTar(CommandOp):
##################################################################
# This function is called when an object of the class is created.
#
# We basically define required parameters and extract relevant
# command line information.
#
##################################################################
def __init__(self, conf_obj):
"""This class returns an object which has the required
configuration values. Please add/delete data from __init__
to suit requirements."""
self.test = 0
self.conf_obj = conf_obj
self.transSftp = Transfer('sftp')
#for next version this should be changed to
#ftp_trans.FtpTrans(self.conf_obj)
self.ftp_obj = ftp_trans.FtpTrans()
self.runid = self.conf_obj.runid
self.backup_dir = self.conf_obj.backup_path
#Change below code to modify logging format
if(not os.path.isdir(self.backup_dir)):
raise error.FnfError(self.backup_dir,
"Error: backup dir not found"+\
self.backup_dir+"; Exiting")
self.root_dir = self.conf_obj.root_path
self.runid_dir = os.getcwd()
self.transCopy = Transfer('copy')
self.summaryName = ""
##################################################################
# Creates directories of users under backup directory
# Calls calcpath for further processing (for every lane and for
# every user
##################################################################
def create_dir(self):
logging.debug("dir list: "+str(self.conf_obj.lane_user_dict))
for lane, name in self.conf_obj.lane_user_dict.iteritems():
if not self.conf_obj.perm_dict[lane][0]:
continue
os.chdir(self.runid_dir)
self.s_what = "s_"+lane+"_"
if not os.path.isdir(name):
os.mkdir(name)
#Recreating dir tree on the ftp server
os.chdir(name)
self.cur_name_dir = os.getcwd()
self.calcpath()
self.add_htm()
##################################################################
#This function populates different path variables and calls concat
#as many times as required. Concat handles further processing
##################################################################
def calcpath(self):
self.Bustard_path = self.conf_obj.Bustard_path
self.Gerald_path = self.conf_obj.Gerald_path
os.chdir(self.Bustard_path)
reqd_list = commands.getoutput("ls "+self.s_what+\
"[1-9]_*_qseq.txt").split("\n")
ind = []
try:
ind.append(reqd_list[0].split('_')[2])
except IndexError:
logging.debug("here is the problem causing ind "+str(ind))
#The below for loop takes care of the case where there could
#be 2 kinds of files for same lane.
#Example:s_2_1_... s_2_2_...
for file in reqd_list:
if file.split('_')[2] not in ind:
ind.append(file.split('_')[2])
#Files which need to be included in the tar file. You could
#directly add to this list to make things easier
for i in ind:
#if there is only one set of files for every lane
if len(ind) == 1:
self.files = {self.s_what+'eland_extended.txt':'',
self.s_what+'sequence.txt':'',
self.s_what+'sorted.txt':'',
self.s_what+'export.txt':''}
#otherwise more than one set of files
else:
self.files = {self.s_what+i+'_'+'eland_extended.txt'\
:'',
self.s_what+i+'_'+'sequence.txt':'',
self.s_what+i+'_'+'sorted.txt':'',
self.s_what+i+'_'+'export.txt':''}
#Add in paths where corresponding files can be found
for file in self.files.iterkeys():
if not file.endswith("qseq.txt"):
self.files[file] = self.Gerald_path
else:
self.files[file] = self.Bustard_path
self.concat(i)
##################################################################
# Concatenates all txt files under Bustard
##################################################################
def concat(self, i_str):
#Separate out the name of the current user.
#Used only for logging purposes
cur_name = self.cur_name_dir.split("/")[-1]
os.chdir(self.Bustard_path)
concat_files = ''
reqd_list = commands.getoutput("ls "+self.s_what+i_str+\
"_*_qseq.txt").split("\n")
logging.info("Starting loop for user "+cur_name)
for file in reqd_list:
concat_files += file +' '
logging.info("Starting concatenation of files")
logging.debug("Starting concatenation of files under "+\
self.Bustard_path)
logging.debug('cat '+self.s_what+i_str+'_qseq.txt ')
if(self.test):
print 'cat '+concat_files+'> '+\
os.path.join(self.cur_name_dir,\
self.s_what+i_str)+'_qseq.txt \n'
else:
os.system('cat '+concat_files+'> '+\
os.path.join(self.cur_name_dir,\
self.s_what+i_str)+'_qseq.txt ')
logging.info("Finished concatenation of files")
logging.debug("Finished concatenation of files under "+\
self.Bustard_path)
self.copy_tar()
##################################################################
# Copies all files under Bustard and Gerald to the path for the
# user. Then tars the files.
##################################################################
def copy_tar(self):
all_paths = ''
#Put all files to be copied in a single string
#Will it be better to do it one by one?
for file, path in self.files.iteritems():
all_paths += os.path.join(path, file) + ' '
logging.info("Starting copying of files")
logging.debug("Starting copying of files under "+\
self.Bustard_path+" and "+ self.Gerald_path+\
" to "+self.cur_name_dir)
logging.debug("cp "+all_paths+" "+self.cur_name_dir)
if(self.test):
print "cp "+all_paths+" "+self.cur_name_dir+"\n"
else:
for f in all_paths.split():
self.transCopy.copy(f.strip(), self.cur_name_dir)
logging.info("Finished copying of files")
os.chdir(self.cur_name_dir)
logging.info("Renaming of files")
for file in os.listdir(os.getcwd()):
if file.endswith(".txt"):
if(self.test):
print "mv "+file+" "+self.runid+'_'+file+"\n"
else:
#TODO: raghu. check error
self.move(file, self.runid+'_'+file)
for file in os.listdir(os.getcwd()):
if file.endswith(".txt"):
logging.info("Starting tarring of file "+file)
logging.debug("gzip "+file)
if(self.test):
print "gzip "+file
else:
os.system("gzip "+file)
logging.info("Finished tarring of file "+file)
##################################################################
#Copies summary, plots and htm from Bustard and Gerald into a temp
#directory. Tars the temp directory and deletes the intermediate
#directory.
##################################################################
def add_htm(self):
self.Bustard_path = self.conf_obj.Bustard_path
self.Gerald_path = self.conf_obj.Gerald_path
logging.info("Starting summary tar creation")
os.chdir(self.runid_dir)
os.mkdir(self.runid+"summary")
os.chdir(self.runid+"summary")
logging.info("Copying htm files from Bustard")
for file in glob.glob(os.path.join(self.Bustard_path,
"*.[hH][Tt][Mm]")):
self.transCopy.copy(file, ".")
logging.info("Copying Plots folder from Bustard")
self.transCopy.copy(os.path.join(self.Bustard_path, "Plots"),
os.path.join(os.getcwd(),"Plots"))
os.mkdir(self.Gerald_path.split('/')[-1])
os.chdir(self.Gerald_path.split('/')[-1])
logging.info("Copying htm files from Gerald ")
for file in glob.glob(os.path.join(self.Gerald_path,
"*.[hH][Tt][Mm]")):
self.transCopy.copy(file, ".")
logging.info("Copying Plots folder from Gerald")
self.transCopy.copy(os.path.join(self.Gerald_path, "Plots"),
os.path.join(os.getcwd(), "Plots"))
logging.info("Starting tarring of Summary")
os.chdir(self.runid_dir)
os.system("tar -czf "+self.runid+"_summary.tar.gz "+\
self.runid+"summary")
logging.info("Finished tarring of Summary")
self.copy_file()
##################################################################
# Copies the required files for different users.
##################################################################
def copy_file(self):
logging.info("Making copies of summary tar")
os.chdir(self.runid_dir)
logging.debug(self.conf_obj.copy_to_runid)
logging.debug(self.conf_obj.copy_to_users)
logging.debug(str(self.conf_obj.lane_user_dict)+" HI")
for file in self.conf_obj.copy_to_runid:
self.transCopy.copy(file, os.getcwd())
for file in self.conf_obj.copy_to_users:
for name in self.conf_obj.lane_user_dict.itervalues():
self.transCopy.copy(file, name)
if file.split('/')[-1] == 'Summary.htm':
os.chdir(name)
logging.debug("Renaming Summary.htm "+os.getcwd())
self.move('Summary.htm', self.runid+'_Summary.htm')
os.chdir('..')
os.chdir(self.runid_dir)
self.removeDir(self.runid+"summary")
def ftp_files(self):
i = 0
#############################################################
#name_folder format
#
#1. For Non-quest
# Length of list = 2
# [<name of user>, <folder to cd to on quest>]
#2. For Quest
# Length of list = 4
# [<name of user>, <folder to cd to on quest>, <sample id>,
# <experiment id>]
#############################################################
for lane, name_folder in self.conf_obj.ftp_user_dict.\
iteritems():
if not self.conf_obj.perm_dict[lane][1]:
continue
os.chdir(self.runid_dir)
os.chdir(name_folder[0])
file_list = commands.getoutput("ls *s_"+lane+"*").\
split("\n")
file_list.append(self.runid+"_summary.tar.gz")
file_list.append(self.runid+"_Summary.htm")
if len(name_folder) < 4:
self.ftp_obj.connect()
self.ftp_obj.chdir(name_folder[1])
try:
self.ftp_obj.mkdir(self.runid)
except ftplib.error_perm:
self.ftp_obj.connect()
self.ftp_obj.chdir(name_folder[1])
logging.debug("runid dir already there")
self.ftp_obj.chdir(self.runid)
try:
self.ftp_obj.mkdir(name_folder[0])
except ftplib.error_perm:
self.ftp_obj.connect()
self.ftp_obj.chdir(name_folder[1])
self.ftp_obj.chdir(self.runid)
logging.debug("user dir also already there")
self.ftp_obj.chdir(name_folder[0])
else:
if self.conf_obj.ftp_quest:
self.ftp_obj.connect()
self.ftp_obj.chdir(name_folder[1])
for file in file_list:
logging.info("Transferring file %s of user %s" \
%(file, name_folder[0]))
if self.conf_obj.ftp_quest:
self.ftp_obj.copy_file(file)
else:
self.transCopy.copy(file, '/cccstor/'+\
name_folder[1]+'/'+file)
logging.debug('cp '+file+' /cccstor/'+\
name_folder[1]+'/'+file)
if self.conf_obj.ftp_quest:
self.ftp_obj.close()
logging.info("Finished transfer !!!")
if len(name_folder) == 4:
sid = name_folder[2]
eid = name_folder[3]
self.conf_obj.server.UpdateQuestSampleResults(sid,\
eid, file_list, self.runid, \
"IlluminaGAII", "heWex45--")
def sftp_files(self):
#name_folder = [uname, passwd, email id]
for lane, name_folder in self.conf_obj.sftp_user_dict.\
iteritems():
if not self.conf_obj.perm_dict[lane][1]:
continue
os.chdir(self.runid_dir)
os.chdir(name_folder[0])
file_list = commands.getoutput("ls *s_"+lane+"*").\
split("\n")
file_list.append(self.runid+"_summary.tar.gz")
file_list.append(self.runid+"_Summary.htm")
self.transSftp.setServerParams(self.conf_obj.sftp_server,
name_folder[0],
name_folder[1])
for file in file_list:
logging.info("Transferring file %s of user %s" \
%(file, name_folder[0]))
self.transSftp.copy(file, "/data/repos/solexa/"+\
name_folder[0]+"/"+file)
self.mail_obj = mail.Mail()
if(name_folder[2] != ""):
email_text = "Results for Illumina run "+self.runid+\
" has been uploaded to our SFTP server.\n"+\
"SFTP server: "+self.conf_obj.sftp_server+\
"\nUsername: "******"\nPassword: "******"\n"
email_text += "Use winscp to download the files\n"+\
"http://winscp.net/eng/download.php\n"
email_text += "\nRegards,\n\n"+\
"Biomedical Informatics Shared Resource\n"+ \
"The Ohio State University Comprehensive Cancer"+\
"Center\n"+\
"210 Biomedical Research Tower\n 460 W 12th Ave\n"+\
"Columbus, OH 43210\n (614) 366-1538\n"
self.mail_obj.mail_diff(name_folder[2], \
"Illumina run "+self.runid, \
email_text)
class MakeGerald(CommandOp):
def __init__(self, logging):
#full path
self.logging = logging
self.transRsync = Transfer('rsync')
def set_params(self, run_folder, gerald_conf, dist_conf, dest, wall):
self.logging.debug("setting params")
if dest == 'OSC':
self.local = 0
self.gerald_pl = "/nfs/03/osu5422/CASAVA/CASAVA_v1.7.0-build/bin/GERALD.pl"
elif dest == 'P01':
self.local = 1
self.gerald_pl = 'GERALD.pl'
self.config_file = gerald_conf
self.runid = run_folder.split('_')[0]
self.ran_pipeline = 0
self.pbs_path = '/nfs/03/osu5422/pbs_scripts/'
self.options = "--EXPT_DIR"
self.root_dir = "/slxdata/solexadata/"
self.clust_server = 'glenn.osc.edu'
self.clust_uname = 'osu5422'
self.clust_passwd = 'A706412O'
if not self.local:
self.all_runs = "/nfs/proj07/BISR/"
else:
self.all_runs = "/slxdata/solexadata/"
if not self.local:
self.expt_dir = self.all_runs+'illumina_runs/'+run_folder+\
"/Data/Intensities/BaseCalls/"
else:
self.expt_dir = self.all_runs+run_folder+\
"/Data/Intensities/BaseCalls/"
self.run_folder = run_folder
self.compress_conf = dist_conf
self.conf_obj = config.Parameters(self.compress_conf, 1)
#############################################################
#Take these from a config???
#############################################################
self.ssh_obj = run_ssh.SSH_cmd(self.clust_server,
self.clust_uname,
self.clust_passwd)
self.transRsync.setServerParams(self.clust_server,
self.clust_uname,
self.clust_passwd)
self.pbs_string = "#PBS -N gerald-job-"+self.runid+"-"+\
str(self.conf_obj.numOfNodes)+"n"+\
str(self.conf_obj.numOfCores)+"p-"+\
"test30tiles\n#PBS -l walltime="+\
str(self.conf_obj.wallTime)+":00:00\n"+\
"#PBS -l nodes="+\
str(self.conf_obj.numOfNodes)+":ppn="+\
str(self.conf_obj.numOfCores)+\
"\n#PBS -j oe\n"+\
"#PBS -S /bin/ksh\n#PBS -m b\n"+\
"#PBS -m e\n\n#tdate=$(date +%T)\n\n"+\
"set -x\n\ncd "
self.backup_dir = self.all_runs+"illumina_runs/\nbackup"
self.comp_conf = "[main]\nrunid="+self.runid+"\n"+\
"[compress]\nroot="+self.all_runs+\
"illumina_runs/\nbackup="+\
self.all_runs+"illumina_runs/backup/\nGerald="
#self.InitiateRsync()
def clean_content(self, in_l):
in_l.reverse()
ret_l = []
for i in in_l:
if i.strip().endswith('directory'):
#error?
raw_input("should not be here")
break
if i.startswith('ls'):
break
ret_l.append(i.split('/')[-1][:-7])
return ret_l
def rsync_back(self):
#if not self.ran_pipeline:
#print "error!!!"
#throw error
#return
self.runid_dir = self.backup_dir+"/"+self.runid
self.runid_dir = "".join(self.runid_dir.split("\n"))
for lane, name_folder in self.conf_obj.ftp_user_dict.\
iteritems():
temp_dir = self.runid_dir
temp_dir = os.path.join(temp_dir, name_folder[0]) + '/'
out_file = open("temp", 'w')
if not test:
self.ssh_obj.run("ls "+temp_dir+"*s_"+lane+"*",
out_file)
else:
print "ls "+temp_dir+"*s_"+lane+"*"
out_file.close()
out_file = open("temp", 'r')
file_cont = out_file.readlines()
file_cont.pop()
out_file.close()
file_cont = self.clean_content(file_cont)
self.remove("temp")
file_list = file_cont
file_list.append(self.runid+"_summary.tar.gz")
file_list.append(self.runid+"_Summary.htm")
for file in file_list:
self.transRsync.copy(temp_dir+file, '/cccstor/'+\
name_folder[1]+'/', 0)
self.logging.debug("transferred "+temp_dir+file+" to /cccstor/"+name_folder[1]+"/")
#tell quest ???
pass
#rsync back to p01
if len(name_folder) == 4:
sid = name_folder[2]
eid = name_folder[3]
self.conf_obj.server.UpdateQuestSampleResults(sid,\
eid, file_list, self.runid, \
"IlluminaGAII", "heWex45--")
#"terry", "chess")
self.logging.debug("updated quest for "+str(file_list))
self.logging.info("Finished run "+self.runid)
def write_comppbs(self):
pbs_file_n = self.runid+'compress.pbs'
self.pbs_file_h = open(pbs_file_n, 'w')
temp = self.run_folder.split("_")[0]
conf_file = self.all_runs+"compression_configs/conf_"+temp+".txt"
self.comp_pbs_string = self.pbs_string+"py_source\n"+\
"source ~/.bashrc\n"+\
"/usr/bin/python cat_gzip.py "+\
conf_file+\
"\n/usr/bin/python osc_quest.py "+\
conf_file+" "+self.run_folder+" 0"
self.pbs_file_h.write(self.comp_pbs_string)
self.pbs_file_h.close()
self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
self.remove(pbs_file_n)
def write_makepbs(self, path):
pbs_file_n = self.runid+'.pbs'
self.pbs_file_h = open(pbs_file_n, 'w')
self.make_pbs_string = self.pbs_string+path+\
"\ntime make -j "+\
str(self.conf_obj.numOfCores)+\
"\ncd $HOME/py_source/\n"+\
"\n/usr/bin/python osc_quest.py "+\
self.all_runs+'compression_configs/conf_'+\
pbs_file_n.strip('.pbs')+\
".txt "+self.run_folder+" 1"
self.pbs_file_h.write(self.make_pbs_string)
self.pbs_file_h.close()
self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
self.remove(pbs_file_n)
def loadConfigObj(self, runFolder):
temp = os.getcwd()
os.chdir(self.codeBasePath)
os.chdir("compression_configs")
localPath = os.path.basename("conf_"+\
runFolder.split("_")[0]+".txt")
print localPath
self.conf_obj = config.Parameters(localPath, 0)
self.conf_obj.popAll()
os.chdir(temp)
def InitiateRsync(self):
self.write_comppbs()
self.logging.info("Finished writing compression pbs") self.logging.debug("Starting Rsync to osc") if not os.access(self.root_dir+self.run_folder, os.F_OK):
raise FatalError("Cannot access "+\
self.root_dir+self.run_folder)
os.chdir(self.root_dir)
if not self.local:
self.transRsync.copy(os.path.join(os.getcwd(),self.run_folder),
self.all_runs+'illumina_runs/', 1)
self.config_file = self.write_config(self.config_file,
'gerald')
self.logging.info("Finished writing Gerald config")
temp = self.gerald_pl+' '+self.config_file+' '+\
self.options+' '+self.expt_dir+' --make'
self.logging.debug("Running command "+temp)
reqd_path = self.gen_make(temp)
self.logging.info("Path for gerald "+reqd_path)
self.comp_conf += reqd_path+"\nBustard="
reqd_path = '/'.join(reqd_path.rstrip('/').split('/')[:-1])
self.comp_conf += reqd_path+"\n"
self.compress_conf = self.write_config(self.comp_conf+\
self.compress_conf,
'compression')
temp = os.getcwd()
os.chdir(self.codeBasePath)
self.conf_obj = config.Parameters(self.compress_conf, 0)
os.chdir(temp)
self.conf_obj.popAll()
#try:
#self.conf_obj.server.\
#UpdateRunStatusJobId(self.run_folder, self.job_id,'')
#except:
#self.logging.info("Quest error")
self.logging.info("Finished writing compression config")
#temp = self.gerald_pl+' '+self.config_file+' '
if not self.local:
self.write_comppbs()
self.logging.info("Finished writing compression pbs")
