class MakeGerald(CommandOp):

    def __init__(self, logging):
        #full path
        self.logging = logging
        self.transRsync = Transfer('rsync')

    def set_params(self, run_folder, gerald_conf, dist_conf, dest, wall):
        self.logging.debug("setting params")
        if dest == 'OSC':
            self.local = 0
            self.gerald_pl = "/nfs/03/osu5422/CASAVA/CASAVA_v1.7.0-build/bin/GERALD.pl"
        elif dest == 'P01':
            self.local = 1
            self.gerald_pl = 'GERALD.pl'
        self.config_file = gerald_conf
        self.runid = run_folder.split('_')[0]
        self.ran_pipeline = 0
        self.pbs_path = '/nfs/03/osu5422/pbs_scripts/'

        self.options = "--EXPT_DIR"
        self.root_dir = "/slxdata/solexadata/"
        self.clust_server = 'glenn.osc.edu' 
        self.clust_uname = 'osu5422' 
        self.clust_passwd = 'A706412O'
        if not self.local:
            self.all_runs = "/nfs/proj06/PAS0447/"
        else:
            self.all_runs = "/slxdata/solexadata/"
        if not self.local:
            self.expt_dir = self.all_runs+'illumina_runs/'+run_folder+\
                            "/Data/Intensities/BaseCalls/"
        else:
            self.expt_dir = self.all_runs+run_folder+\
                            "/Data/Intensities/BaseCalls/"
        self.run_folder = run_folder
        self.compress_conf = dist_conf
        self.conf_obj = config.Parameters(self.compress_conf, 1)
        #############################################################
        #Take these from a config???
        #############################################################
        self.ssh_obj = run_ssh.SSH_cmd(self.clust_server, 
                                       self.clust_uname, 
                                       self.clust_passwd)
        self.transRsync.setServerParams(self.clust_server,
                                        self.clust_uname,
                                        self.clust_passwd)
        self.pbs_string = "#PBS -N gerald-job-"+self.runid+"-"+\
                          str(self.conf_obj.numOfNodes)+"n"+\
                          str(self.conf_obj.numOfCores)+"p-"+\
                          "test30tiles\n#PBS -l walltime="+\
                          str(self.conf_obj.wallTime)+":00:00\n"+\
                          "#PBS -l nodes="+\
                          str(self.conf_obj.numOfNodes)+":ppn="+\
                          str(self.conf_obj.numOfCores)+\
                          "\n#PBS -j oe\n"+\
                          "#PBS -S /bin/ksh\n#PBS -m b\n"+\
                          "#PBS -m e\n\n#tdate=$(date +%T)\n\n"+\
                          "set -x\n\ncd "
        self.backup_dir = self.all_runs+"illumina_runs/\nbackup"
        self.comp_conf = "[main]\nrunid="+self.runid+"\n"+\
                         "[compress]\nroot="+self.all_runs+\
                         "illumina_runs/\nbackup="+\
                         self.all_runs+"illumina_runs/backup/\nGerald="
        #self.InitiateRsync()

    def clean_content(self, in_l):
        in_l.reverse()
        ret_l = []
        for i in in_l:
            if i.strip().endswith('directory'):
                #error?
                raw_input("should not be here")
                break
            if i.startswith('ls'):
                break
            ret_l.append(i.split('/')[-1][:-7])
        return ret_l


    def rsync_back(self):
        #if not self.ran_pipeline:
            #print "error!!!"
            #throw error
            #return
        self.runid_dir = self.backup_dir+"/"+self.runid
        self.runid_dir = "".join(self.runid_dir.split("\n"))
        for lane, name_folder in self.conf_obj.ftp_user_dict.\
                iteritems():
            temp_dir = self.runid_dir
            temp_dir = os.path.join(temp_dir, name_folder[0]) + '/'
            out_file = open("temp", 'w') 
            if not test:
                self.ssh_obj.run("ls "+temp_dir+"*s_"+lane+"*",
                                 out_file)
            else:
                print "ls "+temp_dir+"*s_"+lane+"*"
            out_file.close()
            out_file = open("temp", 'r') 
            file_cont = out_file.readlines()
            file_cont.pop()
            out_file.close()
            file_cont = self.clean_content(file_cont)
            self.remove("temp")
            file_list = file_cont
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            for file in file_list:
                self.transRsync.copy(temp_dir+file, '/cccstor/'+\
                                   name_folder[1]+'/', 0)
                self.logging.debug("transferred "+temp_dir+file+" to /cccstor/"+name_folder[1]+"/")
                #tell quest ???
                pass 
                #rsync back to p01
            if len(name_folder) == 4:
                sid = name_folder[2]
                eid = name_folder[3]
                self.conf_obj.server.UpdateQuestSampleResults(sid,\
                                     eid, file_list, self.runid, \
                                     "IlluminaGAII", "heWex45--")
                                     #"terry", "chess")
        self.logging.info("Finished run "+self.runid)
    def write_comppbs(self):
        pbs_file_n = self.runid+'compress.pbs'
        self.pbs_file_h = open(pbs_file_n, 'w')
        self.comp_pbs_string = self.pbs_string+"py_source\n"+\
                           "PYTHONPATH=$PYTHONPATH:/nfs/03/"+\
                           "osu5422/Solid-programs/"+\
                           "Solid_Corona_Lite_4.2.2/lib/python\n"+\
                           "export PYTHONPATH\n"+\
                           "/usr/bin/python cat_gzip.py "+\
                           self.rem_comp_conf+\
                           "\n/usr/bin/python osc_quest.py "+\
                           self.rem_comp_conf+" "+self.run_folder+" 0"
        self.pbs_file_h.write(self.comp_pbs_string)
        self.pbs_file_h.close()
        self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
        self.remove(pbs_file_n)
        
    def write_makepbs(self, path):
        pbs_file_n = self.runid+'.pbs'
        self.pbs_file_h = open(pbs_file_n, 'w')
        self.make_pbs_string = self.pbs_string+path+\
                           "\ntime make -j "+\
                           str(self.conf_obj.numOfCores)+\
                           "\ncd $HOME/py_source/\n"+\
                           "\n/usr/bin/python osc_quest.py "+\
                           self.all_runs+'compression_configs/conf_'+\
                           pbs_file_n.strip('.pbs')+\
                           ".txt "+self.run_folder+" 1"

        self.pbs_file_h.write(self.make_pbs_string)
        self.pbs_file_h.close()
        self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
        self.remove(pbs_file_n)

    def loadConfigObj(self, runFolder):
        temp = os.getcwd()
        os.chdir(self.codeBasePath)
        os.chdir("compression_configs")
        localPath = os.path.basename(self.rem_comp_conf)
        print localPath
        self.conf_obj = config.Parameters(localPath, 0)
        self.conf_obj.popAll()
        os.chdir(temp)

    def InitiateRsync(self):
        self.logging.debug("Starting Rsync to osc")
        if not os.access(self.root_dir+self.run_folder, os.F_OK):
            raise FatalError("Cannot access "+\
                             self.root_dir+self.run_folder)
        os.chdir(self.root_dir)
        if not self.local:
            self.transRsync.copy(os.path.join(os.getcwd(),self.run_folder),
                               self.all_runs+'illumina_runs/', 1)
        self.config_file = self.write_config(self.config_file,
                                             'gerald')
        self.logging.info("Finished writing Gerald config")
        temp = self.gerald_pl+' '+self.config_file+' '+\
               self.options+' '+self.expt_dir+' --make'
        self.logging.debug("Running command "+temp)
        reqd_path = self.gen_make(temp)
        self.logging.info("Path for gerald "+reqd_path)
        self.comp_conf += reqd_path+"\nBustard="
        reqd_path = '/'.join(reqd_path.rstrip('/').split('/')[:-1])
        self.comp_conf += reqd_path+"\n"
        self.compress_conf = self.write_config(self.comp_conf+\
                                               self.compress_conf,
                                               'compression')
        self.conf_obj = config.Parameters(self.compress_conf, 0)
        self.conf_obj.popAll()
        #try:
	    #self.conf_obj.server.\
		#UpdateRunStatusJobId(self.run_folder, self.job_id,'')
        #except:
	    #self.logging.info("Quest error")
        self.logging.info("Finished writing compression config")
        #temp = self.gerald_pl+' '+self.config_file+' '
        if not self.local:
            self.write_comppbs()
            self.logging.info("Finished writing compression pbs")

    def write_config(self, conf, name):
        file_n = "conf_"+self.runid+".txt"
        gerald_h = open(file_n, 'w')
        gerald_h.write(conf)
        gerald_h.close()
        self.transRsync.copy(file_n, self.all_runs+\
                      name+"_configs", 1)
        if name == "compression":
            self.move(file_n, os.path.join(self.codeBasePath, name+"_configs"))
            self.rem_comp_conf = self.all_runs+name+\
                                 "_configs/"+file_n
            return name+"_configs/"+file_n
        else:
            self.remove(file_n)
        return self.all_runs+name+'_configs/'+file_n

    def gen_compress(self):
        out_file = open("temp", 'w')
        if not test:
            self.ssh_obj.run("qsub pbs_scripts/"+self.runid+\
                             "compress.pbs", out_file)
            print "qsub pbs_scripts/"+self.runid+"compress.pbs"
        self.logging.info("Submitting compression job")
        self.logging.debug("qsub pbs_scripts/"+self.runid+\
                             "compress.pbs")
        out_file.close()
        out_file = open("temp", 'r')
        file_cont = out_file.readlines()
        out_file.close()
        self.remove("temp")
        main_line = file_cont[-2]
        self.job_id = main_line.split('.')[0]
        if test:
            job_id = '123456'
        #self.conf_obj.server.UpdateRunStatusCompressingResults(\
        #       self.run_folder, self.job_id,'')

    def gen_make(self, cmd):
        if not self.local:
            out_file = open("temp", 'w')
            if not test:
                self.ssh_obj.run(cmd, out_file)
            else:
                print cmd
            out_file.close()
        elif self.local:
            if not test:
                os.system(cmd+" > temp")
            else: 
                os.system("touch temp")
                print cmd+" > temp"
        self.logging.info("Finished generating gerald make file")
        out_file = open("temp", 'r')
        file_cont = out_file.readlines()
        out_file.close()
        # -4 because there is a logout line at the end
        main_line = file_cont[-4].strip()
        reqd_path = ""
        if main_line.endswith("completed with no problems"):
            self.remove("temp")
            reqd_path = main_line.split()[4]
            #out_file = open("temp", 'w')
            if not self.local:
                self.write_makepbs(reqd_path)
            #self.ssh_obj.run("python py_source/cluster_side.py "+\
                             #"pbs_scripts/"+self.runid+".pbs"+" "+\
                             #reqd_path+" "+self.runid, out_file)
            #out_file.close()
            #os.system("rm temp")
                out_file = open("temp", 'w')
                if not test:
                    #TODO:raghu. uncomment later
                    self.ssh_obj.run("qsub pbs_scripts/"+self.runid+\
                                     ".pbs", out_file)
                    print "qsub pbs_scripts/"+self.runid+".pbs"
                out_file.close()
                out_file = open("temp", 'r')
                file_cont = out_file.readlines()
                out_file.close()
                main_line = file_cont[-2]
                self.remove("temp")
                self.job_id = main_line.split('.')[0]
                if test:
                    self.job_id = '123456'
                self.logging.info("Submitted gerald job. Jobid = "+self.job_id)
            elif not self.local: #case of compression pbs
                pass
            self.ran_pipeline = 1
            #call quest method to give job_id
            #out_file = open("temp", 'w')
            #self.ssh_obj.run("qstat "+job_id, out_file)
            #out_file.close()
        else:
            self.logging.info("Error!!! in making gerald. "+\
                              "Cannot submit job for make")
            raise myerror
        return reqd_path
        #############################################################
        #raise error if failure and tell quest???
        #############################################################

    def safetyCheck(self):
        pass
Esempio n. 2
0
class CatTar(CommandOp):
    ##################################################################
    # This function is called when an object of the class is created.
    # 
    # We basically define required parameters and extract relevant 
    # command line information.
    # 
    ##################################################################
    def __init__(self, conf_obj):
        """This class returns an object which has the required 
        configuration values. Please add/delete data from __init__ 
        to suit requirements."""
        self.test = 0
        self.conf_obj = conf_obj
        self.transSftp = Transfer('sftp')
        #for next version this should be changed to 
        #ftp_trans.FtpTrans(self.conf_obj)
        self.ftp_obj = ftp_trans.FtpTrans()
        self.runid = self.conf_obj.runid
        self.backup_dir = self.conf_obj.backup_path
        #Change below code to modify logging format
        if(not os.path.isdir(self.backup_dir)):
            raise error.FnfError(self.backup_dir, 
                                 "Error: backup dir not found"+\
                                  self.backup_dir+"; Exiting")
        self.root_dir = self.conf_obj.root_path
        self.runid_dir = os.getcwd()
        self.transCopy = Transfer('copy')
        self.summaryName = ""

    ##################################################################
    # Creates directories of users under backup directory
    # Calls calcpath for further processing (for every lane and for 
    # every user
    ##################################################################
    def create_dir(self):
        logging.debug("dir list: "+str(self.conf_obj.lane_user_dict))
        for lane, name in self.conf_obj.lane_user_dict.iteritems():
            if not self.conf_obj.perm_dict[lane][0]:
                continue
            os.chdir(self.runid_dir)
            self.s_what = "s_"+lane+"_"
            if not os.path.isdir(name):
                os.mkdir(name)
                #Recreating dir tree on the ftp server
            os.chdir(name)
            self.cur_name_dir = os.getcwd()
            self.calcpath()
        self.add_htm()
    
    ##################################################################
    #This function populates different path variables and calls concat
    #as many times as required. Concat handles further processing
    ##################################################################
    def calcpath(self):
        self.Bustard_path = self.conf_obj.Bustard_path
        self.Gerald_path = self.conf_obj.Gerald_path
        os.chdir(self.Bustard_path)
        reqd_list = commands.getoutput("ls "+self.s_what+\
                                       "[1-9]_*_qseq.txt").split("\n")
        ind = []
        try:
            ind.append(reqd_list[0].split('_')[2])
        except IndexError:
            logging.debug("here is the problem causing ind "+str(ind))
        #The below for loop takes care of the case where there could 
        #be 2 kinds of files for same lane. 
        #Example:s_2_1_... s_2_2_...
        for file in reqd_list:
            if file.split('_')[2] not in ind:
                ind.append(file.split('_')[2])
        #Files which need to be included in the tar file. You could 
        #directly add to this list to make things easier
        for i in ind:
            #if there is only one set of files for every lane
            if len(ind) == 1:
                self.files = {self.s_what+'eland_extended.txt':'', 
                              self.s_what+'sequence.txt':'', 
                              self.s_what+'sorted.txt':'', 
                              self.s_what+'export.txt':''}
            #otherwise more than one set of files
            else:
                self.files = {self.s_what+i+'_'+'eland_extended.txt'\
                              :'', 
                              self.s_what+i+'_'+'sequence.txt':'', 
                              self.s_what+i+'_'+'sorted.txt':'', 
                              self.s_what+i+'_'+'export.txt':''}
            #Add in paths where corresponding files can be found
            for file in self.files.iterkeys():
                if not file.endswith("qseq.txt"):
                    self.files[file] = self.Gerald_path
                else:
                    self.files[file] = self.Bustard_path
            self.concat(i)


    ##################################################################
    # Concatenates all txt files under Bustard
    ##################################################################
    def concat(self, i_str):
        #Separate out the name of the current user. 
        #Used only for logging purposes
        cur_name = self.cur_name_dir.split("/")[-1]
        os.chdir(self.Bustard_path)
        concat_files = ''
        reqd_list = commands.getoutput("ls "+self.s_what+i_str+\
                                       "_*_qseq.txt").split("\n")
        logging.info("Starting loop for user "+cur_name)
        for file in reqd_list:
            concat_files += file +' '
        logging.info("Starting concatenation of files")
        logging.debug("Starting concatenation of files under "+\
                      self.Bustard_path)
        logging.debug('cat '+self.s_what+i_str+'_qseq.txt ')
        if(self.test):
            print 'cat '+concat_files+'> '+\
                  os.path.join(self.cur_name_dir,\
                  self.s_what+i_str)+'_qseq.txt \n'
        else:
            os.system('cat '+concat_files+'> '+\
                      os.path.join(self.cur_name_dir,\
                      self.s_what+i_str)+'_qseq.txt ')
        logging.info("Finished concatenation of files")
        logging.debug("Finished concatenation of files under "+\
                      self.Bustard_path)
        self.copy_tar()

    ##################################################################
    # Copies all files under Bustard and Gerald to the path for the 
    # user. Then tars the files.
    ##################################################################
    def copy_tar(self):
        all_paths = ''
        #Put all files to be copied in a single string
        #Will it be better to do it one by one?
        for file, path in self.files.iteritems():
            all_paths += os.path.join(path, file) + ' '
        logging.info("Starting copying of files")
        logging.debug("Starting copying of files under "+\
                      self.Bustard_path+" and "+ self.Gerald_path+\
                      " to "+self.cur_name_dir)
        logging.debug("cp "+all_paths+" "+self.cur_name_dir)
        if(self.test):
            print "cp "+all_paths+" "+self.cur_name_dir+"\n"
        else:
            for f in all_paths.split():
                self.transCopy.copy(f.strip(), self.cur_name_dir)
        logging.info("Finished copying of files")
        os.chdir(self.cur_name_dir)
        logging.info("Renaming of files")
        for file in os.listdir(os.getcwd()):
            if file.endswith(".txt"):
                if(self.test):
                    print "mv "+file+" "+self.runid+'_'+file+"\n"
                else:
                    #TODO: raghu. check error
                    self.move(file, self.runid+'_'+file)
        for file in os.listdir(os.getcwd()):
            if file.endswith(".txt"):
                logging.info("Starting tarring of file "+file)
                logging.debug("gzip "+file)
                if(self.test):
                    print "gzip "+file
                else:
                    os.system("gzip "+file)
                logging.info("Finished tarring of file "+file)
    
    ##################################################################
    #Copies summary, plots and htm from Bustard and Gerald into a temp
    #directory. Tars the temp directory and deletes the intermediate 
    #directory.
    ##################################################################
    def add_htm(self):
        self.Bustard_path = self.conf_obj.Bustard_path
        self.Gerald_path = self.conf_obj.Gerald_path
        logging.info("Starting summary tar creation")
        os.chdir(self.runid_dir)
        os.mkdir(self.runid+"summary")
        os.chdir(self.runid+"summary")
        logging.info("Copying htm files from Bustard")
        for file in glob.glob(os.path.join(self.Bustard_path, 
                  "*.[hH][Tt][Mm]")):
            self.transCopy.copy(file, ".")
        logging.info("Copying Plots folder from Bustard")
        self.transCopy.copy(os.path.join(self.Bustard_path, "Plots"),
                        os.path.join(os.getcwd(),"Plots"))
        os.mkdir(self.Gerald_path.split('/')[-1])
        os.chdir(self.Gerald_path.split('/')[-1])
        logging.info("Copying htm files from Gerald ")
        for file in glob.glob(os.path.join(self.Gerald_path, 
                                     "*.[hH][Tt][Mm]")):
            self.transCopy.copy(file, ".")
        logging.info("Copying Plots folder from Gerald") 
        self.transCopy.copy(os.path.join(self.Gerald_path, "Plots"),
                      os.path.join(os.getcwd(), "Plots"))
        logging.info("Starting tarring of Summary")
        os.chdir(self.runid_dir)
        os.system("tar -czf "+self.runid+"_summary.tar.gz "+\
                  self.runid+"summary")
        logging.info("Finished tarring of Summary")
        self.copy_file()
    
    ##################################################################
    # Copies the required files for different users.
    ##################################################################
    def copy_file(self):
        logging.info("Making copies of summary tar")
        os.chdir(self.runid_dir)
        logging.debug(self.conf_obj.copy_to_runid)
        logging.debug(self.conf_obj.copy_to_users)
        logging.debug(str(self.conf_obj.lane_user_dict)+" HI")
        for file in self.conf_obj.copy_to_runid:
            self.transCopy.copy(file, os.getcwd())
        for file in self.conf_obj.copy_to_users:
            for name in self.conf_obj.lane_user_dict.itervalues():
                self.transCopy.copy(file, name)
                if file.split('/')[-1] == 'Summary.htm':
                    os.chdir(name)
                    logging.debug("Renaming Summary.htm "+os.getcwd())
                    self.move('Summary.htm', self.runid+'_Summary.htm')
                    os.chdir('..')
        os.chdir(self.runid_dir)
        self.removeDir(self.runid+"summary")

    def ftp_files(self):
        i = 0
        #############################################################
        #name_folder format
        #
        #1. For Non-quest
        #   Length of list = 2
        #   [<name of user>, <folder to cd to on quest>]
        #2. For Quest
        #   Length of list = 4
        #   [<name of user>, <folder to cd to on quest>, <sample id>,
        #    <experiment id>]
        #############################################################
        for lane, name_folder in self.conf_obj.ftp_user_dict.\
                iteritems():
            if not self.conf_obj.perm_dict[lane][1]:
                continue
            os.chdir(self.runid_dir)
            os.chdir(name_folder[0])
            file_list = commands.getoutput("ls *s_"+lane+"*").\
                                           split("\n")
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            if len(name_folder) < 4:
                self.ftp_obj.connect()
                self.ftp_obj.chdir(name_folder[1])
                try:
                    self.ftp_obj.mkdir(self.runid)
                except ftplib.error_perm:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
                    logging.debug("runid dir already there")
                self.ftp_obj.chdir(self.runid)
                try:
                    self.ftp_obj.mkdir(name_folder[0])
                except ftplib.error_perm:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
                    self.ftp_obj.chdir(self.runid)
                    logging.debug("user dir also already there")
                self.ftp_obj.chdir(name_folder[0])
            else:
                if self.conf_obj.ftp_quest:
                    self.ftp_obj.connect()
                    self.ftp_obj.chdir(name_folder[1])
            for file in file_list:
                logging.info("Transferring file %s of user %s" \
                        %(file, name_folder[0]))
                if self.conf_obj.ftp_quest:
                    self.ftp_obj.copy_file(file)
                else:
                    self.transCopy.copy(file, '/cccstor/'+\
                              name_folder[1]+'/'+file)
                    logging.debug('cp '+file+' /cccstor/'+\
                              name_folder[1]+'/'+file)
            if self.conf_obj.ftp_quest:
                self.ftp_obj.close()
            logging.info("Finished transfer !!!")
            if len(name_folder) == 4:
                sid = name_folder[2]
                eid = name_folder[3]
                self.conf_obj.server.UpdateQuestSampleResults(sid,\
                                     eid, file_list, self.runid, \
                                     "IlluminaGAII", "heWex45--")
    def sftp_files(self):
        #name_folder = [uname, passwd, email id]
        for lane, name_folder in self.conf_obj.sftp_user_dict.\
                iteritems():
            if not self.conf_obj.perm_dict[lane][1]:
                continue
            os.chdir(self.runid_dir)
            os.chdir(name_folder[0])
            file_list = commands.getoutput("ls *s_"+lane+"*").\
                                           split("\n")
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            self.transSftp.setServerParams(self.conf_obj.sftp_server,
                                           name_folder[0],
                                           name_folder[1])
            for file in file_list:
                logging.info("Transferring file %s of user %s" \
                        %(file, name_folder[0]))
                self.transSftp.copy(file, "/data/repos/solexa/"+\
                        name_folder[0]+"/"+file)
            self.mail_obj = mail.Mail()
            if(name_folder[2] != ""):
                email_text = "Results for Illumina run "+self.runid+\
                             " has been uploaded to our SFTP server.\n"+\
                             "SFTP server: "+self.conf_obj.sftp_server+\
                             "\nUsername: "******"\nPassword: "******"\n"
                email_text += "Use winscp to download the files\n"+\
                              "http://winscp.net/eng/download.php\n"
                email_text += "\nRegards,\n\n"+\
                "Biomedical Informatics Shared Resource\n"+ \
                "The Ohio State University Comprehensive Cancer"+\
                "Center\n"+\
                "210 Biomedical Research Tower\n 460 W 12th Ave\n"+\
                "Columbus, OH 43210\n (614) 366-1538\n"
                self.mail_obj.mail_diff(name_folder[2], \
                                        "Illumina run "+self.runid, \
                                        email_text)
Esempio n. 3
0
class MakeGerald(CommandOp):

    def __init__(self, logging):
        #full path
        self.logging = logging
        self.transRsync = Transfer('rsync')

    def set_params(self, run_folder, gerald_conf, dist_conf, dest, wall):
        self.logging.debug("setting params")
        if dest == 'OSC':
            self.local = 0
            self.gerald_pl = "/nfs/03/osu5422/CASAVA/CASAVA_v1.7.0-build/bin/GERALD.pl"
        elif dest == 'P01':
            self.local = 1
            self.gerald_pl = 'GERALD.pl'
        self.config_file = gerald_conf
        self.runid = run_folder.split('_')[0]
        self.ran_pipeline = 0
        self.pbs_path = '/nfs/03/osu5422/pbs_scripts/'

        self.options = "--EXPT_DIR"
        self.root_dir = "/slxdata/solexadata/"
        self.clust_server = 'glenn.osc.edu' 
        self.clust_uname = 'osu5422' 
        self.clust_passwd = 'A706412O'
        if not self.local:
            self.all_runs = "/nfs/proj07/BISR/"
        else:
            self.all_runs = "/slxdata/solexadata/"
        if not self.local:
            self.expt_dir = self.all_runs+'illumina_runs/'+run_folder+\
                            "/Data/Intensities/BaseCalls/"
        else:
            self.expt_dir = self.all_runs+run_folder+\
                            "/Data/Intensities/BaseCalls/"
        self.run_folder = run_folder
        self.compress_conf = dist_conf
        self.conf_obj = config.Parameters(self.compress_conf, 1)
        #############################################################
        #Take these from a config???
        #############################################################
        self.ssh_obj = run_ssh.SSH_cmd(self.clust_server, 
                                       self.clust_uname, 
                                       self.clust_passwd)
        self.transRsync.setServerParams(self.clust_server,
                                        self.clust_uname,
                                        self.clust_passwd)
        self.pbs_string = "#PBS -N gerald-job-"+self.runid+"-"+\
                          str(self.conf_obj.numOfNodes)+"n"+\
                          str(self.conf_obj.numOfCores)+"p-"+\
                          "test30tiles\n#PBS -l walltime="+\
                          str(self.conf_obj.wallTime)+":00:00\n"+\
                          "#PBS -l nodes="+\
                          str(self.conf_obj.numOfNodes)+":ppn="+\
                          str(self.conf_obj.numOfCores)+\
                          "\n#PBS -j oe\n"+\
                          "#PBS -S /bin/ksh\n#PBS -m b\n"+\
                          "#PBS -m e\n\n#tdate=$(date +%T)\n\n"+\
                          "set -x\n\ncd "
        self.backup_dir = self.all_runs+"illumina_runs/\nbackup"
        self.comp_conf = "[main]\nrunid="+self.runid+"\n"+\
                         "[compress]\nroot="+self.all_runs+\
                         "illumina_runs/\nbackup="+\
                         self.all_runs+"illumina_runs/backup/\nGerald="
        #self.InitiateRsync()

    def clean_content(self, in_l):
        in_l.reverse()
        ret_l = []
        for i in in_l:
            if i.strip().endswith('directory'):
                #error?
                raw_input("should not be here")
                break
            if i.startswith('ls'):
                break
            ret_l.append(i.split('/')[-1][:-7])
        return ret_l


    def rsync_back(self):
        #if not self.ran_pipeline:
            #print "error!!!"
            #throw error
            #return
        self.runid_dir = self.backup_dir+"/"+self.runid
        self.runid_dir = "".join(self.runid_dir.split("\n"))
        for lane, name_folder in self.conf_obj.ftp_user_dict.\
                iteritems():
            temp_dir = self.runid_dir
            temp_dir = os.path.join(temp_dir, name_folder[0]) + '/'
            out_file = open("temp", 'w') 
            if not test:
                self.ssh_obj.run("ls "+temp_dir+"*s_"+lane+"*",
                                 out_file)
            else:
                print "ls "+temp_dir+"*s_"+lane+"*"
            out_file.close()
            out_file = open("temp", 'r') 
            file_cont = out_file.readlines()
            file_cont.pop()
            out_file.close()
            file_cont = self.clean_content(file_cont)
            self.remove("temp")
            file_list = file_cont
            file_list.append(self.runid+"_summary.tar.gz")
            file_list.append(self.runid+"_Summary.htm")
            for file in file_list:
                self.transRsync.copy(temp_dir+file, '/cccstor/'+\
                                   name_folder[1]+'/', 0)
                self.logging.debug("transferred "+temp_dir+file+" to /cccstor/"+name_folder[1]+"/")
                #tell quest ???
                pass 
                #rsync back to p01
            if len(name_folder) == 4:
                sid = name_folder[2]
                eid = name_folder[3]
                self.conf_obj.server.UpdateQuestSampleResults(sid,\
                                     eid, file_list, self.runid, \
                                     "IlluminaGAII", "heWex45--")
                                     #"terry", "chess")
                self.logging.debug("updated quest for "+str(file_list))
        self.logging.info("Finished run "+self.runid)
    def write_comppbs(self):
        pbs_file_n = self.runid+'compress.pbs'
        self.pbs_file_h = open(pbs_file_n, 'w')
        temp = self.run_folder.split("_")[0]
        conf_file = self.all_runs+"compression_configs/conf_"+temp+".txt"
        self.comp_pbs_string = self.pbs_string+"py_source\n"+\
                           "source ~/.bashrc\n"+\
                           "/usr/bin/python cat_gzip.py "+\
                           conf_file+\
                           "\n/usr/bin/python osc_quest.py "+\
                           conf_file+" "+self.run_folder+" 0"
        self.pbs_file_h.write(self.comp_pbs_string)
        self.pbs_file_h.close()
        self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
        self.remove(pbs_file_n)
        
    def write_makepbs(self, path):
        pbs_file_n = self.runid+'.pbs'
        self.pbs_file_h = open(pbs_file_n, 'w')
        self.make_pbs_string = self.pbs_string+path+\
                           "\ntime make -j "+\
                           str(self.conf_obj.numOfCores)+\
                           "\ncd $HOME/py_source/\n"+\
                           "\n/usr/bin/python osc_quest.py "+\
                           self.all_runs+'compression_configs/conf_'+\
                           pbs_file_n.strip('.pbs')+\
                           ".txt "+self.run_folder+" 1"

        self.pbs_file_h.write(self.make_pbs_string)
        self.pbs_file_h.close()
        self.transRsync.copy(pbs_file_n, self.pbs_path, 1)
        self.remove(pbs_file_n)

    def loadConfigObj(self, runFolder):
        temp = os.getcwd()
        os.chdir(self.codeBasePath)
        os.chdir("compression_configs")
        localPath = os.path.basename("conf_"+\
                    runFolder.split("_")[0]+".txt")
        print localPath
        self.conf_obj = config.Parameters(localPath, 0)
        self.conf_obj.popAll()
        os.chdir(temp)

    def InitiateRsync(self):
        self.write_comppbs()
        self.logging.info("Finished writing compression pbs") self.logging.debug("Starting Rsync to osc") if not os.access(self.root_dir+self.run_folder, os.F_OK):
            raise FatalError("Cannot access "+\
                             self.root_dir+self.run_folder)
        os.chdir(self.root_dir)
        if not self.local:
            self.transRsync.copy(os.path.join(os.getcwd(),self.run_folder),
                               self.all_runs+'illumina_runs/', 1)
        self.config_file = self.write_config(self.config_file,
                                             'gerald')
        self.logging.info("Finished writing Gerald config")
        temp = self.gerald_pl+' '+self.config_file+' '+\
               self.options+' '+self.expt_dir+' --make'
        self.logging.debug("Running command "+temp)
        reqd_path = self.gen_make(temp)
        self.logging.info("Path for gerald "+reqd_path)
        self.comp_conf += reqd_path+"\nBustard="
        reqd_path = '/'.join(reqd_path.rstrip('/').split('/')[:-1])
        self.comp_conf += reqd_path+"\n"
        self.compress_conf = self.write_config(self.comp_conf+\
                                               self.compress_conf,
                                               'compression')
        temp = os.getcwd()
        os.chdir(self.codeBasePath)
        self.conf_obj = config.Parameters(self.compress_conf, 0)
        os.chdir(temp)
        self.conf_obj.popAll()
        #try:
	    #self.conf_obj.server.\
		#UpdateRunStatusJobId(self.run_folder, self.job_id,'')
        #except:
	    #self.logging.info("Quest error")
        self.logging.info("Finished writing compression config")
        #temp = self.gerald_pl+' '+self.config_file+' '
        if not self.local:
            self.write_comppbs()
            self.logging.info("Finished writing compression pbs")