예제 #1
0
# -*- coding: utf-8 *-*

import sys
import os
import errno
import time

# global configuration
import vprimer.glv as glv
import vprimer.utils as utl

from vprimer.logging_config import LogConf
log = LogConf.open_log(__name__)

import vcfpy

from vprimer.allele_select import AlleleSelect

class Variant(object):

    def __init__(self):
        pass


    def pick_variant(self):
        """
        """

        # open vcf through vcfpy
        reader = vcfpy.Reader.from_path(glv.conf.vcf_file)
예제 #2
0
    def open_log(self):

        global log
        log = LogConf.open_log(__name__)
예제 #3
0
    def open_log_enzyme(self):

        # in glv
        global log
        log = LogConf.open_log(__name__)

        self.default_enzyme_names = [
            ['# A'],
            ['AluI', 'AGCT', 'AG^_CT', '4'],
            ['ApaI', 'GGGCCC', 'G_GGCC^C', '6'],
            ['AscI', 'GGCGCGCC', 'GG^CGCG_CC', '8'],
            ['AvrII', 'CCTAGG', 'C^CTAG_G', '6'],
            [''],
            ['# B'],
            ['BamHI', 'GGATCC', 'G^GATC_C', '6'],
            ['BbsI', 'GAAGAC', 'GAAGACNN^NNNN_N', '6'],
            ['BclI', 'TGATCA', 'T^GATC_A', '6'],
            ['BglII', 'AGATCT', 'A^GATC_T', '6'],
            ['BsaI', 'GGTCTC', 'GGTCTCN^NNNN_N', '6'],
            ['BsiWI', 'CGTACG', 'C^GTAC_G', '6'],
            ['BsmFI', 'GGGAC', 'GGGACNNNNNNNNNN^NNNN_N', '5'],
            ['BspHI', 'TCATGA', 'T^CATG_A', '6'],
            ['BssHII', 'GCGCGC', 'G^CGCG_C', '6'],
            ['Bst1107I', 'GTATAC', 'GTA^_TAC', '6'],
            ['BstBI', 'TTCGAA', 'TT^CG_AA', '6'],
            ['BstEII', 'GGTNACC', 'G^GTNAC_C', '7'],
            ['BstXI', 'CCANNNNNNTGG', 'CCAN_NNNN^NTGG', '12'],
            [''],
            ['# C'],
            ['ClaI', 'ATCGAT', 'AT^CG_AT', '6'],
            [''],
            ['# D'],
            ['DdeI', 'CTNAG', 'C^TNA_G', '5'],
            ['DpnI', 'GATC', 'GA^_TC', '4'],
            ['DraI', 'TTTAAA', 'TTT^_AAA', '6'],
            ['DraIII', 'CACNNNGTG', 'CAC_NNN^GTG', '9'],
            [''],
            ['# E'],
            ['Eco52I', 'CGGCCG', 'C^GGCC_G', '6'],
            ['EcoO109I', 'RGGNCCY', 'RG^GNC_CY', '7'],
            ['EcoO65I', 'GGTNACC', 'G^GTNAC_C', '7'],
            ['EcoRI', 'GAATTC', 'G^AATT_C', '6'],
            ['EcoRV', 'GATATC', 'GAT^_ATC', '6'],
            ['EcoT14I', 'CCWWGG', 'C^CWWG_G', '6'],
            [''],
            ['# F'],
            ['FseI', 'GGCCGGCC', 'GG_CCGG^CC', '8'],
            [''],
            ['# H'],
            ['HaeII', 'RGCGCY', 'R_GCGC^Y', '6'],
            ['HincII', 'GTYRAC', 'GTY^_RAC', '6'],
            ['HindIII', 'AAGCTT', 'A^AGCT_T', '6'],
            ['HinfI', 'GANTC', 'G^ANT_C', '5'],
            ['HpaI', 'GTTAAC', 'GTT^_AAC', '6'],
            ['HphI', 'GGTGA', 'GGTGANNNNNNN_N^N', '5'],
            [''],
            ['# K'],
            ['KpnI', 'GGTACC', 'G_GTAC^C', '6'],
            [''],
            ['# M'],
            ['MluI', 'ACGCGT', 'A^CGCG_T', '6'],
            ['MseI', 'TTAA', 'T^TA_A', '4'],
            [''],
            ['# N'],
            ['NcoI', 'CCATGG', 'C^CATG_G', '6'],
            ['NdeI', 'CATATG', 'CA^TA_TG', '6'],
            ['NheI', 'GCTAGC', 'G^CTAG_C', '6'],
            ['NlaIII', 'CATG', '_CATG^', '4'],
            ['NotI', 'GCGGCCGC', 'GC^GGCC_GC', '8'],
            ['NruI', 'TCGCGA', 'TCG^_CGA', '6'],
            ['NsiI', 'ATGCAT', 'A_TGCA^T', '6'],
            [''],
            ['# P'],
            ['PacI', 'TTAATTAA', 'TTA_AT^TAA', '8'],
            ['PmeI', 'GTTTAAAC', 'GTTT^_AAAC', '8'],
            ['PmlI', 'CACGTG', 'CAC^_GTG', '6'],
            ['Psp1406I', 'AACGTT', 'AA^CG_TT', '6'],
            ['PstI', 'CTGCAG', 'C_TGCA^G', '6'],
            ['PvuII', 'CAGCTG', 'CAG^_CTG', '6'],
            [''],
            ['# R'],
            ['RsaI', 'GTAC', 'GT^_AC', '4'],
            [''],
            ['# S'],
            ['SacI', 'GAGCTC', 'G_AGCT^C', '6'],
            ['SacII', 'CCGCGG', 'CC_GC^GG', '6'],
            ['SalI', 'GTCGAC', 'G^TCGA_C', '6'],
            ['SapI', 'GCTCTTC', 'GCTCTTCN^NNN_N', '7'],
            ['SbfI', 'CCTGCAGG', 'CC_TGCA^GG', '8'],
            ['ScaI', 'AGTACT', 'AGT^_ACT', '6'],
            ['SfiI', 'GGCCNNNNNGGCC', 'GGCCN_NNN^NGGCC', '13'],
            ['SmaI', 'CCCGGG', 'CCC^_GGG', '6'],
            ['SnaBI', 'TACGTA', 'TAC^_GTA', '6'],
            ['SpeI', 'ACTAGT', 'A^CTAG_T', '6'],
            ['SphI', 'GCATGC', 'G_CATG^C', '6'],
            ['SspI', 'AATATT', 'AAT^_ATT', '6'],
            ['StuI', 'AGGCCT', 'AGG^_CCT', '6'],
            ['SwaI', 'ATTTAAAT', 'ATTT^_AAAT', '8'],
            [''],
            ['# T'],
            ['TaqI', 'TCGA', 'T^CG_A', '4'],
            ['Tth111I', 'GACNNNGTC', 'GACN^N_NGTC', '9'],
            [''],
            ['# X'],
            ['XbaI', 'TCTAGA', 'T^CTAG_A', '6'],
            ['XhoI', 'CTCGAG', 'C^TCGA_G', '6'],
            ['XmaI', 'CCCGGG', 'C^CCGG_G', '6'],
            [''],
            ['# end'],
        ]
예제 #4
0
파일: conf.py 프로젝트: ncod3/vprimer_vorg
    def __init__(self):

        self.init_version = 0.0

        self.ini_file = ''
        self.ini_file_path = ''
        self.ini = configparser.ConfigParser()
        # don't convert to lower case
        self.ini.optionxform = str

        # log
        self.log = LogConf()

        self.thread = 0

        self.parallel_blast_cnt = 0
        self.parallele_full_thread = 0
        self.blast_num_threads = 0

        # default 1000
        self.blast_word_size = 0
        self.use_joblib_threading = 'no'
        self.parallel = False

        # indel_len
        self.min_indel_len = 0
        self.max_indel_len = 0
        # product_size
        self.min_product_size = 0
        self.max_product_size = 0

        # group
        self.regions_dict = dict()
        self.distin_g_list = list()
        self.g_members_dict = dict()

        self.cwd = os.getcwd()

        self.ref_dir = ''
        self.out_dir = ''
        self.out_bak_dir = ''
        self.log_dir = ''

        self.pick_mode = ''
        self.progress = ''
        self.stop = 'no'

        self.ref = ''
        self.vcf = ''

        self.min_indel_len = 0
        self.max_indel_len = 0
        self.min_product_size = 0
        self.max_product_size = 0
        self.fragment_pad_len = 0

        self.PRIMER_THERMODYNAMIC_PARAMETERS_PATH = ''
        self.PRIMER_PRODUCT_SIZE_RANGE = ''
        self.PRIMER_NUM_RETURN = ''
        self.PRIMER_MIN_SIZE = ''
        self.PRIMER_OPT_SIZE = ''
        self.PRIMER_MAX_SIZE = ''
        self.PRIMER_MIN_GC = ''
        self.PRIMER_OPT_GC = ''
        self.PRIMER_MAX_GC = ''
        self.PRIMER_MIN_TM = ''
        self.PRIMER_OPT_TM = ''
        self.PRIMER_MAX_TM = ''
        self.PRIMER_MAX_POLY_X = ''
        self.PRIMER_PAIR_MAX_DIFF_TM = ''

        self.alternate_distance = 0

        # ------------------------
        self.ref_fasta = ''
        self.ref_fasta_chrom_list = ''
        self.ref_fasta_fai = ''
        self.ref_fasta_pickle = ''
        self.ref_fasta_slink_system = ''
        self.ref_fasta_user = ''

        self.vcf_file_user = ''
        self.vcf_file_slink_system = ''
        self.vcf_file = ''

        #-- sample_nickname
        self.vcf_basename_to_fullname = dict()
        self.vcf_nickname_to_fullname = dict()
        self.nickname_to_basename = dict()
        self.basename_to_nickname = dict()

        #-- enzyme
        self.enzyme_files_list = list()
        self.enzyme_files_str = ''

        self.blastdb_title = ''
        self.blastdb = ''
예제 #5
0
파일: conf.py 프로젝트: ncod3/vprimer_vorg
class Conf(object):
    def __init__(self):

        self.init_version = 0.0

        self.ini_file = ''
        self.ini_file_path = ''
        self.ini = configparser.ConfigParser()
        # don't convert to lower case
        self.ini.optionxform = str

        # log
        self.log = LogConf()

        self.thread = 0

        self.parallel_blast_cnt = 0
        self.parallele_full_thread = 0
        self.blast_num_threads = 0

        # default 1000
        self.blast_word_size = 0
        self.use_joblib_threading = 'no'
        self.parallel = False

        # indel_len
        self.min_indel_len = 0
        self.max_indel_len = 0
        # product_size
        self.min_product_size = 0
        self.max_product_size = 0

        # group
        self.regions_dict = dict()
        self.distin_g_list = list()
        self.g_members_dict = dict()

        self.cwd = os.getcwd()

        self.ref_dir = ''
        self.out_dir = ''
        self.out_bak_dir = ''
        self.log_dir = ''

        self.pick_mode = ''
        self.progress = ''
        self.stop = 'no'

        self.ref = ''
        self.vcf = ''

        self.min_indel_len = 0
        self.max_indel_len = 0
        self.min_product_size = 0
        self.max_product_size = 0
        self.fragment_pad_len = 0

        self.PRIMER_THERMODYNAMIC_PARAMETERS_PATH = ''
        self.PRIMER_PRODUCT_SIZE_RANGE = ''
        self.PRIMER_NUM_RETURN = ''
        self.PRIMER_MIN_SIZE = ''
        self.PRIMER_OPT_SIZE = ''
        self.PRIMER_MAX_SIZE = ''
        self.PRIMER_MIN_GC = ''
        self.PRIMER_OPT_GC = ''
        self.PRIMER_MAX_GC = ''
        self.PRIMER_MIN_TM = ''
        self.PRIMER_OPT_TM = ''
        self.PRIMER_MAX_TM = ''
        self.PRIMER_MAX_POLY_X = ''
        self.PRIMER_PAIR_MAX_DIFF_TM = ''

        self.alternate_distance = 0

        # ------------------------
        self.ref_fasta = ''
        self.ref_fasta_chrom_list = ''
        self.ref_fasta_fai = ''
        self.ref_fasta_pickle = ''
        self.ref_fasta_slink_system = ''
        self.ref_fasta_user = ''

        self.vcf_file_user = ''
        self.vcf_file_slink_system = ''
        self.vcf_file = ''

        #-- sample_nickname
        self.vcf_basename_to_fullname = dict()
        self.vcf_nickname_to_fullname = dict()
        self.nickname_to_basename = dict()
        self.basename_to_nickname = dict()

        #-- enzyme
        self.enzyme_files_list = list()
        self.enzyme_files_str = ''

        self.blastdb_title = ''
        self.blastdb = ''
        # ------------------------

    def read_ini(self):

        # ---------------------------------------------------------------
        # ini_file from param.p['config'] absolute or relative path
        self.ini_file = glv.param.p.config
        print("ini_file = {}".format(self.ini_file))
        # ---------------------------------------------------------------

        # ini_file_path
        self.ini_file_path = "{}/{}".format(self.cwd, self.ini_file)
        print("self.ini_file_path = {}".format(self.ini_file_path))

        # read ini_file
        if os.path.exists(self.ini_file_path):
            print("found {}".format(self.ini_file_path))

            # https://docs.python.org/ja/3/library/configparser.html
            with open(self.ini_file_path, encoding='utf-8') as fp:
                self.ini.read_file(fp)

                # adjustment of variable format
                self._rectify_variable()
                # into_variable
                self._ini_into_variable()

                # if want to know vcf
                if (glv.param.p.vcf_samples == True):
                    sample_names = utl.get_vcf_sample_name_list(
                        self.vcf_file_user)
                    print('{}\n'.format('\n'.join(sample_names)))
                    sys.exit(1)

                # several path
                self._set_path_and_all_start()

                # thread
                self._thread_adjusting()

                # nickname
                self._get_nickname()
                # regions
                self._get_regions()
                # distin_g
                self._get_distinguish_groups()
                # g_members
                self._get_members()

                # merge
                self._merge_conf()
                # log all conf
                self._print_conf()

        else:
            print("Not found {}, exit.".format(self.ini_file_path))
            #raise FileNotFoundError(errno.ENOENT,
            #    os.strerror(errno.ENOENT),
            #    self.ini_file_path)
            sys.exit(1)

        return self

    def _print_conf(self):

        log.info("")
        log.info("Adopted configurations:")

        log.info("[global]")
        log.info("init_version={}".format(self.init_version))
        log.info("thread={}".format(self.thread))
        log.info("use_joblib_threading={}".format(self.use_joblib_threading))
        log.info("parallel={}".format(self.parallel))
        log.info("ref={}".format(self.ref))
        log.info("vcf={}".format(self.vcf))
        log.info("min_indel_len={}".format(self.min_indel_len))
        log.info("max_indel_len={}".format(self.max_indel_len))
        log.info("min_product_size={}".format(self.min_product_size))
        log.info("max_product_size={}".format(self.max_product_size))
        log.info("ref_dir={}".format(self.ref_dir))
        log.info("log_dir={}".format(self.log_dir))
        log.info("out_dir={}".format(self.out_dir))
        log.info("out_bak_dir={}".format(self.out_bak_dir))
        log.info("pick_mode={}".format(self.pick_mode))
        log.info("progress={}".format(self.progress))

        log.info("[vprimer']")
        log.info("fragment_pad_len={}".format(self.fragment_pad_len))

        log.info("[caps]")
        log.info("enzyme_files_str={}".format(self.enzyme_files_str))

        log.info("[primer3]")
        log.info("PRIMER_THERMODYNAMIC_PARAMETERS_PATH={}".format(
            self.PRIMER_THERMODYNAMIC_PARAMETERS_PATH))
        log.info("PRIMER_PRODUCT_SIZE_RANGE={}".format(
            self.PRIMER_PRODUCT_SIZE_RANGE))
        log.info("PRIMER_NUM_RETURN={}".format(self.PRIMER_NUM_RETURN))
        log.info("PRIMER_MIN_SIZE={}".format(self.PRIMER_MIN_SIZE))
        log.info("PRIMER_OPT_SIZE={}".format(self.PRIMER_OPT_SIZE))
        log.info("PRIMER_MAX_SIZE={}".format(self.PRIMER_MAX_SIZE))
        log.info("PRIMER_MIN_GC={}".format(self.PRIMER_MIN_GC))
        log.info("PRIMER_OPT_GC={}".format(self.PRIMER_OPT_GC))
        log.info("PRIMER_MAX_GC={}".format(self.PRIMER_MAX_GC))
        log.info("PRIMER_MIN_TM={}".format(self.PRIMER_MIN_TM))
        log.info("PRIMER_OPT_TM={}".format(self.PRIMER_OPT_TM))
        log.info("PRIMER_MAX_TM={}".format(self.PRIMER_MAX_TM))
        log.info("PRIMER_MAX_POLY_X={}".format(self.PRIMER_MAX_POLY_X))
        log.info("PRIMER_PAIR_MAX_DIFF_TM={}".format(
            self.PRIMER_PAIR_MAX_DIFF_TM))

        log.info("[blast]")
        log.info("alternate_distance={}".format(self.alternate_distance))
        log.info("blast_word_size={}".format(self.blast_word_size))
        log.info("")

    def _ini_into_variable(self):

        sect = 'global'

        self.init_version = \
            float(self._set_default(sect, 'init_version', 1.0))

        self.use_joblib_threading = \
             str(self._set_default(sect, 'use_joblib_threading', 'yes'))

        self.parallel = \
            self._conv_joblib(self.use_joblib_threading)

        #log.info("{} {} {}".format(
        #    self.ini[sect]['use_joblib_threading'],
        #    self.use_joblib_threading,
        #    self.parallel))

        self.thread = \
            int(self._set_default(sect, 'thread', 2))

        self.ref = \
            str(self._set_default(sect, 'ref', ''))

        # user's fasta: convert relative path to absolute path based on cwd
        if self.ref.startswith('/'):
            # originally absolute path
            self.ref_fasta_user = self.ref
        else:
            # cwd + relative path
            self.ref_fasta_user = "******".format(self.cwd, self.ref)

        self.vcf = \
            str(self._set_default(sect, 'vcf', ''))

        # user's vcf: convert relative path to absolute path based on cwd
        if self.vcf.startswith('/'):
            # originally absolute path
            self.vcf_file_user = self.vcf
        else:
            # cwd + relative path
            self.vcf_file_user = "******".format(self.cwd, self.vcf)

        self.min_indel_len = \
            int(self._set_default(sect, 'min_indel_len', 50))
        self.max_indel_len = \
            int(self._set_default(sect, 'max_indel_len', 200))
        self.min_product_size = \
            int(self._set_default(sect, 'min_product_size', 200))
        self.max_product_size = \
            int(self._set_default(sect, 'max_product_size', 500))

        self.ref_dir = \
            str(self._set_default(sect, 'ref_dir', 'refs'))
        self.log_dir = \
            str(self._set_default(sect, 'log_dir', 'logs'))
        self.out_dir = \
            str(self._set_default(sect, 'out_dir', 'out_dir'))
        self.out_bak_dir = ''

        self.pick_mode = \
            str(self._set_default(sect, 'pick_mode', 'all'))
        self.progress = \
            str(self._set_default(sect, 'progress', 'all'))

        #sect = 'regions'
        #sect = 'sample_nickname'
        #sect = 'groups'

        sect = 'vprimer'
        self.fragment_pad_len = \
            int(self._set_default(sect, 'fragment_pad_len', 500))

        sect = 'caps'

        self.enzyme_files_str = \
            (self._set_default(sect, 'enzyme_files', ''))

        sect = 'primer3'

        self.PRIMER_THERMODYNAMIC_PARAMETERS_PATH = \
            str(self._set_default(
                sect, 'PRIMER_THERMODYNAMIC_PARAMETERS_PATH', ''))
        self.PRIMER_PRODUCT_SIZE_RANGE = "{}-{}".format(
            self.min_product_size, self.max_product_size)

        self.PRIMER_NUM_RETURN = str(1)

        self.PRIMER_MIN_SIZE = \
            str(self._set_default(sect, 'PRIMER_MIN_SIZE', 23))

        self.PRIMER_OPT_SIZE = \
            str(self._set_default(sect, 'PRIMER_OPT_SIZE', 25))
        self.PRIMER_MAX_SIZE = \
            str(self._set_default(sect, 'PRIMER_MAX_SIZE', 27))
        self.PRIMER_MIN_GC = \
            str(self._set_default(sect, 'PRIMER_MIN_GC', 40))
        self.PRIMER_OPT_GC = \
            str(self._set_default(sect, 'PRIMER_OPT_GC', 50))
        self.PRIMER_MAX_GC = \
            str(self._set_default(sect, 'PRIMER_MAX_GC', 60))
        self.PRIMER_MIN_TM = \
            str(self._set_default(sect, 'PRIMER_MIN_TM', 57.0))
        self.PRIMER_OPT_TM = \
            str(self._set_default(sect, 'PRIMER_OPT_TM', 60.0))
        self.PRIMER_MAX_TM = \
            str(self._set_default(sect, 'PRIMER_MAX_TM', 63.0))
        self.PRIMER_MAX_POLY_X = \
            str(self._set_default(sect, 'PRIMER_MAX_POLY_X', 4))
        self.PRIMER_PAIR_MAX_DIFF_TM = \
            str(self._set_default(
                sect, 'PRIMER_PAIR_MAX_DIFF_TM', 4))

        sect = 'blast'

        self.alternate_distance = \
            int(self._set_default(sect, 'alternate_distance', 10000))
        self.blast_word_size = \
            int(self._set_default(
                sect, 'blast_word_size', self.PRIMER_MIN_SIZE))

    def _set_default(self, sect, key, value):

        if key in self.ini[sect]:
            return self.ini[sect][key]
        else:
            return value

    def _conv_joblib(self, string):

        bool_joblib = False
        if string == 'yes':
            bool_joblib = True

        return bool_joblib

    def _thread_adjusting2(self):

        #log.info("thread={} parallel={}".format(
        #    self.thread, self.parallel))

        self.parallel = \
            self._conv_joblib(self.use_joblib_threading)

        if self.thread < 6:
            self.parallel = False

        if self.parallel == True:

            # unit is 4+1=5
            parallel_base = self.thread
            self.parallele_full_thread = parallel_base

            # blast = 4
            self.parallel_blast_cnt = int(parallel_base / 5)
            self.blast_num_threads = 4

        else:
            # 6 = 5
            full_thread = self.thread - 1
            self.parallele_full_thread = full_thread
            self.blast_num_threads = full_thread

        #log.info("thread={}, parallel={}, parallel_blast_cnt={}".format(
        #    self.parallel, self.thread, self.parallel_blast_cnt))
        #log.info("parallele_full_thread={}, blast_num_threads={}".format(
        #    self.parallele_full_thread, self.blast_num_threads))

    def _thread_adjusting(self):
        ''' in Palallel, if there are 10 threads blast cmd will use at least
            2 cores so par 1 parallel.
            So main python always use 1,
            parallel use 1 thread, blast use 2 threads
        '''

        #self.parallel_blast_cnt = 1
        #self.parallele_full_thread = 1
        #self.blast_num_threads = 1

        #self._thread_adjusting2()
        #return

        #log.info("thread={} use_joblib_threading={}, parallel={}".format(
        #    self.thread, self.use_joblib_threading, self.parallel))

        self.parallel = \
            self._conv_joblib(self.use_joblib_threading)

        if self.thread < 6:
            self.parallel = False

        if self.parallel == True:
            #(7) = 6
            parallel_base = self.thread
            self.parallele_full_thread = parallel_base

            # 6/3=2
            self.parallel_blast_cnt = int(parallel_base / 3)
            self.blast_num_threads = 2

        else:
            # 6 = 5
            full_thread = self.thread - 1
            self.parallele_full_thread = full_thread
            self.blast_num_threads = full_thread

        #log.info("thread={}, use_joblib_threading={}".format(
        #    self.thread, self.use_joblib_threading))
        #log.info("parallel={}, parallel_blast_cnt={}".format(
        #    self.parallel, self.parallel_blast_cnt))
        #log.info("parallele_full_thread={}, blast_num_threads={}".format(
        #    self.parallele_full_thread, self.blast_num_threads))

    def _get_nickname(self):

        # [sample_nickname]
        # nickname    basename
        # sample1   = sample1_sorted.bam
        # sample2   = sample1_sorted.bam

        log.info("[sample_nickname]")

        for nickname in self.ini['sample_nickname']:
            basename = self.ini['sample_nickname'][nickname]

            # fullname will filled in vcf_file get_sample_name_list
            self.nickname_to_basename[nickname] = basename
            self.basename_to_nickname[basename] = nickname

            log.info("{} => {}".format(nickname, basename))

    def _merge_conf(self):

        # debug by param
        if glv.param.p.stop != None:
            glv.conf.stop = glv.param.p.stop

        # update by param
        if glv.param.p.thread != None:
            glv.conf.thread = glv.param.p.thread
            self.use_joblib_threading = 'yes'
            self._thread_adjusting()

        if glv.param.p.joblib != None:
            self.use_joblib_threading = glv.param.p.joblib
            self._thread_adjusting()

        if glv.param.p.ref != None:
            self.ref = glv.param.p.ref

        if glv.param.p.vcf != None:
            self.vcf = glv.param.p.vcf

        if glv.param.p.progress != None:
            self.progress = glv.param.p.progress

        # indel_len
        if glv.param.p.min_indel != None:
            self.min_indel_len = glv.param.p.min_indel

        if glv.param.p.max_indel != None:
            self.max_indel_len = glv.param.p.max_indel

        # product_size
        if glv.param.p.min_product != None:
            self.min_product_size = glv.param.p.min_product

        if glv.param.p.max_product != None:
            self.conf.max_product_size = glv.param.p.max_product

        self.PRIMER_PRODUCT_SIZE_RANGE = "{}-{}".format(
            self.min_product_size, self.max_product_size)

        log.info("glv.param.p={}".format(glv.param.p))

        return self

    def _set_path_and_all_start(self):

        #---------------------------------------------------
        # out_dir
        if glv.param.p.out_dir != None:
            self.out_dir = glv.param.p.out_dir

        # result out dir
        self.out_dir = "{}/{}".format(self.cwd, self.out_dir)

        # logs dir under out dir
        self.log_dir = "{}/{}".format(self.out_dir, self.log_dir)

        # system reference dir
        self.ref_dir = "{}/{}".format(self.cwd, self.ref_dir)

        # out_bak_dir
        self.out_bak_dir = "{}/{}".format(self.out_dir, 'bak')

        # make dir
        self._make_dir_tree()

        # set to LogConf
        global log
        log = self.log.start(__name__, self.out_dir, self.log_dir)

        # log for utils
        utl.start_log()

        # cp ini file to outdir
        self._copy_ini_file()

    def _copy_ini_file(self):

        # ini file
        self.ini_file_path
        # out_dir
        self.out_dir

        # back up
        ini_base = os.path.basename(self.ini_file_path)
        out_dir_ini_file = "{}/{}".format(self.out_dir, ini_base)
        utl.save_to_tmpfile(out_dir_ini_file)

        cmd = "cp {} {}".format(self.ini_file_path, out_dir_ini_file)
        utl.try_exec(cmd)

    def _make_dir_tree(self):

        # already made at conf
        dirs = [
            self.out_dir,
            self.log_dir,
            self.out_bak_dir,
            self.ref_dir,
        ]

        for dir in dirs:
            utl.makedirs(dir)

    def _get_regions(self):

        # chrom:1-1000
        for rg in self.ini['regions']:
            rg_list = self.ini['regions'][rg].split(':')

            if rg_list[0] == '':
                #log.error("error, region={} is empty, exit.".format(rg))
                sys.exit(1)

            if len(rg_list) == 1:
                self.regions_dict[rg] = {
                    'chr': rg_list[0],
                    'start': None,
                    'end': None,
                    'reg': self.ini['regions'][rg]
                }

            else:
                pos = rg_list[1].split('-')
                self.regions_dict[rg] = {
                    'chr': rg_list[0],
                    'start': int(pos[0]),
                    'end': int(pos[1]),
                    'reg': self.ini['regions'][rg]
                }

            log.info("[regions]")
            log.info("{}: {}".format(rg, self.regions_dict[rg]))

    def _get_distinguish_groups(self):

        distins = self.ini['groups']['distinguish_groups'].split(';')
        #log.debug("distins={}".format(distins))

        for distin in distins:

            #log.debug("distin={}".format(distin))
            # [global][pick_mode]
            pick_mode = self.ini['global']['pick_mode']

            # group / reg1, reg2 : pick_mode
            g_pick_mode = distin.split(':')
            if len(g_pick_mode) == 1:
                # not exist pick_mode, only group_regions
                g_region = g_pick_mode
                #log.debug("(1) pick_mode don't exist: {}".format(distin))

            else:
                # exist pick_mode
                if len(g_pick_mode[1]) != 0:
                    pick_mode = g_pick_mode[1]
                    #log.debug("(3) pick_mode exist: {}".format(distin))
                #else:
                #log.debug("(2) pick_mode empty: {}".format(distin))

            # split into group and regions
            g_region = g_pick_mode[0].split('/')

            #log.debug("g_region: {}".format(g_region))

            if len(g_region) == 1:
                # there is no region, only groups separated by <>
                groups = g_region[0].split('<>')
                if groups[0] == '' or groups[1] == '':
                    #log.error(
                    #    "error, empty distinguish_groups {} exit.".format(
                    #    distin))
                    sys.exit(1)

                g_dict = {
                    0: groups[0],
                    1: groups[1],
                    'region': [''],
                    'pick_mode': pick_mode
                }

            else:
                # all staff gathered
                groups = g_region[0].split('<>')
                if groups[0] == '' or groups[1] == '':
                    #log.error(
                    #    "error, empty distinguish_groups {} exit.".format(
                    #    distin))
                    sys.exit(1)

                regions = g_region[1].split(',')

                # gGroup1 <> gGroup2 /
                if len(regions[0]) == 0:
                    g_dict = {
                        0: groups[0],
                        1: groups[1],
                        'region': [''],
                        'pick_mode': pick_mode
                    }
                else:
                    g_dict = {
                        0: groups[0],
                        1: groups[1],
                        'region': regions,
                        'pick_mode': pick_mode
                    }

            self.distin_g_list.append(g_dict)

        log.info("[groups]")
        log.info("{}".format(self.distin_g_list))

    def _get_members(self):
        # group_members =
        #     gHitomebore     : Hitomebore, Sasanishiki
        #     gArroz_da_terra : Arroz_da_terra, Kasalath
        #     gTakanari       : Mochibijin, Tawawakko
        #     gNortai         : Nortai, NERICA1

        group_members_str_org = self.ini['groups']['group_members']
        #log.info("{}".format(group_members_str_org))
        group_members_str = re.sub(r";", ",", group_members_str_org)
        #log.info("{}".format(group_members_str))
        g_members_tmp = group_members_str.split(',')

        group_line = ''
        for gm_tmp in g_members_tmp:
            if ':' in gm_tmp:
                group_line = "{};{},".format(group_line, gm_tmp)
            else:
                group_line = "{},{}".format(group_line, gm_tmp)

        group_line = re.sub(r"^;", "", group_line)
        group_line = re.sub(r",+", ",", group_line)
        group_line = re.sub(r",;", ";", group_line)
        group_line = re.sub(r",$", "", group_line)

        log.info("{}".format(group_line))

        g_members = group_line.split(';')
        #log.info("g_members {}".format(g_members))

        for g_member in g_members:
            gmem_list = g_member.split(':')
            #log.debug("{}".format(gmem_list))

            if len(gmem_list) == 1:
                #log.error("error, empty group_members {} exit.".format(
                #    g_member))
                sys.exit(1)

            elif gmem_list[0] == '' or gmem_list[1] == '':
                #log.error("error, empty group_members {} exit.".format(
                #    g_member))
                sys.exit(1)

            self.g_members_dict[gmem_list[0]] = gmem_list[1].split(',')
            #log.info("{}: {}".format(
            #    gmem_list[0],
            #    self.g_members_dict[gmem_list[0]]))

        log.info("[members]")
        log.info("{}".format(self.g_members_dict))

    def _rectify_variable(self):

        for section in self.ini.sections():
            for key in self.ini[section]:

                val = self.ini[section][key]
                # hash comment remove
                val = utl.strip_hash_comment(val)

                # remove \n at the beginning of value
                val = val.lstrip()

                # replace internal \n to semicolons
                val = val.replace('\n', ';')

                # replace white space to one space
                if key == 'group_members':
                    #log.info("{}".format(val))

                    val = re.sub(r"\s+", " ", val)
                    val = re.sub(r"\s*:\s*", ":", val)
                    val = re.sub(r" ", ",", val)
                    val = re.sub(r",+", ",", val)
                    val = re.sub(r",;", ";", val)
                    # g_members ['group0:TDr2946A_Mal,TDr1489A_Fem',
                    # 'TDr2284A_Mon,TDr1499A_Mon,TDr1509A_Fem',
                    # 'group1:TDr1510A_Fem,TDr3782A_Mal',
                    # 'TDr1533A_Mal,TDr1543A_Fem,TDr1858C_Fem']

                else:
                    #val = re.sub(r"\s+", " ", val)
                    # remove white spaces
                    val = re.sub(r"\s+", "", val)

                # reset
                self.ini[section][key] = val
예제 #6
0
    def __init__(self):

        self.ini = None
        self.param = None

        # ----------------------------------------------------------
        # A dictionary that associates a value with a variable name
        # for all parameters
        self.conf_dict = dict()

        # ----------------------------------------------------------
        self.conf_dict = {
            # show_genotype  no / gt / int
            'show_genotype': {
                'dtype': 'str',
                'default': 'no'
            },

            # debug
            'analyse_caps': {
                'dtype': 'bool',
                'default': 'False'
            },

            # ini, param, ini, easy
            'ini_version': {
                'dtype': 'str',
                'default': glv.ini_version
            },
            'ini_file': {
                'dtype': 'str',
                'default': ''
            },
            'out_dir': {
                'dtype': 'str',
                'default': 'out_vprimer'
            },
            'thread': {
                'dtype': 'int',
                'default': '10'
            },
            'use_joblib_threading':  # param
            {
                'dtype': 'str',
                'default': 'yes'
            },

            #
            'vcf': {
                'dtype': 'str',
                'default': ''
            },
            'ref': {
                'dtype': 'str',
                'default': ''
            },

            #
            'pick_mode': {
                'dtype': 'str',
                'default': 'all'
            },
            'indel_size': {
                'dtype': 'str',
                'default': '20-200'
            },
            'product_size': {
                'dtype': 'str',
                'default': '200-500'
            },

            # list enzyme_file refs/enzyme_names.txt
            'enzyme_file': {
                'dtype': 'str',
                'default': 'no_enzyme'
            },
            # list enzyme
            'enzyme': {
                'dtype': 'str',
                'default': ''
            },

            # list target
            'target': {
                'dtype': 'str',
                'default': ''
            },
            # list a_samples
            'a_sample': {
                'dtype': 'str',
                'default': ''
            },
            # list b_samples
            'b_sample': {
                'dtype': 'str',
                'default': ''
            },

            # list regions
            'regions': {
                'dtype': 'str',
                'default': ''
            },
            # list distinguish_groups
            'distinguish_groups': {
                'dtype': 'str',
                'default': ''
            },
            # list group_members
            'group_members':
            # The default for group_member is group_members_vcf_str
            # read from vcf.
            # Do not initialize the key for safety as we will update it later.
            #                {'dtype': 'str',    'default': ''},
            {
                'dtype': 'str',
            },
            # gen p3_params.txt
            'p3_params': {
                'dtype': 'str',
                'default': 'no_p3_params'
            },
            'fragment_pad_len': {
                'dtype': 'int',
                'default': '500'
            },
            'blast_distance': {
                'dtype': 'int',
                'default': '10000'
            },
            # blast_word_size will be set later after p3 PRIMER_MIN_SIZE
            # is set.
            'blast_word_size':  # PRIMER_MIN_SIZE
            {
                'dtype': 'int',
                'default': '23'
            },

            #
            'show_samples': {
                'dtype': 'bool',
                'default': 'False'
            },
            'show_fasta': {
                'dtype': 'bool',
                'default': 'False'
            },
            'progress': {
                'dtype': 'str',
                'default': 'all'
            },
            'stop': {
                'dtype': 'str',
                'default': 'none'
            },
        }

        # cwd, log --------------------------------------------
        self.cwd = glv.cwd
        self.log = LogConf()

        # for debug
        self.analyse_caps = False

        # show_genotype ---------------------------------------
        self.show_genotype = ""  # ---- INI

        # ini file --------------------------------------------
        self.user_ini_file = ""  # ---- INI
        self.ini_file_path = ""
        self.ini_version_user = ""  # ---- INI
        self.ini_version_system = ""  # ---- INI

        # out_dir ---------------------------------------------
        self.user_out_dir = ""  # ---- INI
        self.out_dir_path = ""
        self.log_dir_path = ""
        self.out_bak_dir_path = ""

        # ref_dir ---------------------------------------------
        self.ref_dir_path = ""  # ---- INI

        # out_curr_setting ------------------------------------
        self.curr_setting_file = ""
        self.curr_setting_file_path = ""

        # thread ----------------------------------------------
        self.thread = 0  # ---- INI
        self.use_joblib_threading = "yes"
        self.parallel = True
        self.parallel_full_thread = 0
        self.parallel_blast_cnt = 0
        self.blast_num_threads = 0

        # vcf -------------------------------------------------
        self.user_vcf_file = ""  # ---- INI
        self.user_vcf_file_path = ""
        self.vcf_file_slink_system = ""
        self.vcf_file_path = ""

        # sample_nickname -------------------------------------
        self.vcf_sample_name_file = ""

        self.vcf_sample_nickname_list = list()
        self.vcf_sample_basename_list = list()
        self.vcf_sample_fullname_list = list()

        self.vcf_sample_nickname_dict = dict()
        self.vcf_sample_basename_dict = dict()
        self.vcf_sample_fullname_dict = dict()

        self.group_members_vcf_str = ""

        # show_samples-----------------------------------------
        self.show_samples = False

        # show_fasta ------------------------------------------
        self.show_fasta = False

        # ref -------------------------------------------------
        self.user_ref_fasta = ""  # ---- INI
        self.user_ref_fasta_path = ""

        # It will be set in main later
        # glv.ref = glv.ref.prepare_ref()
        self.ref_fasta_slink_system = ""
        self.ref_fasta_path = ""
        self.ref_fasta_chrom_dict_list = list()
        self.ref_fasta_chrom_list = list()
        self.ref_fasta_chrom_region_list = list()

        self.ref_fasta_fai = ""
        self.ref_fasta_chrom_txt = ""
        self.ref_fasta_pickle = ""

        # pick_mode -------------------------------------------
        self.pick_mode = ""  # ---- INI

        # indel len -------------------------------------------
        self.indel_size = ""  # ---- INI
        self.min_indel_len = 0
        self.max_indel_len = 0

        # product size ----------------------------------------
        self.product_size = ""  # ---- INI
        self.min_product_size = 0
        self.max_product_size = 0

        # enzyme ----------------------------------------------
        self.enzyme_files_user_str = ""  # ---- INI
        self.enzyme_files_user_list = list()
        self.enzyme_files_list = list()

        self.enzyme_str = ""
        #self.enzyme_list = list()
        self.enzyme_name_list = list()

        # region group member string --------------------------
        # select string by priority, next make dict or list variables
        self.regions_str = ""  # ---- INI
        self.group_members_str = ""  # ---- INI
        self.distinguish_groups_str = ""  # ---- INI

        self.region_name_list = list()
        self.group_name_list = list()

        self.regions_dict = dict()
        self.group_members_dict = dict()
        self.distinguish_groups_list = list()

        # primer3 ---------------------------------------------
        self.p3_params_file = ""  # ---- INI
        self.p3_params_file_path = ""
        self.primer3_header_dict = dict()

        self.fragment_pad_len = 0  # ---- INI

        # primer3 params
        self.p3key = { \
            'PRIMER_MIN_SIZE': 23,
            'PRIMER_OPT_SIZE': 25,
            'PRIMER_MAX_SIZE': 27,
            'PRIMER_MIN_GC': 40,
            'PRIMER_OPT_GC': 50,
            'PRIMER_MAX_GC': 60,
            'PRIMER_MIN_TM': 57.0,
            'PRIMER_OPT_TM': 60.0,
            'PRIMER_MAX_TM': 63.0,
            'PRIMER_MAX_POLY_X': 4,
            'PRIMER_PAIR_MAX_DIFF_TM': 4,
        }

        # blast -----------------------------------------------
        self.blast_distance = 0  # ---- INI

        # not set now -----------------------------------------
        self.blast_word_size = 0
        self.blastdb_title = ""
        self.blastdb = ""

        # start stop ------------------------------------------
        self.progress = ""
        self.stop = ""
예제 #7
0
class ConfBase(object):
    def __init__(self):

        self.ini = None
        self.param = None

        # ----------------------------------------------------------
        # A dictionary that associates a value with a variable name
        # for all parameters
        self.conf_dict = dict()

        # ----------------------------------------------------------
        self.conf_dict = {
            # show_genotype  no / gt / int
            'show_genotype': {
                'dtype': 'str',
                'default': 'no'
            },

            # debug
            'analyse_caps': {
                'dtype': 'bool',
                'default': 'False'
            },

            # ini, param, ini, easy
            'ini_version': {
                'dtype': 'str',
                'default': glv.ini_version
            },
            'ini_file': {
                'dtype': 'str',
                'default': ''
            },
            'out_dir': {
                'dtype': 'str',
                'default': 'out_vprimer'
            },
            'thread': {
                'dtype': 'int',
                'default': '10'
            },
            'use_joblib_threading':  # param
            {
                'dtype': 'str',
                'default': 'yes'
            },

            #
            'vcf': {
                'dtype': 'str',
                'default': ''
            },
            'ref': {
                'dtype': 'str',
                'default': ''
            },

            #
            'pick_mode': {
                'dtype': 'str',
                'default': 'all'
            },
            'indel_size': {
                'dtype': 'str',
                'default': '20-200'
            },
            'product_size': {
                'dtype': 'str',
                'default': '200-500'
            },

            # list enzyme_file refs/enzyme_names.txt
            'enzyme_file': {
                'dtype': 'str',
                'default': 'no_enzyme'
            },
            # list enzyme
            'enzyme': {
                'dtype': 'str',
                'default': ''
            },

            # list target
            'target': {
                'dtype': 'str',
                'default': ''
            },
            # list a_samples
            'a_sample': {
                'dtype': 'str',
                'default': ''
            },
            # list b_samples
            'b_sample': {
                'dtype': 'str',
                'default': ''
            },

            # list regions
            'regions': {
                'dtype': 'str',
                'default': ''
            },
            # list distinguish_groups
            'distinguish_groups': {
                'dtype': 'str',
                'default': ''
            },
            # list group_members
            'group_members':
            # The default for group_member is group_members_vcf_str
            # read from vcf.
            # Do not initialize the key for safety as we will update it later.
            #                {'dtype': 'str',    'default': ''},
            {
                'dtype': 'str',
            },
            # gen p3_params.txt
            'p3_params': {
                'dtype': 'str',
                'default': 'no_p3_params'
            },
            'fragment_pad_len': {
                'dtype': 'int',
                'default': '500'
            },
            'blast_distance': {
                'dtype': 'int',
                'default': '10000'
            },
            # blast_word_size will be set later after p3 PRIMER_MIN_SIZE
            # is set.
            'blast_word_size':  # PRIMER_MIN_SIZE
            {
                'dtype': 'int',
                'default': '23'
            },

            #
            'show_samples': {
                'dtype': 'bool',
                'default': 'False'
            },
            'show_fasta': {
                'dtype': 'bool',
                'default': 'False'
            },
            'progress': {
                'dtype': 'str',
                'default': 'all'
            },
            'stop': {
                'dtype': 'str',
                'default': 'none'
            },
        }

        # cwd, log --------------------------------------------
        self.cwd = glv.cwd
        self.log = LogConf()

        # for debug
        self.analyse_caps = False

        # show_genotype ---------------------------------------
        self.show_genotype = ""  # ---- INI

        # ini file --------------------------------------------
        self.user_ini_file = ""  # ---- INI
        self.ini_file_path = ""
        self.ini_version_user = ""  # ---- INI
        self.ini_version_system = ""  # ---- INI

        # out_dir ---------------------------------------------
        self.user_out_dir = ""  # ---- INI
        self.out_dir_path = ""
        self.log_dir_path = ""
        self.out_bak_dir_path = ""

        # ref_dir ---------------------------------------------
        self.ref_dir_path = ""  # ---- INI

        # out_curr_setting ------------------------------------
        self.curr_setting_file = ""
        self.curr_setting_file_path = ""

        # thread ----------------------------------------------
        self.thread = 0  # ---- INI
        self.use_joblib_threading = "yes"
        self.parallel = True
        self.parallel_full_thread = 0
        self.parallel_blast_cnt = 0
        self.blast_num_threads = 0

        # vcf -------------------------------------------------
        self.user_vcf_file = ""  # ---- INI
        self.user_vcf_file_path = ""
        self.vcf_file_slink_system = ""
        self.vcf_file_path = ""

        # sample_nickname -------------------------------------
        self.vcf_sample_name_file = ""

        self.vcf_sample_nickname_list = list()
        self.vcf_sample_basename_list = list()
        self.vcf_sample_fullname_list = list()

        self.vcf_sample_nickname_dict = dict()
        self.vcf_sample_basename_dict = dict()
        self.vcf_sample_fullname_dict = dict()

        self.group_members_vcf_str = ""

        # show_samples-----------------------------------------
        self.show_samples = False

        # show_fasta ------------------------------------------
        self.show_fasta = False

        # ref -------------------------------------------------
        self.user_ref_fasta = ""  # ---- INI
        self.user_ref_fasta_path = ""

        # It will be set in main later
        # glv.ref = glv.ref.prepare_ref()
        self.ref_fasta_slink_system = ""
        self.ref_fasta_path = ""
        self.ref_fasta_chrom_dict_list = list()
        self.ref_fasta_chrom_list = list()
        self.ref_fasta_chrom_region_list = list()

        self.ref_fasta_fai = ""
        self.ref_fasta_chrom_txt = ""
        self.ref_fasta_pickle = ""

        # pick_mode -------------------------------------------
        self.pick_mode = ""  # ---- INI

        # indel len -------------------------------------------
        self.indel_size = ""  # ---- INI
        self.min_indel_len = 0
        self.max_indel_len = 0

        # product size ----------------------------------------
        self.product_size = ""  # ---- INI
        self.min_product_size = 0
        self.max_product_size = 0

        # enzyme ----------------------------------------------
        self.enzyme_files_user_str = ""  # ---- INI
        self.enzyme_files_user_list = list()
        self.enzyme_files_list = list()

        self.enzyme_str = ""
        #self.enzyme_list = list()
        self.enzyme_name_list = list()

        # region group member string --------------------------
        # select string by priority, next make dict or list variables
        self.regions_str = ""  # ---- INI
        self.group_members_str = ""  # ---- INI
        self.distinguish_groups_str = ""  # ---- INI

        self.region_name_list = list()
        self.group_name_list = list()

        self.regions_dict = dict()
        self.group_members_dict = dict()
        self.distinguish_groups_list = list()

        # primer3 ---------------------------------------------
        self.p3_params_file = ""  # ---- INI
        self.p3_params_file_path = ""
        self.primer3_header_dict = dict()

        self.fragment_pad_len = 0  # ---- INI

        # primer3 params
        self.p3key = { \
            'PRIMER_MIN_SIZE': 23,
            'PRIMER_OPT_SIZE': 25,
            'PRIMER_MAX_SIZE': 27,
            'PRIMER_MIN_GC': 40,
            'PRIMER_OPT_GC': 50,
            'PRIMER_MAX_GC': 60,
            'PRIMER_MIN_TM': 57.0,
            'PRIMER_OPT_TM': 60.0,
            'PRIMER_MAX_TM': 63.0,
            'PRIMER_MAX_POLY_X': 4,
            'PRIMER_PAIR_MAX_DIFF_TM': 4,
        }

        # blast -----------------------------------------------
        self.blast_distance = 0  # ---- INI

        # not set now -----------------------------------------
        self.blast_word_size = 0
        self.blastdb_title = ""
        self.blastdb = ""

        # start stop ------------------------------------------
        self.progress = ""
        self.stop = ""

    # --- before log file open
    def collect_param_ini(self, param, ini):
        ''' aggregate all parameters into one dictionary
        '''

        self.param = param
        self.ini = ini

        for vname in self.conf_dict:

            param_value = self._get_param_value(vname, param)
            ini_value = None

            if param.p['ini_file'] is not None:
                # ini
                ini_value = self._get_ini_value(vname, ini)

            self.conf_dict[vname]['param'] = param_value
            self.conf_dict[vname]['ini'] = ini_value

    def _get_param_value(self, vname, param):
        ''' get data from parameter
        param handles values with the correct data type
        '''

        ret = None

        # a_sample="DRS_013.all.rd DRS_084.all.rd,DRS_099.all.rd ref"
        # b_sample="DRS_025.all.rd, DRS_061.all.rd DRS_101.all.rd"

        # {'a_sample': ['DRS_013.all.rd',
        #   'DRS_084.all.rd,DRS_099.all.rd', 'ref'],
        # 'b_sample': ['DRS_025.all.rd,', 'DRS_061.all.rd',
        #   'DRS_101.all.rd'],

        if vname in param.p:

            val = param.p[vname]

            if type(val) == list:
                # 'DRS_084.all.rd,DRS_099.all.rd'
                # 'DRS_025.all.rd,'
                #print("{}={}".format(vname, val))

                mod_list = list()
                for item in val:
                    sep_list = item.split(",")
                    for sep in sep_list:
                        if sep != "":
                            mod_list.append(sep)

                #ret = ','.join(val)
                ret = ','.join(mod_list)
                #print("\t{}={}".format(vname, ret))

            else:
                ret = val

        return ret

    def _get_ini_value(self, vname, ini):
        ''' get data from ini file
        ini handles values as strings
        '''

        ret = None

        if vname in ini.ini['vprimer']:

            val = ini.ini['vprimer'][vname]
            if type(val) == list:
                ret = ','.join(val)
            else:
                ret = self._cast_val(val, self.conf_dict[vname]['dtype'])

        return ret

    def _cast_val(self, value, dtype):
        ''' for ini file data, casting data to fit data type
        '''

        #print("_cast_val: value={}, dtype={}".format(value, dtype))
        #print("type(value={}".format(type(value)))
        #print("type(dtype={}".format(type(dtype)))

        if dtype == 'int':
            #print("int")
            return int(value)

        elif dtype == 'float':
            #print("float")
            return float(value)

        elif dtype == 'bool':

            # for param
            if type(value) == bool:
                return value
            else:
                # form ini
                if value == "True":
                    return True
                elif value == "False":
                    return False
                else:
                    return None

        elif dtype == 'str':
            #print("str")
            return str(value)

        else:
            #print("else")
            return str(value)

    def _make_out_dir_tree(self):
        '''
        '''

        # if already made at conf
        dirs = [self.out_dir_path, self.log_dir_path, self.out_bak_dir_path]

        for dir in dirs:
            if os.path.isdir(dir):
                # prelog
                utl.prelog("exist dir {}.".format(dir), __name__)
            else:
                utl.prelog("not exist dir {}.".format(dir), __name__)
                utl.makedirs(dir)

    def out_dir_logging_start(self):
        '''
        '''

        # user defined path
        self.user_out_dir = self._value_choice('out_dir')

        # absolute path
        self.out_dir_path = utl.full_path(self.user_out_dir)
        # log_dir
        self.log_dir_path = "{}/{}".format(self.out_dir_path, "logs")
        # bak_dir
        self.out_bak_dir_path = "{}/{}".format(self.out_dir_path, "bak")

        # out_dir and log_dir
        self._make_out_dir_tree()

        # for conf
        global log
        log = self.log.logging_start(__name__, self.out_dir_path,
                                     self.log_dir_path)

        # for utl
        utl.open_log()

    # --- after log file open
    def choice_variables(self):
        ''' decide variable values
        '''

        # print param and ini variables
        self._print_param_ini()

        # for debug
        self.analyse_caps = self._value_choice('analyse_caps')

        # out_dir ---------------------------------------------
        self.user_out_dir = self._value_choice('out_dir')
        self.out_dir_path = utl.full_path(self.user_out_dir)

        # vcf -------------------------------------------------
        self.user_vcf_file = self._value_choice('vcf')
        self.user_vcf_file_path = utl.full_path(self.user_vcf_file)

        # ref -------------------------------------------------
        self.user_ref_fasta = self._value_choice('ref')
        self.user_ref_fasta_path = utl.full_path(self.user_ref_fasta)

        # thread ----------------------------------------------
        self.thread = self._value_choice('thread')


        if self.out_dir_path == "" or \
            self.user_vcf_file_path == "" or \
            self.user_ref_fasta_path == "":
            err_mes = "out_dir={} and vcf={} and ref={} ".format(
                self.out_dir_path, self.user_vcf_file_path,
                self.user_ref_fasta_path)
            err_mes += "are all required. exit."
            log.error(err_mes)
            sys.exit(1)

        log.info("thread={}".format(self.thread))

        # out_dir ---------------------------------------------
        self.out_dir_path = utl.full_path(self.user_out_dir)
        self.log_dir_path = "{}/{}".format(self.out_dir_path, "logs")
        self.out_bak_dir_path = "{}/{}".format(self.out_dir_path, "bak")

        #pprint.pprint(self.conf_dict)
        # INI show_genotype
        self.show_genotype = self._value_choice('show_genotype')
        if self.show_genotype == "":
            self.show_genotype = "gt"

        if not self.show_genotype in glv.show_genotype_list:
            err_mes = "show_genotype is selected from one of "
            err_mes += ", ".join(glv.show_genotype_list)
            log.error("{}. exit.".format(err_mes))
            log.error("show_genotype={}".format(self.show_genotype))
            sys.exit(1)

        # ini_file --------------------------------------------
        # INI
        self.ini_version_user = self._value_choice('ini_version')
        self.ini_version_system = self.conf_dict['ini_version']['default']

        self.user_ini_file = self.conf_dict['ini_file']['param']
        self.ini_file_path = utl.full_path(self.user_ini_file)

        # ref_dir ---------------------------------------------
        self.ref_dir_path = utl.full_path("refs")
        # make ref_dir
        utl.makedirs(self.ref_dir_path)

        # out_curr_setting ------------------------------------
        self.curr_setting_file = "current_setting_ini.txt"
        self.curr_setting_file_path = "{}/{}".format(self.out_dir_path,
                                                     self.curr_setting_file)

        # thread ----------------------------------------------
        self.use_joblib_threading = self._value_choice('use_joblib_threading')

        if not self.use_joblib_threading in ['yes', 'no']:
            err_mes = "use_joblib_threading Choose from Yes or No."
            log.error("{} exit.".format(err_mes))
            log.error("use_joblib_threading={}".format(
                self.use_joblib_threading))
            sys.exit(1)

        # thread adjust
        self.parallel, \
        self.parallel_full_thread, \
        self.parallel_blast_cnt, \
        self.blast_num_threads \
            = self._thread_adjusting()

        # vcf -------------------------------------------------
        basename_user_vcf = os.path.basename(self.user_vcf_file_path)
        self.vcf_file_slink_system = "{}/{}{}".format(self.ref_dir_path,
                                                      'slink_',
                                                      basename_user_vcf)

        # gtonly.gz
        self.vcf_file_path = "{}/{}{}".format(self.ref_dir_path,
                                              basename_user_vcf,
                                              "_GTonly.vcf.gz")

        # read
        self.prepare_vcf()

        # sample_nickname -------------------------------------
        basename_vcf_file = os.path.basename(self.vcf_file_path)
        self.vcf_sample_name_file = "{}/sample_name_{}.txt".format(
            self.ref_dir_path, basename_vcf_file)
        self.save_vcf_sample_name_txt()

        self.vcf_sample_nickname_list, \
        self.vcf_sample_basename_list, \
        self.vcf_sample_fullname_list, \
        self.vcf_sample_nickname_dict, \
        self.vcf_sample_basename_dict, \
        self.vcf_sample_fullname_dict, \
        self.group_members_vcf_str \
            = self.make_vcf_sample_variable()

        # illegal
        self.conf_dict['group_members']['default'] = \
            self.group_members_vcf_str

        # show_fasta-------------------------------------------
        self.show_fasta = self._value_choice('show_fasta')

        # show_samples-----------------------------------------
        self.show_samples = self._value_choice('show_samples')

        # Because it stops at show_fasta
        if self.show_fasta != True and self.show_samples == True:
            log.info("only show_samples mode, exit.")
            log.info("program finished {}\n".format(
                utl.elapsed_time(time.time(), glv.now_epochtime)))
            sys.exit(1)

        # pick_mode
        self.pick_mode = self._value_choice('pick_mode')

        # indel len
        self.indel_size = self._value_choice('indel_size')
        self.min_indel_len, self.max_indel_len = \
            [int(i) for i in self.indel_size.split('-')]

        # product size
        self.product_size = self._value_choice('product_size')
        self.min_product_size, self.max_product_size = \
            [int(i) for i in self.product_size.split('-')]

        # ref -------------------------------------------------
        # It will be set in main later
        # glv.ref = glv.ref.prepare_ref()
        self.ref_fasta_slink_system = ""
        self.ref_fasta_path = ""
        self.ref_fasta_chrom_list = []
        self.ref_fasta_fai = ""
        self.ref_fasta_chrom_txt = ""
        self.ref_fasta_pickle = ""

        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # enzyme
        self.enzyme_files_user_str = self._value_choice('enzyme_file')
        #self.enzyme_files_user_list = list()
        #self.enzyme_files_list = list()
        self.enzyme_str = self._value_choice('enzyme')
        #self.enzyme_name_list = list()

        # start stop
        self.progress = self._value_choice('progress')
        self.stop = self._value_choice('stop')

        # primer3
        self.fragment_pad_len = self._value_choice('fragment_pad_len')
        self.p3_params_file = self._value_choice('p3_params')

        # blast
        self.blast_distance = self._value_choice('blast_distance')

        # not set now
        self.blast_word_size = 0
        self.blastdb_title = ""
        self.blastdb = ""

        # region group member string ---------------------------------
        # select string by priority, next make dict or list variables

        self.regions_str = self.set_regions_str()
        self.group_members_str = self.set_group_members_str()
        self.distinguish_groups_str = self.set_distinguish_groups_str()

    def setup_variables(self):
        '''
        '''

        # setup only regions and members
        if self.show_genotype != "no":
            self._setup_genotype_variables()

        # Satisfy three structural variables
        #   1) regions_dict
        #   2) distinguish_groups_list
        #   3) group_members_dict

        # 1.1) Adjusting regions_str in easy mode --------------------------
        #log.debug("org self.regions_str={}".format(self.regions_str))

        #   Easy mode lacks the region name, so make up for it here.
        if '<EASY_MODE>' in self.regions_str:
            # <EASY_MODE>chrom_01:1-200000,chrom_02,chrom_03:all
            regions_str = ""
            rg_cnt = 1
            for region in self.regions_str.split(','):
                region = re.sub(r"^<EASY_MODE>", "", region)
                # region_name = region1
                region_name = "easy_region{}".format(rg_cnt)
                regions_str += "{}:{},".format(region_name, region)
                rg_cnt += 1

            # complete
            self.regions_str = re.sub(r",$", "", regions_str)

        # ====================
        # 1.2) make regions dictionary from regions_str ....................
        self.regions_str, \
        self.regions_dict, \
        self.region_name_list \
            = self.set_regions_dict(self.regions_str)

        self._set_chosen_value('regions', self.regions_str)
        log.info("chosen self.regions_str={}".format(self.regions_str))

        #log.debug("glv.conf.regions_dict={}".format(self.regions_dict))
        #log.debug("glv.conf.region_name_list={}".format(
        #    self.region_name_list))

        # 2.1) group_members_str -------------------------------------------
        #log.debug("org self.group_members_str={}".format(
        #    self.group_members_str))

        # 2.2) make group_members dictionary from group_members_str ........
        self.group_members_dict, \
        self.group_name_list \
            = self.set_group_members_dict(self.group_members_str)

        self._set_chosen_value('group_members', self.group_members_str)
        log.info("chosen self.group_members_str={}".format(
            self.group_members_str))

        #log.debug("glv.conf.group_members_dict={}".format(
        #    self.group_members_dict))
        #log.debug("glv.conf.group_name_list={}".format(
        #    self.group_name_list))

        #print("self.group_members_str={}".format(self.group_members_str))
        #print("self.group_members_dict={}".format(self.group_members_dict))
        #print("self.group_name_list={}".format(self.group_name_list))

        # 3.1) distinguish_groups_str --------------------------------------
        #log.debug("org self.distinguish_groups_str={}".format(
        #    self.distinguish_groups_str))

        #   Easy mode lacks the region name, so make up for it here.
        if "<EASY_MODE>" in self.distinguish_groups_str:
            region_names_str = "+".join(self.region_name_list)
            self.distinguish_groups_str = re.sub(r"<EASY_MODE>",
                                                 region_names_str,
                                                 self.distinguish_groups_str)

        # 3.2) make distinguish_groups_list from distinguish_groups_str ....
        # Avoid checking when show_genotype
        if self.show_genotype == "no":
            self.distinguish_groups_str, \
            self.distinguish_groups_list \
                = self.set_distinguish_groups_list(
                    self.distinguish_groups_str)

        self._set_chosen_value('distinguish_groups',
                               self.distinguish_groups_str)
        log.info("chosen self.distinguish_groups_str={}".format(
            self.distinguish_groups_str))

        #log.debug("distinguish_groups_list=\n{}".format(
        #    pprint.pformat(self.distinguish_groups_list)))

        # 4.1) Create a file to set the environment of primer3
        if self.p3_params_file == "no_p3_params":
            self.p3_params_file = "{}/{}".format(self.ref_dir_path,
                                                 "p3_params.txt")

        self.p3_params_file_path = utl.full_path(self.p3_params_file)
        #log.debug("self.p3_params_file_path={}".format(
        #    self.p3_params_file_path))

        # primer3, make and read parameters
        self.primer3_header_dict = self._set_primer3_header_dict()

        # 4.2) blast_word_size = PRIMER_MAX_SIZE
        self.blast_word_size = self.primer3_header_dict['PRIMER_MIN_SIZE']

        # 5.1) File that describes the enzyme name to be handled
        # enzyme_files_list
        self.enzyme_files_user_list = list()
        for file in self.enzyme_files_user_str.split(','):
            # user file full path list
            file_path_user = utl.full_path(file)
            self.enzyme_files_user_list.append(file_path_user)

            basename_user = os.path.basename(file_path_user)
            enzyme_file_slink_system = "{}/{}{}".format(
                self.ref_dir_path, "slink_", basename_user)

            self.enzyme_files_list.append(enzyme_file_slink_system)

        #pprint.pprint(self.enzyme_files_user_list)
        #pprint.pprint(self.enzyme_files_list)

        # 5.2) enzyme
        self.enzyme_files_list, self.enzyme_name_list \
            = self.read_enzyme_file()

        log.info("enzyme_files_list={}".format(self.enzyme_files_list))
        log.info("enzyme_name_list={}".format(self.enzyme_name_list))

        # 6) progress, stop
        if not self._check_progress_stop():
            if self.progress != "all" and self.stop != "no":
                err_mes = "The progress or stop settings are incorrect."
                log.error("{} exit.".format(err_mes))
                log.error("progress={}, stop={}".format(
                    self.progress, self.stop))
                log.error("{}".format(", ".join(
                    glv.outlist.outf_prefix.keys())))
                sys.exit(1)

    def _setup_genotype_variables(self):
        '''
        '''

        if self.regions_str == "all":
            self.regions_str = "{}".format(",".join(self.ref_fasta_chrom_list))

        if self.group_members_str == "all":
            self.group_members_str = "all:{}".format(",".join(
                self.vcf_sample_nickname_list))
        '''
        print("regions_str={}".format(
            self.regions_str))
        print("group_members_str={}".format(
            self.group_members_str))
        print("distinguish_groups_str={}".format(
            self.distinguish_groups_str))


        print("region_name_list={}".format(
            self.region_name_list))
        print("group_name_list={}".format(
            self.group_name_list))

        print("regions_dict={}".format(
            self.regions_dict))
        print("group_members_dict={}".format(
            self.group_members_dict))
        print("distinguish_groups_list={}".format(
            self.distinguish_groups_list))
        #sys.exit(1)
        '''

    def _check_progress_stop(self):
        '''
        '''

        ret = True

        if self.progress == "all" and self.stop == "no":
            ret = True

        elif self.progress not in glv.outlist.outf_prefix.keys():
            ret = False

        elif self.stop not in glv.outlist.outf_prefix.keys():
            ret = False

        return ret

    def _thread_adjusting(self):
        ''' in Parallel, if there are 10 threads blast cmd will use at least
            2 cores so par 1 parallel.
            main python always use 1,
            parallel use 1 thread, blast use 2 threads
        '''

        parallel = True

        parallel_full_thread = 0
        parallel_blast_cnt = 0
        blast_num_threads = 0

        if self.thread < 6 or self.use_joblib_threading != "yes":
            parallel = False

        if parallel == True:

            # unit is 4+1=5
            parallel_base = self.thread
            parallel_full_thread = parallel_base

            # blast = 4
            parallel_blast_cnt = int(parallel_base / 5)
            blast_num_threads = 4

        else:
            # 6 = 5
            full_thread = self.thread - 1
            parallel_full_thread = full_thread
            blast_num_threads = full_thread

        return parallel, parallel_full_thread,\
            parallel_blast_cnt, blast_num_threads

    def _value_choice(self, vname):
        '''
        '''

        chosen_value = ""

        # If neither param nor ini has a key, use the default value
        if self.conf_dict[vname]['param'] is None and \
            self.conf_dict[vname]['ini'] is None:
            #print("cv 1")
            chosen_value = self.conf_dict[vname]['default']

        elif self.conf_dict[vname]['param'] is not None:
            #print("cv 2")
            chosen_value = self.conf_dict[vname]['param']

        elif self.conf_dict[vname]['ini'] is not None:
            #print("cv 3")
            chosen_value = self.conf_dict[vname]['ini']

        elif self.conf_dict[vname]['default'] is not None:
            #print("cv 4")
            chosen_value = self.conf_dict[vname]['default']

        else:
            utl.prelog("not found value of key {}.".format(vname), __name__)
            sys.exit(1)

        #print("vname={}".format(vname))
        #print("dtype={}".format(self.conf_dict[vname]['dtype']))
        #print(chosen_value)
        #print(type(chosen_value))

        # cast by dtype
        chosen_value = self._cast_val(chosen_value,
                                      self.conf_dict[vname]['dtype'])

        #print(type(chosen_value))

        # chosen value
        self._set_chosen_value(vname, chosen_value)
        #print("{}={}".format(vname, self.conf_dict[vname]['chosen']))

        return chosen_value

    def _set_chosen_value(self, vname, chosen_value):

        # Assuming the chosen key does not exist
        self.conf_dict[vname]['chosen'] = chosen_value

    def _is_chrom_name(self, chrom_name):
        '''
        '''
        ret = False
        #print("_is_chrom_name, {}, {}".format(
        #    chrom_name, self.ref_fasta_chrom_list))
        if chrom_name in self.ref_fasta_chrom_list:
            ret = True
        #print("{}".format(ret))
        return ret

    def _is_region_name(self, region_name):
        '''
        '''
        ret = False
        #print("_is_region_name, {}, {}".format(
        #    region_name, self.region_name_list))
        if region_name in self.region_name_list:
            ret = True
        #print("{}".format(ret))
        return ret

    def _is_group_name(self, group_name):
        '''
        '''
        ret = False
        #print("_is_group_name, {}, {}".format(
        #    group_name, self.group_name_list))
        if group_name in self.group_name_list:
            ret = True
        #print("{}".format(ret))
        return ret

    def _is_valid_int_range(self, range_str):
        '''
        '''
        ret = True

        if "-" not in range_str:
            ret = False
        else:
            min_size, max_size = range_str.split("-")

            if not min_size.isdecimal() or \
                not max_size.isdecimal():
                ret = False
            elif int(min_size) > int(max_size):
                ret = False

        return ret

    def _is_valid_chrom_range(self, range_str):

        ret = True

        #print("_is_valid_chrom_range, range_str={}".format(range_str))
        chrom_name, rg_str = range_str.split(':')
        #print("_is_valid_chrom_range, chrom_name={}, rg_str={}".format(
        #    chrom_name, rg_str))

        min_pos, max_pos = [int(i) for i in rg_str.split('-')]
        #print("_is_valid_chrom_range, min_pos={}, max_pos={}".format(
        #    min_pos, max_pos))

        region_str, start, end, length = self._get_chrom_info(chrom_name)
        #print("_is_valid_chrom_range, _get_chrom_info={}, {}, {}, {}".format(
        #    region_str, start, end, length))

        if min_pos < start or end < max_pos:
            ret = False
        #print("_is_valid_chrom_range, ret={}".format(ret))

        return ret

    def _get_chrom_info(self, chrom_name):
        '''
        '''

        # glv.conf.ref_fasta_chrom_dict_list
        start = 0
        end = 0
        length = 0

        for d in self.ref_fasta_chrom_dict_list:
            #end': 30583384, 'length': 30583384, 'start': 1
            if chrom_name == d.get('chrom'):
                start = d.get('start')
                end = d.get('end')
                length = d.get('length')

        region_str = chrom_name

        if start is not None:
            region_str = "{}:{}-{}".format(region_str, start, end)

        #pprint.pprint(glv.conf.ref_fasta_chrom_dict_list)
        #print("{}, {}".format(region_str, length))

        return region_str, start, end, length

    def _is_easy_mode(self):
        ''' On the command line, determine if we are currently in easy mode
        '''

        easy_mode = False
        a_sample = False
        b_sample = False

        #print("self.conf_dict['a_sample']['param']={}".format(
        #    self.conf_dict['a_sample']['param']))
        #print("self.conf_dict['b_sample']['param']={}".format(
        #    self.conf_dict['b_sample']['param']))

        if self.conf_dict['a_sample']['param'] is not None and \
            self.conf_dict['a_sample']['param'] != "":
            a_sample = True

        if self.conf_dict['b_sample']['param'] is not None and \
            self.conf_dict['b_sample']['param'] != "":
            b_sample = True

        #print("a_sample={}".format(a_sample))
        #print("b_sample={}".format(b_sample))

        if a_sample == True or b_sample == True:
            easy_mode = True

        #print("easy_mode={}".format(easy_mode))

        return easy_mode

    def _print_param_ini(self):
        ''' for debug
        '''
        # parameter and ini_file variables

        #utl.prelog
        log.info("\n======== param.p ====================")
        log.info("self.param.p=\n\n{}\n".format(pprint.pformat(self.param.p)))

        log.info("\n======== ini.ini['vprimer'] =========")

        if self.param.p['ini_file'] is not None:
            log.info("\nini_file={}\n\n{}\n".format(
                self.param.p['ini_file'],
                pprint.pformat(dict(self.ini.ini['vprimer']))))
        else:
            log.info("ini_file not specified.")
        log.info("\n=====================================\n")

    def _set_primer3_header_dict(self):
        '''
        '''

        primer3_header_dict = dict()

        if os.path.isfile(self.p3_params_file_path):
            log.info("found {}.".format(self.p3_params_file_path))
            # This file may have been edited by the user, so copy it
            utl.save_to_tmpfile(self.p3_params_file_path, True, True)

        else:
            log.info("not found {}.".format(self.p3_params_file_path))
            with open(self.p3_params_file_path, mode='w') as f:
                f.write("{}={}\n".format('#PARAM', 'VALUE'))

                for key, value in list(self.p3key.items()):
                    f.write("{}={}\n".format(key, value))

        # 1.1) open and read parameters
        with open(self.p3_params_file_path, mode='r') as f:
            # iterator
            for r_liner in f:
                r_line = r_liner.strip()  # cr, ws

                if r_line.startswith('#') or r_line == '':
                    continue

                r_line = utl.strip_hash_comment(r_line)
                vname, value = r_line.split('=')
                if vname == 'PRIMER_PRODUCT_SIZE_RANGE' or \
                    vname == 'PRIMER_NUM_RETURN':
                    continue

                primer3_header_dict[vname] = value

        # constant value for primer3

        # PRIMER_FIRST_BASE_INDEX=1
        primer3_header_dict['PRIMER_FIRST_BASE_INDEX'] = str(1)
        # PRIMER_PRODUCT_SIZE_RANGE=???-???
        primer3_header_dict['PRIMER_PRODUCT_SIZE_RANGE'] = \
            "{}-{}".format(self.min_product_size, self.max_product_size)
        # PRIMER_NUM_RETURN=1
        primer3_header_dict['PRIMER_NUM_RETURN'] = str(1)

        return primer3_header_dict
예제 #8
0
    def open_log_vcffile(self):

        # in glv
        global log
        log = LogConf.open_log(__name__)