def coords(connection, table, algnmt, index, coords):
    cursor = connection.cursor
    logger.info('calculating alignment coordinates for {} in table {}'.format(
        algnmt, table))
    sql = "select avg(length({})) from {}"
    logger.debug('calculate alignment length sql query:{}'.format(
        sql.format(algnmt, table)))
    cursor.execute(sql.format(algnmt, table))
    #    cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`")
    n = cursor.fetchall()[0]
    n = str(n[0])
    n = float(n)
    i = 1
    sql = "update {} set {} = if(substring({},{},1) not like '-',concat({},' ',substring({},{},1),',',substring_index(substring_index({},' ',{}),' ',-1),',',{}),{})"
    logger.debug('update coords statement for table {}:{}'.format(
        table,
        sql.format(table, coords, algnmt, str(i), coords, algnmt, str(i),
                   index, str(i), str(i), coords)))
    logger.info('calculating alignment coordinates for table {}'.format(table))
    while i < n:
        print 'coordinate calculation ' + str(100 * i / n) + '% complete'
        cursor.execute(
            sql.format(table, coords, algnmt, str(i), coords, algnmt, str(i),
                       index, str(i), str(i), coords))
        #        cursor.execute("update `" + table + "` set `" + coords + "` = if(substring("+algnmt+","+str(i)+
        #        ",1) not like '-',concat("+coords+",' ',substring("+algnmt+","+str(i)+",1),',',substring_index(substring_index("
        #        +index+",' ',"+str(i)+"),' ',-1),',',"+str(i)+"),"+coords+")")
        i = i + 1
        connection.commit()
def index_alignment(connection,table,algnmt,index,indexing_col):
    cursor=connection.cursor
    sql="select avg(length({})) from {}"
    logger.debug('select alignment length sql query: {}'.format(sql.format(algnmt,table)))
    cursor.execute(sql.format(algnmt,table))
#    cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`")
    n=cursor.fetchall()[0]
    n=str(n[0])
    n=float(n)
    logger.info('alignment total length: {}'.format(str(n)))
    i=1
    print 'calculating alignment index.'
    sql1="update {} set {} = if(substring({},{},1) not like '-',concat({},' ',{}),concat({},' ','-'))"
    sql2="update {} set {}=if(substring({},{},1) not like '-',{}+1,{})"
    logger.debug=('update index sql statement'.format(sql1.format(table,index,algnmt,str(i),index,indexing_col,index)))
    logger.debug=('update indexing_col sql statement'.format(sql2.format(table,indexing_col,algnmt,str(i),indexing_col,indexing_col)))
    while i<n:        
        sys.stdout.write( 'indexing ' + str(100*i/n) + '% complete')
        cursor.execute(sql1.format(table,index,algnmt,str(i),index,indexing_col,index))
#        cursor.execute("update `" + table + "` set `" + index + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',concat("+
#        index+",' ',"+indexing_col+"),concat("+index+",' ','-'))")
        cursor.execute(sql2.format(table,indexing_col,algnmt,str(i),indexing_col,indexing_col))
#        cursor.execute("update `" + table + "` set `" + indexing_col + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',"
#        +indexing_col+"+1,"+indexing_col+")")
        i=i+1
        connection.commit()
    check_sql="select max(length{})"
    cursor.execute(check_sql.format(index))
    check_results = cursor.fetchall()[0]
    logger.info('alignment indexing complete for table {}. {} positions indexed for an alignment {} characters long'
        .format(table,check_results,str(n)))
Exemplo n.º 3
0
	def __initPoll(cls, user):
		"""
		Tries to reinitialize poll if needed in 10 times (each 10 seconds)
		As soon as poll initialized user will be removed from buffer
		"""
		for x in xrange(10):
			if user.source not in Transport:
				logger.debug("longpoll: while we were wasting our time"
					", the user has left (jid: %s)" % user.source)
				with cls.__lock:
					if user in cls.__buff:
						cls.__buff.remove(user)
				return None

			if Transport[user.source].vk.initPoll():
				with cls.__lock:
					logger.debug("longpoll: successfully initialized longpoll (jid: %s)"
						% user.source)
					if user not in cls.__buff:
						return None
					cls.__buff.remove(user)
					# Check if user still in transport when we finally came down here
					if user.source in Transport:
						cls.__add(Transport[user.source])
					break
			time.sleep(10)
		else:
			with cls.__lock:
				if user not in cls.__buff:
					return None
				cls.__buff.remove(user)
			logger.error("longpoll: failed to add user to poll in 10 retries (jid: %s)"
				% user.source)
Exemplo n.º 4
0
	def __addToBuff(cls, user):
		"""
		Adds user to the list of "bad" users
		The list is mostly contain users whose poll
			request was failed for some reasons
		"""
		cls.__buff.add(user)
		logger.debug("longpoll: adding user to watcher (jid: %s)" % user.source)
		utils.runThread(cls.__initPoll, (user,), "__initPoll-%s" % user.source)
def format_algnmt(algnmt, inform, outext):
    com = "convalign -i {} {} {}".format(inform, outext, algnmt)
    logger.debug('convert alignment command: {}'.format(com))
    p = sp.Popen([com], stderr=sys.stderr, stdout=sys.stdout)
    p.wait()
    algnmt = glob.glob(os.path.split(algnmt)[0] + '*.' + outext)
    if len(algnmt):
        logger.info('alignment conversion successful')
        return algnmt
    else:
        logger.error('conversion failed')
        return
def format_algnmt(algnmt,inform,outext):
    com = "convalign -i {} {} {}".format(inform,outext,algnmt)
    logger.debug('convert alignment command: {}'.format(com))
    p = sp.Popen([com],stderr=sys.stderr,stdout=sys.stdout)
    p.wait()
    algnmt = glob.glob(os.path.split(algnmt)[0] + '*.' + outext)
    if len(algnmt):
        logger.info('alignment conversion successful')
        return algnmt
    else:
        logger.error('conversion failed')
        return
Exemplo n.º 7
0
	def process(cls):
		"""
		Processes poll sockets by select.select()
		As soon as socket will be ready to be read
			will be called user.processPollResult() function
		Read processPollResult.__doc__ to learn more about status codes
		"""
		while ALIVE:
			socks = cls.__list.keys()
			if not socks:
				time.sleep(0.02)
				continue
			try:
				ready, error = select.select(socks, [], socks, 2)[::2]
			except (select.error, socket.error) as e:
				logger.error("longpoll: %s" % (e.message))  # debug?

			for sock in error:
				with cls.__lock:
					# We will just re-add the user to poll
					# in case if anything weird happen to the socket
					try:
						cls.__add(cls.__list.pop(sock)[0])
					except KeyError:
						continue

			for sock in ready:
				with cls.__lock:
					try:
						user, opener = cls.__list.pop(sock)
					except KeyError:
						continue

					# Check if user is still in the memory
					user = Transport.get(user.source)
					if not user:
						continue
					# Check if the user haven't left yet
					if not user.vk.online:
						continue
					utils.runThread(cls.processResult, (user, opener),
						"poll.processResult-%s" % user.source)

			with cls.__lock:
				for sock, (user, opener) in cls.__list.items():
					if not user.vk.online:
						logger.debug("longpoll: user is not online, so removing their from poll"
							" (jid: %s)" % user.source)
						try:
							del cls.__list[sock]
						except KeyError:
							pass
Exemplo n.º 8
0
	def __add(cls, user):
		"""
		Issues readable socket to use it in select()
		Adds user in buffer on error occurred
		Adds user in self.__list if no errors
		"""
		if DEBUG_POLL:
			logger.debug("longpoll: really adding user to poll (jid: %s)", user.source)
		opener = user.vk.makePoll()
		if DEBUG_POLL:
			logger.debug("longpoll: user has been added to poll (jid: %s)", user.source)
		cls.__list[opener.sock] = (user, opener)
		return opener
Exemplo n.º 9
0
def xhtml_handler(cl, msg):
    destination = msg.getTo().getStripped()
    source = msg.getFrom().getStripped()
    if source in Users and msg.getType() == "chat":
        user = Users[source]
        html = msg.getTag("html")
        if html and html.getTag("body"):  # XHTML-IM!
            logger.debug("fetched xhtml image from %s", source)
            try:
                xhtml = mod_xhtml.parseXHTML(user, html, source, destination)
            except Exception:
                xhtml = False
            if xhtml:
                raise xmpp.NodeProcessed()
Exemplo n.º 10
0
def xhtml_handler(cl, msg):
	destination = msg.getTo().getStripped()
	source = msg.getFrom().getStripped()
	if source in Transport and msg.getType() == "chat":
		user = Transport[source]
		html = msg.getTag("html")
		if html and html.getTag("body"):  # XHTML-IM!
			logger.debug("fetched xhtml image from %s", source)
			try:
				xhtml = mod_xhtml.parseXHTML(user, html, source, destination)
			except Exception:
				xhtml = False
			if xhtml:
				raise xmpp.NodeProcessed()
Exemplo n.º 11
0
	def add(cls, some_user):
		"""
		Adds the User class object to poll
		"""
		if DEBUG_POLL:
			logger.debug("longpoll: adding user to poll (jid: %s)", some_user.source)
		with cls.__lock:
			if some_user in cls.__buff:
				return None
			for sock, (user, opener) in cls.__list.iteritems():
				if some_user == user:
					break
			else:
				cls.__add(some_user)
Exemplo n.º 12
0
	def processResult(cls, user, opener):
		"""
		Processes the select result (see above)
		Handles answers from user.processPollResult()
		Decides if need to add user to poll or not
		"""
		result = utils.execute(user.processPollResult, (opener,))
		if DEBUG_POLL:
			logger.debug("longpoll: result=%d (jid: %s)" % (result, user.source))
		if result == -1:
			return None
		# if we'll set user.vk.pollInitialized to False
		# then an exception will be raised
		# if we do that user will be reinitialized
		if not result:
			user.vk.pollInitialzed = False
		cls.add(user)
Exemplo n.º 13
0
	def processResult(cls, user, opener):
		"""
		Processes the select result (see above)
		Handles answers from user.processPollResult()
		Decides if need to add user to poll or not
		"""
		result = utils.execute(user.processPollResult, (opener,))
		if DEBUG_POLL:
			logger.debug("longpoll: result=%s (jid: %s)", result, user.source)
		if result == -1:
			return None
		# if we'll set user.vk.pollInitialized to False
		# then makePoll() will raise an exception
		# by doing that, we force user's poll reinitialization
		if not result:
			user.vk.pollInitialzed = False
		cls.add(user)
Exemplo n.º 14
0
def xhtml_handler(cl, msg):
    destination = msg.getTo().getStripped()
    source = msg.getFrom()
    if isinstance(source, (str, unicode)):
        logger.warning("Received message did not contain a valid jid: %s", msg)
        raise xmpp.NodeProcessed()
    source = source.getStripped()
    if source in Users and msg.getType() == "chat":
        user = Users[source]
        html = msg.getTag("html")
        if html and html.getTag("body"):  # XHTML-IM!
            logger.debug("fetched xhtml image from %s", source)
            try:
                xhtml = mod_xhtml.parseXHTML(user, html, source, destination)
            except Exception:
                xhtml = False
            if xhtml:
                raise xmpp.NodeProcessed()
Exemplo n.º 15
0
	def add(cls, some_user):
		"""
		Adds the User class object to poll
		"""
		if DEBUG_POLL:
			logger.debug("longpoll: adding user to poll (jid: %s)", some_user.source)
		with cls.__lock:
			if some_user in cls.__buff:
				return None
			for sock, (user, opener) in cls.__list.iteritems():
				if some_user == user:
					break
			else:
				try:
					cls.__add(some_user)
				except Exception as e:
					if not isinstance(e, api.LongPollError):
						crashLog("poll.add")
					logger.error("longpoll: failed to make poll (jid: %s)", some_user.source)
					cls.__addToBuff(some_user)
Exemplo n.º 16
0
	def __add(cls, user):
		"""
		Issues readable socket to use it in select()
		Adds user in buffer on error occurred
		Adds user in self.__list if no errors
		"""
		if DEBUG_POLL:
			logger.debug("longpoll: really adding user to poll (jid: %s)", user.source)
		try:
			opener = user.vk.makePoll()
		except Exception as e:
			if not isinstance(e, api.LongPollError):
				crashLog("poll.add")
			logger.error("longpoll: failed to make poll (jid: %s)", user.source)
			cls.__addToBuff(user)
			return False
		else:
			if DEBUG_POLL:
				logger.debug("longpoll: user has been added to poll (jid: %s)", user.source)
			cls.__list[opener.sock] = (user, opener)
		return opener
def remove_gap_positions(connection,table,source,dest,source_index,dest_index,positions):
    i=1
    logger.info('calculating gap-reduced alignment for table {}.'.format(table))
    sql1="update {} set {} = concat({},substring({},{},1) where {} is not null)"
    sql2="update {} set {} = concat({},' ',substring_index(substring_index({},' ',{}),' ',-1) where {} is not null)"
    logger.debug('update gap-reduced alignment sql statement: {}'.format(sql1.format(table,dest,dest,source,str(positions[0]),source)))
    logger.debug('update gap-reduced alignment index statement: {}'
        .format(sql2.format(table,dest_index,dest_index,source_index,str(positions[0]+1),source)))
    cursor=connection.cursor()
    logger.info('calculating gap-reduced alignment for table {}, {} positions to be removed'.format(table,str(len(positions))))
    for p in positions:
        cursor.execute(sql1.format(sql1.format(table,dest,dest,source,str(p),source)))
#        cursor.execute("update `" + table + "` set `" + dest + "` = concat(`" + dest + "`, substring(`" + source + "`, " + str(p) 
#        + ", 1)) where " + source +" is not Null")
        cursor.execute(sql2.format(table,dest_index,dest_index,source_index,str(p+1),source))
#        cursor.execute("update `" + table + "` set `" + dest_index + "` = concat(`" + dest_index + "`, ' ', substring_index(substring_index(`" 
#        + source_index + "`,' '," + str(p+1) + "),' ',-1)) where " + source +" is not Null")
        print 'calculating gap-reduced alignment, current position is ' + str(p)
        i=i+1
        connection.commit()
    check_sql = "select avg(length({})),avg(length({})) from {} where {} is not null"
    logger.debug('check sql query: {}'.format(source,dest,table,source))
    cursor.execute(check_sql.format(dest,source,table,source))
    check_results=dict(cursor.fetchall())
    for k,v in check_results.iteritems():
        logger.info('finished calculating gap-reduced alignment for table {}. Removed {} positions.'.format(table,str(float(k)-float(v))))
    return
def index_alignment(connection, table, algnmt, index, indexing_col):
    cursor = connection.cursor
    sql = "select avg(length({})) from {}"
    logger.debug('select alignment length sql query: {}'.format(
        sql.format(algnmt, table)))
    cursor.execute(sql.format(algnmt, table))
    #    cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`")
    n = cursor.fetchall()[0]
    n = str(n[0])
    n = float(n)
    logger.info('alignment total length: {}'.format(str(n)))
    i = 1
    print 'calculating alignment index.'
    sql1 = "update {} set {} = if(substring({},{},1) not like '-',concat({},' ',{}),concat({},' ','-'))"
    sql2 = "update {} set {}=if(substring({},{},1) not like '-',{}+1,{})"
    logger.debug = ('update index sql statement'.format(
        sql1.format(table, index, algnmt, str(i), index, indexing_col, index)))
    logger.debug = ('update indexing_col sql statement'.format(
        sql2.format(table, indexing_col, algnmt, str(i), indexing_col,
                    indexing_col)))
    while i < n:
        sys.stdout.write('indexing ' + str(100 * i / n) + '% complete')
        cursor.execute(
            sql1.format(table, index, algnmt, str(i), index, indexing_col,
                        index))
        #        cursor.execute("update `" + table + "` set `" + index + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',concat("+
        #        index+",' ',"+indexing_col+"),concat("+index+",' ','-'))")
        cursor.execute(
            sql2.format(table, indexing_col, algnmt, str(i), indexing_col,
                        indexing_col))
        #        cursor.execute("update `" + table + "` set `" + indexing_col + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',"
        #        +indexing_col+"+1,"+indexing_col+")")
        i = i + 1
        connection.commit()
    check_sql = "select max(length{})"
    cursor.execute(check_sql.format(index))
    check_results = cursor.fetchall()[0]
    logger.info(
        'alignment indexing complete for table {}. {} positions indexed for an alignment {} characters long'
        .format(table, check_results, str(n)))
def identify_gap_positions(table,column,ratio,l):
    sql="select avg(length(`{}`)) from `{}`"
    logger.debug('alignment length query: {}'.format(sql.format(column,table)))
    cursor.execute(sql.format(column,table))
#    cursor.execute("select avg(length(`" + column + "`)) from `" + table + "`")
    n=cursor.fetchall()[0]
    n=str(n[0])
    n=float(n)
    sql="select count(*) from `{}` where `{}` is not null"
    logger.debug('row count sql query: {}'.format(sql.format(table,column)))
    cursor.execute(sql.format(table,column))
#    cursor.execute("select count(*) from`" + table + "` where `" + column + "` is not Null")
    cutoff=cursor.fetchone()[0]
    cutoff = str(cutoff)
    cutoff = float(cutoff)*ratio
    i=1
    logger.info('examining alignment in table {} for mostly gap positions.'.format(table))
    sql="select substring({},{},1) count(*) from `{}` group by substring({},{},1)"
    logger.debug('gap character count statement: {}'.format(sql.format(column,str(i),table,column,str(i))))
    while i<n:
        print str(100*i/n) + '% of alignment examined.'
#        cursor.execute("select substring(" + column + ", " + str(i) + ",  1), count(*) from `" + table + "` group by substring(" + column 
#        + ", " + str(i) + ",  1)")
        cursor.execute(sql.format(column,str(i),table,column,str(i)))        
        results = dict(cursor.fetchall())
        try:
            count = results['-']
        except KeyError:
                count = 0
        if count <= cutoff:
            l.append(i)
        i = i + 1
    return l
def upload(temp_t,dest_t,connection,algnmt,algnmt_col,field_sep='\t',line_sep=None):
    cursor = connection.cursor()
    check_sql="select Accession_Number,'' from {} where {} is null"
    cursor.execute(check_sql.format(dest_t))
    rows=dict(cursor.fetchall())
    cnt0 = []
    for k,v in rows.iteritems():
        cnt0.append(k)
    if line_sep:
        sql = "load data local infile '{}' ignore into table {} fields terminated by {} {} (Header,{})"
        logger.debug('load alignment data statement: {}'.format(sql.format(algnmt,temp_t,field_sep,line_sep,algnmt_col)))
        cursor.execute(sql.format(algnmt,temp_t,field_sep,line_sep))
    else:
        sql = "load data local infile '{}' ignore into table {} fields terminated by {} (Header,{})"
        logger.debug('load alignment data statement: {}'.format(sql.format(algnmt,temp_t,field_sep,algnmt_col)))
        cursor.execute(sql.format(algnmt,temp_t,field_sep))
    connection.commit()
    sql = "update {} a, {} b set a.{}=b.{} where a.Accession_Number=b.Accession_Number"
    logger.debug('update aligned seqs to table statement: {}'.format(sql.format(dest_t,temp_t,algnmt_col,algnmt_col)))
    cursor.execute(sql.format(dest_t,temp_t))
    connection.commit()
    cursor.execute(check_sql.format(dest_t))
    rows=dict(cursor.fetchall())
    cnt1 = []
    for k,v in rows.iteritems():
        cnt1.append(k)
    updated = len(cnt1) - len (cnt0)
    if 0 >= updated:
        logger.error('alignment upload failed.')
        return
    else:
        logger.info('{} aligned sequences uploaded into table {}'.format(updated,dest_t))
        cursor.execute("truncate {}".format(temp_t))
        connection.commit()
def coords(connection,table,algnmt,index,coords):
    cursor=connection.cursor
    logger.info('calculating alignment coordinates for {} in table {}'.format(algnmt,table))
    sql="select avg(length({})) from {}"
    logger.debug('calculate alignment length sql query:{}'.format(sql.format(algnmt,table)))
    cursor.execute(sql.format(algnmt,table))
#    cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`")
    n=cursor.fetchall()[0]
    n=str(n[0])
    n=float(n)
    i=1
    sql="update {} set {} = if(substring({},{},1) not like '-',concat({},' ',substring({},{},1),',',substring_index(substring_index({},' ',{}),' ',-1),',',{}),{})"
    logger.debug('update coords statement for table {}:{}'
    .format(table,sql.format(table,coords,algnmt,str(i),coords,algnmt,str(i),index,str(i),str(i),coords)))    
    logger.info('calculating alignment coordinates for table {}'.format(table))      
    while i<n:
        print 'coordinate calculation ' + str(100*i/n) + '% complete'
        cursor.execute(sql.format(table,coords,algnmt,str(i),coords,algnmt,str(i),index,str(i),str(i),coords))
#        cursor.execute("update `" + table + "` set `" + coords + "` = if(substring("+algnmt+","+str(i)+
#        ",1) not like '-',concat("+coords+",' ',substring("+algnmt+","+str(i)+",1),',',substring_index(substring_index("
#        +index+",' ',"+str(i)+"),' ',-1),',',"+str(i)+"),"+coords+")")
        i=i+1
        connection.commit()
def upload(temp_t,
           dest_t,
           connection,
           algnmt,
           algnmt_col,
           field_sep='\t',
           line_sep=None):
    cursor = connection.cursor()
    check_sql = "select Accession_Number,'' from {} where {} is null"
    cursor.execute(check_sql.format(dest_t))
    rows = dict(cursor.fetchall())
    cnt0 = []
    for k, v in rows.iteritems():
        cnt0.append(k)
    if line_sep:
        sql = "load data local infile '{}' ignore into table {} fields terminated by {} {} (Header,{})"
        logger.debug('load alignment data statement: {}'.format(
            sql.format(algnmt, temp_t, field_sep, line_sep, algnmt_col)))
        cursor.execute(sql.format(algnmt, temp_t, field_sep, line_sep))
    else:
        sql = "load data local infile '{}' ignore into table {} fields terminated by {} (Header,{})"
        logger.debug('load alignment data statement: {}'.format(
            sql.format(algnmt, temp_t, field_sep, algnmt_col)))
        cursor.execute(sql.format(algnmt, temp_t, field_sep))
    connection.commit()
    sql = "update {} a, {} b set a.{}=b.{} where a.Accession_Number=b.Accession_Number"
    logger.debug('update aligned seqs to table statement: {}'.format(
        sql.format(dest_t, temp_t, algnmt_col, algnmt_col)))
    cursor.execute(sql.format(dest_t, temp_t))
    connection.commit()
    cursor.execute(check_sql.format(dest_t))
    rows = dict(cursor.fetchall())
    cnt1 = []
    for k, v in rows.iteritems():
        cnt1.append(k)
    updated = len(cnt1) - len(cnt0)
    if 0 >= updated:
        logger.error('alignment upload failed.')
        return
    else:
        logger.info('{} aligned sequences uploaded into table {}'.format(
            updated, dest_t))
        cursor.execute("truncate {}".format(temp_t))
        connection.commit()
def remove_gap_positions(connection, table, source, dest, source_index,
                         dest_index, positions):
    i = 1
    logger.info(
        'calculating gap-reduced alignment for table {}.'.format(table))
    sql1 = "update {} set {} = concat({},substring({},{},1) where {} is not null)"
    sql2 = "update {} set {} = concat({},' ',substring_index(substring_index({},' ',{}),' ',-1) where {} is not null)"
    logger.debug('update gap-reduced alignment sql statement: {}'.format(
        sql1.format(table, dest, dest, source, str(positions[0]), source)))
    logger.debug('update gap-reduced alignment index statement: {}'.format(
        sql2.format(table, dest_index, dest_index, source_index,
                    str(positions[0] + 1), source)))
    cursor = connection.cursor()
    logger.info(
        'calculating gap-reduced alignment for table {}, {} positions to be removed'
        .format(table, str(len(positions))))
    for p in positions:
        cursor.execute(
            sql1.format(sql1.format(table, dest, dest, source, str(p),
                                    source)))
        #        cursor.execute("update `" + table + "` set `" + dest + "` = concat(`" + dest + "`, substring(`" + source + "`, " + str(p)
        #        + ", 1)) where " + source +" is not Null")
        cursor.execute(
            sql2.format(table, dest_index, dest_index, source_index,
                        str(p + 1), source))
        #        cursor.execute("update `" + table + "` set `" + dest_index + "` = concat(`" + dest_index + "`, ' ', substring_index(substring_index(`"
        #        + source_index + "`,' '," + str(p+1) + "),' ',-1)) where " + source +" is not Null")
        print 'calculating gap-reduced alignment, current position is ' + str(
            p)
        i = i + 1
        connection.commit()
    check_sql = "select avg(length({})),avg(length({})) from {} where {} is not null"
    logger.debug('check sql query: {}'.format(source, dest, table, source))
    cursor.execute(check_sql.format(dest, source, table, source))
    check_results = dict(cursor.fetchall())
    for k, v in check_results.iteritems():
        logger.info(
            'finished calculating gap-reduced alignment for table {}. Removed {} positions.'
            .format(table, str(float(k) - float(v))))
    return
def identify_gap_positions(table, column, ratio, l):
    sql = "select avg(length(`{}`)) from `{}`"
    logger.debug('alignment length query: {}'.format(sql.format(column,
                                                                table)))
    cursor.execute(sql.format(column, table))
    #    cursor.execute("select avg(length(`" + column + "`)) from `" + table + "`")
    n = cursor.fetchall()[0]
    n = str(n[0])
    n = float(n)
    sql = "select count(*) from `{}` where `{}` is not null"
    logger.debug('row count sql query: {}'.format(sql.format(table, column)))
    cursor.execute(sql.format(table, column))
    #    cursor.execute("select count(*) from`" + table + "` where `" + column + "` is not Null")
    cutoff = cursor.fetchone()[0]
    cutoff = str(cutoff)
    cutoff = float(cutoff) * ratio
    i = 1
    logger.info(
        'examining alignment in table {} for mostly gap positions.'.format(
            table))
    sql = "select substring({},{},1) count(*) from `{}` group by substring({},{},1)"
    logger.debug('gap character count statement: {}'.format(
        sql.format(column, str(i), table, column, str(i))))
    while i < n:
        print str(100 * i / n) + '% of alignment examined.'
        #        cursor.execute("select substring(" + column + ", " + str(i) + ",  1), count(*) from `" + table + "` group by substring(" + column
        #        + ", " + str(i) + ",  1)")
        cursor.execute(sql.format(column, str(i), table, column, str(i)))
        results = dict(cursor.fetchall())
        try:
            count = results['-']
        except KeyError:
            count = 0
        if count <= cutoff:
            l.append(i)
        i = i + 1
    return l
Exemplo n.º 25
0
def setup(bot):
    logger.debug("| Loaded Netflix | ")
    bot.add_cog(Netflix(bot))
Exemplo n.º 26
0
def teardown(bot):
    logger.debug("| Unloaded Netflix | ")
    bot.remove_cog(Netflix(bot))