def coords(connection, table, algnmt, index, coords): cursor = connection.cursor logger.info('calculating alignment coordinates for {} in table {}'.format( algnmt, table)) sql = "select avg(length({})) from {}" logger.debug('calculate alignment length sql query:{}'.format( sql.format(algnmt, table))) cursor.execute(sql.format(algnmt, table)) # cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`") n = cursor.fetchall()[0] n = str(n[0]) n = float(n) i = 1 sql = "update {} set {} = if(substring({},{},1) not like '-',concat({},' ',substring({},{},1),',',substring_index(substring_index({},' ',{}),' ',-1),',',{}),{})" logger.debug('update coords statement for table {}:{}'.format( table, sql.format(table, coords, algnmt, str(i), coords, algnmt, str(i), index, str(i), str(i), coords))) logger.info('calculating alignment coordinates for table {}'.format(table)) while i < n: print 'coordinate calculation ' + str(100 * i / n) + '% complete' cursor.execute( sql.format(table, coords, algnmt, str(i), coords, algnmt, str(i), index, str(i), str(i), coords)) # cursor.execute("update `" + table + "` set `" + coords + "` = if(substring("+algnmt+","+str(i)+ # ",1) not like '-',concat("+coords+",' ',substring("+algnmt+","+str(i)+",1),',',substring_index(substring_index(" # +index+",' ',"+str(i)+"),' ',-1),',',"+str(i)+"),"+coords+")") i = i + 1 connection.commit()
def index_alignment(connection,table,algnmt,index,indexing_col): cursor=connection.cursor sql="select avg(length({})) from {}" logger.debug('select alignment length sql query: {}'.format(sql.format(algnmt,table))) cursor.execute(sql.format(algnmt,table)) # cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`") n=cursor.fetchall()[0] n=str(n[0]) n=float(n) logger.info('alignment total length: {}'.format(str(n))) i=1 print 'calculating alignment index.' sql1="update {} set {} = if(substring({},{},1) not like '-',concat({},' ',{}),concat({},' ','-'))" sql2="update {} set {}=if(substring({},{},1) not like '-',{}+1,{})" logger.debug=('update index sql statement'.format(sql1.format(table,index,algnmt,str(i),index,indexing_col,index))) logger.debug=('update indexing_col sql statement'.format(sql2.format(table,indexing_col,algnmt,str(i),indexing_col,indexing_col))) while i<n: sys.stdout.write( 'indexing ' + str(100*i/n) + '% complete') cursor.execute(sql1.format(table,index,algnmt,str(i),index,indexing_col,index)) # cursor.execute("update `" + table + "` set `" + index + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',concat("+ # index+",' ',"+indexing_col+"),concat("+index+",' ','-'))") cursor.execute(sql2.format(table,indexing_col,algnmt,str(i),indexing_col,indexing_col)) # cursor.execute("update `" + table + "` set `" + indexing_col + "` = if(substring("+algnmt+","+str(i)+",1) not like '-'," # +indexing_col+"+1,"+indexing_col+")") i=i+1 connection.commit() check_sql="select max(length{})" cursor.execute(check_sql.format(index)) check_results = cursor.fetchall()[0] logger.info('alignment indexing complete for table {}. {} positions indexed for an alignment {} characters long' .format(table,check_results,str(n)))
def __initPoll(cls, user): """ Tries to reinitialize poll if needed in 10 times (each 10 seconds) As soon as poll initialized user will be removed from buffer """ for x in xrange(10): if user.source not in Transport: logger.debug("longpoll: while we were wasting our time" ", the user has left (jid: %s)" % user.source) with cls.__lock: if user in cls.__buff: cls.__buff.remove(user) return None if Transport[user.source].vk.initPoll(): with cls.__lock: logger.debug("longpoll: successfully initialized longpoll (jid: %s)" % user.source) if user not in cls.__buff: return None cls.__buff.remove(user) # Check if user still in transport when we finally came down here if user.source in Transport: cls.__add(Transport[user.source]) break time.sleep(10) else: with cls.__lock: if user not in cls.__buff: return None cls.__buff.remove(user) logger.error("longpoll: failed to add user to poll in 10 retries (jid: %s)" % user.source)
def __addToBuff(cls, user): """ Adds user to the list of "bad" users The list is mostly contain users whose poll request was failed for some reasons """ cls.__buff.add(user) logger.debug("longpoll: adding user to watcher (jid: %s)" % user.source) utils.runThread(cls.__initPoll, (user,), "__initPoll-%s" % user.source)
def format_algnmt(algnmt, inform, outext): com = "convalign -i {} {} {}".format(inform, outext, algnmt) logger.debug('convert alignment command: {}'.format(com)) p = sp.Popen([com], stderr=sys.stderr, stdout=sys.stdout) p.wait() algnmt = glob.glob(os.path.split(algnmt)[0] + '*.' + outext) if len(algnmt): logger.info('alignment conversion successful') return algnmt else: logger.error('conversion failed') return
def format_algnmt(algnmt,inform,outext): com = "convalign -i {} {} {}".format(inform,outext,algnmt) logger.debug('convert alignment command: {}'.format(com)) p = sp.Popen([com],stderr=sys.stderr,stdout=sys.stdout) p.wait() algnmt = glob.glob(os.path.split(algnmt)[0] + '*.' + outext) if len(algnmt): logger.info('alignment conversion successful') return algnmt else: logger.error('conversion failed') return
def process(cls): """ Processes poll sockets by select.select() As soon as socket will be ready to be read will be called user.processPollResult() function Read processPollResult.__doc__ to learn more about status codes """ while ALIVE: socks = cls.__list.keys() if not socks: time.sleep(0.02) continue try: ready, error = select.select(socks, [], socks, 2)[::2] except (select.error, socket.error) as e: logger.error("longpoll: %s" % (e.message)) # debug? for sock in error: with cls.__lock: # We will just re-add the user to poll # in case if anything weird happen to the socket try: cls.__add(cls.__list.pop(sock)[0]) except KeyError: continue for sock in ready: with cls.__lock: try: user, opener = cls.__list.pop(sock) except KeyError: continue # Check if user is still in the memory user = Transport.get(user.source) if not user: continue # Check if the user haven't left yet if not user.vk.online: continue utils.runThread(cls.processResult, (user, opener), "poll.processResult-%s" % user.source) with cls.__lock: for sock, (user, opener) in cls.__list.items(): if not user.vk.online: logger.debug("longpoll: user is not online, so removing their from poll" " (jid: %s)" % user.source) try: del cls.__list[sock] except KeyError: pass
def __add(cls, user): """ Issues readable socket to use it in select() Adds user in buffer on error occurred Adds user in self.__list if no errors """ if DEBUG_POLL: logger.debug("longpoll: really adding user to poll (jid: %s)", user.source) opener = user.vk.makePoll() if DEBUG_POLL: logger.debug("longpoll: user has been added to poll (jid: %s)", user.source) cls.__list[opener.sock] = (user, opener) return opener
def xhtml_handler(cl, msg): destination = msg.getTo().getStripped() source = msg.getFrom().getStripped() if source in Users and msg.getType() == "chat": user = Users[source] html = msg.getTag("html") if html and html.getTag("body"): # XHTML-IM! logger.debug("fetched xhtml image from %s", source) try: xhtml = mod_xhtml.parseXHTML(user, html, source, destination) except Exception: xhtml = False if xhtml: raise xmpp.NodeProcessed()
def xhtml_handler(cl, msg): destination = msg.getTo().getStripped() source = msg.getFrom().getStripped() if source in Transport and msg.getType() == "chat": user = Transport[source] html = msg.getTag("html") if html and html.getTag("body"): # XHTML-IM! logger.debug("fetched xhtml image from %s", source) try: xhtml = mod_xhtml.parseXHTML(user, html, source, destination) except Exception: xhtml = False if xhtml: raise xmpp.NodeProcessed()
def add(cls, some_user): """ Adds the User class object to poll """ if DEBUG_POLL: logger.debug("longpoll: adding user to poll (jid: %s)", some_user.source) with cls.__lock: if some_user in cls.__buff: return None for sock, (user, opener) in cls.__list.iteritems(): if some_user == user: break else: cls.__add(some_user)
def processResult(cls, user, opener): """ Processes the select result (see above) Handles answers from user.processPollResult() Decides if need to add user to poll or not """ result = utils.execute(user.processPollResult, (opener,)) if DEBUG_POLL: logger.debug("longpoll: result=%d (jid: %s)" % (result, user.source)) if result == -1: return None # if we'll set user.vk.pollInitialized to False # then an exception will be raised # if we do that user will be reinitialized if not result: user.vk.pollInitialzed = False cls.add(user)
def processResult(cls, user, opener): """ Processes the select result (see above) Handles answers from user.processPollResult() Decides if need to add user to poll or not """ result = utils.execute(user.processPollResult, (opener,)) if DEBUG_POLL: logger.debug("longpoll: result=%s (jid: %s)", result, user.source) if result == -1: return None # if we'll set user.vk.pollInitialized to False # then makePoll() will raise an exception # by doing that, we force user's poll reinitialization if not result: user.vk.pollInitialzed = False cls.add(user)
def xhtml_handler(cl, msg): destination = msg.getTo().getStripped() source = msg.getFrom() if isinstance(source, (str, unicode)): logger.warning("Received message did not contain a valid jid: %s", msg) raise xmpp.NodeProcessed() source = source.getStripped() if source in Users and msg.getType() == "chat": user = Users[source] html = msg.getTag("html") if html and html.getTag("body"): # XHTML-IM! logger.debug("fetched xhtml image from %s", source) try: xhtml = mod_xhtml.parseXHTML(user, html, source, destination) except Exception: xhtml = False if xhtml: raise xmpp.NodeProcessed()
def add(cls, some_user): """ Adds the User class object to poll """ if DEBUG_POLL: logger.debug("longpoll: adding user to poll (jid: %s)", some_user.source) with cls.__lock: if some_user in cls.__buff: return None for sock, (user, opener) in cls.__list.iteritems(): if some_user == user: break else: try: cls.__add(some_user) except Exception as e: if not isinstance(e, api.LongPollError): crashLog("poll.add") logger.error("longpoll: failed to make poll (jid: %s)", some_user.source) cls.__addToBuff(some_user)
def __add(cls, user): """ Issues readable socket to use it in select() Adds user in buffer on error occurred Adds user in self.__list if no errors """ if DEBUG_POLL: logger.debug("longpoll: really adding user to poll (jid: %s)", user.source) try: opener = user.vk.makePoll() except Exception as e: if not isinstance(e, api.LongPollError): crashLog("poll.add") logger.error("longpoll: failed to make poll (jid: %s)", user.source) cls.__addToBuff(user) return False else: if DEBUG_POLL: logger.debug("longpoll: user has been added to poll (jid: %s)", user.source) cls.__list[opener.sock] = (user, opener) return opener
def remove_gap_positions(connection,table,source,dest,source_index,dest_index,positions): i=1 logger.info('calculating gap-reduced alignment for table {}.'.format(table)) sql1="update {} set {} = concat({},substring({},{},1) where {} is not null)" sql2="update {} set {} = concat({},' ',substring_index(substring_index({},' ',{}),' ',-1) where {} is not null)" logger.debug('update gap-reduced alignment sql statement: {}'.format(sql1.format(table,dest,dest,source,str(positions[0]),source))) logger.debug('update gap-reduced alignment index statement: {}' .format(sql2.format(table,dest_index,dest_index,source_index,str(positions[0]+1),source))) cursor=connection.cursor() logger.info('calculating gap-reduced alignment for table {}, {} positions to be removed'.format(table,str(len(positions)))) for p in positions: cursor.execute(sql1.format(sql1.format(table,dest,dest,source,str(p),source))) # cursor.execute("update `" + table + "` set `" + dest + "` = concat(`" + dest + "`, substring(`" + source + "`, " + str(p) # + ", 1)) where " + source +" is not Null") cursor.execute(sql2.format(table,dest_index,dest_index,source_index,str(p+1),source)) # cursor.execute("update `" + table + "` set `" + dest_index + "` = concat(`" + dest_index + "`, ' ', substring_index(substring_index(`" # + source_index + "`,' '," + str(p+1) + "),' ',-1)) where " + source +" is not Null") print 'calculating gap-reduced alignment, current position is ' + str(p) i=i+1 connection.commit() check_sql = "select avg(length({})),avg(length({})) from {} where {} is not null" logger.debug('check sql query: {}'.format(source,dest,table,source)) cursor.execute(check_sql.format(dest,source,table,source)) check_results=dict(cursor.fetchall()) for k,v in check_results.iteritems(): logger.info('finished calculating gap-reduced alignment for table {}. Removed {} positions.'.format(table,str(float(k)-float(v)))) return
def index_alignment(connection, table, algnmt, index, indexing_col): cursor = connection.cursor sql = "select avg(length({})) from {}" logger.debug('select alignment length sql query: {}'.format( sql.format(algnmt, table))) cursor.execute(sql.format(algnmt, table)) # cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`") n = cursor.fetchall()[0] n = str(n[0]) n = float(n) logger.info('alignment total length: {}'.format(str(n))) i = 1 print 'calculating alignment index.' sql1 = "update {} set {} = if(substring({},{},1) not like '-',concat({},' ',{}),concat({},' ','-'))" sql2 = "update {} set {}=if(substring({},{},1) not like '-',{}+1,{})" logger.debug = ('update index sql statement'.format( sql1.format(table, index, algnmt, str(i), index, indexing_col, index))) logger.debug = ('update indexing_col sql statement'.format( sql2.format(table, indexing_col, algnmt, str(i), indexing_col, indexing_col))) while i < n: sys.stdout.write('indexing ' + str(100 * i / n) + '% complete') cursor.execute( sql1.format(table, index, algnmt, str(i), index, indexing_col, index)) # cursor.execute("update `" + table + "` set `" + index + "` = if(substring("+algnmt+","+str(i)+",1) not like '-',concat("+ # index+",' ',"+indexing_col+"),concat("+index+",' ','-'))") cursor.execute( sql2.format(table, indexing_col, algnmt, str(i), indexing_col, indexing_col)) # cursor.execute("update `" + table + "` set `" + indexing_col + "` = if(substring("+algnmt+","+str(i)+",1) not like '-'," # +indexing_col+"+1,"+indexing_col+")") i = i + 1 connection.commit() check_sql = "select max(length{})" cursor.execute(check_sql.format(index)) check_results = cursor.fetchall()[0] logger.info( 'alignment indexing complete for table {}. {} positions indexed for an alignment {} characters long' .format(table, check_results, str(n)))
def identify_gap_positions(table,column,ratio,l): sql="select avg(length(`{}`)) from `{}`" logger.debug('alignment length query: {}'.format(sql.format(column,table))) cursor.execute(sql.format(column,table)) # cursor.execute("select avg(length(`" + column + "`)) from `" + table + "`") n=cursor.fetchall()[0] n=str(n[0]) n=float(n) sql="select count(*) from `{}` where `{}` is not null" logger.debug('row count sql query: {}'.format(sql.format(table,column))) cursor.execute(sql.format(table,column)) # cursor.execute("select count(*) from`" + table + "` where `" + column + "` is not Null") cutoff=cursor.fetchone()[0] cutoff = str(cutoff) cutoff = float(cutoff)*ratio i=1 logger.info('examining alignment in table {} for mostly gap positions.'.format(table)) sql="select substring({},{},1) count(*) from `{}` group by substring({},{},1)" logger.debug('gap character count statement: {}'.format(sql.format(column,str(i),table,column,str(i)))) while i<n: print str(100*i/n) + '% of alignment examined.' # cursor.execute("select substring(" + column + ", " + str(i) + ", 1), count(*) from `" + table + "` group by substring(" + column # + ", " + str(i) + ", 1)") cursor.execute(sql.format(column,str(i),table,column,str(i))) results = dict(cursor.fetchall()) try: count = results['-'] except KeyError: count = 0 if count <= cutoff: l.append(i) i = i + 1 return l
def upload(temp_t,dest_t,connection,algnmt,algnmt_col,field_sep='\t',line_sep=None): cursor = connection.cursor() check_sql="select Accession_Number,'' from {} where {} is null" cursor.execute(check_sql.format(dest_t)) rows=dict(cursor.fetchall()) cnt0 = [] for k,v in rows.iteritems(): cnt0.append(k) if line_sep: sql = "load data local infile '{}' ignore into table {} fields terminated by {} {} (Header,{})" logger.debug('load alignment data statement: {}'.format(sql.format(algnmt,temp_t,field_sep,line_sep,algnmt_col))) cursor.execute(sql.format(algnmt,temp_t,field_sep,line_sep)) else: sql = "load data local infile '{}' ignore into table {} fields terminated by {} (Header,{})" logger.debug('load alignment data statement: {}'.format(sql.format(algnmt,temp_t,field_sep,algnmt_col))) cursor.execute(sql.format(algnmt,temp_t,field_sep)) connection.commit() sql = "update {} a, {} b set a.{}=b.{} where a.Accession_Number=b.Accession_Number" logger.debug('update aligned seqs to table statement: {}'.format(sql.format(dest_t,temp_t,algnmt_col,algnmt_col))) cursor.execute(sql.format(dest_t,temp_t)) connection.commit() cursor.execute(check_sql.format(dest_t)) rows=dict(cursor.fetchall()) cnt1 = [] for k,v in rows.iteritems(): cnt1.append(k) updated = len(cnt1) - len (cnt0) if 0 >= updated: logger.error('alignment upload failed.') return else: logger.info('{} aligned sequences uploaded into table {}'.format(updated,dest_t)) cursor.execute("truncate {}".format(temp_t)) connection.commit()
def coords(connection,table,algnmt,index,coords): cursor=connection.cursor logger.info('calculating alignment coordinates for {} in table {}'.format(algnmt,table)) sql="select avg(length({})) from {}" logger.debug('calculate alignment length sql query:{}'.format(sql.format(algnmt,table))) cursor.execute(sql.format(algnmt,table)) # cursor.execute("select avg(length(`" + algnmt + "`)) from `" + table + "`") n=cursor.fetchall()[0] n=str(n[0]) n=float(n) i=1 sql="update {} set {} = if(substring({},{},1) not like '-',concat({},' ',substring({},{},1),',',substring_index(substring_index({},' ',{}),' ',-1),',',{}),{})" logger.debug('update coords statement for table {}:{}' .format(table,sql.format(table,coords,algnmt,str(i),coords,algnmt,str(i),index,str(i),str(i),coords))) logger.info('calculating alignment coordinates for table {}'.format(table)) while i<n: print 'coordinate calculation ' + str(100*i/n) + '% complete' cursor.execute(sql.format(table,coords,algnmt,str(i),coords,algnmt,str(i),index,str(i),str(i),coords)) # cursor.execute("update `" + table + "` set `" + coords + "` = if(substring("+algnmt+","+str(i)+ # ",1) not like '-',concat("+coords+",' ',substring("+algnmt+","+str(i)+",1),',',substring_index(substring_index(" # +index+",' ',"+str(i)+"),' ',-1),',',"+str(i)+"),"+coords+")") i=i+1 connection.commit()
def upload(temp_t, dest_t, connection, algnmt, algnmt_col, field_sep='\t', line_sep=None): cursor = connection.cursor() check_sql = "select Accession_Number,'' from {} where {} is null" cursor.execute(check_sql.format(dest_t)) rows = dict(cursor.fetchall()) cnt0 = [] for k, v in rows.iteritems(): cnt0.append(k) if line_sep: sql = "load data local infile '{}' ignore into table {} fields terminated by {} {} (Header,{})" logger.debug('load alignment data statement: {}'.format( sql.format(algnmt, temp_t, field_sep, line_sep, algnmt_col))) cursor.execute(sql.format(algnmt, temp_t, field_sep, line_sep)) else: sql = "load data local infile '{}' ignore into table {} fields terminated by {} (Header,{})" logger.debug('load alignment data statement: {}'.format( sql.format(algnmt, temp_t, field_sep, algnmt_col))) cursor.execute(sql.format(algnmt, temp_t, field_sep)) connection.commit() sql = "update {} a, {} b set a.{}=b.{} where a.Accession_Number=b.Accession_Number" logger.debug('update aligned seqs to table statement: {}'.format( sql.format(dest_t, temp_t, algnmt_col, algnmt_col))) cursor.execute(sql.format(dest_t, temp_t)) connection.commit() cursor.execute(check_sql.format(dest_t)) rows = dict(cursor.fetchall()) cnt1 = [] for k, v in rows.iteritems(): cnt1.append(k) updated = len(cnt1) - len(cnt0) if 0 >= updated: logger.error('alignment upload failed.') return else: logger.info('{} aligned sequences uploaded into table {}'.format( updated, dest_t)) cursor.execute("truncate {}".format(temp_t)) connection.commit()
def remove_gap_positions(connection, table, source, dest, source_index, dest_index, positions): i = 1 logger.info( 'calculating gap-reduced alignment for table {}.'.format(table)) sql1 = "update {} set {} = concat({},substring({},{},1) where {} is not null)" sql2 = "update {} set {} = concat({},' ',substring_index(substring_index({},' ',{}),' ',-1) where {} is not null)" logger.debug('update gap-reduced alignment sql statement: {}'.format( sql1.format(table, dest, dest, source, str(positions[0]), source))) logger.debug('update gap-reduced alignment index statement: {}'.format( sql2.format(table, dest_index, dest_index, source_index, str(positions[0] + 1), source))) cursor = connection.cursor() logger.info( 'calculating gap-reduced alignment for table {}, {} positions to be removed' .format(table, str(len(positions)))) for p in positions: cursor.execute( sql1.format(sql1.format(table, dest, dest, source, str(p), source))) # cursor.execute("update `" + table + "` set `" + dest + "` = concat(`" + dest + "`, substring(`" + source + "`, " + str(p) # + ", 1)) where " + source +" is not Null") cursor.execute( sql2.format(table, dest_index, dest_index, source_index, str(p + 1), source)) # cursor.execute("update `" + table + "` set `" + dest_index + "` = concat(`" + dest_index + "`, ' ', substring_index(substring_index(`" # + source_index + "`,' '," + str(p+1) + "),' ',-1)) where " + source +" is not Null") print 'calculating gap-reduced alignment, current position is ' + str( p) i = i + 1 connection.commit() check_sql = "select avg(length({})),avg(length({})) from {} where {} is not null" logger.debug('check sql query: {}'.format(source, dest, table, source)) cursor.execute(check_sql.format(dest, source, table, source)) check_results = dict(cursor.fetchall()) for k, v in check_results.iteritems(): logger.info( 'finished calculating gap-reduced alignment for table {}. Removed {} positions.' .format(table, str(float(k) - float(v)))) return
def identify_gap_positions(table, column, ratio, l): sql = "select avg(length(`{}`)) from `{}`" logger.debug('alignment length query: {}'.format(sql.format(column, table))) cursor.execute(sql.format(column, table)) # cursor.execute("select avg(length(`" + column + "`)) from `" + table + "`") n = cursor.fetchall()[0] n = str(n[0]) n = float(n) sql = "select count(*) from `{}` where `{}` is not null" logger.debug('row count sql query: {}'.format(sql.format(table, column))) cursor.execute(sql.format(table, column)) # cursor.execute("select count(*) from`" + table + "` where `" + column + "` is not Null") cutoff = cursor.fetchone()[0] cutoff = str(cutoff) cutoff = float(cutoff) * ratio i = 1 logger.info( 'examining alignment in table {} for mostly gap positions.'.format( table)) sql = "select substring({},{},1) count(*) from `{}` group by substring({},{},1)" logger.debug('gap character count statement: {}'.format( sql.format(column, str(i), table, column, str(i)))) while i < n: print str(100 * i / n) + '% of alignment examined.' # cursor.execute("select substring(" + column + ", " + str(i) + ", 1), count(*) from `" + table + "` group by substring(" + column # + ", " + str(i) + ", 1)") cursor.execute(sql.format(column, str(i), table, column, str(i))) results = dict(cursor.fetchall()) try: count = results['-'] except KeyError: count = 0 if count <= cutoff: l.append(i) i = i + 1 return l
def setup(bot): logger.debug("| Loaded Netflix | ") bot.add_cog(Netflix(bot))
def teardown(bot): logger.debug("| Unloaded Netflix | ") bot.remove_cog(Netflix(bot))