Пример #1
0
 def saveText(self, user_id, text, length = 200):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''SET NAMES utf8''')
         cur.execute('''REPLACE INTO b_text VALUES (%s, %s, %s)''',
                 (user_id, text, length))
Пример #2
0
 def saveBio(self, user_id, bio):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''SET NAMES utf8''')
         cur.execute('''REPLACE INTO b_bio VALUES (%s, %s)''',
                 (user_id, bio))
    def populateFriends(self, user_id):
        ''' Discover friends of user_id in mentioned table. This method should
        be called after populating all the mentions of this user and all of its
        potential friends. We get additional parameter user_id although it's not
        necessary. Finally, we return a list of friends so the caller can do
        further job when needed. '''

        con = db.con()
        with con:
            cur = con.cursor()

            # first get the nodes that this node mentioned
            outgoing = []
            cur.execute('''SELECT mentioned FROM m_mentioned
                           WHERE author = %s''', (user_id))
            rownums = int(cur.rowcount)
            for i in range(rownums):
                outgoing.append(cur.fetchone()[0])

            # second get the nodes that this node was mentioned by
            incoming = []
            cur.execute('''SELECT author FROM m_mentioned
                           WHERE mentioned = %s''', (user_id))
            rownums = int(cur.rowcount)
            for i in range(rownums):
                incoming.append(cur.fetchone()[0])

            # get the intersection of them and populate into friends table
            friends = set(outgoing) & set(incoming)
            for friend in friends:
                cur.execute('''REPLACE INTO m_friends
                               VALUES (%s, %s)''', (user_id, friend))

        return friends      # this is 'set' type
def main_nameCrawler():
    seeds = []

    con = db.con()
    with con:
        cur = con.cursor()
        cur.execute('''SELECT DISTINCT root FROM m_cluster
                       WHERE distance = 1''')
        numrows = int(cur.rowcount)
        for i in range(numrows):
            seeds.append(cur.fetchone()[0])

        cur.execute('''SELECT DISTINCT friend FROM m_cluster
                       WHERE distance = 1''')
        numrows = int(cur.rowcount)
        for i in range(numrows):
            seeds.append(cur.fetchone()[0])

    seeds = list(set(seeds))

    q = TokenQueue("../data/tokens40.csv", 0.25)
    c = CrawlWorker(q)
    l = Logger("../data/users2.7_cluster_dist1.jsonarr")

    with futures.ThreadPoolExecutor(max_workers=8) as executor:
        while seeds:
            # create a single job list
            nsize = 100
            user_ids = popmany(seeds, nsize)

            def call_logger(future):
                l.logJsonArr(future.result())

            future = executor.submit(c.getGuaranteedLookup, user_ids)
            future.add_done_callback(call_logger)
 def createCovered():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_covered (
                        node_id BIGINT,
                        PRIMARY KEY (node_id)
                        ) ENGINE=MyISAM''')
 def createAge():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_ages (
                        user_id BIGINT,
                        age INT,
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM''')
Пример #7
0
 def create_follower(self):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS b_follower (
                         user_id BIGINT,
                         follower BIGINT,
                         PRIMARY KEY (user_id, follower)
                        ) ENGINE = MyISAM''')
 def createFriends():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_friends (
                        me BIGINT,
                        friend BIGINT,
                        PRIMARY KEY (me, friend)
                        ) ENGINE=MyISAM''')
Пример #9
0
 def createBioTable(self):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS b_bio (
                        user_id BIGINT,
                        bio TEXT,
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM DEFAULT CHARSET=utf8''')
    def populateCovered(self, user_id):
        ''' Sub module of process() method; userd to populate data into the
        covered table. '''

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''REPLACE INTO m_covered VALUES (%s)''',
                    (user_id))
 def createText():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_text (
                        user_id BIGINT,
                        text LONGTEXT,
                        length INT,
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM''')
 def createMentioned():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_mentioned (
                        author BIGINT,
                        mentioned BIGINT,
                        PRIMARY KEY (author, mentioned),
                        KEY (mentioned)
                        ) ENGINE=MyISAM''')
Пример #13
0
    def count_following(self, user_id):
        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT count(friend)
                           FROM b_following
                           WHERE user_id = %s''', (user_id))
            following_count = cur.fetchone()[0]

        return following_count
Пример #14
0
    def getAge(self, user_id):

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT age FROM m_ages WHERE user_id = %s''',
                    (user_id))
            age = cur.fetchone()[0]

            return age
Пример #15
0
 def createFCounts(self):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS b_fcounts (
                        user_id BIGINT,
                        friends_count INT,
                        followers_count INT,
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM''')
    def populateMentioned(self, user_id, mentioned):
        ''' Sub module of process() method; used to populate the data into the
        mentioned table. '''

        con = db.con()
        with con:
            cur = con.cursor()
            for a_mentioned in mentioned:
                cur.execute('''REPLACE INTO m_mentioned VALUES (%s, %s)''',
                        (user_id, a_mentioned))
Пример #17
0
 def createTextTable(self):
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS b_text (
                        user_id BIGINT,
                        text LONGTEXT,
                        length INT,
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM DEFAULT CHARSET=utf8''')
 def purgeDB():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''DELETE FROM m_covered''')
         cur.execute('''DELETE FROM m_mentioned''')
         cur.execute('''DELETE FROM m_friends''')
         cur.execute('''DELETE FROM m_cluster''')
         cur.execute('''DELETE FROM m_names''')
         cur.execute('''DELETE FROM m_ages''')
         cur.execute('''DELETE FROM m_text''')
 def createCluster():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_cluster (
                        root BIGINT,
                        friend BIGINT,
                        distance INT,
                        PRIMARY KEY (root, friend),
                        KEY (root, distance)
                        ) ENGINE=MyISAM''')
 def createName():
     con = db.con()
     with con:
         cur = con.cursor()
         cur.execute('''CREATE TABLE IF NOT EXISTS m_names (
                        user_id BIGINT,
                        name VARCHAR(50),
                        lang VARCHAR(10),
                        screen_name VARCHAR(50),
                        PRIMARY KEY (user_id)
                        ) ENGINE=MyISAM''')
Пример #21
0
def getScreenName(user_id):
    '''
    Return the screen_name of the user specified by the user_id.
    '''
    con = db.con()
    with con:
        cur = con.cursor()
        cur.execute('''SELECT screen_name FROM users WHERE user_id = %s''', (user_id))
        row = cur.fetchone()
        sn = row[0]

    return sn
Пример #22
0
def getAge(user_id):
    '''
    Return a age of the user specified by the user_id.
    '''
    con = db.con()
    with con:
        cur = con.cursor()
        cur.execute('''SELECT age FROM users WHERE user_id = %s''', (user_id))
        row = cur.fetchone()
        age = row[0]

    return age
    def checkFriendsNumber(self):
        ''' Sub method of run(). Used to check the number of collected friends
        of the node we are working on now. '''

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT COUNT(*) FROM m_cluster
                           WHERE root = %s''', (self.user_id))
            count = cur.fetchone()[0]

        return count
Пример #24
0
def probYearGivenDomainInYearAfterSampling(start_year, end_year):
    """
    While another method probYearGivenDomainInYear() computes the value
    considering entire set available, this method performs sampling from the
    set, and computes the probability of years given the sampled users.

    As in probYearGivenDomainInYear(), the range searched is [start_year,
    end_year). In other words, the conditioning is performed on the users
    included in the range[start_year, end_year).
    """
    assert start_year < end_year

    n_sample = 100

    return_range = (2012 - 50, 2012 - 11)  # this is inclusive

    q = """SELECT b.name AS name, sum(b.num) AS count
           FROM babyname b
           INNER JOIN popularname p
           ON b.name = p.name
           WHERE year BETWEEN %s AND %s
           GROUP BY name
           ORDER BY count
           """ % (
        start_year,
        end_year - 1,
    )  # sql between is inclusive

    name_and_weights = []

    con = db.con()
    with con:
        cur = con.cursor()
        cur.execute(q)
        numrows = int(cur.rowcount)
        for i in range(numrows):
            row = cur.fetchone()
            name = row[0]
            count = row[1]
            name_and_weights.append([name, count])

    names, weights = zip(*name_and_weights)
    sum_weights = sum(weights)
    weights = [x / sum_weights for x in weights]

    sampled_counts = np.random.multinomial(n_sample, weights)

    assert len(names) == len(sampled_counts)

    prob_year = probYearGivenWeightedNames(zip(names, sampled_counts), return_range)

    return prob_year
 def populateCluster(self, friends):
     ''' We should not insert the data if it, a pair of (me, friend) already
     exists, since it means the data has been inserted before this, or there
     is a shorter path from the root. That's the reason why we use IGNORE
     syntax. '''
     con = db.con()
     with con:
         cur = con.cursor()
         for friend in friends:
             distance = self.step + 1
             cur.execute('''INSERT IGNORE INTO m_cluster
                            VALUES (%s, %s, %s)''',
                            (self.root, friend, distance))
    def _getseeds():
        seeds = []

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT DISTINCT friend
                           FROM m_cluster''')
            numrows = int(cur.rowcount)
            for i in range(numrows):
                seeds.append(cur.fetchone()[0])

        return seeds
Пример #27
0
def getFirstName(user_id):
    '''
    Return a first name of the user specified by the user_id.
    '''
    con = db.con()
    with con:
        cur = con.cursor()
        cur.execute('''SELECT name FROM users WHERE user_id = %s''', (user_id))
        row = cur.fetchone()
        name = row[0]

    firstname = firstNameOf(name)

    return firstname
Пример #28
0
    def get_following(self, user_id):
        friends = []

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT friend FROM b_following WHERE user_id = %s''',
                    (user_id))
            numrows = int(cur.rowcount)
            for i in range(numrows):
                friend = cur.fetchone()[0]
                friends.append(friend)

        return friends
Пример #29
0
    def get_allFollowing(self):
        friends = []

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT DISTINCT friend
                           FROM b_following''')
            numrows = int(cur.rowcount)
            for i in range(numrows):
                friend = cur.fetchone()[0]
                friends.append(friend)

        return friends
Пример #30
0
    def get_follower(self, user_id):
        followers = []

        con = db.con()
        with con:
            cur = con.cursor()
            cur.execute('''SELECT follower FROM b_follower WHERE user_id = %s''',
                    (user_id))
            numrows = int(cur.rowcount)
            for i in range(numrows):
                follower = cur.fetchone()[0]
                followers.append(follower)

        return followers
Пример #31
0
def getListing():
    #datatable parameters
    draw = request.args.get('draw')
    start = request.args.get("start")
    rowperpage = request.args.get("length")

    columnIndex_arr = request.args.get('order')
    columnName_arr = request.args.get('columns')
    order_arr = request.args.get('order')
    searchValue = request.args.get('search[value]')
    searchValue = "%" + searchValue + "%"
    # columnIndex = columnIndex_arr[0]['column']
    # columnName = columnName_arr[columnIndex]['data']
    # columnSortOrder = order_arr[0]['dir']
    # searchValue = search_arr['value']
    print(draw)

    cnn = con()
    cr = cnn.cursor
    query = """SELECT `title`,`image_url`, `cost_price`,`catalogue_value`,`sold_at`, `stock_list`, `new`,`parent_page`,stock_link, `created_at`, `updated_at`
        FROM `stocks` where title like %s  or cost_price like %s order by created_at limit %s offset %s"""
    allQuery = """SELECT `title`, `image_url`, `cost_price`, `catalogue_value`,
         `sold_at`, `stock_list`, `new`,`parent_page`,stock_link, `created_at`, `updated_at` 
        
        
        FROM `stocks` order by created_at limit %s offset %s"""
    allparam = (int(rowperpage), int(start))
    rrd = (searchValue, searchValue, int(rowperpage), int(start))
    res = False
    try:
        if searchValue != '':
            cr.execute(query, rrd)
        else:
            cr.execute(allQuery, allparam)
        res = cr.with_rows
    except Exception as ex:
        cnn.connection.rollback()
    print(res)
    data = cr.fetchall()
    listings = []

    for title, image_url, cost_price, catalogue_value, sold_at, stock_list, new, parent_page, stock_link, created_at, updated_at in data:
        listings.append({
            "title": title,
            "image_url": image_url,
            "cost_price": cost_price,
            "catalogue_value": catalogue_value,
            "sold_at": sold_at,
            "stock_list": stock_list,
            "new": new,
            "parent_page": parent_page,
            "stock_link": stock_link,
            "created_at": created_at,
            "updated_at": updated_at
        })

    print(listings)
    itotalQ = """select count(*) from stocks where title like %s or cost_price like %s"""
    param = (searchValue, searchValue)
    try:
        cr.execute(itotalQ, param)
    except Exception as ex:
        print(ex)
        cnn.connection.rollback()
    itotalrecords = cr.fetchall()
    itotalr = itotalrecords[0][0]
    total = len(listings)
    response = {
        "draw": int(draw),
        "iTotalRecords": total,
        "iTotalDisplayRecords": itotalr,
        "aaData": listings
    }

    return jsonify(response)
Пример #32
0
import telebot
from service import Spam
import os
from threading import Thread
from datetime import datetime, timedelta
from config import token, admin_id
from db import con, Global, WhiteNumber
import keyboard
import text
import time
import psutil as ps
import function
from proxy.proxyParsing import ParsingProxy

con()

startime = datetime.now()


class Telegs:
    def __init__(self, token, admin_id):
        self.admin_id = admin_id
        self.bot = telebot.TeleBot(token)
        self.send = self.bot.send_message
        self.next = self.bot.register_next_step_handler
        self.attack = []
        self.proxys = []

        @self.bot.message_handler(commands=['start'])
        def start(message):
            userInfo = Global.select().where(