def saveText(self, user_id, text, length = 200):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SET NAMES utf8''')
cur.execute('''REPLACE INTO b_text VALUES (%s, %s, %s)''',
(user_id, text, length))
def saveBio(self, user_id, bio):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SET NAMES utf8''')
cur.execute('''REPLACE INTO b_bio VALUES (%s, %s)''',
(user_id, bio))
def populateFriends(self, user_id):
''' Discover friends of user_id in mentioned table. This method should
be called after populating all the mentions of this user and all of its
potential friends. We get additional parameter user_id although it's not
necessary. Finally, we return a list of friends so the caller can do
further job when needed. '''
con = db.con()
with con:
cur = con.cursor()
# first get the nodes that this node mentioned
outgoing = []
cur.execute('''SELECT mentioned FROM m_mentioned
WHERE author = %s''', (user_id))
rownums = int(cur.rowcount)
for i in range(rownums):
outgoing.append(cur.fetchone()[0])
# second get the nodes that this node was mentioned by
incoming = []
cur.execute('''SELECT author FROM m_mentioned
WHERE mentioned = %s''', (user_id))
rownums = int(cur.rowcount)
for i in range(rownums):
incoming.append(cur.fetchone()[0])
# get the intersection of them and populate into friends table
friends = set(outgoing) & set(incoming)
for friend in friends:
cur.execute('''REPLACE INTO m_friends
VALUES (%s, %s)''', (user_id, friend))
return friends # this is 'set' type
def main_nameCrawler():
seeds = []
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT DISTINCT root FROM m_cluster
WHERE distance = 1''')
numrows = int(cur.rowcount)
for i in range(numrows):
seeds.append(cur.fetchone()[0])
cur.execute('''SELECT DISTINCT friend FROM m_cluster
WHERE distance = 1''')
numrows = int(cur.rowcount)
for i in range(numrows):
seeds.append(cur.fetchone()[0])
seeds = list(set(seeds))
q = TokenQueue("../data/tokens40.csv", 0.25)
c = CrawlWorker(q)
l = Logger("../data/users2.7_cluster_dist1.jsonarr")
with futures.ThreadPoolExecutor(max_workers=8) as executor:
while seeds:
# create a single job list
nsize = 100
user_ids = popmany(seeds, nsize)
def call_logger(future):
l.logJsonArr(future.result())
future = executor.submit(c.getGuaranteedLookup, user_ids)
future.add_done_callback(call_logger)
def createCovered():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_covered (
node_id BIGINT,
PRIMARY KEY (node_id)
) ENGINE=MyISAM''')
def createAge():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_ages (
user_id BIGINT,
age INT,
PRIMARY KEY (user_id)
) ENGINE=MyISAM''')
def create_follower(self):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS b_follower (
user_id BIGINT,
follower BIGINT,
PRIMARY KEY (user_id, follower)
) ENGINE = MyISAM''')
def createFriends():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_friends (
me BIGINT,
friend BIGINT,
PRIMARY KEY (me, friend)
) ENGINE=MyISAM''')
def createBioTable(self):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS b_bio (
user_id BIGINT,
bio TEXT,
PRIMARY KEY (user_id)
) ENGINE=MyISAM DEFAULT CHARSET=utf8''')
def populateCovered(self, user_id):
''' Sub module of process() method; userd to populate data into the
covered table. '''
con = db.con()
with con:
cur = con.cursor()
cur.execute('''REPLACE INTO m_covered VALUES (%s)''',
(user_id))
def createText():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_text (
user_id BIGINT,
text LONGTEXT,
length INT,
PRIMARY KEY (user_id)
) ENGINE=MyISAM''')
def createMentioned():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_mentioned (
author BIGINT,
mentioned BIGINT,
PRIMARY KEY (author, mentioned),
KEY (mentioned)
) ENGINE=MyISAM''')
def count_following(self, user_id):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT count(friend)
FROM b_following
WHERE user_id = %s''', (user_id))
following_count = cur.fetchone()[0]
return following_count
def getAge(self, user_id):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT age FROM m_ages WHERE user_id = %s''',
(user_id))
age = cur.fetchone()[0]
return age
def createFCounts(self):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS b_fcounts (
user_id BIGINT,
friends_count INT,
followers_count INT,
PRIMARY KEY (user_id)
) ENGINE=MyISAM''')
def populateMentioned(self, user_id, mentioned):
''' Sub module of process() method; used to populate the data into the
mentioned table. '''
con = db.con()
with con:
cur = con.cursor()
for a_mentioned in mentioned:
cur.execute('''REPLACE INTO m_mentioned VALUES (%s, %s)''',
(user_id, a_mentioned))
def createTextTable(self):
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS b_text (
user_id BIGINT,
text LONGTEXT,
length INT,
PRIMARY KEY (user_id)
) ENGINE=MyISAM DEFAULT CHARSET=utf8''')
def purgeDB():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''DELETE FROM m_covered''')
cur.execute('''DELETE FROM m_mentioned''')
cur.execute('''DELETE FROM m_friends''')
cur.execute('''DELETE FROM m_cluster''')
cur.execute('''DELETE FROM m_names''')
cur.execute('''DELETE FROM m_ages''')
cur.execute('''DELETE FROM m_text''')
def createCluster():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_cluster (
root BIGINT,
friend BIGINT,
distance INT,
PRIMARY KEY (root, friend),
KEY (root, distance)
) ENGINE=MyISAM''')
def createName():
con = db.con()
with con:
cur = con.cursor()
cur.execute('''CREATE TABLE IF NOT EXISTS m_names (
user_id BIGINT,
name VARCHAR(50),
lang VARCHAR(10),
screen_name VARCHAR(50),
PRIMARY KEY (user_id)
) ENGINE=MyISAM''')
def getScreenName(user_id):
'''
Return the screen_name of the user specified by the user_id.
'''
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT screen_name FROM users WHERE user_id = %s''', (user_id))
row = cur.fetchone()
sn = row[0]
return sn
def getAge(user_id):
'''
Return a age of the user specified by the user_id.
'''
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT age FROM users WHERE user_id = %s''', (user_id))
row = cur.fetchone()
age = row[0]
return age
def checkFriendsNumber(self):
''' Sub method of run(). Used to check the number of collected friends
of the node we are working on now. '''
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT COUNT(*) FROM m_cluster
WHERE root = %s''', (self.user_id))
count = cur.fetchone()[0]
return count
def probYearGivenDomainInYearAfterSampling(start_year, end_year):
"""
While another method probYearGivenDomainInYear() computes the value
considering entire set available, this method performs sampling from the
set, and computes the probability of years given the sampled users.
As in probYearGivenDomainInYear(), the range searched is [start_year,
end_year). In other words, the conditioning is performed on the users
included in the range[start_year, end_year).
"""
assert start_year < end_year
n_sample = 100
return_range = (2012 - 50, 2012 - 11) # this is inclusive
q = """SELECT b.name AS name, sum(b.num) AS count
FROM babyname b
INNER JOIN popularname p
ON b.name = p.name
WHERE year BETWEEN %s AND %s
GROUP BY name
ORDER BY count
""" % (
start_year,
end_year - 1,
) # sql between is inclusive
name_and_weights = []
con = db.con()
with con:
cur = con.cursor()
cur.execute(q)
numrows = int(cur.rowcount)
for i in range(numrows):
row = cur.fetchone()
name = row[0]
count = row[1]
name_and_weights.append([name, count])
names, weights = zip(*name_and_weights)
sum_weights = sum(weights)
weights = [x / sum_weights for x in weights]
sampled_counts = np.random.multinomial(n_sample, weights)
assert len(names) == len(sampled_counts)
prob_year = probYearGivenWeightedNames(zip(names, sampled_counts), return_range)
return prob_year
def populateCluster(self, friends):
''' We should not insert the data if it, a pair of (me, friend) already
exists, since it means the data has been inserted before this, or there
is a shorter path from the root. That's the reason why we use IGNORE
syntax. '''
con = db.con()
with con:
cur = con.cursor()
for friend in friends:
distance = self.step + 1
cur.execute('''INSERT IGNORE INTO m_cluster
VALUES (%s, %s, %s)''',
(self.root, friend, distance))
def _getseeds():
seeds = []
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT DISTINCT friend
FROM m_cluster''')
numrows = int(cur.rowcount)
for i in range(numrows):
seeds.append(cur.fetchone()[0])
return seeds
def getFirstName(user_id):
'''
Return a first name of the user specified by the user_id.
'''
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT name FROM users WHERE user_id = %s''', (user_id))
row = cur.fetchone()
name = row[0]
firstname = firstNameOf(name)
return firstname
def get_following(self, user_id):
friends = []
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT friend FROM b_following WHERE user_id = %s''',
(user_id))
numrows = int(cur.rowcount)
for i in range(numrows):
friend = cur.fetchone()[0]
friends.append(friend)
return friends
def get_allFollowing(self):
friends = []
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT DISTINCT friend
FROM b_following''')
numrows = int(cur.rowcount)
for i in range(numrows):
friend = cur.fetchone()[0]
friends.append(friend)
return friends
def get_follower(self, user_id):
followers = []
con = db.con()
with con:
cur = con.cursor()
cur.execute('''SELECT follower FROM b_follower WHERE user_id = %s''',
(user_id))
numrows = int(cur.rowcount)
for i in range(numrows):
follower = cur.fetchone()[0]
followers.append(follower)
return followers
def getListing():
#datatable parameters
draw = request.args.get('draw')
start = request.args.get("start")
rowperpage = request.args.get("length")
columnIndex_arr = request.args.get('order')
columnName_arr = request.args.get('columns')
order_arr = request.args.get('order')
searchValue = request.args.get('search[value]')
searchValue = "%" + searchValue + "%"
# columnIndex = columnIndex_arr[0]['column']
# columnName = columnName_arr[columnIndex]['data']
# columnSortOrder = order_arr[0]['dir']
# searchValue = search_arr['value']
print(draw)
cnn = con()
cr = cnn.cursor
query = """SELECT `title`,`image_url`, `cost_price`,`catalogue_value`,`sold_at`, `stock_list`, `new`,`parent_page`,stock_link, `created_at`, `updated_at`
FROM `stocks` where title like %s or cost_price like %s order by created_at limit %s offset %s"""
allQuery = """SELECT `title`, `image_url`, `cost_price`, `catalogue_value`,
`sold_at`, `stock_list`, `new`,`parent_page`,stock_link, `created_at`, `updated_at`
FROM `stocks` order by created_at limit %s offset %s"""
allparam = (int(rowperpage), int(start))
rrd = (searchValue, searchValue, int(rowperpage), int(start))
res = False
try:
if searchValue != '':
cr.execute(query, rrd)
else:
cr.execute(allQuery, allparam)
res = cr.with_rows
except Exception as ex:
cnn.connection.rollback()
print(res)
data = cr.fetchall()
listings = []
for title, image_url, cost_price, catalogue_value, sold_at, stock_list, new, parent_page, stock_link, created_at, updated_at in data:
listings.append({
"title": title,
"image_url": image_url,
"cost_price": cost_price,
"catalogue_value": catalogue_value,
"sold_at": sold_at,
"stock_list": stock_list,
"new": new,
"parent_page": parent_page,
"stock_link": stock_link,
"created_at": created_at,
"updated_at": updated_at
})
print(listings)
itotalQ = """select count(*) from stocks where title like %s or cost_price like %s"""
param = (searchValue, searchValue)
try:
cr.execute(itotalQ, param)
except Exception as ex:
print(ex)
cnn.connection.rollback()
itotalrecords = cr.fetchall()
itotalr = itotalrecords[0][0]
total = len(listings)
response = {
"draw": int(draw),
"iTotalRecords": total,
"iTotalDisplayRecords": itotalr,
"aaData": listings
}
return jsonify(response)
import telebot
from service import Spam
import os
from threading import Thread
from datetime import datetime, timedelta
from config import token, admin_id
from db import con, Global, WhiteNumber
import keyboard
import text
import time
import psutil as ps
import function
from proxy.proxyParsing import ParsingProxy
con()
startime = datetime.now()
class Telegs:
def __init__(self, token, admin_id):
self.admin_id = admin_id
self.bot = telebot.TeleBot(token)
self.send = self.bot.send_message
self.next = self.bot.register_next_step_handler
self.attack = []
self.proxys = []
@self.bot.message_handler(commands=['start'])
def start(message):
userInfo = Global.select().where(