def main():
tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass,
tracker2_db)
countid = 0
countid1 = 1000000
while countid < 123000000:
select = """SELECT b.id, c.key_id FROM tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE b.company_id = 14 AND c.company_id = 14 AND b.matchedVideo_id = c.id and b.id > %s and b.id <= %s""" % (
countid, countid1)
countid += 1000000
countid1 += 1000000
print select
select_result = tracker2.query_sql_cmd(select)
print "fetched!"
retrytracker2 = 0
while retrytracker2 < 10:
try:
print "retrytracker2:"
print retrytracker2
target = MySQLHelper(t_host, t_user, t_pass, t_db)
print "start insert!"
print datetime.datetime.now()
break
except Exception, e:
print e
retrytracker2 += 1
#target = MySQLHelper(t_host,t_user,t_pass,t_db)
insert_result = "insert into infback_Tmp " + " values " + str(
select_result)[1:-1].replace("L", "").replace("u", "")
target.insert_sql_cmd(insert_result)
print "finished!"
print datetime.datetime.now()
def NovelCrawl_Main(count):
url = r'http://www.yousuu.com/booklist'
proxies = fetchproxies()
for i in range(count):
try:
header = get_header()
proxy = random.choice(proxies)
url = CrawlNovelList(url, header, proxy)
except Exception as e:
print 'error'
print e.message
pass
sql = 'select distinct novellisturl from pagenovel '
a = MySQLHelper()
NovelListUrl = a.SqlFecthAll(sql)
a.CloseCon()
print len(NovelListUrl)
#proxies = fetchproxies()
for i in NovelListUrl:
try:
header = get_header()
proxy = random.choice(proxies)
print i[0]
CrawlNovel(i[0], header, proxy)
except Exception as e:
print 'error'
print e.message
pass
a = MySQLHelper()
sql = 'select distinct novelurl from novelurl'
NovelUrl = a.SqlFecthAll(sql)
a.CloseCon()
print len(NovelUrl)
#proxies = fetchproxies()
for i in NovelUrl:
try:
header = get_header()
proxy = random.choice(proxies)
print i[0]
CrawlNovelData(i[0], header, proxy)
except Exception as e:
print 'error'
print e.message
pass
def main():
helper = MySQLHelper('127.0.0.1', 'root', '', 'pymysql')
helper.connect()
# 添加
helper.cud(sql='insert into `tb_stu`(`name`) values(%s)', params=["mary"])
# 查询
results = helper.query(sql="select * from `tb_stu`")
for result in results:
id = result[0]
name = result[1]
print("id:%s,name:%s" % (id, name))
# 关闭
helper.close()
def Main_CrawlNovelData(pp):
sql = 'select distinct novelurl from novelurl order by rundate desc limit %s offset %s' % (
300, int(pp) * 300)
a = MySQLHelper()
NovelUrl = a.SqlFecthAll(sql)
a.CloseCon()
proxies = fetchproxies(40)
for i in NovelUrl:
try:
header = get_header()
proxy = random.choice(proxies)
print i[0]
CrawlNovelData(i[0], header, proxy)
except Exception as e:
print 'error'
print e.message
pass
def __init__(self, run): self.__run = run self.__hash = str(uuid.uuid4()) self.__raven = Raven() self.__db = MySQLHelper()
#cursor.execute("drop index meta_isp on infringingP2P_Tmp")
cursor.execute("drop index meta_isp on allP2P_Tmp")
pre_DASHBOARD.close()
print "truncate finished!"
'''tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db)
print datetime.datetime.now()
infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" %(ETL_DTE,ETL_DTE)
print infP2P_Tmp_select
infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select)
print "inf fetched"'''
retryVIADASHBOARD = 0
while retryVIADASHBOARD < 10:
try:
print "retryVIADASHBOARD:"
print retryVIADASHBOARD
VIACOM_DASHBOARD = MySQLHelper(DASHBOARD_host, DASHBOARD_user,
DASHBOARD_pass, DASHBOARD_db)
break
except Exception, e:
print e
retryVIADASHBOARD += 1
'''infP2P_Tmp_insert = "insert into infringingP2P_Tmp " + " values " + str(infP2P_result)[1:-1].replace("L", "").replace("u","")
VIACOM_DASHBOARD.insert_sql_cmd(infP2P_Tmp_insert)
print datetime.datetime.now()'''
retryTime_ware = 0
while retryTime_ware < 10:
try:
print "retryTime_ware:"
print retryTime_ware
p2pwarehouse = MySQLHelper(p2pwarehouse_host, p2pwarehouse_user,
p2pwarehouse_pass, p2pwarehouse_db)
except Exception, e:
print e
retryTime += 1
cursor.execute("truncate table infringingP2P_Tmp")
#cursor.execute("truncate table allP2P_Tmp")
cursor.execute("drop index meta_isp on infringingP2P_Tmp")
#cursor.execute("drop index meta_isp on allP2P_Tmp")
pre_DASHBOARD.close()
print "truncate finished!"
retrytracker2 = 0
while retrytracker2 < 10:
try:
print "retrytracker2:"
print retrytracker2
tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass,
tracker2_db)
break
except Exception, e:
print e
retrytracker2 += 1
#tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db)
print datetime.datetime.now()
infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" % (
ETL_DTE, ETL_DTE)
print infP2P_Tmp_select
infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select)
print "inf fetched"
retryVIADASHBOARD = 0
while retryVIADASHBOARD < 10:
try:
print "retryVIADASHBOARD:"
#coding= utf-8
import re
import time
import cx_Oracle
from MySQLHelper import MySQLHelper
a = MySQLHelper()
sql = 'select * from fun.noveldata order by rundate desc '
rawdata = a.SqlFecthAll(sql)
a.CloseCon()
conn = cx_Oracle.connect('system/Syy19930119@localhost:1521/orcl')
cursor = conn.cursor()
try:
cursor.execute('drop table SYY_NOVELDATA')
except:
pass
sql = """
CREATE TABLE system.SYY_NOVELDATA (\
RUNDATE DATE\
,NOVELURL VARCHAR2(250)\
,NOVELNAME VARCHAR2(250)\
,NOVEL_ORGURL VARCHAR2(250)\
,TAG_CATEGORY VARCHAR2(250)\
,AUTHOR VARCHAR2(250)\
,NOVELRANK VARCHAR2(250)\
,WORLDCOUNT VARCHAR2(250)\
,SECTIONCOUNT VARCHAR2(250)\
retryTime += 1
cursor.execute(
"truncate table infringingP2P_Tmp;truncate table allP2P_Tmp;drop index meta_isp on infringingP2P_Tmp;drop index meta_isp on allP2P_Tmp"
)
#truncate table meta_Tmp;truncate table isp_Tmp;drop index meta_isp on infringingP2P_Tmp;drop index meta_isp on allP2P_Tmp)
cursor.close()
#DASHBOARD.commit()
print "truncate finished!"
#fetch the VTWeb infringing IPs and hashes begin
retrytracker2 = 0
while retrytracker2 < 10:
try:
print "retrytracker2:"
print retrytracker2
tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass,
tracker2_db)
break
except Exception, e:
print e
retrytracker2 += 1
#tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db)
print datetime.datetime.now()
infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" % (
ETL_DTE, ETL_DTE)
print infP2P_Tmp_select
set_NamesUTF8 = """SET NAMES UTF8"""
print set_NamesUTF8
infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select)
print "inf fetched"
retryVIADASHBOARD = 0
while retryVIADASHBOARD < 10:
def __init__(self, run):
self.__run = run
self.__raven = Raven()
self.__db = MySQLHelper()
self.__running = False
