def main(): tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass, tracker2_db) countid = 0 countid1 = 1000000 while countid < 123000000: select = """SELECT b.id, c.key_id FROM tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE b.company_id = 14 AND c.company_id = 14 AND b.matchedVideo_id = c.id and b.id > %s and b.id <= %s""" % ( countid, countid1) countid += 1000000 countid1 += 1000000 print select select_result = tracker2.query_sql_cmd(select) print "fetched!" retrytracker2 = 0 while retrytracker2 < 10: try: print "retrytracker2:" print retrytracker2 target = MySQLHelper(t_host, t_user, t_pass, t_db) print "start insert!" print datetime.datetime.now() break except Exception, e: print e retrytracker2 += 1 #target = MySQLHelper(t_host,t_user,t_pass,t_db) insert_result = "insert into infback_Tmp " + " values " + str( select_result)[1:-1].replace("L", "").replace("u", "") target.insert_sql_cmd(insert_result) print "finished!" print datetime.datetime.now()
def NovelCrawl_Main(count): url = r'http://www.yousuu.com/booklist' proxies = fetchproxies() for i in range(count): try: header = get_header() proxy = random.choice(proxies) url = CrawlNovelList(url, header, proxy) except Exception as e: print 'error' print e.message pass sql = 'select distinct novellisturl from pagenovel ' a = MySQLHelper() NovelListUrl = a.SqlFecthAll(sql) a.CloseCon() print len(NovelListUrl) #proxies = fetchproxies() for i in NovelListUrl: try: header = get_header() proxy = random.choice(proxies) print i[0] CrawlNovel(i[0], header, proxy) except Exception as e: print 'error' print e.message pass a = MySQLHelper() sql = 'select distinct novelurl from novelurl' NovelUrl = a.SqlFecthAll(sql) a.CloseCon() print len(NovelUrl) #proxies = fetchproxies() for i in NovelUrl: try: header = get_header() proxy = random.choice(proxies) print i[0] CrawlNovelData(i[0], header, proxy) except Exception as e: print 'error' print e.message pass
def main(): helper = MySQLHelper('127.0.0.1', 'root', '', 'pymysql') helper.connect() # 添加 helper.cud(sql='insert into `tb_stu`(`name`) values(%s)', params=["mary"]) # 查询 results = helper.query(sql="select * from `tb_stu`") for result in results: id = result[0] name = result[1] print("id:%s,name:%s" % (id, name)) # 关闭 helper.close()
def Main_CrawlNovelData(pp): sql = 'select distinct novelurl from novelurl order by rundate desc limit %s offset %s' % ( 300, int(pp) * 300) a = MySQLHelper() NovelUrl = a.SqlFecthAll(sql) a.CloseCon() proxies = fetchproxies(40) for i in NovelUrl: try: header = get_header() proxy = random.choice(proxies) print i[0] CrawlNovelData(i[0], header, proxy) except Exception as e: print 'error' print e.message pass
def __init__(self, run): self.__run = run self.__hash = str(uuid.uuid4()) self.__raven = Raven() self.__db = MySQLHelper()
#cursor.execute("drop index meta_isp on infringingP2P_Tmp") cursor.execute("drop index meta_isp on allP2P_Tmp") pre_DASHBOARD.close() print "truncate finished!" '''tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db) print datetime.datetime.now() infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" %(ETL_DTE,ETL_DTE) print infP2P_Tmp_select infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select) print "inf fetched"''' retryVIADASHBOARD = 0 while retryVIADASHBOARD < 10: try: print "retryVIADASHBOARD:" print retryVIADASHBOARD VIACOM_DASHBOARD = MySQLHelper(DASHBOARD_host, DASHBOARD_user, DASHBOARD_pass, DASHBOARD_db) break except Exception, e: print e retryVIADASHBOARD += 1 '''infP2P_Tmp_insert = "insert into infringingP2P_Tmp " + " values " + str(infP2P_result)[1:-1].replace("L", "").replace("u","") VIACOM_DASHBOARD.insert_sql_cmd(infP2P_Tmp_insert) print datetime.datetime.now()''' retryTime_ware = 0 while retryTime_ware < 10: try: print "retryTime_ware:" print retryTime_ware p2pwarehouse = MySQLHelper(p2pwarehouse_host, p2pwarehouse_user, p2pwarehouse_pass, p2pwarehouse_db)
except Exception, e: print e retryTime += 1 cursor.execute("truncate table infringingP2P_Tmp") #cursor.execute("truncate table allP2P_Tmp") cursor.execute("drop index meta_isp on infringingP2P_Tmp") #cursor.execute("drop index meta_isp on allP2P_Tmp") pre_DASHBOARD.close() print "truncate finished!" retrytracker2 = 0 while retrytracker2 < 10: try: print "retrytracker2:" print retrytracker2 tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass, tracker2_db) break except Exception, e: print e retrytracker2 += 1 #tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db) print datetime.datetime.now() infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" % ( ETL_DTE, ETL_DTE) print infP2P_Tmp_select infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select) print "inf fetched" retryVIADASHBOARD = 0 while retryVIADASHBOARD < 10: try: print "retryVIADASHBOARD:"
#coding= utf-8 import re import time import cx_Oracle from MySQLHelper import MySQLHelper a = MySQLHelper() sql = 'select * from fun.noveldata order by rundate desc ' rawdata = a.SqlFecthAll(sql) a.CloseCon() conn = cx_Oracle.connect('system/Syy19930119@localhost:1521/orcl') cursor = conn.cursor() try: cursor.execute('drop table SYY_NOVELDATA') except: pass sql = """ CREATE TABLE system.SYY_NOVELDATA (\ RUNDATE DATE\ ,NOVELURL VARCHAR2(250)\ ,NOVELNAME VARCHAR2(250)\ ,NOVEL_ORGURL VARCHAR2(250)\ ,TAG_CATEGORY VARCHAR2(250)\ ,AUTHOR VARCHAR2(250)\ ,NOVELRANK VARCHAR2(250)\ ,WORLDCOUNT VARCHAR2(250)\ ,SECTIONCOUNT VARCHAR2(250)\
retryTime += 1 cursor.execute( "truncate table infringingP2P_Tmp;truncate table allP2P_Tmp;drop index meta_isp on infringingP2P_Tmp;drop index meta_isp on allP2P_Tmp" ) #truncate table meta_Tmp;truncate table isp_Tmp;drop index meta_isp on infringingP2P_Tmp;drop index meta_isp on allP2P_Tmp) cursor.close() #DASHBOARD.commit() print "truncate finished!" #fetch the VTWeb infringing IPs and hashes begin retrytracker2 = 0 while retrytracker2 < 10: try: print "retrytracker2:" print retrytracker2 tracker2 = MySQLHelper(tracker2_host, tracker2_user, tracker2_pass, tracker2_db) break except Exception, e: print e retrytracker2 += 1 #tracker2 = MySQLHelper(tracker2_host,tracker2_user,tracker2_pass,tracker2_db) print datetime.datetime.now() infP2P_Tmp_select = """SELECT a.meta_id, a.isp_id, c.key_id AS protocol_hash, count(a.IP) AS infringingIPs FROM tracker2.takedownNoticeItemP2PDetail AS a, tracker2.matchedVideoP2PItem AS b, tracker2.matchedVideo AS c WHERE a.company_id = 14 AND b.company_id = 14 AND c.company_id = 14 AND a.matchedVideoP2PItem_id = b.id AND b.matchedVideo_id = c.id AND a.first_notice_send_time >= DATE_SUB('%s 08:00:00', INTERVAL +1 DAY) AND a.first_notice_send_time < DATE_SUB('%s 08:00:00', INTERVAL 0 DAY) GROUP BY a.meta_id, a.isp_id, c.key_id""" % ( ETL_DTE, ETL_DTE) print infP2P_Tmp_select set_NamesUTF8 = """SET NAMES UTF8""" print set_NamesUTF8 infP2P_result = tracker2.query_sql_cmd(infP2P_Tmp_select) print "inf fetched" retryVIADASHBOARD = 0 while retryVIADASHBOARD < 10:
def __init__(self, run): self.__run = run self.__raven = Raven() self.__db = MySQLHelper() self.__running = False