def shoulu_func(huoqu_shoulu_time_stamp, set_url_data):
shoulu_canshu = 1
while True:
now_time = int(time.time())
time_stamp = now_time + 20
sql = """select * from shoulu_Linshi_List where is_zhixing = '0' and time_stamp='{huoqu_shoulu_time_stamp}' and (shijianchuo is NULL or shijianchuo < '{time_stamp}') limit 1;""".format(
time_stamp=now_time,
huoqu_shoulu_time_stamp=huoqu_shoulu_time_stamp)
objs_data = database_create_data.operDB(sql, 'select')
for obj_data in objs_data['data']:
tid = obj_data[0]
print('tid========> ', tid)
search = obj_data[5]
lianjie = obj_data[1]
huoqu_shoulu_time_stamp = obj_data[3]
if threading.active_count() <= 6:
sql = """update shoulu_Linshi_List set shijianchuo ='{time_stamp}' where id = {detail_id};""".format(
time_stamp=time_stamp, detail_id=tid)
database_create_data.operDB(sql, 'update')
threadObj = threading.Thread(target=zhongzhuanqi.shouluChaxun,
args=(lianjie, tid, search))
threadObj.start()
else:
sleep(0.5)
continue
count_sql = """select count(id) from shoulu_Linshi_List where is_zhixing = '1' and time_stamp='{huoqu_shoulu_time_stamp}';""".format(
huoqu_shoulu_time_stamp=huoqu_shoulu_time_stamp)
count_objs = database_create_data.operDB(count_sql, 'select')
if count_objs['data'][0][0] == set_url_data:
break
def shouluChaxun(lianjie, tid, search):
# 'shoulu': shoulu,
# 'title': title,
# 'kuaizhao_time': kuaizhao_time,
# 'status_code': status_code
# pc端百度
if str(search) == '1':
resultObj = shouluORfugaiChaxun.baiduShouLuPC(lianjie)
# 移动端百度
elif str(search) == '4':
resultObj = shouluORfugaiChaxun.baiduShouLuMobeil(lianjie)
# pc360
elif str(search) == '3':
resultObj = shouluORfugaiChaxun.pcShoulu360(lianjie)
# 移动端360
elif str(search) == '6':
resultObj = shouluORfugaiChaxun.mobielShoulu360(lianjie)
# pc搜狗
elif str(search) == '2':
# print('搜狗pc---------')
resultObj = shouluORfugaiChaxun.sougouShouLuPC(lianjie)
# 移动端搜狗
else:
# print('搜狗 mobile =--------')
resultObj = shouluORfugaiChaxun.sougouShouLuMOBILE(lianjie)
# print('resultObj-----> ',resultObj)
sql = """update shoulu_Linshi_List set is_shoulu='{shoulu}', title='{title}', kuaizhao_time='{kuaizhao}', status_code='{status_code}', is_zhixing='{is_zhixing}' where id ={id};""".format(
shoulu=resultObj['shoulu'],
title=resultObj['title'],
kuaizhao=resultObj['kuaizhao_time'],
status_code=resultObj['status_code'],
id=tid,
is_zhixing='1')
database_create_data.operDB(sql, 'update')
def fugai_func(huoqu_fugai_time_stamp, set_keyword_data):
while True:
now_time = int(time.time())
time_stamp = now_time + 30
sql = """select * from fugai_Linshi_List where is_zhixing = '0' and time_stamp='{huoqu_fugai_time_stamp}' and (shijianchuo < '{time_stamp}' or shijianchuo is NULL) limit 1;""".format(
huoqu_fugai_time_stamp=huoqu_fugai_time_stamp, time_stamp=now_time)
objs_data = database_create_data.operDB(sql, 'select')
for obj_data in objs_data['data']:
tid = obj_data[0]
search = obj_data[2]
mohu_pipei = obj_data[3]
keyword = obj_data[10]
if threading.active_count() <= 6:
# 更改数据库时间戳 二十秒可执行下一次
sql = """update fugai_Linshi_List set shijianchuo ='{time_stamp}' where id = '{id}';""".format(
time_stamp=time_stamp, id=tid)
database_create_data.operDB(sql, 'update')
# 启动线程
fugai_thread4 = threading.Thread(
target=zhongzhuanqi.fugaiChaxun,
args=(tid, search, keyword, mohu_pipei,
huoqu_fugai_time_stamp))
fugai_thread4.start()
else:
sleep(0.5)
continue
count_sql = """select count(id) from fugai_Linshi_List where is_zhixing = '1' and time_stamp='{huoqu_fugai_time_stamp}';""".format(
huoqu_fugai_time_stamp=huoqu_fugai_time_stamp)
count_objs = database_create_data.operDB(count_sql, 'select')
if count_objs['data'][0][0] == set_keyword_data:
break
def dingshi_timer():
global timer_flag
if timer_flag:
return
else:
timer_flag = True
sql = """select * from task_Detail where is_perform='1';"""
is_run_flag = False # 表示是否有任务要运行
objs_list = database_create_data.operDB(sql, 'select')
if objs_list['data']:
for obj in objs_list['data']:
detail_id = obj[0]
tid = obj[1]
search_engine = obj[2]
lianjie = obj[3]
keywords = obj[4]
mohupipei = obj[5]
time_stamp_obj = obj[8]
shijianchuo = time.time()
now_date = datetime.datetime.today().strftime('%Y-%m-%d 23-59-59')
if time_stamp_obj:
time_stamp_obj = int(time_stamp_obj)
if time_stamp_obj < int(shijianchuo):
is_run_flag = True
else:
time_stamp = int(shijianchuo) + 300
sql = """update task_Detail set time_stamp ='{time_stamp}' where id = {detail_id};""".format(
time_stamp=time_stamp, detail_id=detail_id)
database_create_data.operDB(sql, 'update')
is_run_flag = True
if is_run_flag:
thread_obj = pool.get_thread()
if lianjie:
# print('定时器二 线程启动 收录pc端',int(time.time()))
thread_mobile_url = thread_obj(target=thread_url_shoulu,
args=(detail_id, keywords,
lianjie,
search_engine))
thread_mobile_url.start()
else:
# print('定时器二 线程启动 收录移动端',int(time.time()))
thread_mobile_mohupipei = thread_obj(
target=thread_pcmohupipei_fugai,
args=(search_engine, detail_id, keywords, mohupipei))
thread_mobile_mohupipei.start()
# 所有线程执行完毕 只剩主线程 则退出
while True:
# if threading.active_count() == 1:
if ThreadPool().queue1.empty():
break
else:
time.sleep(1)
timer_flag = False
def lijijiankong(json_data):
print('立即监控id ---- >', json_data)
for data in json_data:
if threading.active_count() <= 6:
jiankong = threading.Thread(target=get_task_list, args=(data, ))
jiankong.start()
select_sql = """select id from task_Detail where tid='{}'""".format(
data)
objs = database_create_data.operDB(select_sql, 'select')
sql_data_list = []
for obj in objs['data']:
delete_sql = """delete from task_Detail_Data where tid='{}'""".format(
obj[0])
sql_data_list.append(delete_sql)
database_create_data.operDB('', 'delete', True, sql_data_list)
else:
continue
def insert_into(data, time_stamp, shibiecanshu):
if shibiecanshu == 'zhongdianci':
json_data = json.loads(data)
task_name = json_data['task_name']
mohupipei = json_data['mohupipei']
keyword_list = list(set(json_data['keywords'].split('\n')))
search_engine_list = ','.join(json_data['search_engine'])
sql_list = []
sql_two = """select id from Task_List where task_name='{}';""".format(task_name)
objs = database_create_data.operDB(sql_two, 'select')
tid = objs['data'][0][0]
create_time = time.strftime("%Y-%m-%d %H-%M-%S", time.localtime())
for search_engine in search_engine_list.split(','):
for keyword in keyword_list:
if keyword:
if 'http' in keyword:
new_keyword = re.findall("(.*)http", keyword)[0].replace('\t', '')
if new_keyword:
lianjie_list = keyword.split(new_keyword)
for lianjie in lianjie_list:
if lianjie:
lianjie = lianjie.replace('\t', '')
data_insert = (tid, search_engine, lianjie, new_keyword, mohupipei, create_time)
sql_three_sql = """insert into task_Detail (tid, search_engine, lianjie, keywords, mohupipei, create_time) values {data_insert};""".format(
data_insert=data_insert)
sql_list.append(sql_three_sql)
else:
lianjie = ''
data_insert = (tid, search_engine, lianjie, keyword, mohupipei, create_time)
sql_three = """insert into task_Detail (tid, search_engine, lianjie, keywords, mohupipei, create_time) values {data_insert};""".format(
data_insert=data_insert)
sql_list.append(sql_three)
database_create_data.operDB('', 'insert', True, sql_list)
if shibiecanshu == 'shoulu':
json_data_list = json.loads(data)
data_url_list = json_data_list['editor_content'].strip().split('\n')
sql_list = []
if json_data_list['searchEngineModel']:
print('引擎----> ', json_data_list['searchEngineModel'])
for search in json_data_list['searchEngineModel']:
for dataurl in set(data_url_list):
data_url = dataurl.replace('\n', '')
if 'http' in dataurl:
url_data = data_url
else:
url_data = 'http://' + data_url
lianjie = url_data.strip().replace('\t', '')
insert_sql = """insert into shoulu_Linshi_List (url, time_stamp, search, is_zhixing) values('{url}', '{time_stamp}', '{search}', '{is_zhixing}');""".format(
url=lianjie, time_stamp=time_stamp, search=search, is_zhixing='0')
sql_list.append(insert_sql)
database_create_data.operDB('', 'insert', True, sql_list)
if shibiecanshu == 'fugai':
tiaojian_list = []
json_data_list = data
sql_list = []
for tiaojian in json_data_list['fugai_tiaojian'].split('|'):
if not tiaojian.isspace():
tiaojian = tiaojian.strip().replace('\n', '')
if tiaojian not in tiaojian_list:
tiaojian_list.append(tiaojian)
str_tiaojian = ",".join(str(i) for i in tiaojian_list)
for search in json_data_list['searchEngineModel']:
for keyword in set(json_data_list['editor_content'].strip().split('\n')):
keyword = keyword.strip()
insert_sql = """insert into fugai_Linshi_List (keyword, search_engine, sousuo_guize, time_stamp, is_zhixing) values ('{keyword}', '{search_engine}', '{sousuo_guize}', '{time_stamp}', '0');""".format(
keyword=keyword, search_engine=search, sousuo_guize=str_tiaojian, time_stamp=time_stamp)
sql_list.append(insert_sql)
database_create_data.operDB('', 'insert', True, sql_list)
def get_task_list(data=None):
xiaoyu_dengyu_date = datetime.datetime.today().strftime(
'%Y-%m-%d 23-59-59')
start_time = datetime.datetime.today().strftime('%Y-%m-%d %H:%M:59')
if data:
sql = """select id,next_datetime from task_List where qiyong_status = '1' and id = '{}';""".format(
data)
update_sql = """update task_Detail set time_stamp='' where tid = '{}'""".format(
data)
database_create_data.operDB(update_sql, 'update')
else:
sql = """select id,next_datetime from task_List where next_datetime <='{start_time}' and qiyong_status = '1' limit 1;""".format(
start_time=start_time)
objs = database_create_data.operDB(sql, 'select')
if objs['data']:
data_id = objs['data'][0][0]
select_sql = """select id from task_Detail where tid='{data}'""".format(
data=data_id)
select_objs = database_create_data.operDB(select_sql, 'select')
if select_objs['data']:
next_time = str(objs['data'][0][1])
next_datetime = datetime.datetime.strptime(next_time,
'%Y-%m-%d %H:%M:%S')
# print('next_datetime-----------> ',next_datetime, type(next_datetime))
update_status_sql = """update task_List set task_status = '0', zhixing = '1' where id = '{}'""".format(
data_id)
database_create_data.operDB(update_status_sql, 'update')
if next_datetime.strftime('%Y-%m-%d') <= datetime.datetime.today(
).strftime('%Y-%m-%d'):
# 修改下一次执行时间
next_datetime_addoneday = (
next_datetime +
datetime.timedelta(days=1)).strftime('%Y-%m-%d %H:%M:%S')
next_sql = """update task_List set next_datetime = '{next_datetime}' where id = '{id}';""".format(
next_datetime=next_datetime_addoneday, id=data_id)
database_create_data.operDB(next_sql, 'update')
# 修改 任务详情为 启用
sql_status = """update task_Detail set is_perform = '1', task_start_time = '{time}' where tid = '{task_id}';""".format(
time=start_time, task_id=data_id)
database_create_data.operDB(sql_status, 'update')
else:
update_sql = """update task_List set qiyong_status = '0' where id = '{data_id}';""".format(
data_id=data_id)
database_create_data.operDB(update_sql, 'update')
def fugaiChaxun(tid, search, keyword, mohu_pipei, huoqu_fugai_time_stamp=None):
"""
'paiming':int(rank_num),
'title':title,
'title_url':ret_two_url,
'sousuo_guize':mohu_pipei,
'status_code':status_code
"""
order_list = []
# pc端百度
if str(search) == '1':
# print('pc端 -- 覆盖百度', int(time.time()))
resultObj = shouluORfugaiChaxun.baiduFuGaiPC(keyword, mohu_pipei)
# 移动端百度
elif str(search) == '4':
# print('移动端 -- 覆盖百度', int(time.time()))
resultObj = shouluORfugaiChaxun.baiduFuGaiMOBIEL(keyword, mohu_pipei)
# pc360
elif str(search) == '3':
# print('pc端 -- 覆盖360', int(time.time()))
resultObj = shouluORfugaiChaxun.pcFugai360(keyword, mohu_pipei)
# # 移动端360
elif str(search) == '6':
# print('移动端 -- 覆盖360 ', int(time.time()))
resultObj = shouluORfugaiChaxun.mobielFugai360(keyword, mohu_pipei)
else:
pass
# print('resultObj-------> ',resultObj)
if huoqu_fugai_time_stamp:
json_detail_data = []
for result in resultObj:
order_list.append(result['paiming'])
zhanwei = 0
if result['paiming']:
zhanwei = 1
json_detail_data.append({
'rank':
result['paiming'],
'title':
result['title'].replace('\'', '').replace('"', ''),
'url':
result['title_url'],
'guize':
result['sousuo_guize'],
'keyword':
keyword,
'zhanwei':
zhanwei,
'search_engine':
search
})
json_data = ''
if len(json_detail_data):
json_data = json.dumps(json_detail_data)
str_order = '0'
if order_list:
str_order = ','.join(str(i) for i in set(order_list))
sql_two = """update fugai_Linshi_List set paiming_detail='{paiming_detail}', json_detail_data='{json_detail_data}', chaxun_status='1', is_zhixing='{is_zhixing}' where id = '{id}';""".format(
paiming_detail=str_order,
is_zhixing='1',
id=tid,
json_detail_data=json_data)
database_create_data.operDB(sql_two, 'update')
else:
# 给点词监控返回 排名
for result in resultObj:
order_list.append(result['paiming'])
return order_list