def update_china_history():
cursor = None
conn = None
try:
context = get_china_history()
print(f"{time.asctime()}开始更新国内历史数据")
conn, cursor = get_conn()
sql = "INSERT INTO c_history(updateTime,createTime,`date`,confirm,suspect,dead,heal,nowConfirm,importedCase,healRate,deadRate) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.executemany(sql, context) # 插入数据
print('execute successfully')
conn.commit() # 提交事务保存数据
print(f"{time.asctime()}数据更新完毕")
except:
traceback.print_exc()
finally:
close_conn(conn, cursor)
def update_china_status():
cursor = None
conn = None
try:
context = get_china_status()
print(f"{time.asctime()}开始更新国内疫情数据")
conn, cursor = get_conn()
sql = "INSERT INTO data_china(updateTime,country,province,city,confirm,suspect,heal,dead,nowConfirm,healRate,deadRate) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.executemany(sql, context) # 插入数据
print('execute successfully')
conn.commit() # 提交事务保存数据
print(f"{time.asctime()}数据更新完毕")
except:
traceback.print_exc()
finally:
close_conn(conn, cursor)
def update_hotsearch():
cursor = None
conn = None
try:
context = get_baidu_hot()
print(f"{time.asctime()}开始更新热搜数据")
conn, cursor = get_conn()
sql = "insert into topic(board_id,title,content,user_id,link,createTime,updateTime) values(1,%s,%s,1,%s,%s,%s)"
ts = datetime.now().strftime("%Y-%m-%d %X")
for i in context:
cursor.execute(sql,
(i['title'], i['title'], i['link'], ts, ts)) # 插入数据
print('execute successfully')
conn.commit() # 提交事务保存数据
print(f"{time.asctime()}数据更新完毕")
except:
traceback.print_exc()
finally:
close_conn(conn, cursor)
def update_global_data():
cursor = None
conn = None
try:
context, history = get_global_status()
print(f"{time.asctime()}开始更新全球疫情数据")
conn, cursor = get_conn()
ts = time.strftime("%Y-%m-%d %X")
status_sql = "INSERT INTO data_global(updateTime,continent,country,province,confirm,heal,dead,nowConfirm) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)"
his_sql = "INSERT INTO g_history(updateTime,createTime,`date`,confirm,dead,heal,newAddConfirm,deadRate,healRate) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.executemany(status_sql, context) # 插入全球疫情数据
for his_item in history:
print(his_item)
cursor.execute(his_sql, his_item) # 插入全球历史数据
print('execute successfully')
conn.commit() # 提交事务保存数据
print(f"{time.asctime()}数据更新完毕")
except:
traceback.print_exc()
finally:
close_conn(conn, cursor)
import time
import json
import traceback
import sys
from db_init_prog.config import get_conn, close_conn, APIs
from datetime import datetime
from db_init_prog.china_history_spider import update_china_history
from db_init_prog.china_status_spider import update_china_status
from db_init_prog.global_spider import update_global_data
from db_init_prog.news_spider import update_hotsearch
if __name__ == '__main__':
cursor = None
conn = None
conn, cursor = get_conn()
cursor.execute('TRUNCATE TABLE `c_history`')
cursor.execute('TRUNCATE TABLE `g_history`')
cursor.execute('TRUNCATE TABLE `data_china`')
cursor.execute('TRUNCATE TABLE `data_global`')
print('all table have been truncated')
conn.commit()
conn.close()
update_hotsearch()
print('hot search updated')
update_global_data()
print('global data updated')
print('global history updated')
update_china_status()
update_china_history()