def base_jy_detail_pipeline(data):
print('---------base_jy_detail_pipeline----------')
# 传入连接数据库
conn, cursor = get_cnn_cursor(BT_MYSQL_DB)
# 保存附件信息
if int(data['has_attach']):
attach_sql_list = make_attach_sql_list(province=province_name,
data=data)
try_save_to_db(attach_sql_list, conn, cursor)
# 保存详情
# 传入省份缩写，类别，数据，要存储的item，不进行保存的字段，获取sql语句
sql_txt = get_insert_sql_txt(province=province_name,
category='jy',
data=data,
item=bt_jy_detail_item,
ex_fields=[
'notice_id', 'attach_url_list',
'attach_location_list', 'attach_id_list',
'pipeline_func'
])
result = try_save_to_db(sql_txt, conn, cursor)
# 详情存成功了, 如果是完结了的, 就保存连接到url list, 下次就不会再请求这个连接了
if result == 1:
save_url(conn, cursor, province=province_name, link=data.get('link'))
cursor.close()
conn.close()
def tj_cg_detail_pipeline(data):
"""天津政府采购网详情页处理"""
print(f'---------{province_name}_cg_detail_pipeline----------')
conn, cursor = get_cnn_cursor(TJ_MYSQL_DB)
# 保存附件信息
if int(data['has_attach']):
attach_sql_list = make_attach_sql_list(province=province_name,
data=data)
try_save_to_db(attach_sql_list, conn, cursor)
# 保存详情
sql_txt, values = get_sql_txt_and_value(province=province_name,
category='cg',
data=data,
item=tj_cg_detail_item,
ex_fields=[
'notice_id', 'attach_url_list',
'attach_location_list',
'attach_id_list',
'pipeline_func'
])
result = try_save_to_db2(sql_txt, values, conn, cursor)
# 详情存成功了, 如果是完结了的, 就保存连接到url list, 下次就不会再请求这个连接了
if result == 1:
save_url(conn, cursor, province=province_name, link=data.get('link'))
cursor.close()
conn.close()
def task_main():
# 每天的任务获取
all_everyday_tasks = collect_provinces_tasks()
# 连接task数据库
task_conn, task_cursor = get_cnn_cursor(
WITHOUT_MYSQL_DB, cursorclass=pymysql.cursors.DictCursor)
# 连接tender数据库
# tender_conn,tender_cursor=get_cnn_cursor(TENDER_MYSQL_DB)
# 对 任务url 去重后的列表
url_list = list(set(all_everyday_tasks))
sql = "insert ignore into task_tender(url,pri_level) values(%s,%s)"
par = url_list
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '\n', sql,
'添加', len(url_list), '条')
try:
task_cursor.executemany(sql, par)
task_conn.commit()
except Exception as e:
print(e)
task_cursor.close()
task_conn.close()