def loadFoodMaterialListPage(url, class1_id, class2_id):
divList = getContent(url).xpath('//div[@class="listtyle1"]')
for div in divList:
html_url = div.xpath('div[@class="img"]/a/@href')[0]
logo_url = div.xpath("div/a/img/@src")[0]
title = div.xpath('div[@class="info1"]/h3/a/text()')[0]
description = ""
descriptionList = div.xpath('div[@class="info1"]/div/span/text()')
for d in descriptionList:
description = d
sql = "insert into lb_food_material ( name , class1_id , class2_id ,description, html_url," \
" logo_url) values ( %s, %s, %s, %s, %s, %s)"
params = (title, class1_id, class2_id, description, html_url, logo_url)
# print(sql % params)
MysqlHelper.MysqlHelper().cud(sql, params)
def spiderComment(food_id, content):
# 爬取评论数据
comlist = content.xpath("//div[@class='cp_comlist_w']/ul/li")
count = 0
for c in comlist:
content = c.xpath(".//p/strong")[0].tail.encode("utf-8").strip()
commentTime = ""
comm_time = c.xpath(".//div/span/text()")[0].encode("utf-8")
m = re.compile(r'(\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2})来自').findall(comm_time)
if len(m) > 0:
commentTime = m[0]
comment_time = datetime.datetime.strptime(commentTime, '%Y-%m-%d %H:%M:%S')
user_name = ""
user_name_list = c.xpath("a/h5/text()")
if len(user_name_list) > 0:
user_name = user_name_list[0]
user_avatar_url = c.xpath("a/img/@src")[0]
comHref = c.xpath("a/@href")
user_id = 0
user_url = ""
if len(comHref) > 0:
com_user_url = comHref[0]
m = re.compile(r'.*id=(\d*)').match(com_user_url)
if m is not None:
user_id = m.group(1)
user_url = com_user_url
# print(comment)
# print(comment_time)
# print(user_name)
# print(user_avatar_url)
# print(user_id)
# print(user_url)
sql = "insert into lb_food_comment ( food_id , user_name , user_id , user_avatar_url ," \
" user_url, content, comment_time) values ( %s, %s, %s, %s, %s, %s, %s)"
params = (food_id, user_name, user_id, user_avatar_url, user_url, content, comment_time)
# print(sql % params)
if MysqlHelper.MysqlHelper(isTest).cud(sql, params) == 1:
count += 1
return count == len(comlist)
def loadFoodClass1():
"""
爬取第一分类
"""
url = "https://www.meishij.net/jiankang/"
dlList = getContent(url).xpath('//div[@class="nav"]/ul/li[2]/div/div/div/dl')
for dl in dlList:
aList = dl.xpath("dt/a")
for a in aList:
text = a.text
class_url = a.attrib.get("href")
sql = "insert into lb_food_class ( name , level , class_url ) values ( %s, %s, %s)"
params = (text, 1, class_url)
print(sql % params)
MysqlHelper.MysqlHelper().cud(sql, params)
def spiderMaterial(food_id, content):
divList = content.xpath("//div[@class='materials_box']/div")
count = 0
allCount = 0
for div in divList:
goods_ad_list = div.xpath("./@class")
if len(goods_ad_list) > 0 and goods_ad_list[0] == "goods_ad":
continue
tag = div.xpath("h3/a/text()")[0]
liList = div.xpath("ul/li")
allCount += len(liList)
for li in liList:
thumbnail_url = ""
name = ""
dosage = ""
thumbnail_url_list = li.xpath("a/img/@src")
if len(thumbnail_url_list) > 0:
thumbnail_url = thumbnail_url_list[0]
name_list = li.xpath("div/h4/a/text()")
if len(name_list) > 0:
name = name_list[0]
if name == "":
name_list = li.xpath("h4/a/text()")
if len(name_list) > 0:
name = name_list[0]
dosage_list = li.xpath("div/h4/span/text()")
if len(dosage_list) > 0:
dosage = dosage_list[0]
if dosage == "":
dosage_list = li.xpath("span/text()")
if len(dosage_list) > 0:
dosage = dosage_list[0]
sql = "insert into lb_food_material_assoc ( food_id , tag, name, thumbnail_url, dosage)" \
" values ( %s, %s, %s, %s, %s )"
params = (food_id, tag, name, thumbnail_url, dosage)
# print(sql % params)
if MysqlHelper.MysqlHelper(isTest).cud(sql, params) == 1:
count += 1
return allCount == count
def loadFoodDetail():
sql = "select id, html_url from lb_food where is_spider_detail = %s limit %s, %s "
params = (0, pageSize * (page - 1), pageSize * page)
print(sql % params)
rows = MysqlHelper.MysqlHelper(isTest).fetchall(sql, params)
for row in rows:
food_id, html_url = row
try:
content = getContent(html_url)
# global isTest
# isTest = True
# content = getContent("https://www.meishij.net/zuofa/liangbankugua_37.html")
if spiderBaseInfo(food_id, content):
print("爬取基本信息成功")
if spiderComment(food_id, content):
print("爬取评论数据成功")
if spiderProcess(food_id, content):
print("爬取做法成功")
if spiderMaterial(food_id, content):
print("爬取用料成功")
# 更新状态
updateSpiderDetail("爬取成功", food_id, True)
sleepRandom()
continue
else:
s = "爬取用料出错"
print(s)
updateSpiderDetail(s, food_id)
else:
error_msg = "爬取做法出错"
print(error_msg)
updateSpiderDetail(error_msg, food_id)
else:
s1 = "爬取评论数据出错"
print(s1)
updateSpiderDetail(s1, food_id)
else:
s2 = "爬取基本信息出错"
print(s2)
updateSpiderDetail(s2, food_id)
except Exception as e:
updateSpiderDetail(e.message, food_id)
sleepRandom()
def truncate_table_sql(model, batch_size):
"""
init
"""
try:
cf = ConfigParser.ConfigParser()
cf.read("./conf/load_config.conf")
mysql_host = cf.get("db", "mysql_host")
mysql_port = cf.getint("db", "mysql_port")
mysql_user = cf.get("db", "mysql_user")
mysql_passwd = cf.get("db", "mysql_passwd")
conf_name = "conf_%s" % model
try:
test_db = cf.get(conf_name, "test_db") % batch_size
except Exception as e:
sys.exit(1)
mysql = mysql_helper.MysqlHelper(host = mysql_host,\
port = mysql_port, user = mysql_user, \
passwd = mysql_passwd, db = test_db)
except Exception as exception:
print exception
return
table_name=["anakin2_yolo_time_satistic_k1200", "anakin2_yolo_time_satistic_p4", \
"anakin_tensorrt_time_satistic_k1200", "anakin_tensorrt_time_satistic_p4", \
"nvidia_list_1sec_k1200", "nvidia_list_1sec_p4", \
"nvidia_list_1sec_version_k1200", "nvidia_list_1sec_version_p4", \
"nvidia_list_1sec_version_tensorRT_k1200", "nvidia_list_1sec_version_tensorRT_p4", \
"top_list_1sec_avg_k1200", "top_list_1sec_avg_p4", \
"top_list_1sec_avg_tensorRT_k1200", "top_list_1sec_avg_tensorRT_p4", \
"top_list_1sec_k1200", "top_list_1sec_p4", \
"log_monitor_k1200", "log_monitor_p4"]
for item in table_name:
truncate_sql = "truncate table %s" % (item)
print "[INFO]: start truncate the sql"
try:
truncate_result = mysql.executes(truncate_sql)
print("[INFO]: truncate %s success!!!" % item)
print truncate_result
except Exception as exception:
print("[ERROR]: truncate %s error!!!" % item)
def __init__(self, db_name):
"""
init
"""
self.config = {}
try:
cf = ConfigParser.ConfigParser()
cf.read("../../conf/load_config.conf")
self.mysql_host = cf.get("db", "mysql_host")
self.mysql_port = cf.getint("db", "mysql_port")
self.mysql_user = cf.get("db", "mysql_user")
self.mysql_passwd = cf.get("db", "mysql_passwd")
self.test_db = db_name
self.mysql = mysql_helper.MysqlHelper(host = self.mysql_host,\
port = self.mysql_port, user = self.mysql_user, \
passwd = self.mysql_passwd, db = self.test_db)
except Exception as exception:
print exception
return
def getNewsDetail(self, news_url):
"""
获取新闻具体细节
Args:
search_url:每条新闻具体链接
Returns:
新闻链接、标题、正文、时间、日期
"""
result = {}
res = requests.get(news_url)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
if (len(soup.select('.article-title h2')) > 0
and len(soup.select('.date')[0]) > 0
and len(soup.select('.time')[0]) > 0):
result['url'] = news_url
result["title"] = soup.select('.article-title h2')[0].text
result["date"] = soup.select('.date')[0].text.lstrip('发布时间：')
result["time"] = soup.select('.time')[0].text
#source = soup.select('.account-authentication')[0].text
#print(title,date,time,source)
if len(result["date"]) == 5:
result["datetime"] = "2020-" + result["date"] + " " + result[
"time"]
elif len(result["date"]) == 8:
result[
"datetime"] = "20" + result["date"] + " " + result["time"]
else:
result["datetime"] = result["date"] + " " + result["time"]
result["article"] = self.getArticle(news_url)
msh = MysqlHelper.MysqlHelper(host="localhost",
username="******",
password="******",
db="baiduSearchNews",
charset="utf8",
port=3306)
msh.connect()
sql = "insert into disease values('%s','%s','%s','%s')" % (
result["article"], result["datetime"], result["title"],
result['url'])
msh.insert(sql)
def __init__(self, model, batch_size=1):
"""
init
"""
self.config = {}
try:
cf = ConfigParser.ConfigParser()
cf.read("../conf/load_config.conf")
self.mysql_host = cf.get("db", "mysql_host")
self.mysql_port = cf.getint("db", "mysql_port")
self.mysql_user = cf.get("db", "mysql_user")
self.mysql_passwd = cf.get("db", "mysql_passwd")
#TODO
conf_name = "conf_%s" % model
try:
self.test_db = cf.get(conf_name, "test_db") % batch_size
except Exception as e:
self.test_db = cf.get("db", "test_db")
self.mysql = mysql_helper.MysqlHelper(host = self.mysql_host,\
port = self.mysql_port, user = self.mysql_user, \
passwd = self.mysql_passwd, db = self.test_db)
except Exception as exception:
print exception
return
body = browser.find_element_by_xpath("/html/body/pre").text
print("start " + stock)
data = json.loads(body)
chouma = data["data"]["result"]["result"]
if len(chouma) > 0:
chouma_data = chouma[0]
row = helper.fetchone(sql_select, [stock])
if not row:
helper.insert(sql_insert, chouma_data)
print("insert " + stock)
else:
chouma_data.append(stock)
helper.update(sql_update, chouma_data)
time.sleep(0.01)
print("update " + stock)
except Exception as e:
print(e)
# raise e
helper = MysqlHelper('okaiok.com', 'root', 'qq84607952', 'walle')
helper.connect()
sql_select = "SELECT stock FROM chouma WHERE stock= %s "
sql_update = "UPDATE chouma SET CODE=%s,stock=%s,price=%s,zhangfu=%s,huoli=%s,costdown90=%s,costup90=%s,costavg=%s,focus70=%s,focus90=%s WHERE stock = %s "
sql_insert = "insert into chouma (code,stock,price,zhangfu,huoli,costdown90,costup90,costavg,focus70,focus90) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
for stock in list:
url_name = quote(stock + "筹码分布")
url = "http://www.iwencai.com/stockpick/load-data?typed=0&preParams=&ts=1&f=1&qs=result_original&selfsectsn=&querytype=stock&searchfilter=&tid=stockpick&w=" + url_name + "&queryarea="
spider_chouma(url, stock)
i = 0
progress = 0
book_info_url = "https://c4240.818tu.com/api/novels/catalog/"
#数据库配置
# database= {'host': '120.26.204.210', 'user': '******', 'password': '******','database':'shucheng'}
database = {
'host': 'rm-bp128gc7rp6s4g1pvo.mysql.rds.aliyuncs.com',
'user': '******',
'password': '******',
'database': 'shucheng'
}
params = [1, 2]
sql = 'select * from book_pool where supplier_id=100024 and progress=%s order by id asc ' % (
progress)
mysqlHelper = MysqlHelper(database['host'], 3306, database['database'],
database['user'], database['password'])
list = mysqlHelper.get_all(sql)
for item in list:
book_info = get_book_info(item[2])
chapter_count = item[10]
print(chapter_count)
print(book_info['article_count'])
if (book_info['article_count'] != chapter_count):
#书源章节已更新,采集最新章节
# 计算最后一章是多少
print(item)
table = 'book_chapter_%s' % (item[0] % CHAPTER_TABLE_NUM)
print(table)
sql = "select chapter_order from %s where book_id= '%s' order by chapter_order desc limit 1 " % (
table, item[0])
print(sql)
def get_sha1_mysql(self,sql,param):
#连接数据库，查询数据库sha1加密密码
mysqltest = MysqlHelper(host=self.host,user=self.user,password=self.pwd,database=self.database,port=self.port,charset=self.charset)
ret = mysqltest.all(sql,param)
return ret
#!/usr/bin/env python # encoding: utf-8 #version: 1.0 #author: wanghaiyun #contact: [email protected] #file: test_helper.py #time: 2017/8/11 13:00 import MysqlHelper #name = 'bbb' #birthday = '1889-7-9' #id = 1 #params = [name, birthday, id] #params = [name] #print(params) #sql = 'update students set name=%s where id=%s' #sql = "insert into students(name,birthday,gender) values(%s,%s,%s)" #sql = 'delete from students where name=%s' sql = "select * from students where id=1" mysqlhelper = MysqlHelper.MysqlHelper(host='localhost', port=3306, user='******', password='******', database='aa') #mysqlhelper.cud(sql, params) res = mysqlhelper.one(sql) print(type(res)) print(res)
from MysqlHelper import *
# param1 = input('请输入ID：')
# param2 = input('请输入设备种类：')
# param3 = input('请输入类别ID：')
# sql = 'insert into classify values (16,%s,%s)'
# mh = MysqlHelper('localhost',3306,'root','root','classify')
# mh.cud(sql,[param2,param3])
sql = 'select * from classify where pid=2'
mh = MysqlHelper('localhost',3306,'root','root','classify')
result=mh.read(sql)
print(result,end='')
print(result)
def __init__(self):
self.mysql_helper = MysqlHelper(host="localhost",
user="******",
passwd="123456",
port=3306,
db='python')
# -*- coding:utf-8 -*-
import MySQLdb
# 引用mysql
from MysqlHelper import *
mysql = MysqlHelper(host="123.206.30.117",
user="******",
passwd="111",
db="1024",
port=3306)
def selectVideoUrl(params):
"""
查询指定索引的视频索引
:param params: 参数list
:return:
"""
sql = "select count(vid) from huli11 where isdowlond is null limit %s , %s"
return mysql.get_all(sql=sql, params=params)
if __name__ == "__main__":
# sql = "insert into student(name,age,passwd) values(%s,%s,%s)"
# name = raw_input("请输入您的名字：")
# age = raw_input("请输入您的年龄:")
# passwd = raw_input("请输入您的密码:")
# params =[name,age,passwd]
# print(params)
#
# mysqlconn = MysqlHelper(host="123.206.30.117",user="******",passwd="111",db="test",port=3306)
def __init__(self):
self.browser = share_browser.share_browser()
# self.l = []
self.mysql = MysqlHelper('localhost', 3306, 'lianjia', 'root',
'xiangjia')
import MysqlHelper
import hashlib
helper = MysqlHelper.MysqlHelper(host='192.168.153.134')
name = input("请输入你注册的账号:")
password = input("请输入你的密码:")
password = hashlib.sha1(password.encode("utf-8")).hexdigest()
result = helper.get_one('select uname,upwd from userinfos where uname=%s',[name])
try:
if name==result[0] and password==result[1]:
print("登录成功")
else:
print("密码或者账号错误")
except Exception as result:
print(result)
print("账号不存在")
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018/6/26 4:08 PM
# @Author : playboy
from MysqlHelper import *
# 连接person数据库
my_db = MysqlHelper('10.211.55.3', 'root', 'chaoer1314', 'person')
# 新建person表
# 如果表存在先删除
my_db.create_table("drop table if exists person")
my_db.create_table(
"create table if not exists person(name char(20), age int, PRIMARY KEY (name))"
)
# 在person表中插入数据
my_db.insert_data("insert into person(name, age) VALUES ('playboy', 20)")
my_db.insert_data("insert into person(name, age) VALUES ('aaa',37)")
my_db.insert_data("insert into person(name, age) VALUES ('bbb',17)")
# 修改数据
my_db.update_data("update person set age='25' where name = '%s' " %
('playboy'))
# 查询数据
ret = my_db.find_data("select * from person WHERE age > '%d'" % (20))
print(ret)
# 删除数据
# coding=utf-8 import MysqlHelper helper = MysqlHelper(db='test2') sql = 'select count(*) from stu where isdelete=0' row = helper.fetchone(sql) print(row[0])
# coding:utf-8
from MysqlHelper import *
params = ["fedddr", "2"]
sql = "UPDATE xiaoliu SET name=%s WHERE id<%s"
mysql_helper = MysqlHelper(host="localhost",
user="******",
passwd="123456",
port=3306,
db='python')
# mysqls.cud(sql,params)
# sql = "select *from xiaoliu;"
# params=[]
# print(mysqls.find_get(sql,params))
mysql_helper.cud("update xiaoliu set age=3;")
mysql_helper.find_get("select *from xiaoliu;")
mysql_helper.show_result()
#!/usr/bin/env python2
# -*- coding:utf-8 -*-
# Hello world - 西蒙.科泽斯 这是“向编程之神所称颂的传统咒语，愿他帮助并保佑你更好的学习这门语言
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import time
from MysqlHelper import *
from GengzinavUtils import *
# 初始化mysql
mysql = MysqlHelper(host="127.0.0.1",
user="******",
passwd="111",
db="1024",
port=3306)
utils = GengzinavUtils()
class GengzinavDAO():
def saveLinkInfo(self, linktext, linkhref, isnav, ispaqu):
'''
保存数据到数据库
:param linktext: 链接的标题
:param linkhref: 链接
:param isnav: 是否为导航 默认 0
:param ispaqu: 是否爬取了 默认 0
:return:
# coding=utf-8
import MysqlHelper
import hashlib
# 接收用户输入：
name = input("请输入用户名：")
pwd = input('请输入密码：')
# 对密码加密：
s1 = hashlib.sha1()
s1.update(pwd.encode('utf-8'))
# 此处加上一句：encode('utf-8'),否则提示：TypeError: Unicode-objects must be encoded before hashing
pwd2 = s1.hexdigest()
# 根据用户名查询密码：
sql = 'select passwd from users where name=%s'
helper = MysqlHelper.MysqlHelper('192.168.85.20', 3306, 'follow', 'root', '123456')
result = helper.get_all(sql, [name])
if len(result) == 0:
print("用户名错误。")
elif result[0][0] == pwd2:
print("登录成功。")
else:
print("密码错误。")
# print(result)
#encoding=utf-8
from MysqlHelper import *
sql = 'insert into students(sid,sname) values(%s,%s)'
name = raw_input("please input name:")
id = raw_input("please input id:")
params = [int(id),name]
mysqlHelper = MysqlHelper('192.168.1.107',3306,'db_name', 'root', 'root')
count = mysqlHelper.insert(sql,params)
if count == 1:
print 'ok'
else:
print 'error'
sql = 'select * from students order by sid desc'
all = mysqlHelper.get_all(sql)
print all
@version: v1.0 @author: young @license: Apache Licence @contact: [email protected] @site: @software: PyCharm @file: testLogin.py @time: 2017/7/21 14:12 """ from MysqlHelper import * from hashlib import sha1 sname = raw_input("请输入用户名：") spwd = raw_input("请输入密码：") s1 = sha1() s1.update(spwd) spwdSha1 = s1.hexdigest() sql = "select upwd from userinfos where uname=%s" params = [sname] sqlhelper = MysqlHelper('localhost', 3306, 'python', 'root', 'root') userinfo = sqlhelper.get_one(sql, params) if userinfo == None: print '用户名错误' elif userinfo[0] == spwdSha1: print '登录成功' else: print '密码错误'
# 设置tushare pro的token并获取连接
ts.set_token('c1e2a05c71936f78e19e37807638e73fe1a6e32dccd428d5deab8701')
pro = ts.pro_api()
# 设定获取日线行情的初始日期和终止日期，其中终止日期设定为昨天。
start_dt = '20100101'
time_temp = datetime.datetime.now() - datetime.timedelta(days=1)
end_dt = time_temp.strftime('%Y%m%d')
# 查询当前所有正常上市交易的股票列表
data = pro.stock_basic(exchange='', list_status='L',
fields='ts_code,symbol,name,area,industry,fullname'\
',enname,market,exchange,'\
'curr_type,list_status,list_date,delist_date,is_hs')
try:
conn = MysqlHelper(host="127.0.0.1", user="******", password="******", charset="utf8", database="stock", port=3306)
# 根据字段统计count, join>>AND,OR,可以不传，默认为AND0-
total = len(data)
for i in range(total):
ts = data.iloc[i]["ts_code"]
cnt = conn.findKeySql(const.COUNT, table="stock_basic", params={"ts_code": ts}, join="AND")
if cnt == 0:
print("处理%d/%d：：插入：：%s" % (int(i), int(total), ts))
conn.findKeySql(const.INSERT, table="stock_basic",
data={"ts_code": data.iloc[i]["ts_code"],
"symbol": data.iloc[i]["symbol"],
"symbol": data.iloc[i]["symbol"],
"name": data.iloc[i]["name"],
"area": data.iloc[i]["area"],
"industry": data.iloc[i]["industry"],
# coding=utf-8
from MysqlHelper import *
try:
uname = input("请输入用户名：")
upwd = input("请输入密码：")
redis = RedisHelper()
upwd3 = redis.get(uname)
if upwd3 != None:
if upwd == upwd3.decode():
print('redis ok')
else:
print('密码错误')
else:
mysql = MysqlHelper()
sql = 'select password from user where name=%s'
params = [uname]
result = mysql.fetchone(sql, params)
if result == None:
print('用户名不存在')
elif result[0] == upwd:
print('mysql ok')
redis.set(uname, upwd)
else:
print('密码错误')
except Exception as e:
print(e)
#encodeing=utf-8
from MysqlHelper import *
from hashlib import sha1
name = input('请输入用户名：')
pwd = input('请输入密码')
#加密
sha = sha1()
sha.update(pwd.encode('utf-8'))
pwd2 = sha.hexdigest
print(str(pwd2))
#根据用户名查询
query = 'select passwd from users where name=%s'
sqlhelper = MysqlHelper('localhost', 3306, 'pymysql', 'root', 'kcy000')
result = sqlhelper.all(query, [name])
if (len(result) == 0):
print('用户不存才')
elif result[0][0] == pwd2:
print('登录成功')
else:
print('密码错误')
print(result)
import MysqlHelper
helper = MysqlHelper.MysqlHelper(host='192.168.153.134',
user='******',
password=1122,
db='python2')
# 查找
helper.get_all('select * from students', prams=[])
# 增加一条数据
helper.change("insert into students(id,name) values(%s,%s)", ['0', 'jack'])
# 删除
helper.change("delete from students where name=%s", ['张良'])
# 更改
helper.change("update students set name = %s where name=%s", ['王二', '张飞'])
helper.get_all('select * from students', prams=[])
def spiderBaseInfo(food_id, content):
# 爬取基本信息
logo_url = ""
logo_url_list = content.xpath("//div/div[@class='cp_headerimg_w']/img/@src")
if len(logo_url_list) > 0:
logo_url = logo_url_list[0]
# favorite_num = ""
suitable_label = ""
cooking_method = ""
cooking_difficulty = ""
people_num = ""
cooking_taste = ""
ready_time = ""
cooking_time = ""
user_avatar_url = ""
user_id = 0
user_url = ""
user_grade = ""
final_img_url = ""
infoDivList = content.xpath("//div/div[@class='cp_main_info_w']")
for div in infoDivList:
# xpath = div.xpath("div/span[@class='favbtns']/a/span")[0]
# favorite_num = xpath.text
tongjiList = div.xpath("div[@class='info1']//dt/a/text()")
for tongji in tongjiList:
suitable_label += tongji + ","
suitable_label = suitable_label[0:-1]
cooking_method_list = div.xpath("div[@class='info2']/ul/li[1]//a/text()")
if len(cooking_method_list) > 0:
cooking_method = cooking_method_list[0]
cooking_difficulty_list = div.xpath("div[@class='info2']/ul/li[2]//a/text()")
if len(cooking_difficulty_list) > 0:
cooking_difficulty = cooking_difficulty_list[0]
people_num_list = div.xpath("div[@class='info2']/ul/li[3]//a/text()")
if len(people_num_list) > 0:
people_num = people_num_list[0]
cooking_taste_list = div.xpath("div[@class='info2']/ul/li[4]//a/text()")
if len(cooking_taste_list) > 0:
cooking_taste = cooking_taste_list[0]
ready_time_list = div.xpath("div[@class='info2']/ul/li[5]//a/text()")
if len(ready_time_list) > 0:
ready_time = ready_time_list[0]
cooking_time_list = div.xpath("div[@class='info2']/ul/li[6]//a/text()")
if len(cooking_time_list) > 0:
cooking_time = cooking_time_list[0]
user_avatar_url = div.xpath("div[@class='info3']/div[@class='user']/a/img/@src")[0]
user_url = div.xpath("div[@class='info3']/div[@class='user']/a/@href")[0]
if user_url is not None:
m = re.compile(r'.*id=(\d*)').match(user_url)
if m is not None:
user_id = m.group(1)
ug = div.xpath("div[@class='info3']//div[@class='info']/h4/a/@title")
if len(ug) > 0:
user_grade = ug[0]
materials_desc = content.xpath("string(//div/div/div[@class='materials']/p)")
# 爬取成品图
final_img_url_list = list(
set(content.xpath("//div[@class='swiper-wrapper swiper-wrapper1']/div/img/@src")))
for fiu in final_img_url_list:
final_img_url += fiu + ","
final_img_url = final_img_url[0:-1]
cooking_skill = content.xpath("string(//div[@class='editnew edit']/p)")
# print (logo_url)
# print (favorite_num)
# print (suitable_label)
# print (cooking_method)
# print (cooking_difficulty)
# print (people_num)
# print (cooking_taste)
# print (ready_time)
# print (cooking_time)
# print (user_url)
# print (user_avatar_url)
# print (user_id)
# print (user_grade)
# print (materials_desc)
# print (final_img_url)
# print (cooking_skill)
sql = "update lb_food set logo_url = %s, suitable_label = %s, cooking_method = %s, cooking_difficulty = %s," \
" people_num = %s, cooking_taste = %s, ready_time = %s, cooking_time = %s, user_url = %s, " \
"user_avatar_url = %s, user_id = %s, user_grade = %s, materials_desc = %s, final_img_url = %s," \
" cooking_skill = %s where id = %s"
params = (
logo_url, suitable_label, cooking_method, cooking_difficulty, people_num, cooking_taste,
ready_time, cooking_time, user_url, user_avatar_url, user_id, user_grade, materials_desc,
final_img_url, cooking_skill, food_id)
# print(sql % params)
return MysqlHelper.MysqlHelper(isTest).cud(sql, params) == 1