def alter_vegetable():
"""
增添蔬菜信息
:return:
"""
veg_name = request.json["vegetable_name"]
veg_information = request.json["vegetable_information"]
operate_type = request.json["operate_type"]
print(operate_type)
if veg_name and operate_type is not None:
if operate_type is True:
if veg_information:
for i in range(len(veg_name)):
VegetableModelDao.add_vegetable(veg_name[i],
veg_information[i])
# 添加成功
response_data = response[200]
else:
# 缺少参数
response_data = response[20101]
else:
for i in range(len(veg_name)):
VegetableModelDao.delete_vegetable(veg_name[i])
# 删除成功
response_data = response[200]
else:
# 缺少参数
response_data = response[20101]
return json.dumps(response_data, ensure_ascii=False)
def network_train():
"""
模型的训练
:return:
"""
req_json = request.json
model_name = req_json['model_name']
veg_list = req_json['vegetable_list']
if not (model_name and veg_list):
return json.dumps(response[20101], ensure_ascii=False)
model_id = PredictModelModelDao.get_id_by_name(model_name)
if model_id == -1:
return json.dumps(response[20501], ensure_ascii=False)
for veg_name in veg_list:
veg_id = VegetableModelDao.get_id_by_name(veg_name)
veg_model_list = VegetablePriceModelDao.query_vegetable_price_data(
1, veg_id)
price_list = [veg_model.price
for veg_model in veg_model_list] # [-1060:]
if model_id == 1:
# 异步加入进程池
pool.apply_async(bp_train, (
price_list,
veg_name,
))
else:
pool.apply_async(lstm_train, (
price_list,
veg_id,
veg_name,
))
return json.dumps(response[200])
def predict_price():
"""
选择模型进行预测, 控制好id为1是指bp, id为2是指lstm
:return:
"""
new_pool = Pool(processes=4)
req_json = request.json
model_name = req_json['model_name']
veg_name = req_json['vegetable_name']
start_date = req_json['start_date']
if not (model_name and veg_name and start_date):
return json.dumps(response[20101], ensure_ascii=False)
model_id = PredictModelModelDao.get_id_by_name(model_name)
if model_id == -1:
return json.dumps(response[20501], ensure_ascii=False)
veg_id = VegetableModelDao.get_id_by_name(veg_name)
pre_date = day_decrease(start_date, 150)[:10]
veg_model_list = VegetablePriceModelDao.query_vegetable_price_data(
2, veg_id, pre_date, start_date)
price_list = [veg_model.price
for veg_model in veg_model_list][-100:] # 只取100个
if model_id == 1:
# new_price_list = bp_predict(price_list, veg_name)
# 以这个解决本地训练然后传到服务器进行测试的网络模型
result = new_pool.apply_async(bp_predict, (
price_list,
veg_name,
))
new_price_list = result.get()
elif model_id == 2:
# new_price_list = lstm_predict(price_list, veg_id, veg_name)
# 另开一个进程解决 <class 'ValueError'> Variable 1/rnn/basic_lstm_cell/kernel already exists, disallowed.
result = new_pool.apply_async(lstm_predict, (
price_list,
veg_name,
))
new_price_list = result.get()
else:
# todo: ARIMA模型的数据获取，new_price_list为十个价格的数组
new_price_list = None
date_list = []
the_date = start_date
for i in range(10):
the_date = day_increase(the_date, 1)[:10]
date_list.append(the_date)
data = {'date': date_list, 'predict_price': new_price_list}
response_data = {'data': data}
response_data.update(response[200])
new_pool.close()
new_pool.join()
return json.dumps(response_data)
def vegetable_info():
"""
获取蔬菜信息
:return:
"""
vegetable_name = request.json['vegetable_name']
if vegetable_name:
if VegetableModelDao.get_id_by_name(vegetable_name):
vegetable_information = VegetableModelDao.get_information(
vegetable_name)
if vegetable_information is None:
# 无蔬菜信息
response_data = response[20503]
else:
# 获取信息成功
response_data = {'vegetable_info': vegetable_information}
response_data.update(response[200])
else:
# 缺少蔬菜
response_data = response[20401]
else:
# 缺少参数
response_data = response[20101]
return json.dumps(response_data, ensure_ascii=False)
def all_vegetable():
"""
获取所有蔬菜
:return:
"""
data = []
vegetable_list = VegetableModelDao.query_vegetable()
for i in range(len(vegetable_list)):
vegetable = vegetable_list[i]
one_data_1 = {
'name': vegetable.veg_name,
'img_url': vegetable.veg_img_url,
'description': vegetable.veg_information
}
# one_data = {str(i + 1): one_data_1}
data.append(one_data_1)
response_data = {'data': data}
response_data.update(response[200])
return json.dumps(response_data, ensure_ascii=False)
def get_accuracy():
"""
模型的训练
:return:
"""
new_pool = Pool(processes=4)
req_json = request.json
model_name = req_json['model_name']
veg_name = req_json['vegetable_name']
if not (model_name and veg_name):
return json.dumps(response[20101], ensure_ascii=False)
model_id = PredictModelModelDao.get_id_by_name(model_name)
if model_id == -1:
return json.dumps(response[20501], ensure_ascii=False)
veg_id = VegetableModelDao.get_id_by_name(veg_name)
veg_model_list = VegetablePriceModelDao.query_vegetable_price_data(
1, veg_id)
price_list = [veg_model.price for veg_model in veg_model_list] # [-1060:]
if model_id == 1:
# response_data = bp_get_accuracy(price_list, veg_name)
price_list = manager.list(price_list)
result = new_pool.apply_async(bp_get_accuracy, (
price_list,
veg_name,
))
response_data = result.get()
else:
# response_data = lstm_get_accuracy(price_list, veg_id, veg_name)
# 另开一个进程解决 <class 'ValueError'> Variable 1/rnn/basic_lstm_cell/kernel already exists, disallowed.
price_list = manager.list(price_list)
result = new_pool.apply_async(lstm_get_accuracy, (
price_list,
veg_id,
veg_name,
))
response_data = result.get()
# response_data = {}
response_data.update(response[200])
new_pool.close()
new_pool.join()
return json.dumps(response_data)
def get_today_price(today_time):
data = redis_client.get('REDIS_TODAY_VEG_PRICE' + today_time)
if not data:
data = []
veg_model_list = VegetablePriceModelDao.query_vegetable_price_data(
5, start_date=today_time)
for veg_model in veg_model_list:
veg_name = VegetableModelDao.get_name_by_id(veg_model.veg_id)
one_price = [
veg_name, veg_model.date, veg_model.price, veg_model.place
]
data.append(one_price)
if len(data) > 0:
redis_client.set('REDIS_TODAY_VEG_PRICE' + today_time, str(data))
# 设置过期时间
redis_client.expire('REDIS_TODAY_VEG_PRICE' + today_time,
3600 * 24)
else:
data = eval(data)
return data
def get_k_line():
"""
获取蔬菜k线图
:return:
"""
req_json = request.json
vegetable_name = req_json['vegetable_name']
date = req_json['date']
if not (vegetable_name and date):
return json.dumps(response[20101])
price = []
veg_id = VegetableModelDao.get_id_by_name(vegetable_name)
veg_model_list = VegetablePriceModelDao.query_vegetable_price_data(
2, veg_id, date[0], date[1])
for veg_model in veg_model_list:
one_price = [
vegetable_name, veg_model.date, veg_model.price, veg_model.place
]
price.append(one_price)
response_data = {'vegetable_name': vegetable_name, 'data': price}
response_data.update(response[200])
return json.dumps(response_data)
veg_all_data.append(
('芫茜', '芫茜，一种蔬菜，又称为野香草、假香荽。中医认为，香菜辛温香窜，内通心脾，外达四肢，辟一切不正之气，为温中健胃'
'养生食品。日常食之，有消食下气，醒脾调中，壮阳助兴等功效，适于寒性体质、胃弱体质以及肠腑壅滞者食用，可用来治疗胃'
'脘冷痛、消化不良、麻疹不透等症状。 香菜含有丰富的矿物质及维生素,常少量食用，对身体有益。', ' '))
veg_all_data.append(
('园椒', '青圆椒又名青柿子椒、菜椒、甜椒等。它的维生素C含量是番茄含量的7~15倍，在蔬菜中占首位。它特有的味道有刺激唾液分泌的作用;'
'所含的辣椒素能增进食欲，帮助消化，防止便秘。因此，成为餐桌上的常见菜肴。', ' '))
veg_all_data.append(
('云南小瓜', '云南小瓜又名角瓜、西葫芦。营养价值：西葫芦含有较多维生素C、葡萄糖等营养物质，尤其是钙的含量极高。不同品种每100g'
'可食部分(鲜重)营养物质含量如下：蛋白质0.6－0.9g，脂肪0.1－0.2g，纤维素0.8－0.9g，糖类2.5－3.3g，胡萝卜素'
'20－40微克，维生素C2.5－9毫克，钙22－29毫克。', ' '))
veg_all_data.append(
('紫椰菜', '紫椰菜一般指紫甘蓝。紫甘蓝含有多种的营养物质，尤其是紫甘蓝中含有的多种维生素都是非常丰富的，例如维生素C、维生素'
'E、维生素B族等都是非常丰富的。紫甘蓝含有丰富的硫元素，这种元素的主要作用是杀虫止痒，对于各种皮肤瘙痒，湿疹等疾患'
'具有一定疗效，因而经常吃这类蔬菜对于维护皮肤健康十分有益。', ' '))
VegetableModelDao.add_many_data(veg_all_data)
print('初始数据插入完毕，开始爬虫')
csv_save_path = 'csv_save_path/'
""" 爬虫部分 """
# 请求头
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/57.0.2987.133 Safari/537.36'
}
# 初始化访问地址
base_url = "http://www.jnmarket.net/import/list-1_"
offset = 1
# strs = '2019/02/10'
# strs = strs.replace('/', '-')
# print(strs)
import os
csv_save_path = 'csv_save_path/'
# 将蔬菜数据录入数据库
i = 1
start_t = time.time()
data = []
for root, dirs, files in os.walk(csv_save_path):
for file in files:
# print(i)
veg_name = file.split('.')[0]
veg_id = VegetableModelDao.get_id_by_name(veg_name)
if veg_id is not None:
i = i + 1
data.append(veg_name)
uri = root + file
# veg_data = pd.read_csv(uri)
# VegetablePriceModelDao.add_many_data(veg_id, veg_data)
print(data)
print(len(data))
stop_t = time.time()
print(stop_t - start_t, 's')
data = [
"葫芦瓜", "鲜冬菇", "云南小瓜", "芦笋", "西芹", "槟芋", "西兰花", "韭菜", "荷兰豆", "西生菜", "青瓜",
"土豆", "椰菜花", "莴笋", "鲜淮山", "玉米", "韭菜花", "生菜", "青尖椒", "鲜百合", "莲藕", "园椒",
"菠菜", "芫茜", "白豆角", "红葱头", "本地芹菜", "本地菜心", "葱", "韭黄", "鲜人参", "南瓜", "蒜心",
"芥兰", "玉豆", "大蒜", "鲜虫草", "甜墨豆", "茄瓜", "苦瓜", "马蹄", "紫椰菜", "油麦菜", "红萝卜",
def spider_vegetable():
print("SPIDER START!!")
name = [] # 蔬菜名字
place = [] # 蔬菜产地
price = [] # 蔬菜价格
date = [] # 日期
# 获取今天的日期
today = time.strftime('%y-%m-%d', time.localtime(time.time()))
# 请求头
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/57.0.2987.133 Safari/537.36'
}
# 初始化访问地址
base_url = "http://www.jnmarket.net/import/list-1_"
offset = 1
# 爬取数据并添加到字典中
while 1:
# 获取响应并定义选择器
url = base_url + str(offset) + ".html"
response = requests.get(url, headers=headers)
html = response.text
selector = etree.HTML(html)
spider_date = selector.xpath('//table/tbody/tr/td[5]/text()')
spider_name = selector.xpath('//table/tbody/tr/td[1]/text()')
spider_place = selector.xpath('//table/tbody/tr/td[2]/text()')
spider_price = selector.xpath('//table/tbody/tr/td[3]/text()')
if len(spider_date) == spider_date.count(today):
# 格式化日期，如2016/01/01
for i in range(len(spider_date)):
time_struct = time.strptime(spider_date[i], "%y-%m-%d")
spider_date[i] = time.strftime("%Y/%m/%d", time_struct)
name += spider_name
place += spider_place
price += spider_price
date += spider_date
offset += 1
else:
# 截取长度
length = spider_date.count(today)
# 格式化日期，如2016/01/01
for i in range(len(spider_date)):
time_struct = time.strptime(spider_date[i], "%y-%m-%d")
spider_date[i] = time.strftime("%Y/%m/%d", time_struct)
name += spider_name[:length]
place += spider_place[:length]
price += spider_price[:length]
date += spider_date[:length]
break
# 添加到数据库中
for i in range(len(name)):
veg_id = VegetableModelDao.get_id_by_name(name[i])
str_time = date[i].replace('/', '-')
VegetablePriceModelDao.add_one_data(veg_id, str_time, price[i],
place[i])
print('SPIDER OVER')