def alter_vegetable(): """ 增添蔬菜信息 :return: """ veg_name = request.json["vegetable_name"] veg_information = request.json["vegetable_information"] operate_type = request.json["operate_type"] print(operate_type) if veg_name and operate_type is not None: if operate_type is True: if veg_information: for i in range(len(veg_name)): VegetableModelDao.add_vegetable(veg_name[i], veg_information[i]) # 添加成功 response_data = response[200] else: # 缺少参数 response_data = response[20101] else: for i in range(len(veg_name)): VegetableModelDao.delete_vegetable(veg_name[i]) # 删除成功 response_data = response[200] else: # 缺少参数 response_data = response[20101] return json.dumps(response_data, ensure_ascii=False)
def network_train(): """ 模型的训练 :return: """ req_json = request.json model_name = req_json['model_name'] veg_list = req_json['vegetable_list'] if not (model_name and veg_list): return json.dumps(response[20101], ensure_ascii=False) model_id = PredictModelModelDao.get_id_by_name(model_name) if model_id == -1: return json.dumps(response[20501], ensure_ascii=False) for veg_name in veg_list: veg_id = VegetableModelDao.get_id_by_name(veg_name) veg_model_list = VegetablePriceModelDao.query_vegetable_price_data( 1, veg_id) price_list = [veg_model.price for veg_model in veg_model_list] # [-1060:] if model_id == 1: # 异步加入进程池 pool.apply_async(bp_train, ( price_list, veg_name, )) else: pool.apply_async(lstm_train, ( price_list, veg_id, veg_name, )) return json.dumps(response[200])
def predict_price(): """ 选择模型进行预测, 控制好id为1是指bp, id为2是指lstm :return: """ new_pool = Pool(processes=4) req_json = request.json model_name = req_json['model_name'] veg_name = req_json['vegetable_name'] start_date = req_json['start_date'] if not (model_name and veg_name and start_date): return json.dumps(response[20101], ensure_ascii=False) model_id = PredictModelModelDao.get_id_by_name(model_name) if model_id == -1: return json.dumps(response[20501], ensure_ascii=False) veg_id = VegetableModelDao.get_id_by_name(veg_name) pre_date = day_decrease(start_date, 150)[:10] veg_model_list = VegetablePriceModelDao.query_vegetable_price_data( 2, veg_id, pre_date, start_date) price_list = [veg_model.price for veg_model in veg_model_list][-100:] # 只取100个 if model_id == 1: # new_price_list = bp_predict(price_list, veg_name) # 以这个解决本地训练然后传到服务器进行测试的网络模型 result = new_pool.apply_async(bp_predict, ( price_list, veg_name, )) new_price_list = result.get() elif model_id == 2: # new_price_list = lstm_predict(price_list, veg_id, veg_name) # 另开一个进程解决 <class 'ValueError'> Variable 1/rnn/basic_lstm_cell/kernel already exists, disallowed. result = new_pool.apply_async(lstm_predict, ( price_list, veg_name, )) new_price_list = result.get() else: # todo: ARIMA模型的数据获取,new_price_list为十个价格的数组 new_price_list = None date_list = [] the_date = start_date for i in range(10): the_date = day_increase(the_date, 1)[:10] date_list.append(the_date) data = {'date': date_list, 'predict_price': new_price_list} response_data = {'data': data} response_data.update(response[200]) new_pool.close() new_pool.join() return json.dumps(response_data)
def vegetable_info(): """ 获取蔬菜信息 :return: """ vegetable_name = request.json['vegetable_name'] if vegetable_name: if VegetableModelDao.get_id_by_name(vegetable_name): vegetable_information = VegetableModelDao.get_information( vegetable_name) if vegetable_information is None: # 无蔬菜信息 response_data = response[20503] else: # 获取信息成功 response_data = {'vegetable_info': vegetable_information} response_data.update(response[200]) else: # 缺少蔬菜 response_data = response[20401] else: # 缺少参数 response_data = response[20101] return json.dumps(response_data, ensure_ascii=False)
def all_vegetable(): """ 获取所有蔬菜 :return: """ data = [] vegetable_list = VegetableModelDao.query_vegetable() for i in range(len(vegetable_list)): vegetable = vegetable_list[i] one_data_1 = { 'name': vegetable.veg_name, 'img_url': vegetable.veg_img_url, 'description': vegetable.veg_information } # one_data = {str(i + 1): one_data_1} data.append(one_data_1) response_data = {'data': data} response_data.update(response[200]) return json.dumps(response_data, ensure_ascii=False)
def get_accuracy(): """ 模型的训练 :return: """ new_pool = Pool(processes=4) req_json = request.json model_name = req_json['model_name'] veg_name = req_json['vegetable_name'] if not (model_name and veg_name): return json.dumps(response[20101], ensure_ascii=False) model_id = PredictModelModelDao.get_id_by_name(model_name) if model_id == -1: return json.dumps(response[20501], ensure_ascii=False) veg_id = VegetableModelDao.get_id_by_name(veg_name) veg_model_list = VegetablePriceModelDao.query_vegetable_price_data( 1, veg_id) price_list = [veg_model.price for veg_model in veg_model_list] # [-1060:] if model_id == 1: # response_data = bp_get_accuracy(price_list, veg_name) price_list = manager.list(price_list) result = new_pool.apply_async(bp_get_accuracy, ( price_list, veg_name, )) response_data = result.get() else: # response_data = lstm_get_accuracy(price_list, veg_id, veg_name) # 另开一个进程解决 <class 'ValueError'> Variable 1/rnn/basic_lstm_cell/kernel already exists, disallowed. price_list = manager.list(price_list) result = new_pool.apply_async(lstm_get_accuracy, ( price_list, veg_id, veg_name, )) response_data = result.get() # response_data = {} response_data.update(response[200]) new_pool.close() new_pool.join() return json.dumps(response_data)
def get_today_price(today_time): data = redis_client.get('REDIS_TODAY_VEG_PRICE' + today_time) if not data: data = [] veg_model_list = VegetablePriceModelDao.query_vegetable_price_data( 5, start_date=today_time) for veg_model in veg_model_list: veg_name = VegetableModelDao.get_name_by_id(veg_model.veg_id) one_price = [ veg_name, veg_model.date, veg_model.price, veg_model.place ] data.append(one_price) if len(data) > 0: redis_client.set('REDIS_TODAY_VEG_PRICE' + today_time, str(data)) # 设置过期时间 redis_client.expire('REDIS_TODAY_VEG_PRICE' + today_time, 3600 * 24) else: data = eval(data) return data
def get_k_line(): """ 获取蔬菜k线图 :return: """ req_json = request.json vegetable_name = req_json['vegetable_name'] date = req_json['date'] if not (vegetable_name and date): return json.dumps(response[20101]) price = [] veg_id = VegetableModelDao.get_id_by_name(vegetable_name) veg_model_list = VegetablePriceModelDao.query_vegetable_price_data( 2, veg_id, date[0], date[1]) for veg_model in veg_model_list: one_price = [ vegetable_name, veg_model.date, veg_model.price, veg_model.place ] price.append(one_price) response_data = {'vegetable_name': vegetable_name, 'data': price} response_data.update(response[200]) return json.dumps(response_data)
veg_all_data.append( ('芫茜', '芫茜,一种蔬菜,又称为野香草、假香荽。中医认为,香菜辛温香窜,内通心脾,外达四肢,辟一切不正之气,为温中健胃' '养生食品。日常食之,有消食下气,醒脾调中,壮阳助兴等功效,适于寒性体质、胃弱体质以及肠腑壅滞者食用,可用来治疗胃' '脘冷痛、消化不良、麻疹不透等症状。 香菜含有丰富的矿物质及维生素,常少量食用,对身体有益。', ' ')) veg_all_data.append( ('园椒', '青圆椒又名青柿子椒、菜椒、甜椒等。它的维生素C含量是番茄含量的7~15倍,在蔬菜中占首位。它特有的味道有刺激唾液分泌的作用;' '所含的辣椒素能增进食欲,帮助消化,防止便秘。因此,成为餐桌上的常见菜肴。', ' ')) veg_all_data.append( ('云南小瓜', '云南小瓜又名角瓜、西葫芦。营养价值:西葫芦含有较多维生素C、葡萄糖等营养物质,尤其是钙的含量极高。不同品种每100g' '可食部分(鲜重)营养物质含量如下:蛋白质0.6-0.9g,脂肪0.1-0.2g,纤维素0.8-0.9g,糖类2.5-3.3g,胡萝卜素' '20-40微克,维生素C2.5-9毫克,钙22-29毫克。', ' ')) veg_all_data.append( ('紫椰菜', '紫椰菜一般指紫甘蓝。紫甘蓝含有多种的营养物质,尤其是紫甘蓝中含有的多种维生素都是非常丰富的,例如维生素C、维生素' 'E、维生素B族等都是非常丰富的。紫甘蓝含有丰富的硫元素,这种元素的主要作用是杀虫止痒,对于各种皮肤瘙痒,湿疹等疾患' '具有一定疗效,因而经常吃这类蔬菜对于维护皮肤健康十分有益。', ' ')) VegetableModelDao.add_many_data(veg_all_data) print('初始数据插入完毕,开始爬虫') csv_save_path = 'csv_save_path/' """ 爬虫部分 """ # 请求头 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/57.0.2987.133 Safari/537.36' } # 初始化访问地址 base_url = "http://www.jnmarket.net/import/list-1_" offset = 1
# strs = '2019/02/10' # strs = strs.replace('/', '-') # print(strs) import os csv_save_path = 'csv_save_path/' # 将蔬菜数据录入数据库 i = 1 start_t = time.time() data = [] for root, dirs, files in os.walk(csv_save_path): for file in files: # print(i) veg_name = file.split('.')[0] veg_id = VegetableModelDao.get_id_by_name(veg_name) if veg_id is not None: i = i + 1 data.append(veg_name) uri = root + file # veg_data = pd.read_csv(uri) # VegetablePriceModelDao.add_many_data(veg_id, veg_data) print(data) print(len(data)) stop_t = time.time() print(stop_t - start_t, 's') data = [ "葫芦瓜", "鲜冬菇", "云南小瓜", "芦笋", "西芹", "槟芋", "西兰花", "韭菜", "荷兰豆", "西生菜", "青瓜", "土豆", "椰菜花", "莴笋", "鲜淮山", "玉米", "韭菜花", "生菜", "青尖椒", "鲜百合", "莲藕", "园椒", "菠菜", "芫茜", "白豆角", "红葱头", "本地芹菜", "本地菜心", "葱", "韭黄", "鲜人参", "南瓜", "蒜心", "芥兰", "玉豆", "大蒜", "鲜虫草", "甜墨豆", "茄瓜", "苦瓜", "马蹄", "紫椰菜", "油麦菜", "红萝卜",
def spider_vegetable(): print("SPIDER START!!") name = [] # 蔬菜名字 place = [] # 蔬菜产地 price = [] # 蔬菜价格 date = [] # 日期 # 获取今天的日期 today = time.strftime('%y-%m-%d', time.localtime(time.time())) # 请求头 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/57.0.2987.133 Safari/537.36' } # 初始化访问地址 base_url = "http://www.jnmarket.net/import/list-1_" offset = 1 # 爬取数据并添加到字典中 while 1: # 获取响应并定义选择器 url = base_url + str(offset) + ".html" response = requests.get(url, headers=headers) html = response.text selector = etree.HTML(html) spider_date = selector.xpath('//table/tbody/tr/td[5]/text()') spider_name = selector.xpath('//table/tbody/tr/td[1]/text()') spider_place = selector.xpath('//table/tbody/tr/td[2]/text()') spider_price = selector.xpath('//table/tbody/tr/td[3]/text()') if len(spider_date) == spider_date.count(today): # 格式化日期,如2016/01/01 for i in range(len(spider_date)): time_struct = time.strptime(spider_date[i], "%y-%m-%d") spider_date[i] = time.strftime("%Y/%m/%d", time_struct) name += spider_name place += spider_place price += spider_price date += spider_date offset += 1 else: # 截取长度 length = spider_date.count(today) # 格式化日期,如2016/01/01 for i in range(len(spider_date)): time_struct = time.strptime(spider_date[i], "%y-%m-%d") spider_date[i] = time.strftime("%Y/%m/%d", time_struct) name += spider_name[:length] place += spider_place[:length] price += spider_price[:length] date += spider_date[:length] break # 添加到数据库中 for i in range(len(name)): veg_id = VegetableModelDao.get_id_by_name(name[i]) str_time = date[i].replace('/', '-') VegetablePriceModelDao.add_one_data(veg_id, str_time, price[i], place[i]) print('SPIDER OVER')