def guessword_list(self,city, g_type, keyword, other_id, addr=''): """ 生成插入mongo.guessword的数据 :param g_type: 联想词类型 3:小区 :param keyword: 联想词 :param other_id: 小区id :param addr: 地址 :return: 拼装好的数据 """ if city == "bj": mongo_db = "spider" else: mongo_db = "spider_%s"%(city) index_name = 'online_' + city + '_house_sell' guessword_dict = {} guessword_dict['type_id'] = g_type guessword_dict['type_name'] = self.guessdao.guessword_type[g_type] guessword_dict['other_id'] = other_id guessword_dict['keyword'] = keyword guessword_dict['addr'] = addr guessword_dict['pinyin'] = BaseUtils.getFpy(keyword, delimiter='') guessword_dict['logogram'] = BaseUtils.getSpy(keyword, delimiter='').lower() guessword_dict['count_broker'] = self.es_search(city=city,index_name=index_name, table=mongo_db,key_v=self.guessdao.guessword_key[g_type], value_search=other_id, type_id=g_type) guessword_dict['created'] = int(time.time()) guessword_dict['updated'] = int(time.time()) return guessword_dict
value = {"_index": index, "_type": table, "_id": id, "_source": result} actions.append(value) return actions def format_data(self, data): format_houses = dict() format_houses['_id'] = str(data[0]['_id']) if 'addr' in data[0]: format_houses['addr'] = str(data[0]['addr']) if 'count_broker' in data[0]: format_houses['count_broker'] = data[0]['count_broker'] if data[0].get('type_id'): format_houses['type_id'] = data[0]['type_id'] if data[0].get('created'): format_houses['created'] = int(data[0]['created']) if data[0].get('updated'): format_houses['updated'] = int(data[0]['updated']) if data[0].get('keyword'): format_houses['keyword'] = str(data[0]['keyword']) if data[0].get('logogram'): format_houses['logogram'] = str(data[0]['logogram']) if data[0].get('other_id'): format_houses['other_id'] = int(data[0]['other_id']) if data[0].get('pinyin'): format_houses['pinyin'] = str(data[0]['pinyin']) if data[0].get('type_name'): format_houses['type_name'] = str(data[0]['type_name']) return format_houses if __name__ == '__main__': print(BaseUtils.getFpy("数据",delimiter=' '))
def add_cityarea2(self, city, cityarea_id, new_cityarea2): add_dict = {} new_cityarea2_dict = {} current_time = BaseUtils.getIntTime() new_cityarea2_dict['_id'] = self.max_id.getMaxId(type="cityarea2", city=city) new_cityarea2_dict['loc'] = new_cityarea2.get("loc", "") new_cityarea2_dict['name'] = new_cityarea2.get("name", "").strip() new_cityarea2_dict['area2_pinyin'] = BaseUtils.getFpy( new_cityarea2_dict['name']) new_cityarea2_dict['polygons'] = new_cityarea2.get("polygons", "") new_cityarea2_dict['statu'] = 1 new_cityarea2_dict['created'] = current_time new_cityarea2_dict['updated'] = current_time is_find = self.dao.find_one( city=city, filter={"name": { "$regex": new_cityarea2_dict['name'] }}) if is_find: cityarea2_id = is_find.get("_id", "") add_dict['master_info'] = { "status": 0, "cityarea2_id": False, "remark": "商圈id%s存在相似商圈" % (cityarea2_id) } return add_dict else: insert_id = self.dao.insert_one(city=city, datas=new_cityarea2_dict) if not insert_id: add_dict['master_info'] = { "status": 0, "cityarea2_id": insert_id, "remark": "商圈添加失败" } return add_dict find_cityarea = self.cityarea_dao.find_one( city=city, filter={"_id": cityarea_id}) if not find_cityarea: add_dict['master_info'] = { "status": 0, "cityarea2_id": insert_id, "remark": "添加到城区失败" } return add_dict list_cityarea2 = find_cityarea[0].get("list_cityarea2", "") list_cityarea2[ new_cityarea2_dict['name']] = new_cityarea2_dict['_id'] cityarea_update_action = self.cityarea_dao.update( city=city, filter={'_id': cityarea_id}, datas={ "$set": { "list_cityarea2": list_cityarea2, "updated": current_time } }) if cityarea_update_action['nModified'] == 1: # 添加商圈联想词 #guess_status = self.guessword.add_guessword(city=city, guessword_list=[new_cityarea2_dict['name']], type_id=2) add_dict['master_info'] = { "status": 1, "cityarea2_id": insert_id, "remark": "添加商圈并同步到城区表成功" } else: add_dict['master_info'] = { "status": 0, "cityarea2_id": insert_id, "remark": "添加商圈成功,同步到城区表失败" } return add_dict
def merge_cityarea(self, city, master_cityarea, slave_cityarea): try: # 商圈表 -----> 商圈的归并 # 支持跨城区合并 同城区相当于删除商圈 current_time = BaseUtils.getIntTime() delete_dict = {} update_dict = {} error_dict = {} mater_cityarea_id = master_cityarea.get("cityarea_id", "") mater_cityarea_name = master_cityarea.get("cityarea_name", "") mater_cityarea_py = BaseUtils.getFpy(mater_cityarea_name, delimiter='') mater_cityarea2_id = master_cityarea.get("cityarea2_id", "") mater_cityarea2_name = master_cityarea.get("cityarea2_name", "") mater_cityarea2_py = BaseUtils.getFpy(mater_cityarea2_name, delimiter='') slave_cityarea_id = slave_cityarea.get("cityarea_id", "") slave_cityarea_name = slave_cityarea.get("cityarea_name", "") slave_cityarea2_id = slave_cityarea.get("cityarea2_id", "") slave_cityarea2_name = slave_cityarea.get("cityarea2_name", "") slave_cityarea2_py = BaseUtils.getFpy(slave_cityarea2_name, delimiter='') master_cityarea2 = [{ "cityarea2_id": mater_cityarea2_id, "cityarea2_name": mater_cityarea2_name, "cityarea2_py": mater_cityarea2_py }] master_cityarea_data = self.cityarea_service.find( city=city, filter={'_id': mater_cityarea_id}) slave_cityarea_data = self.cityarea_service.find( city=city, filter={'_id': slave_cityarea_id}) if not master_cityarea_data: error_dict = {"status": 1, "remark": "主城区的城区找不到数据"} return { "update_dict": update_dict, "delete_dict": delete_dict, "error_dict": error_dict } if not slave_cityarea_data: error_dict = {"status": 1, "remark": "从城区的城区找不到数据"} return { "update_dict": update_dict, "delete_dict": delete_dict, "error_dict": error_dict } find_cityarea2 = self.find(city=city, filter={'_id': slave_cityarea2_id}) if find_cityarea2: self.cityarea2_recycle.insert_one(city=city, datas=find_cityarea2) delete_status = self.delete_by_id(city=city, cityarea2_id=slave_cityarea2_id) if delete_status['ok'] == 1: delete_dict = { "id": slave_cityarea2_id, "status": 1, "remark": "商圈删除成功" } else: delete_dict = { "id": slave_cityarea2_id, "status": 0, "remark": "商圈删除失败" } slave_list_cityarea2 = slave_cityarea_data[0].get( "list_cityarea2", "") for k in list(slave_list_cityarea2.keys()): if slave_list_cityarea2[k] == slave_cityarea2_id: slave_list_cityarea2.pop(k) continue datas = { "$set": { "list_cityarea2": slave_list_cityarea2, "updated": current_time } } slave_cityarea_update_action = self.cityarea_service.update_filter( filter={'_id': slave_cityarea_id}, city=city, datas=datas) if slave_cityarea_update_action['nModified'] == 1: update_dict = { "id": slave_cityarea_id, "status": 1, "remark": "更新slave城区表成功" } else: update_dict = { "id": slave_cityarea_id, "status": 0, "remark": "更新slave城区表失败" } # 更改小区的城区商圈 master_cityarea_info = { "cityarea_id": mater_cityarea_id, "cityarea_name": mater_cityarea_name, "area_pinyin": mater_cityarea_py, "cityarea2_id": mater_cityarea2_id, "cityarea2_name": mater_cityarea2_name, "area2_pinyin": mater_cityarea2_py } slave_cityarea_info = { "cityarea2_id": slave_cityarea2_id, "cityarea2_name": slave_cityarea2_name, "area2_pinyin": slave_cityarea2_py } borough_online_status = self.change_borough_online_cityarea( master_cityarea=master_cityarea_info, slave_cityarea=slave_cityarea_info, city=city) if not borough_online_status: error_dict = {"status": 1, "remark": "borough_online的城区商圈更改失败"} # 更改小区缓存 self.update_area_cache(city=city, cityarea2={"id": slave_cityarea2_id}) # 小区列表es拼装的数据 updateBody = { "query": { "bool": { "must": [{ "nested": { "path": "cityarea2", "query": { "bool": { "must": [{ "term": { "cityarea2.cityarea2_id": slave_cityarea2_id } }] } } } }], } }, "script": { "inline": "ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name;ctx._source.cityarea2 = params.cityarea2", "params": { "cityarea_id": mater_cityarea_id, "cityarea_name": mater_cityarea_name, "cityarea2": master_cityarea2 }, "lang": "painless" } } if city == 'bj': index_doc = "spider" else: index_doc = "spider_" + city borough_index = "online_" + city + "_borough" borough_list_status = self.essearch.update_by_query( index=borough_index, doc_type=index_doc, body=updateBody) # 房源的批量修改 house_index = "active_" + city + "_house_sell" house_list_status = self.essearch.update_by_query( index=house_index, doc_type=index_doc, body=updateBody) update_info = { "delete_dict": delete_dict, "update_dict": update_dict, "error_dict": error_dict, "borough_es": borough_list_status, "house_es": house_list_status } return update_info except Exception as e: print("update_fail", e) return False
def mod_cityarea2_relation(self, city, before_cityarea_info, after_cityarea_info): current_time = BaseUtils.getIntTime() update_dict = {} before_cityarea_id = before_cityarea_info.get("cityarea_id", "") before_cityarea2_id = before_cityarea_info.get("cityarea2_id", "") before_cityarea2_name = before_cityarea_info.get("cityarea2_name", "") before_cityarea2_py = BaseUtils.getFpy(before_cityarea2_name) after_cityarea_id = after_cityarea_info.get("cityarea_id", "") after_cityarea_name = after_cityarea_info.get("cityarea_name", "") after_cityarea_py = BaseUtils.getFpy(after_cityarea_name) if before_cityarea_id: find_before_cityarea = self.cityarea_dao.find_one( city=city, filter={"_id": before_cityarea_id}) if not find_before_cityarea: update_dict['master_info'] = { "status": 0, "cityarea2_id": before_cityarea_id, "remark": "城区id为%s的城区找不到" % (before_cityarea_id) } return update_dict list_cityarea2 = find_before_cityarea.get("list_cityarea2", "") for k, v in list_cityarea2.items(): if k == before_cityarea2_name: list_cityarea2.pop(k) break cityarea_update_action = self.cityarea_dao.update( city=city, filter={'_id': before_cityarea_id}, datas={ "$set": { "list_cityarea2": list_cityarea2, "updated": current_time } }) if cityarea_update_action['nModified'] == 0: update_dict['master_info'] = { "status": 0, "cityarea2_id": before_cityarea_id, "remark": "城区更新失败" } return update_dict find_after_cityarea = self.cityarea_dao.find_one( city=city, filter={"_id": after_cityarea_id}) after_list_cityarea2 = find_after_cityarea.get( "list_cityarea2", "") after_list_cityarea2[before_cityarea2_name] = before_cityarea2_id after_cityarea_update_action = self.cityarea_dao.update( filter={'_id': after_cityarea_id}, datas={ "$set": { "list_cityarea2": after_list_cityarea2, "updated": current_time } }) if after_cityarea_update_action['nModified'] == 1: # 二手房小区缓存 # cityarea = {'id': after_cityarea_id} # self.borough_online.update_area_cache(cityarea=cityarea) updateBody = { "query": { "bool": { "must": [{ "nested": { "path": "cityarea2", "query": { "bool": { "must": [{ "term": { "cityarea2.cityarea2_id": before_cityarea2_id } }] } } } }], } }, "script": { "inline": "ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name", "params": { "cityarea_id": after_cityarea_id, "cityarea_name": after_cityarea_name }, "lang": "painless" } } if city == 'bj': index_doc = "spider" else: index_doc = "spider_" + city borough_index = "online_" + city + "_borough" borough_list_status = self.essearch.update_by_query( index=borough_index, doc_type=index_doc, body=updateBody) # borough_list_status = True # 房源的批量修改 house_index = "online_" + city + "_house_sell" house_list_status = self.essearch.update_by_query( index=house_index, doc_type=index_doc, body=updateBody) # 改变商圈所属城区 filter = { "cityarea.cityarea2.cityarea2_id": before_cityarea2_id } master_cityarea_info = { "cityarea": { "cityarea_name": after_cityarea_name, "cityarea_id": after_cityarea_id, "area_pinyin": after_cityarea_py, "cityarea_py": after_cityarea_py }, "cityarea2": [{ "cityarea2_name": before_cityarea2_name, "cityarea2_py": before_cityarea2_py, "cityarea2_id": before_cityarea2_id, "area2_pinyin": before_cityarea2_py }] } datas = { "$set": { "cityarea": master_cityarea_info, "borough_ctype": 8, "updated": current_time } } update_action = self.boroughline_dao.updateAll(city=city, filter=filter, datas=datas) # 更新缓存 cityarea2 = {'id': before_cityarea2_id} self.update_area_cache(city=city, cityarea2=cityarea2) update_dict['master_info'] = { "status": 1, "cityarea2_id": before_cityarea_id, "remark": "更新城区表成功" } else: update_dict['master_info'] = { "status": 0, "cityarea2_id": before_cityarea_id, "remark": "更新城区表失败" } return update_dict else: update_dict['master_info'] = { "status": 0, "cityarea2_id": False, "remark": "城区id传的有误" } return update_dict
def mod_cityarea2_name(self, city, cityarea_id, cityarea2_info): current_time = BaseUtils.getIntTime() cityarea2_id = cityarea2_info.get("cityarea2_id", "") name = cityarea2_info.get("new_name", "") cityarea2_py = BaseUtils.getFpy(name) update_dict = {} if cityarea2_id: cityarea2_action = self.dao.update( city=city, filter={'_id': cityarea2_id}, datas={"$set": { "name": name, "updated": current_time }}) if cityarea2_action['nModified'] == 0: update_dict["master_info"] = { "status": 0, "cityarea2_id": cityarea2_id, "remark": "商圈id为%s的更新失败" % (cityarea2_id) } return update_dict find_cityarea = self.cityarea_dao.find_one( city=city, filter={"_id": cityarea_id}) if not find_cityarea: update_dict['master_info'] = { "status": 0, "cityarea2_id": cityarea2_id, "remark": "城区id为%s的城区找不到" % (cityarea_id) } return update_dict list_cityarea2 = find_cityarea.get("list_cityarea2", "") for k, v in list_cityarea2.items(): if v == cityarea2_id: list_cityarea2.pop(k) list_cityarea2[name] = v break cityarea_update_action = self.cityarea_dao.update( city=city, filter={'_id': cityarea_id}, datas={ "$set": { "list_cityarea2": list_cityarea2, "updated": current_time } }) cityarea2_update_action = self.dao.update( city=city, filter={'_id': cityarea2_id}, datas={ "$set": { "name": name, "area2_pinyin": cityarea2_py, "updated": current_time } }) if cityarea_update_action['nModified'] == 1: # 小区的缓存修改 cityarea2 = {'id': cityarea2_id} # 修改商圈的小区 filter = {"cityarea.cityarea2.cityarea2_id": cityarea2_id} cityarea_name = find_cityarea.get("name", "") cityarea_py = find_cityarea.get("area_pinyin", "") cityarea_id = find_cityarea.get("_id", "") master_cityarea_info = { "cityarea": { "cityarea_name": cityarea_name, "cityarea_id": cityarea_id, "area_pinyin": cityarea_py, "cityarea_py": cityarea_py }, "cityarea2": [{ "cityarea2_name": name, "cityarea2_py": cityarea2_py, "cityarea2_id": cityarea2_id, "area2_pinyin": cityarea2_py }] } find_borough_by_cityarea2 = self.boroughline_dao.find_one( city=city, filter={"cityarea.cityarea2.cityarea2_id": cityarea2_id}) if not find_borough_by_cityarea2: update_dict['master_info'] = { "status": 1, "cityarea2_id": cityarea2_id, "remark": "更新商圈并同步到城区表成功-该商圈下无小区" } return update_dict datas = { "$set": { "cityarea": master_cityarea_info, "borough_ctype": 8, "updated": current_time } } update_action = self.boroughline_dao.updateAll(city=city, filter=filter, datas=datas) self.update_area_cache(city=city, cityarea2=cityarea2) # 小区es列表页 master_cityarea2 = [{ "cityarea2_id": cityarea2_id, "cityarea2_name": name, "cityarea2_py": cityarea2_py }] updateBody = { "query": { "bool": { "must": [{ "nested": { "path": "cityarea2", "query": { "bool": { "must": [{ "term": { "cityarea2.cityarea2_id": cityarea2_id } }] } } } }], } }, "script": { "inline": "ctx._source.cityarea2 = params.cityarea2", "params": { "cityarea2": master_cityarea2 }, "lang": "painless" } } if city == 'bj': index_doc = "spider" else: index_doc = "spider_" + city borough_index = "online_" + city + "_borough" borough_list_status = self.essearch.update_by_query( index=borough_index, doc_type=index_doc, body=updateBody) # borough_list_status = True # 房源的批量修改 house_index = "online_" + city + "_house_sell" house_list_status = self.essearch.update_by_query( index=house_index, doc_type=index_doc, body=updateBody) # house_list_status = True # 商圈联想词 变更 # guess_status = self.guessword.modify_borough_name(keyword_dict={origin_name: master_name}) # publish信息 message = { "city": city, "operate": "update", "type": "cityarea2", "pms": { "master": { "id": cityarea2_id, "name": name } } } push_status = self.publish_message("dm_info_change", message) update_dict['master_info'] = { "status": 1, "cityarea2_id": cityarea2_id, "remark": "更新商圈并同步到城区表成功" } else: update_dict['master_info'] = { "status": 0, "cityarea2_id": cityarea2_id, "remark": "更新商圈成功,同步到城区表失败" } return update_dict else: update_dict['master_info'] = { "status": 0, "cityarea2_id": False, "remark": "商圈id传的有误" } return update_dict