class BoroughGuesswordService(BaseMgoService):
def __init__(self, *args, **kwargs):
# city = kwargs.get('city')
# db_name = f"newhouse_{city}"
self.guessdao = GuessWordDao(*args, **kwargs)
self.onlinedao = BoroughOnlineDao(*args, **kwargs)
self.complexsearchservice = ComplexSearchService()
self.area_list = {
1: CityareaService(),
2: Cityarea2Service(),
3: self.onlinedao
}
def add_guessword(self, *args, **kwargs):
city = kwargs.get('city')
guessword_list = kwargs.get('guessword_list')
type_id = kwargs.get('type_id')
result = dict()
for guessword in guessword_list:
guessword = guessword
borough_id = self.get_borough_id(city=city, guessword=guessword, type_id=type_id)
if borough_id:
guessword_status = self.get_guessword(city=city, guessword=guessword, type_id=type_id)
# 如果联想词不存在添加联想词同步到es
if not guessword_status:
self.save_mongo(city,type_id, guessword, borough_id)
self.push_es(city,type_id, guessword)
result.setdefault(guessword, True)
else:
# 如果联想词存在更新联想词同步到es
# result.setdefault(guessword, False)
data = self.guessword_list(city, type_id, guessword, borough_id)
data.pop('created')
self.guessdao.update(
filter={'other_id': borough_id, 'keyword': guessword, 'type_id': type_id},
datas={'$set': {'datas': data}}, city=city)
self.push_es(city, type_id, guessword)
result.setdefault(guessword, True)
else:
result.setdefault(guessword, False)
return result
def update_guessword(self, *args, **kwargs):
city = kwargs.get('city')
keyword_list = kwargs.get('guessword_list')
type_id = kwargs.get('type_id')
result = dict()
for keyword in keyword_list:
keyword = keyword
borough_id = self.get_borough_id(city=city, guessword=keyword, type_id=type_id)
guessword_status = self.get_guessword(city=city, guessword=keyword, type_id=type_id)
if borough_id and guessword_status:
data = self.guessword_list(city, type_id, keyword, borough_id)
data.pop('created')
self.guessdao.update(
filter={'other_id': borough_id, 'keyword': keyword, 'type_id': type_id}, datas={'$set': {'datas': data}}, city=city)
self.push_es(city,type_id, keyword)
result.setdefault(keyword, True)
else:
result.setdefault(keyword, False)
return result
def delete_guessword(self, *args, **kwargs):
city = kwargs.get('city')
keyword_list = kwargs.get('guessword_list')
type_id = kwargs.get('type_id')
result = dict()
for keyword_info in keyword_list:
keyword = keyword_info.get("name","")
other_id = keyword_info.get("id", 0)
where = {'type_id': type_id}
if other_id:
where.setdefault('other_id', other_id,)
else:
where.setdefault('keyword', keyword)
if not other_id and not keyword:
result.setdefault(keyword, False)
return result
guessword_status = self.guessdao.find_one(city=city, filter=where)
if guessword_status:
_id = guessword_status[0]['_id']
self.guessdao.delete_by_id(city=city, filter={'_id':_id})
self.delete_es(city,self.complexsearchservice.online_keyword_alias, str(_id))
result.setdefault(keyword, True)
else:
result.setdefault(keyword, False)
return result
def modify_borough_name(self, *args, **kwargs):
city = kwargs.get('city')
keyword_dict = kwargs.get('guessword_dict')
result = dict()
for old_borough_name, new_borough_name in keyword_dict.items():
borough = self.onlinedao.find_one(city=city,filter={'borough_name': old_borough_name})
guessword_status = self.get_guessword(city=city, guessword=old_borough_name)
if borough and guessword_status:
borough_id = borough[0]['_id']
data = self.guessword_list(city=city,g_type=3, keyword=new_borough_name, other_id=borough_id)
data.pop('created')
self.onlinedao.update(city = city,filter={'_id': borough_id},
datas={'$set': {'borough_name': new_borough_name}})
if not self.get_guessword(city=city,guessword=new_borough_name):
self.guessdao.update(city=city,
filter={'other_id': borough_id, 'keyword': old_borough_name, 'type_id': 3},
datas={'$set': data})
self.push_es(city=city , g_type=3 , keyword=new_borough_name)
result.setdefault("%s->%s" % (old_borough_name, new_borough_name), '修改成功')
else:
result.setdefault(new_borough_name, '联想词已存在')
else:
result.setdefault(old_borough_name, '联想词或者小区不存在')
return result
def get_borough_id(self, city, guessword, type_id):
"""
查询联想词对应的小区id (borough_byname or borough_hide)
:param keyword:联想词
:return:小区id
"""
if type_id == 3:
where = {
'$or': [
{
'borough_byname': guessword
},
{
'borough_hide': guessword
},
]
}
else:
where = {
"name": guessword
}
result = self.area_list[type_id].get_one(city=city,filter=where)
if result:
return result[0]['_id']
else:
return ''
def get_guessword(self, city, guessword, type_id=3):
"""
查询联想词是否存在
:param keyword:联想词
:return:true or false
"""
where = dict()
where.setdefault('keyword', guessword)
where.setdefault('type_id', type_id)
result = self.guessdao.find_one(city=city,filter=where)
if result:
return True
else:
return False
def save_mongo(self, city,g_type, keyword, borough_id):
try:
result = self.guessword_list(city,g_type, keyword, borough_id)
self.guessdao.update(city=city,datas=result)
return '成功存入mongo'
except:
return '存入mongo失败'
def guessword_list(self,city, g_type, keyword, other_id, addr=''):
"""
生成插入mongo.guessword的数据
:param g_type: 联想词类型 3:小区
:param keyword: 联想词
:param other_id: 小区id
:param addr: 地址
:return: 拼装好的数据
"""
if city == "bj":
mongo_db = "spider"
else:
mongo_db = "spider_%s"%(city)
index_name = 'online_' + city + '_house_sell'
guessword_dict = {}
guessword_dict['type_id'] = g_type
guessword_dict['type_name'] = self.guessdao.guessword_type[g_type]
guessword_dict['other_id'] = other_id
guessword_dict['keyword'] = keyword
guessword_dict['addr'] = addr
guessword_dict['pinyin'] = BaseUtils.getFpy(keyword, delimiter='')
guessword_dict['logogram'] = BaseUtils.getSpy(keyword, delimiter='').lower()
guessword_dict['count_broker'] = self.es_search(city=city,index_name=index_name, table=mongo_db,key_v=self.guessdao.guessword_key[g_type], value_search=other_id, type_id=g_type)
guessword_dict['created'] = int(time.time())
guessword_dict['updated'] = int(time.time())
return guessword_dict
def es_search(self, city, index_name, table, key_v, value_search, type_id):
# 商圈数据结构和 小区城区不同 需要嵌套查询
if type_id == 2:
query_json = {
"query": {
"bool": {
"must": [
{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [
{
"term": {"cityarea2.cityarea2_id": value_search}
}
]
}
}
}
},
{
"nested": {
"path": "house_info",
"query": {
"bool": {
"must_not": [
{
"term": {"house_info.source": 10}
}
]
}
}
}
}
]
}
}
}
else:
query_json = {
"query": {
"bool": {
"must": [
{
"term": {key_v: value_search}
},
{
"nested": {
"path": "house_info",
"query": {
"bool": {
"must_not": [
{
"term": {"house_info.source": 10}
}
]
}
}
}
}
]
}
}
}
# import json
# print(json.dumps(query_json))
res = self.complexsearchservice.search_filter(city=city, dsl=query_json, index_name=index_name, doc_name=table)['total'] # 获取所有数据
# 获取第一条数据，得分最高。
top_10_recodes = res
return top_10_recodes
def push_es(self, city, g_type, keyword):
try:
actions = self.generate_es_data(city, g_type, self.complexsearchservice.online_keyword_alias, keyword)
self.complexsearchservice.search_bulk(actions=actions)
return '成功推送es'
except Exception as e:
return '推送es失败'
def delete_es(self, city,alias, id):
try:
if city == "bj":
mongo_db = "spider"
else:
mongo_db = "spider_%s" % (city)
index_name = str(self.complexsearchservice.get_alias_es(alias=alias))
self.complexsearchservice.del_es(index_name=index_name, doc_name=mongo_db, id=id)
return '删除es成功'
except:
return '删除es失败'
def generate_es_data(self, city, g_type, alias, keyword):
"""
拼装插入es的数据
:param alias:线上联想词es别名
:param type:es type名称(相当于mysql的表)
:return: 返回拼装好的es数据
"""
where = dict()
where.setdefault('keyword', keyword)
where.setdefault('type_id', g_type)
field = ['addr','count_broker','type_id','created','updated','keyword','logogram','other_id','pinyin','type_name']
result = self.guessdao.find_one(city=city, filter=where,field=field)
result = self.format_data(result)
index = str(self.complexsearchservice.get_alias_es(alias=alias))
actions = []
id = result.pop("_id")
if city == "bj":
table = "spider"
else:
table = "spider_%s"%(city)
value = {"_index": index, "_type": table, "_id": id, "_source": result}
actions.append(value)
return actions
def format_data(self, data):
format_houses = dict()
format_houses['_id'] = str(data[0]['_id'])
if 'addr' in data[0]:
format_houses['addr'] = str(data[0]['addr'])
if 'count_broker' in data[0]:
format_houses['count_broker'] = data[0]['count_broker']
if data[0].get('type_id'):
format_houses['type_id'] = data[0]['type_id']
if data[0].get('created'):
format_houses['created'] = int(data[0]['created'])
if data[0].get('updated'):
format_houses['updated'] = int(data[0]['updated'])
if data[0].get('keyword'):
format_houses['keyword'] = str(data[0]['keyword'])
if data[0].get('logogram'):
format_houses['logogram'] = str(data[0]['logogram'])
if data[0].get('other_id'):
format_houses['other_id'] = int(data[0]['other_id'])
if data[0].get('pinyin'):
format_houses['pinyin'] = str(data[0]['pinyin'])
if data[0].get('type_name'):
format_houses['type_name'] = str(data[0]['type_name'])
return format_houses
class CommunityService(BaseMgoService):
def __init__(self, *args, **kwargs):
self.borough_online_dao = BoroughOnlineDao(*args, **kwargs)
self.community_dao = CommunityDao(*args, **kwargs)
self.house_sell_gov_dao = HouseSellGovDao(*args, **kwargs)
# 根据指定的城区id查询数据
def get_community_list(self, *args, **kwargs):
result = {}
city = kwargs.get('city')
pms = kwargs.get('pms')
sort = pms.get('sort')
page = pms.get('page')
index = page.get('index')
size = page.get('size')
filter = pms.get('filter', {})
filters = {'name': {'$regex': filter.get('name', '')}}
field = {
'_id': 1,
'name': 1,
'loc': 1,
'pinyin': 1,
'created': 1,
'updated': 1
} # 返回字段
total = self.community_dao.find_count(city=city, filter=filters)
data = self.community_dao.find_page(city=city,
filter=filters,
page=page,
sort=sort,
field=field)
for community_info in data:
boroughList = []
id = community_info.get('_id')
filter = {'community.community_id': id}
#获取gov表里所属社区的房源数量
gov_filter = 'community_id = ' + str(id)
community_info['gov_total'] = self.house_sell_gov_dao.select_count(
city=city, filter=gov_filter)
borough_infos = self.borough_online_dao.get_one(city=city,
filter=filter)
for borough_info in borough_infos:
borough = {}
borough['id'] = borough_info.get('_id')
borough['name'] = borough_info.get('borough_name')
boroughList.append(borough)
community_info['boroughList'] = boroughList
result.setdefault('data', data)
result.setdefault('page', {
"index": index,
"size": size,
"total": total
})
return result
def create_community(self, *args, **kwargs):
result = {}
community_info = {}
city = kwargs.get('city')
pms = kwargs.get('pms')
name = pms.get('community_name')
community_page = {"index": 1, "size": 1}
community_sort = [("_id", -1)]
community = self.community_dao.find_page(city=city,
sort=community_sort,
page=community_page)[0]
community_max_id = community.get('_id')
community_info['_id'] = community_max_id + 1
community_info['name'] = name
community_info['pinyin'] = BaseUtils.getPiyin(value=name)
community_info['created'] = int(time.time())
community_info['updated'] = int(time.time())
self.community_dao.insert_one(city=city, datas=community_info)
result.setdefault(
'data', {
"status": 1,
"comunity_id": community_info['_id'],
"remark": "新建社区id为%s的社区成功" % (community_info['_id'])
})
return result
def deleteCommunity(self, *args, **kwargs):
result = {}
community_info = {}
city = kwargs.get('city')
pms = kwargs.get('pms')
community_id = pms.get('community_id')
borough_info = self.borough_online_dao.find_one(
city=city, filter={"community.community_id": community_id})
#判断社区里是否还有小区
if borough_info:
result.setdefault('data', {
"status": -1,
"comunity_id": community_id,
"remark": "社区里存在小区无法删除"
})
else:
self.community_dao.delete_by_id(city=city,
filter={'_id': community_id})
result.setdefault('data', {
"status": 1,
"comunity_id": community_id,
"remark": "社区删除成功"
})
return result
def modifyCommunityName(self, *args, **kwargs):
result = {}
community_info = {}
borough_community_info = {}
city = kwargs.get('city')
pms = kwargs.get('pms')
id = pms.get('community_id')
community_info['name'] = pms.get('community_name')
community_info['pinyin'] = BaseUtils.getPiyin(
value=community_info['name'])
community_info['updated'] = int(time.time())
#修改社区表信息
self.community_dao.update(city=city,
filter={'_id': id},
datas={'$set': community_info})
#修改小区表里社区信息
borough_community_info['community.community_name'] = pms.get(
'community_name')
borough_community_info['community.community_py'] = community_info[
'pinyin']
self.borough_online_dao.update(city=city,
filter={'community.community_id': id},
datas={'$set': borough_community_info})
# 更新小区缓存
# self.update_borough_cache(city=city, master={"id": borough_id})
# 删除小区详情页缓存
# self.delete_borough_detail_cache(borough_id=borough_id, city=city)
# 更新小区es社区
# 更新房源的社区
result.setdefault('data', {
"status": 1,
"comunity_id": id,
"remark": "社区主名修改成功"
})
return result
class Cityarea2Service(BaseMgoService):
def __init__(self, *args, **kwargs):
self.cityarea_dao = CityareaDao(*args, **kwargs)
self.dao = Cityarea2Dao(*args, **kwargs)
self.boroughline_dao = BoroughOnlineDao(*args, **kwargs)
self.cityarea_service = CityareaService()
self.max_id = BoroughMaxIdService()
self.price = BoroughMonthPriceService()
# self.boroug_online = BoroughOnlineService()
# 更新小区缓存的链接库
redis_config = redis_conf.get('borough_info_redis')
self.redis_conn = dbfactory.create_db(db_config=redis_config,
db_type='db_redis')
self.essearch = BoroughSearchService()
self.cityarea2_recycle = Cityarea2RecycleService()
# 根据指定商圈id获取数据
def getcityarea2_data(self, city, cityarea2_id, field):
filter = {'_id': cityarea2_id}
data = self.get_page(city=city, field=field, filter=filter)
return data
# 根据指定id删除数据
def delete_by_id(self, city, cityarea2_id):
filter = {'_id': cityarea2_id}
data = self.remove_by_id(city=city, filter=filter)
return data
def update_filterall(self, filter, datas, city):
result = self.update_by_filterall(filter=filter,
datas=datas,
city=city)
return result
def find(self, city, filter):
data = self.get_one(city=city, filter=filter)
return data
# 增加商圈
def add_cityarea2(self, city, cityarea_id, new_cityarea2):
add_dict = {}
new_cityarea2_dict = {}
current_time = BaseUtils.getIntTime()
new_cityarea2_dict['_id'] = self.max_id.getMaxId(type="cityarea2",
city=city)
new_cityarea2_dict['loc'] = new_cityarea2.get("loc", "")
new_cityarea2_dict['name'] = new_cityarea2.get("name", "").strip()
new_cityarea2_dict['area2_pinyin'] = BaseUtils.getFpy(
new_cityarea2_dict['name'])
new_cityarea2_dict['polygons'] = new_cityarea2.get("polygons", "")
new_cityarea2_dict['statu'] = 1
new_cityarea2_dict['created'] = current_time
new_cityarea2_dict['updated'] = current_time
is_find = self.dao.find_one(
city=city, filter={"name": {
"$regex": new_cityarea2_dict['name']
}})
if is_find:
cityarea2_id = is_find.get("_id", "")
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "商圈id%s存在相似商圈" % (cityarea2_id)
}
return add_dict
else:
insert_id = self.dao.insert_one(city=city,
datas=new_cityarea2_dict)
if not insert_id:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "商圈添加失败"
}
return add_dict
find_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": cityarea_id})
if not find_cityarea:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "添加到城区失败"
}
return add_dict
list_cityarea2 = find_cityarea[0].get("list_cityarea2", "")
list_cityarea2[
new_cityarea2_dict['name']] = new_cityarea2_dict['_id']
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 1:
# 添加商圈联想词
#guess_status = self.guessword.add_guessword(city=city, guessword_list=[new_cityarea2_dict['name']], type_id=2)
add_dict['master_info'] = {
"status": 1,
"cityarea2_id": insert_id,
"remark": "添加商圈并同步到城区表成功"
}
else:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "添加商圈成功,同步到城区表失败"
}
return add_dict
# 删除商圈
def delete_cityarea2(self, city, cityarea, cityarea2):
cityarea_id = cityarea.get("id", "")
cityarea2_id = cityarea2.get("id", "")
cityarea2_name = cityarea2.get("name", "")
current_time = BaseUtils.getIntTime()
update_dict = {}
is_find = self.boroughline_dao.find_one(
city=city,
filter={"cityarea.cityarea2.cityarea2_id": cityarea2_id})
if is_find:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "该商圈下有小区数据" % (cityarea2_id)
}
return update_dict
if cityarea2_id:
find_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": cityarea_id})
if not find_cityarea:
update_dict['master_info'] = {
"status": 0,
"cityarea_id": cityarea_id,
"remark": "城区id为%s的城区找不到" % (cityarea_id)
}
return update_dict
find_cityarea2 = self.dao.find_one(city=city,
filter={"_id": cityarea2_id})
if not find_cityarea2:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "城区id为%s的城区找不到" % (cityarea_id)
}
return update_dict
cityarea2_action = self.dao.delete_by_id(
city=city, filter={"_id": cityarea2_id})
insert_action = self.cityarea2_recycle.insert_one(
city=city, datas=find_cityarea2)
if cityarea2_action['ok'] == 0:
update_dict["master_info"] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "商圈id为%s的删除失败" % (cityarea2_id)
}
return update_dict
list_cityarea2 = find_cityarea[0].get("list_cityarea2", "")
for k, v in list_cityarea2.items():
if v == cityarea2_id:
list_cityarea2.pop(k)
break
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 1:
# 删除商圈联想词
#guess_status = self.guessword.delete_guessword(keyword_list=[cityarea2_name],type_id=2)
# publish信息
message = {
"city": city,
"operate": "delete",
"type": "cityarea2",
"pms": {
"slave": [{
"id": cityarea2_id,
"name": cityarea2_name
}]
}
}
push_status = self.publish_message("dm_info_change", message)
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": cityarea2_id,
"remark": "删除商圈并同步到城区表成功",
"push_status": push_status
}
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "删除商圈成功,同步到城区表失败"
}
return update_dict
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "商圈id传的有误"
}
return update_dict
# 修改商圈名称
def mod_cityarea2_name(self, city, cityarea_id, cityarea2_info):
current_time = BaseUtils.getIntTime()
cityarea2_id = cityarea2_info.get("cityarea2_id", "")
name = cityarea2_info.get("new_name", "")
cityarea2_py = BaseUtils.getFpy(name)
update_dict = {}
if cityarea2_id:
cityarea2_action = self.dao.update(
city=city,
filter={'_id': cityarea2_id},
datas={"$set": {
"name": name,
"updated": current_time
}})
if cityarea2_action['nModified'] == 0:
update_dict["master_info"] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "商圈id为%s的更新失败" % (cityarea2_id)
}
return update_dict
find_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": cityarea_id})
if not find_cityarea:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "城区id为%s的城区找不到" % (cityarea_id)
}
return update_dict
list_cityarea2 = find_cityarea.get("list_cityarea2", "")
for k, v in list_cityarea2.items():
if v == cityarea2_id:
list_cityarea2.pop(k)
list_cityarea2[name] = v
break
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
cityarea2_update_action = self.dao.update(
city=city,
filter={'_id': cityarea2_id},
datas={
"$set": {
"name": name,
"area2_pinyin": cityarea2_py,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 1:
# 小区的缓存修改
cityarea2 = {'id': cityarea2_id}
# 修改商圈的小区
filter = {"cityarea.cityarea2.cityarea2_id": cityarea2_id}
cityarea_name = find_cityarea.get("name", "")
cityarea_py = find_cityarea.get("area_pinyin", "")
cityarea_id = find_cityarea.get("_id", "")
master_cityarea_info = {
"cityarea": {
"cityarea_name": cityarea_name,
"cityarea_id": cityarea_id,
"area_pinyin": cityarea_py,
"cityarea_py": cityarea_py
},
"cityarea2": [{
"cityarea2_name": name,
"cityarea2_py": cityarea2_py,
"cityarea2_id": cityarea2_id,
"area2_pinyin": cityarea2_py
}]
}
find_borough_by_cityarea2 = self.boroughline_dao.find_one(
city=city,
filter={"cityarea.cityarea2.cityarea2_id": cityarea2_id})
if not find_borough_by_cityarea2:
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈并同步到城区表成功-该商圈下无小区"
}
return update_dict
datas = {
"$set": {
"cityarea": master_cityarea_info,
"borough_ctype": 8,
"updated": current_time
}
}
update_action = self.boroughline_dao.updateAll(city=city,
filter=filter,
datas=datas)
self.update_area_cache(city=city, cityarea2=cityarea2)
# 小区es列表页
master_cityarea2 = [{
"cityarea2_id": cityarea2_id,
"cityarea2_name": name,
"cityarea2_py": cityarea2_py
}]
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline": "ctx._source.cityarea2 = params.cityarea2",
"params": {
"cityarea2": master_cityarea2
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# borough_list_status = True
# 房源的批量修改
house_index = "online_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
# house_list_status = True
# 商圈联想词 变更
# guess_status = self.guessword.modify_borough_name(keyword_dict={origin_name: master_name})
# publish信息
message = {
"city": city,
"operate": "update",
"type": "cityarea2",
"pms": {
"master": {
"id": cityarea2_id,
"name": name
}
}
}
push_status = self.publish_message("dm_info_change", message)
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈并同步到城区表成功"
}
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈成功,同步到城区表失败"
}
return update_dict
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "商圈id传的有误"
}
return update_dict
# 修改商圈所属城区
def mod_cityarea2_relation(self, city, before_cityarea_info,
after_cityarea_info):
current_time = BaseUtils.getIntTime()
update_dict = {}
before_cityarea_id = before_cityarea_info.get("cityarea_id", "")
before_cityarea2_id = before_cityarea_info.get("cityarea2_id", "")
before_cityarea2_name = before_cityarea_info.get("cityarea2_name", "")
before_cityarea2_py = BaseUtils.getFpy(before_cityarea2_name)
after_cityarea_id = after_cityarea_info.get("cityarea_id", "")
after_cityarea_name = after_cityarea_info.get("cityarea_name", "")
after_cityarea_py = BaseUtils.getFpy(after_cityarea_name)
if before_cityarea_id:
find_before_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": before_cityarea_id})
if not find_before_cityarea:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "城区id为%s的城区找不到" % (before_cityarea_id)
}
return update_dict
list_cityarea2 = find_before_cityarea.get("list_cityarea2", "")
for k, v in list_cityarea2.items():
if k == before_cityarea2_name:
list_cityarea2.pop(k)
break
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': before_cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 0:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "城区更新失败"
}
return update_dict
find_after_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": after_cityarea_id})
after_list_cityarea2 = find_after_cityarea.get(
"list_cityarea2", "")
after_list_cityarea2[before_cityarea2_name] = before_cityarea2_id
after_cityarea_update_action = self.cityarea_dao.update(
filter={'_id': after_cityarea_id},
datas={
"$set": {
"list_cityarea2": after_list_cityarea2,
"updated": current_time
}
})
if after_cityarea_update_action['nModified'] == 1:
# 二手房小区缓存
# cityarea = {'id': after_cityarea_id}
# self.borough_online.update_area_cache(cityarea=cityarea)
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
before_cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline":
"ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name",
"params": {
"cityarea_id": after_cityarea_id,
"cityarea_name": after_cityarea_name
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# borough_list_status = True
# 房源的批量修改
house_index = "online_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
# 改变商圈所属城区
filter = {
"cityarea.cityarea2.cityarea2_id": before_cityarea2_id
}
master_cityarea_info = {
"cityarea": {
"cityarea_name": after_cityarea_name,
"cityarea_id": after_cityarea_id,
"area_pinyin": after_cityarea_py,
"cityarea_py": after_cityarea_py
},
"cityarea2": [{
"cityarea2_name": before_cityarea2_name,
"cityarea2_py": before_cityarea2_py,
"cityarea2_id": before_cityarea2_id,
"area2_pinyin": before_cityarea2_py
}]
}
datas = {
"$set": {
"cityarea": master_cityarea_info,
"borough_ctype": 8,
"updated": current_time
}
}
update_action = self.boroughline_dao.updateAll(city=city,
filter=filter,
datas=datas)
# 更新缓存
cityarea2 = {'id': before_cityarea2_id}
self.update_area_cache(city=city, cityarea2=cityarea2)
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": before_cityarea_id,
"remark": "更新城区表成功"
}
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "更新城区表失败"
}
return update_dict
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "城区id传的有误"
}
return update_dict
# 合并商圈
def merge_cityarea(self, city, master_cityarea, slave_cityarea):
try:
# 商圈表 -----> 商圈的归并
# 支持跨城区合并 同城区相当于删除商圈
current_time = BaseUtils.getIntTime()
delete_dict = {}
update_dict = {}
error_dict = {}
mater_cityarea_id = master_cityarea.get("cityarea_id", "")
mater_cityarea_name = master_cityarea.get("cityarea_name", "")
mater_cityarea_py = BaseUtils.getFpy(mater_cityarea_name,
delimiter='')
mater_cityarea2_id = master_cityarea.get("cityarea2_id", "")
mater_cityarea2_name = master_cityarea.get("cityarea2_name", "")
mater_cityarea2_py = BaseUtils.getFpy(mater_cityarea2_name,
delimiter='')
slave_cityarea_id = slave_cityarea.get("cityarea_id", "")
slave_cityarea_name = slave_cityarea.get("cityarea_name", "")
slave_cityarea2_id = slave_cityarea.get("cityarea2_id", "")
slave_cityarea2_name = slave_cityarea.get("cityarea2_name", "")
slave_cityarea2_py = BaseUtils.getFpy(slave_cityarea2_name,
delimiter='')
master_cityarea2 = [{
"cityarea2_id": mater_cityarea2_id,
"cityarea2_name": mater_cityarea2_name,
"cityarea2_py": mater_cityarea2_py
}]
master_cityarea_data = self.cityarea_service.find(
city=city, filter={'_id': mater_cityarea_id})
slave_cityarea_data = self.cityarea_service.find(
city=city, filter={'_id': slave_cityarea_id})
if not master_cityarea_data:
error_dict = {"status": 1, "remark": "主城区的城区找不到数据"}
return {
"update_dict": update_dict,
"delete_dict": delete_dict,
"error_dict": error_dict
}
if not slave_cityarea_data:
error_dict = {"status": 1, "remark": "从城区的城区找不到数据"}
return {
"update_dict": update_dict,
"delete_dict": delete_dict,
"error_dict": error_dict
}
find_cityarea2 = self.find(city=city,
filter={'_id': slave_cityarea2_id})
if find_cityarea2:
self.cityarea2_recycle.insert_one(city=city,
datas=find_cityarea2)
delete_status = self.delete_by_id(city=city,
cityarea2_id=slave_cityarea2_id)
if delete_status['ok'] == 1:
delete_dict = {
"id": slave_cityarea2_id,
"status": 1,
"remark": "商圈删除成功"
}
else:
delete_dict = {
"id": slave_cityarea2_id,
"status": 0,
"remark": "商圈删除失败"
}
slave_list_cityarea2 = slave_cityarea_data[0].get(
"list_cityarea2", "")
for k in list(slave_list_cityarea2.keys()):
if slave_list_cityarea2[k] == slave_cityarea2_id:
slave_list_cityarea2.pop(k)
continue
datas = {
"$set": {
"list_cityarea2": slave_list_cityarea2,
"updated": current_time
}
}
slave_cityarea_update_action = self.cityarea_service.update_filter(
filter={'_id': slave_cityarea_id}, city=city, datas=datas)
if slave_cityarea_update_action['nModified'] == 1:
update_dict = {
"id": slave_cityarea_id,
"status": 1,
"remark": "更新slave城区表成功"
}
else:
update_dict = {
"id": slave_cityarea_id,
"status": 0,
"remark": "更新slave城区表失败"
}
# 更改小区的城区商圈
master_cityarea_info = {
"cityarea_id": mater_cityarea_id,
"cityarea_name": mater_cityarea_name,
"area_pinyin": mater_cityarea_py,
"cityarea2_id": mater_cityarea2_id,
"cityarea2_name": mater_cityarea2_name,
"area2_pinyin": mater_cityarea2_py
}
slave_cityarea_info = {
"cityarea2_id": slave_cityarea2_id,
"cityarea2_name": slave_cityarea2_name,
"area2_pinyin": slave_cityarea2_py
}
borough_online_status = self.change_borough_online_cityarea(
master_cityarea=master_cityarea_info,
slave_cityarea=slave_cityarea_info,
city=city)
if not borough_online_status:
error_dict = {"status": 1, "remark": "borough_online的城区商圈更改失败"}
# 更改小区缓存
self.update_area_cache(city=city,
cityarea2={"id": slave_cityarea2_id})
# 小区列表es拼装的数据
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
slave_cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline":
"ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name;ctx._source.cityarea2 = params.cityarea2",
"params": {
"cityarea_id": mater_cityarea_id,
"cityarea_name": mater_cityarea_name,
"cityarea2": master_cityarea2
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# 房源的批量修改
house_index = "active_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
update_info = {
"delete_dict": delete_dict,
"update_dict": update_dict,
"error_dict": error_dict,
"borough_es": borough_list_status,
"house_es": house_list_status
}
return update_info
except Exception as e:
print("update_fail", e)
return False
# 根据商圈找小区,更新小区的数据
def change_borough_online_cityarea(self, master_cityarea, slave_cityarea,
city):
current_time = BaseUtils.getIntTime()
mater_cityarea_id = master_cityarea.get("cityarea_id", "")
mater_cityarea_name = master_cityarea.get("cityarea_name", "")
mater_area_pinyin = master_cityarea.get("area_pinyin", "")
mater_cityarea2_id = master_cityarea.get("cityarea2_id", "")
mater_cityarea2_name = master_cityarea.get("cityarea2_name", "")
mater_area2_pinyin = master_cityarea.get("area2_pinyin", "")
slave_cityarea_cityarea2_id = slave_cityarea.get("cityarea2_id", "")
filter = {
"cityarea.cityarea2.cityarea2_id": slave_cityarea_cityarea2_id
}
master_cityarea_info = {
"cityarea": {
"cityarea_name": mater_cityarea_name,
"cityarea_id": mater_cityarea_id,
"area_pinyin": mater_area_pinyin,
"cityarea_py": mater_area_pinyin
},
"cityarea2": [{
"cityarea2_name": mater_cityarea2_name,
"cityarea2_py": mater_area2_pinyin,
"cityarea2_id": mater_cityarea2_id,
"area2_pinyin": mater_area2_pinyin
}]
}
datas = {
"$set": {
"cityarea": master_cityarea_info,
"borough_ctype": 8,
"updated": current_time
}
}
filter = {
"cityarea.cityarea2.cityarea2_id": slave_cityarea_cityarea2_id
}
find_borough_by_cityarea2 = self.boroughline_dao.find_one(
city=city, filter=filter)
if not find_borough_by_cityarea2:
return True
update_action = self.boroughline_dao.updateAll(filter=filter,
datas=datas,
city=city)
if update_action['nModified'] > 0:
return True
else:
return False
# 生成小区缓存数据
def generate_cache_data(self, name, city):
data_info = {}
if isinstance(name, int):
find_borough = self.boroughline_dao.find_one(filter={"_id": name},
city=city)[0]
else:
find_borough = self.boroughline_dao.find_one(
filter={'borough_byname': name}, city=city)[0]
if not find_borough:
data_info['status'] = 0
data_info['message'] = '找不到该小区'
return data_info
data_info['id'] = find_borough.get('_id', 0)
data_info['name'] = find_borough.get('borough_name', '')
data_info['loc'] = find_borough.get('loc', [])
cityarea_id = find_borough['cityarea']['cityarea']['cityarea_id']
cityarea_name = find_borough['cityarea']['cityarea']['cityarea_name']
data_info['cityarea'] = {
"cityarea_id": cityarea_id,
"cityarea_name": cityarea_name
}
if 'cityarea2' in find_borough['cityarea']:
data_info['cityarea2'] = find_borough['cityarea']['cityarea2']
if "traffic" in find_borough:
if not isinstance(find_borough['traffic'], dict):
data_info['subway'] = False
elif 'subway' not in find_borough['traffic']:
data_info['subway'] = False
elif len(find_borough['traffic']['subway']) > 0:
data_info['subway'] = True
else:
data_info['subway'] = False
if "borough_config" in find_borough:
data_info['config'] = find_borough['borough_config']
else:
data_info['config'] = []
data_info['price'] = self.price.getPriceByBoroughId(
borough_id=data_info['id'], city=city)
borough_byname = find_borough.get("borough_byname", [])
borough_hide = find_borough.get("borough_hide", [])
name_list = borough_byname + borough_hide
data_info['name_list'] = name_list
return data_info
# 更新单个小区缓存
def update_borough_cache(self, city, master={}, slave={}):
try:
key = "%s_borough" % city_mapping.get(city)['fpy']
if slave == {}:
if master.get('id', ''):
datas = self.generate_cache_data(name=master.get('id'),
city=city)
else:
datas = self.generate_cache_data(name=master.get('name'),
city=city)
name_list = datas.pop('name_list')
for borough_name in name_list:
self.redis_conn.hset(key, BaseUtils.getMd5(borough_name),
json.dumps(datas))
else:
self.redis_conn.hdel(key, BaseUtils.getMd5(slave.get('name')))
return True
except Exception as e:
print(e)
return False
# 城区商圈变化后更新小区缓存
def update_area_cache(self, city, cityarea={}, cityarea2={}):
try:
cityarea_id = cityarea.get('id', '')
cityarea_name = cityarea.get('name', '')
cityarea2_id = cityarea2.get('id', '')
cityarea2_name = cityarea2.get('name', '')
if cityarea2_id:
datas = self.boroughline_dao.find_one(
filter={"cityarea.cityarea2.cityarea2_id": cityarea2_id},
city=city)
elif cityarea_id:
datas = self.boroughline_dao.find_one(
filter={"cityarea.cityarea.cityarea_id": cityarea_id},
city=city)
elif cityarea_name:
datas = self.boroughline_dao.find_one(
filter={"cityarea.cityarea.cityarea_name": cityarea_name},
city=city)
elif cityarea2_name:
datas = self.boroughline_dao.find_one(filter={
"cityarea.cityarea2.cityarea2_name":
cityarea2_name
},
city=city)
else:
datas = []
for i in datas:
self.update_borough_cache(
city=city, master={'name': i.get('borough_name')})
except Exception as e:
print(e)
return False
# 向各方发布消息
def publish_message(self, channel, message):
status = False
try:
status = self.redis_conn.publish(channel, message)
if status:
return True
return status
except Exception as e:
print(e)
return status
#获取商圈列表
def getCityarea2List(self, *args, **kwargs):
result = {}
city = kwargs.get('city')
pms = kwargs.get('pms')
sort = pms.get('sort', {})
#处理sort
sort = [('_id', 1)] if sort.get('id', 1) == 1 else [('_id', -1)]
page = pms.get('page')
index = page.get('index')
size = page.get('size')
filter = pms.get('filter', {})
field = {'_id': 1, 'name': 1, 'loc': 1, 'area2_pinyin': 1} # 返回字段
total = self.dao.find_count(city=city, pms=pms)
data = self.dao.getCityarea2ListByCity(city=city,
filter=filter,
page=page,
sort=sort,
field=field)
result.setdefault('data', data)
result.setdefault('page', {
"index": index,
"size": size,
"total": total
})
return result
# 根据id获取商圈详情
def getCityarea2InfoById(self, *args, **kwargs):
result = {}
city = kwargs.get('city')
cityarea2_id = kwargs.get('id')
cityarea_info = self.dao.getCityarea2InfoById(
city=city, cityarea2_id=cityarea2_id)
result.setdefault('data', cityarea_info)
return result