def guessword_list(self,city, g_type, keyword, other_id, addr=''):
"""
生成插入mongo.guessword的数据
:param g_type: 联想词类型 3:小区
:param keyword: 联想词
:param other_id: 小区id
:param addr: 地址
:return: 拼装好的数据
"""
if city == "bj":
mongo_db = "spider"
else:
mongo_db = "spider_%s"%(city)
index_name = 'online_' + city + '_house_sell'
guessword_dict = {}
guessword_dict['type_id'] = g_type
guessword_dict['type_name'] = self.guessdao.guessword_type[g_type]
guessword_dict['other_id'] = other_id
guessword_dict['keyword'] = keyword
guessword_dict['addr'] = addr
guessword_dict['pinyin'] = BaseUtils.getFpy(keyword, delimiter='')
guessword_dict['logogram'] = BaseUtils.getSpy(keyword, delimiter='').lower()
guessword_dict['count_broker'] = self.es_search(city=city,index_name=index_name, table=mongo_db,key_v=self.guessdao.guessword_key[g_type], value_search=other_id, type_id=g_type)
guessword_dict['created'] = int(time.time())
guessword_dict['updated'] = int(time.time())
return guessword_dict
value = {"_index": index, "_type": table, "_id": id, "_source": result}
actions.append(value)
return actions
def format_data(self, data):
format_houses = dict()
format_houses['_id'] = str(data[0]['_id'])
if 'addr' in data[0]:
format_houses['addr'] = str(data[0]['addr'])
if 'count_broker' in data[0]:
format_houses['count_broker'] = data[0]['count_broker']
if data[0].get('type_id'):
format_houses['type_id'] = data[0]['type_id']
if data[0].get('created'):
format_houses['created'] = int(data[0]['created'])
if data[0].get('updated'):
format_houses['updated'] = int(data[0]['updated'])
if data[0].get('keyword'):
format_houses['keyword'] = str(data[0]['keyword'])
if data[0].get('logogram'):
format_houses['logogram'] = str(data[0]['logogram'])
if data[0].get('other_id'):
format_houses['other_id'] = int(data[0]['other_id'])
if data[0].get('pinyin'):
format_houses['pinyin'] = str(data[0]['pinyin'])
if data[0].get('type_name'):
format_houses['type_name'] = str(data[0]['type_name'])
return format_houses
if __name__ == '__main__':
print(BaseUtils.getFpy("数据",delimiter=' '))
def add_cityarea2(self, city, cityarea_id, new_cityarea2):
add_dict = {}
new_cityarea2_dict = {}
current_time = BaseUtils.getIntTime()
new_cityarea2_dict['_id'] = self.max_id.getMaxId(type="cityarea2",
city=city)
new_cityarea2_dict['loc'] = new_cityarea2.get("loc", "")
new_cityarea2_dict['name'] = new_cityarea2.get("name", "").strip()
new_cityarea2_dict['area2_pinyin'] = BaseUtils.getFpy(
new_cityarea2_dict['name'])
new_cityarea2_dict['polygons'] = new_cityarea2.get("polygons", "")
new_cityarea2_dict['statu'] = 1
new_cityarea2_dict['created'] = current_time
new_cityarea2_dict['updated'] = current_time
is_find = self.dao.find_one(
city=city, filter={"name": {
"$regex": new_cityarea2_dict['name']
}})
if is_find:
cityarea2_id = is_find.get("_id", "")
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "商圈id%s存在相似商圈" % (cityarea2_id)
}
return add_dict
else:
insert_id = self.dao.insert_one(city=city,
datas=new_cityarea2_dict)
if not insert_id:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "商圈添加失败"
}
return add_dict
find_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": cityarea_id})
if not find_cityarea:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "添加到城区失败"
}
return add_dict
list_cityarea2 = find_cityarea[0].get("list_cityarea2", "")
list_cityarea2[
new_cityarea2_dict['name']] = new_cityarea2_dict['_id']
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 1:
# 添加商圈联想词
#guess_status = self.guessword.add_guessword(city=city, guessword_list=[new_cityarea2_dict['name']], type_id=2)
add_dict['master_info'] = {
"status": 1,
"cityarea2_id": insert_id,
"remark": "添加商圈并同步到城区表成功"
}
else:
add_dict['master_info'] = {
"status": 0,
"cityarea2_id": insert_id,
"remark": "添加商圈成功,同步到城区表失败"
}
return add_dict
def merge_cityarea(self, city, master_cityarea, slave_cityarea):
try:
# 商圈表 -----> 商圈的归并
# 支持跨城区合并 同城区相当于删除商圈
current_time = BaseUtils.getIntTime()
delete_dict = {}
update_dict = {}
error_dict = {}
mater_cityarea_id = master_cityarea.get("cityarea_id", "")
mater_cityarea_name = master_cityarea.get("cityarea_name", "")
mater_cityarea_py = BaseUtils.getFpy(mater_cityarea_name,
delimiter='')
mater_cityarea2_id = master_cityarea.get("cityarea2_id", "")
mater_cityarea2_name = master_cityarea.get("cityarea2_name", "")
mater_cityarea2_py = BaseUtils.getFpy(mater_cityarea2_name,
delimiter='')
slave_cityarea_id = slave_cityarea.get("cityarea_id", "")
slave_cityarea_name = slave_cityarea.get("cityarea_name", "")
slave_cityarea2_id = slave_cityarea.get("cityarea2_id", "")
slave_cityarea2_name = slave_cityarea.get("cityarea2_name", "")
slave_cityarea2_py = BaseUtils.getFpy(slave_cityarea2_name,
delimiter='')
master_cityarea2 = [{
"cityarea2_id": mater_cityarea2_id,
"cityarea2_name": mater_cityarea2_name,
"cityarea2_py": mater_cityarea2_py
}]
master_cityarea_data = self.cityarea_service.find(
city=city, filter={'_id': mater_cityarea_id})
slave_cityarea_data = self.cityarea_service.find(
city=city, filter={'_id': slave_cityarea_id})
if not master_cityarea_data:
error_dict = {"status": 1, "remark": "主城区的城区找不到数据"}
return {
"update_dict": update_dict,
"delete_dict": delete_dict,
"error_dict": error_dict
}
if not slave_cityarea_data:
error_dict = {"status": 1, "remark": "从城区的城区找不到数据"}
return {
"update_dict": update_dict,
"delete_dict": delete_dict,
"error_dict": error_dict
}
find_cityarea2 = self.find(city=city,
filter={'_id': slave_cityarea2_id})
if find_cityarea2:
self.cityarea2_recycle.insert_one(city=city,
datas=find_cityarea2)
delete_status = self.delete_by_id(city=city,
cityarea2_id=slave_cityarea2_id)
if delete_status['ok'] == 1:
delete_dict = {
"id": slave_cityarea2_id,
"status": 1,
"remark": "商圈删除成功"
}
else:
delete_dict = {
"id": slave_cityarea2_id,
"status": 0,
"remark": "商圈删除失败"
}
slave_list_cityarea2 = slave_cityarea_data[0].get(
"list_cityarea2", "")
for k in list(slave_list_cityarea2.keys()):
if slave_list_cityarea2[k] == slave_cityarea2_id:
slave_list_cityarea2.pop(k)
continue
datas = {
"$set": {
"list_cityarea2": slave_list_cityarea2,
"updated": current_time
}
}
slave_cityarea_update_action = self.cityarea_service.update_filter(
filter={'_id': slave_cityarea_id}, city=city, datas=datas)
if slave_cityarea_update_action['nModified'] == 1:
update_dict = {
"id": slave_cityarea_id,
"status": 1,
"remark": "更新slave城区表成功"
}
else:
update_dict = {
"id": slave_cityarea_id,
"status": 0,
"remark": "更新slave城区表失败"
}
# 更改小区的城区商圈
master_cityarea_info = {
"cityarea_id": mater_cityarea_id,
"cityarea_name": mater_cityarea_name,
"area_pinyin": mater_cityarea_py,
"cityarea2_id": mater_cityarea2_id,
"cityarea2_name": mater_cityarea2_name,
"area2_pinyin": mater_cityarea2_py
}
slave_cityarea_info = {
"cityarea2_id": slave_cityarea2_id,
"cityarea2_name": slave_cityarea2_name,
"area2_pinyin": slave_cityarea2_py
}
borough_online_status = self.change_borough_online_cityarea(
master_cityarea=master_cityarea_info,
slave_cityarea=slave_cityarea_info,
city=city)
if not borough_online_status:
error_dict = {"status": 1, "remark": "borough_online的城区商圈更改失败"}
# 更改小区缓存
self.update_area_cache(city=city,
cityarea2={"id": slave_cityarea2_id})
# 小区列表es拼装的数据
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
slave_cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline":
"ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name;ctx._source.cityarea2 = params.cityarea2",
"params": {
"cityarea_id": mater_cityarea_id,
"cityarea_name": mater_cityarea_name,
"cityarea2": master_cityarea2
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# 房源的批量修改
house_index = "active_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
update_info = {
"delete_dict": delete_dict,
"update_dict": update_dict,
"error_dict": error_dict,
"borough_es": borough_list_status,
"house_es": house_list_status
}
return update_info
except Exception as e:
print("update_fail", e)
return False
def mod_cityarea2_relation(self, city, before_cityarea_info,
after_cityarea_info):
current_time = BaseUtils.getIntTime()
update_dict = {}
before_cityarea_id = before_cityarea_info.get("cityarea_id", "")
before_cityarea2_id = before_cityarea_info.get("cityarea2_id", "")
before_cityarea2_name = before_cityarea_info.get("cityarea2_name", "")
before_cityarea2_py = BaseUtils.getFpy(before_cityarea2_name)
after_cityarea_id = after_cityarea_info.get("cityarea_id", "")
after_cityarea_name = after_cityarea_info.get("cityarea_name", "")
after_cityarea_py = BaseUtils.getFpy(after_cityarea_name)
if before_cityarea_id:
find_before_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": before_cityarea_id})
if not find_before_cityarea:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "城区id为%s的城区找不到" % (before_cityarea_id)
}
return update_dict
list_cityarea2 = find_before_cityarea.get("list_cityarea2", "")
for k, v in list_cityarea2.items():
if k == before_cityarea2_name:
list_cityarea2.pop(k)
break
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': before_cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 0:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "城区更新失败"
}
return update_dict
find_after_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": after_cityarea_id})
after_list_cityarea2 = find_after_cityarea.get(
"list_cityarea2", "")
after_list_cityarea2[before_cityarea2_name] = before_cityarea2_id
after_cityarea_update_action = self.cityarea_dao.update(
filter={'_id': after_cityarea_id},
datas={
"$set": {
"list_cityarea2": after_list_cityarea2,
"updated": current_time
}
})
if after_cityarea_update_action['nModified'] == 1:
# 二手房小区缓存
# cityarea = {'id': after_cityarea_id}
# self.borough_online.update_area_cache(cityarea=cityarea)
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
before_cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline":
"ctx._source.cityarea_id = params.cityarea_id;ctx._source.cityarea_name = params.cityarea_name",
"params": {
"cityarea_id": after_cityarea_id,
"cityarea_name": after_cityarea_name
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# borough_list_status = True
# 房源的批量修改
house_index = "online_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
# 改变商圈所属城区
filter = {
"cityarea.cityarea2.cityarea2_id": before_cityarea2_id
}
master_cityarea_info = {
"cityarea": {
"cityarea_name": after_cityarea_name,
"cityarea_id": after_cityarea_id,
"area_pinyin": after_cityarea_py,
"cityarea_py": after_cityarea_py
},
"cityarea2": [{
"cityarea2_name": before_cityarea2_name,
"cityarea2_py": before_cityarea2_py,
"cityarea2_id": before_cityarea2_id,
"area2_pinyin": before_cityarea2_py
}]
}
datas = {
"$set": {
"cityarea": master_cityarea_info,
"borough_ctype": 8,
"updated": current_time
}
}
update_action = self.boroughline_dao.updateAll(city=city,
filter=filter,
datas=datas)
# 更新缓存
cityarea2 = {'id': before_cityarea2_id}
self.update_area_cache(city=city, cityarea2=cityarea2)
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": before_cityarea_id,
"remark": "更新城区表成功"
}
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": before_cityarea_id,
"remark": "更新城区表失败"
}
return update_dict
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "城区id传的有误"
}
return update_dict
def mod_cityarea2_name(self, city, cityarea_id, cityarea2_info):
current_time = BaseUtils.getIntTime()
cityarea2_id = cityarea2_info.get("cityarea2_id", "")
name = cityarea2_info.get("new_name", "")
cityarea2_py = BaseUtils.getFpy(name)
update_dict = {}
if cityarea2_id:
cityarea2_action = self.dao.update(
city=city,
filter={'_id': cityarea2_id},
datas={"$set": {
"name": name,
"updated": current_time
}})
if cityarea2_action['nModified'] == 0:
update_dict["master_info"] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "商圈id为%s的更新失败" % (cityarea2_id)
}
return update_dict
find_cityarea = self.cityarea_dao.find_one(
city=city, filter={"_id": cityarea_id})
if not find_cityarea:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "城区id为%s的城区找不到" % (cityarea_id)
}
return update_dict
list_cityarea2 = find_cityarea.get("list_cityarea2", "")
for k, v in list_cityarea2.items():
if v == cityarea2_id:
list_cityarea2.pop(k)
list_cityarea2[name] = v
break
cityarea_update_action = self.cityarea_dao.update(
city=city,
filter={'_id': cityarea_id},
datas={
"$set": {
"list_cityarea2": list_cityarea2,
"updated": current_time
}
})
cityarea2_update_action = self.dao.update(
city=city,
filter={'_id': cityarea2_id},
datas={
"$set": {
"name": name,
"area2_pinyin": cityarea2_py,
"updated": current_time
}
})
if cityarea_update_action['nModified'] == 1:
# 小区的缓存修改
cityarea2 = {'id': cityarea2_id}
# 修改商圈的小区
filter = {"cityarea.cityarea2.cityarea2_id": cityarea2_id}
cityarea_name = find_cityarea.get("name", "")
cityarea_py = find_cityarea.get("area_pinyin", "")
cityarea_id = find_cityarea.get("_id", "")
master_cityarea_info = {
"cityarea": {
"cityarea_name": cityarea_name,
"cityarea_id": cityarea_id,
"area_pinyin": cityarea_py,
"cityarea_py": cityarea_py
},
"cityarea2": [{
"cityarea2_name": name,
"cityarea2_py": cityarea2_py,
"cityarea2_id": cityarea2_id,
"area2_pinyin": cityarea2_py
}]
}
find_borough_by_cityarea2 = self.boroughline_dao.find_one(
city=city,
filter={"cityarea.cityarea2.cityarea2_id": cityarea2_id})
if not find_borough_by_cityarea2:
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈并同步到城区表成功-该商圈下无小区"
}
return update_dict
datas = {
"$set": {
"cityarea": master_cityarea_info,
"borough_ctype": 8,
"updated": current_time
}
}
update_action = self.boroughline_dao.updateAll(city=city,
filter=filter,
datas=datas)
self.update_area_cache(city=city, cityarea2=cityarea2)
# 小区es列表页
master_cityarea2 = [{
"cityarea2_id": cityarea2_id,
"cityarea2_name": name,
"cityarea2_py": cityarea2_py
}]
updateBody = {
"query": {
"bool": {
"must": [{
"nested": {
"path": "cityarea2",
"query": {
"bool": {
"must": [{
"term": {
"cityarea2.cityarea2_id":
cityarea2_id
}
}]
}
}
}
}],
}
},
"script": {
"inline": "ctx._source.cityarea2 = params.cityarea2",
"params": {
"cityarea2": master_cityarea2
},
"lang": "painless"
}
}
if city == 'bj':
index_doc = "spider"
else:
index_doc = "spider_" + city
borough_index = "online_" + city + "_borough"
borough_list_status = self.essearch.update_by_query(
index=borough_index, doc_type=index_doc, body=updateBody)
# borough_list_status = True
# 房源的批量修改
house_index = "online_" + city + "_house_sell"
house_list_status = self.essearch.update_by_query(
index=house_index, doc_type=index_doc, body=updateBody)
# house_list_status = True
# 商圈联想词 变更
# guess_status = self.guessword.modify_borough_name(keyword_dict={origin_name: master_name})
# publish信息
message = {
"city": city,
"operate": "update",
"type": "cityarea2",
"pms": {
"master": {
"id": cityarea2_id,
"name": name
}
}
}
push_status = self.publish_message("dm_info_change", message)
update_dict['master_info'] = {
"status": 1,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈并同步到城区表成功"
}
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": cityarea2_id,
"remark": "更新商圈成功,同步到城区表失败"
}
return update_dict
else:
update_dict['master_info'] = {
"status": 0,
"cityarea2_id": False,
"remark": "商圈id传的有误"
}
return update_dict