def do_fetch(city: City, communities_queue: Queue):
# 抓取, 直到没东西可抓
# http://docs.sqlalchemy.org/en/latest/orm/session_basics.html#is-the-session-thread-safe
db_session = Session()
while not communities_queue.empty():
a_community = communities_queue.get()
try:
fetch_page(city, a_community.id)
a_community.page_fetched_at = datetime.now()
except Exception as e:
_counts['failed'] += 1
logging.error(
f'# 抓取失败, community_id={a_community.id}, message="{e}"')
else:
db_session.add(a_community)
db_session.commit()
_counts['completed'] += 1
if _counts['completed'] % 10 == 0:
count_remaining = _counts["total"] - _counts["completed"]
logging.info(
f'进度={_counts["completed"]}/{_counts["total"]}, 剩余={count_remaining}'
)
communities_queue.task_done()
db_session.close()
def update_city(city_id):
"""
初始化/更新城市信息
"""
logging.info('初始化/更新城市信息... city_id={}'.format(city_id))
city_info = get_city_info(city_id)
city = City(city_info)
db_session = Session()
db_session.merge(city)
for district_info in city_info['district']:
district = District(city.id, district_info)
logging.info(
'城市={}, 区域={}, 商圈数={}'.format(
city.name, district.name, district.biz_circles_count
)
)
DISTRICT_MAP[district.name] = district.id
db_session.merge(district)
for biz_circle_info in district_info['bizcircle']:
biz_circle = db_session.query(BizCircle).filter(
BizCircle.id == int(biz_circle_info['bizcircle_id'])
).first()
if biz_circle:
# 记录已存在，可能需要更新 district_id
if district.id not in biz_circle.district_id:
# biz_circle.district_id.append()、district_id += 等方式都不能更新表
biz_circle.district_id = biz_circle.district_id + [
district.id
]
else:
biz_circle = BizCircle(city.id, district.id, biz_circle_info)
db_session.add(biz_circle)
db_session.commit()
db_session.close()
logging.info('初始化/更新城市信息结束.')
