def do_fetch(city: City, communities_queue: Queue):
# 抓取, 直到没东西可抓
# http://docs.sqlalchemy.org/en/latest/orm/session_basics.html#is-the-session-thread-safe
db_session = Session()
while not communities_queue.empty():
a_community = communities_queue.get()
try:
fetch_page(city, a_community.id)
a_community.page_fetched_at = datetime.now()
except Exception as e:
_counts['failed'] += 1
logging.error(
f'# 抓取失败, community_id={a_community.id}, message="{e}"')
else:
db_session.add(a_community)
db_session.commit()
_counts['completed'] += 1
if _counts['completed'] % 10 == 0:
count_remaining = _counts["total"] - _counts["completed"]
logging.info(
f'进度={_counts["completed"]}/{_counts["total"]}, 剩余={count_remaining}'
)
communities_queue.task_done()
db_session.close()
def update_city(city_id):
"""
初始化/更新城市信息
"""
logging.info('初始化/更新城市信息... city_id={}'.format(city_id))
city_info = get_city_info(city_id)
city = City(city_info)
db_session = Session()
db_session.merge(city)
for district_info in city_info['district']:
district = District(city.id, district_info)
logging.info(
'城市={}, 区域={}, 商圈数={}'.format(
city.name, district.name, district.biz_circles_count
)
)
DISTRICT_MAP[district.name] = district.id
db_session.merge(district)
for biz_circle_info in district_info['bizcircle']:
biz_circle = db_session.query(BizCircle).filter(
BizCircle.id == int(biz_circle_info['bizcircle_id'])
).first()
if biz_circle:
# 记录已存在，可能需要更新 district_id
if district.id not in biz_circle.district_id:
# biz_circle.district_id.append()、district_id += 等方式都不能更新表
biz_circle.district_id = biz_circle.district_id + [
district.id
]
else:
biz_circle = BizCircle(city.id, district.id, biz_circle_info)
db_session.add(biz_circle)
db_session.commit()
db_session.close()
logging.info('初始化/更新城市信息结束.')
def parse_all_communities(city_id):
db_session = Session()
communities = db_session.query(Community).filter(
Community.city_id == city_id, Community.detail == None,
Community.page_fetched_at != None).all()
total_count = len(communities)
logging.info(f'city_id={city_id}, 待分析={total_count}')
for i, a_community in enumerate(communities):
detail = parse_community_detail(a_community.id)
if detail:
a_community.detail = detail
if (i + 1) % 100 == 0 or (i == total_count - 1):
logging.info(f'进度={i + 1}/{total_count}, 剩余={total_count - i - 1}')
db_session.commit()
logging.info('已全部分析完成.')
db_session.close()