def fetch_list_pages(db, city_index):
logging.info('抓取区域列表')
areas_obj = db.query(Area).filter(Area.city_index == city_index,
Area.insert == 0).all()
for area in areas_obj:
page = 1
while True:
logging.info(f'正在抓取第{page}页')
api_url = f'http://m.58.com/xiaoquweb/getXiaoquList/?city={area.area_index}&page={page}'
res = requests.get(api_url,
headers=config.config.headers,
timeout=5)
res.raise_for_status()
res_json_obj = json.loads(res.content)
if res_json_obj['code'] == "0":
page_dto = res_json_obj['data']['pageDTO']
info_list = res_json_obj['data']['infoList']
if page_dto['currentPageNo'] > page_dto['totalPage']:
break
for community in info_list:
add_community = Community(city_index, area.area_index,
community)
db.add(add_community)
logging.info('新增1条小区信息')
page += 1
area.insert = 1
db.commit()
logging.info('抓取区域列表数据完毕')
db.close()
def add_community(self):
'''
Create list_of_community built of Community objects. Except Vallue Error when data isn't integer.
'''
first_index_community = 0
for row in Territory.list_of_territory:
try:
index_community = int(row[2])
if index_community != first_index_community and self.number == int(
row[1]):
community = Community(row[4], row[5], index_community)
self.list_of_community.append(community)
first_index_community = index_community
except ValueError:
continue
def load_file():
if not os.path.isfile('static/malopolska.csv'):
raise FileNotFoundError
else:
with open('static/malopolska.csv', 'r') as csvfile:
for row in csvfile:
row = row.strip()
row = row.split('\t')
if row[0].isdigit():
data = (row[4], row[5])
if not row[1]:
voivodeship = Voivodeship(row[0], *data)
elif not row[2]:
county = County(row[1], *data)
voivodeship.add_county(county)
elif row[3]:
community = Community(row[2], row[3], *data)
county.add_community(community)
