def save_buslines(city,
name_list,
left_bottom,
right_top,
dbName,
collection,
user='******',
port=27017,
auth_name=None,
pwd=None,
start_num=0):
geoobj = str(left_bottom[0]) + '|' + str(left_bottom[1]) + '|' + str(
right_top[0]) + '|' + str(right_top[1])
params['city'] = city
params['geoobj'] = geoobj
params['pagenum'] = '1'
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
for i in range(start_num, len(name_list)):
line_name = name_list[i].encode('utf-8')
line_name = line_name.replace('(停运)', '')
params['keywords'] = line_name
line_url = 'https://www.amap.com/service/poiInfo?'
response = requests.get(url=line_url, params=params).json()
if response[u'status'] == u'1':
col.insert_one(response)
print '第%d条线路%s数据采集完成，该线路包含%d条数据，一共%d条线路！' % (
i, line_name, int(
response['data']['busline_count']), len(name_list) - 1)
else:
break
def get_buslines(dbName,
collection,
encode='utf-8',
user='******',
port=27017,
auth_name=None,
pwd=None):
# user = '******'; port = 27017; dbName = 'bus_data'; collection = 'lines'
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
lines_list = col.find()
keys = list(lines_list[0]['data']['busline_list'][0].keys())
keys.remove('stations')
buslines = []
for lines in lines_list:
if lines['data']['busline_count'] != 0:
busline_list = lines['data']['busline_list']
for busline in busline_list:
del busline['stations']
buslines.append(busline)
buslines_df = pd.DataFrame(buslines)
keys_new = ['key_name', 'name']
for key in keys_new:
keys.remove(key)
keys_new.extend(keys)
buslines_df = buslines_df[keys_new]
buslines_df.columns = keys_new
# 编码转换
buslines_df = cc.df_code_conver(buslines_df, encode)
return buslines_df
def get_busline_names(dbName,
collection,
user='******',
port=27017,
auth_name=None,
pwd=None):
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
line_list = col.find()
name_list = []
for i in range(line_list.count()):
line = line_list[i]['data']['poi_list']
for j in range(len(line)):
name_list.extend(line[j]['address'].split(';'))
name_list = list(set(name_list))
name_list.sort()
return name_list
def get_residential(dbName,
collection,
encode='utf-8',
user='******',
port=27017,
auth_name=None,
pwd=None):
# residential_df:点df数据
# residential_area_df:区域df数据
keys = [
'name', 'disp_name', 'id', 'adcode', 'cityname', 'areacode', 'address',
'typecode', 'shape_region', 'longitude', 'latitude'
]
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
all_page = col.find()
residential_list = []
for i in range(all_page.count()):
page = all_page[i]
poi_list = page['data']['poi_list']
for j in range(len(poi_list)):
poi = poi_list[j]
temp_list = []
for key in keys:
temp_list.append(poi[key])
try:
polyline = poi['domain_list'][-1]['value']
polyline_convent = polyline.replace('_', ';')
except:
polyline_convent = 'NaN'
temp_list.append(polyline_convent)
residential_list.append(temp_list)
keys.append('polyline')
residential_df = pd.DataFrame(residential_list)
residential_df.columns = keys
residential_df.rename(columns={
'longitude': 'x_coord',
'latitude': 'y_coord'
},
inplace=True)
residential_df = residential_df.drop_duplicates()
residential_area_df = residential_df[residential_df['polyline'] != 'NaN']
residential_area_df.index = range(residential_area_df.shape[0])
# 转码
residential_df = cc.df_code_conver(residential_df, encode)
residential_area_df = cc.df_code_conver(residential_area_df, encode)
return residential_df, residential_area_df
def save_polygon(polygon_df,
dbName,
collection,
user='******',
port=27017,
auth_name=None,
pwd=None,
start_num=0):
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
request_url = 'https://restapi.amap.com/v3/config/district?'
params['subdistrict'] = '0'
polygon_num = polygon_df.shape[0]
for i in range(start_num, polygon_num):
keywords = polygon_df['adcode'][i]
params['keywords'] = keywords
response = requests.get(url=request_url, params=params).json()
col.insert_one(response)
print '第%d个区域入库成功，一共%d个区域！' % (i, polygon_num - 1)
def get_polygon_df(polygon_df,
dbName,
collection,
encode='utf-8',
user='******',
port=27017,
auth_name=None,
pwd=None):
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
response_list = col.find()
polygon_df['polyline'] = 'NaN'
polygon_num = response_list.count()
for i in range(polygon_num):
response = response_list[i]
polyline = response['districts'][0]['polyline']
polygon_df['polyline'][i] = polyline
print '第%d个区域数据获取成功，一共%d个区域！' % (i, polygon_num - 1)
polygon_df = cc.df_code_conver(polygon_df, encode)
return polygon_df
def get_stations(dbName,
collection,
encode='utf-8',
user='******',
port=27017,
auth_name=None,
pwd=None):
keys = [
'name', 'line_name', 'company', 'poiid1', 'poiid2', 'start_time',
'end_time', 'spell', 'status', 'trans_flag', 'code', 'station_id',
'station_num'
]
col = db.MongoDB_login(dbName, collection, user, port, auth_name, pwd)
lines_list = col.find()
stations_list = []
for i in range(lines_list.count()):
if lines_list[i]['data']['busline_count'] != 0:
busline_list = lines_list[i]['data']['busline_list']
for j in range(len(busline_list)):
line_name = busline_list[j]['name']
company = busline_list[j]['company']
line_stations = busline_list[j]['stations']
for k in range(len(line_stations)):
station = line_stations[k]
station['line_name'] = line_name
station['company'] = company
station_list = []
for key in keys:
station_list.append(station[key])
station_list.extend(
map(float, station['xy_coords'].split(';')))
stations_list.append(station_list)
print '%d条线路转换完成！' % (i + 1)
keys.extend(['x_coord', 'y_coord'])
stations_df = pd.DataFrame(stations_list)
stations_df.columns = keys
stations_df = cc.df_code_conver(stations_df, encode)
return stations_df