def get_citylist_to_mongo(citylist_xml):
save_to_db = DB_Utils()
soup = BeautifulSoup(citylist_xml, 'lxml')
#print(soup.prettify())
city_info = soup.select('d')
for i in city_info:
data = {
'province': i.get('d4'),
'city_name': i.get('d2'),
'city_code': i.get('d1'),
'city_piny': i.get('d3')
}
if i.get('d4') == '韩国':
break
save_to_db.save_one_to_mongo('citylist', data)
print('正在将城市列表存储到mongodb: #', i.get('d4'), i.get('d2'))
def get_1day_weather_data(web_data):
soup = BeautifulSoup(web_data, 'lxml')
html = etree.HTML(web_data)
# print(html)
# print(soup)
try:
datas = html.xpath('//script')
# print(datas)
abc = soup.select('.ctop .crumbs')
db_utils = DB_Utils()
for data in datas:
# print(data.text)
# print(data.text())
if 'hour3data' in str(data.text):
print(data.text)
data1 = {}
# print('###########', data.text)
data1 = data.text.split('=')[-1]
# print('aaaa',data1)
data2 = json.loads(data1)
# print(data2['1d'])
print(abc[0].text.replace('\n', '').replace(' ', ''), '实时天气：')
city = abc[0].text.replace('\n', '').replace(' ', '')
city_info = city + '实时天气：\n'
data4 = []
for i in data2['1d']:
# print(i.split(','))
#print(i.split(',')[0], end=' ')
# print(i.split(',')[1])
#print(i.split(',')[2], end=' ')
#print(i.split(',')[3], end=' ')
#print(i.split(',')[4], end=' ')
#print(i.split(',')[5])
data3 = [
i.split(',')[0],
i.split(',')[2],
i.split(',')[3],
i.split(',')[4],
i.split(',')[5]
]
print(data3)
data4.append(data3)
weather_data = {'city': city, '24hour_weather': data4}
db_utils.save_one_to_mongo('24hour_weather', weather_data)
print('save to mongo')
except Exception as e:
print(e)
def get_1day_weather_data(web_data):
soup = BeautifulSoup(web_data, 'lxml')
datas = soup.select('script')
abc = soup.select('.ctop .crumbs')
db_utils = DB_Utils()
for data in datas:
if 'hour3data' in str(data):
data1 = {}
data1 = data.text.split('=')[-1]
# print(data1)
data2 = json.loads(data1)
# print(data2['1d'])
print(abc[0].text.replace('\n', '').replace(' ', ''), '实时天气：')
city = abc[0].text.replace('\n', '').replace(' ', '')
city_info = city + '实时天气：\n'
data4 = []
for i in data2['1d']:
# print(i.split(','))
#print(i.split(',')[0], end=' ')
# print(i.split(',')[1])
#print(i.split(',')[2], end=' ')
#print(i.split(',')[3], end=' ')
#print(i.split(',')[4], end=' ')
#print(i.split(',')[5])
data3 = [
i.split(',')[0],
i.split(',')[2],
i.split(',')[3],
i.split(',')[4],
i.split(',')[5]
]
print(data3)
data4.append(data3)
weather_data = {'city': city, '24hour_weather': data4}
db_utils.save_one_to_mongo('24hour_weather', weather_data)
print('save to mongo')
def get_1day_weather_data(province, city_name, city_code, city_piny, web_data):
try:
soup = BeautifulSoup(web_data, 'lxml')
datas = soup.select('script')
address = soup.select('.ctop .crumbs')
day_wea = soup.select('.t .clearfix .wea')[0].text
night_wea = soup.select('.t .clearfix .wea')[1].text
day_tem = soup.select('.t .clearfix .tem')[0].text.replace('\n', '')
night_tem = soup.select('.t .clearfix .tem')[1].text.replace('\n', '')
day_win = soup.select('.t .clearfix .win span')[0].get(
'title') + soup.select('.t .clearfix .win span')[0].text
night_win = soup.select('.t .clearfix .win span')[1].get(
'title') + soup.select('.t .clearfix .win span')[1].text
sun_up = soup.select('.t .clearfix .sunUp')[0].text.replace('\n', '')
sun_down = soup.select('.t .clearfix .sunDown')[0].text.replace(
'\n', '')
day_wea_list = [day_wea, day_tem, day_win, sun_up]
night_wea_list = [night_wea, night_tem, night_win, sun_down]
print(day_wea_list)
print(night_wea_list)
db_utils = DB_Utils()
for data in datas:
if 'hour3data' in str(data):
data1 = {}
data1 = data.text.split('=')[-1]
# print(data1)
data2 = json.loads(data1)
# print(data2['1d'])
print(address[0].text.replace('\n', '').replace(' ', ''),
'实时天气：')
city = address[0].text.replace('\n', '').replace(' ', '')
# city_info = city + '实时天气：\n'
data4 = []
for i in data2['1d']:
data3 = [
i.split(',')[0],
i.split(',')[2],
i.split(',')[3],
i.split(',')[4],
i.split(',')[5]
]
print(data3)
data4.append(data3)
weather_data_of_1day = {
'province': province,
'city_name': city_name,
'city_code': city_code,
'city_piny': city_piny,
'city': city,
'day_wea': day_wea_list,
'night_wea': night_wea_list,
'24hour_weather': data4
}
db_utils.save_one_to_mongo('1day_weather_data',
weather_data_of_1day)
print('save to mongo')
except:
print(province, city_name, city_code)
db_utils1 = DB_Utils()
error_city = {
'province': province,
'city_name': city_name,
'city_code': city_code,
}
db_utils1.save_one_to_mongo('error', error_city)
print('save error to mongo')
pass