def parse_doctor_info_detail(self, response):
self.logger.info('>>>>>>正在抓取{}:医生详细信息>>>>>>'.format(
self.hospital_name))
loader = response.meta['loader']
dept_name1 = custom_remove_tags(''.join(response.meta['dept_name']))
doctor_level2 = response.xpath(
'//div[@class="FrontProducts_detail02-'
'1482202997396_htmlbreak"]/p[1]/strong/text()').extract_first('')
doctor_level1 = response.meta['doctor_level']
dept_name2 = response.xpath(
'//div[@id="FrontPublic_breadCrumb01-1482202386120"]/div/'
'a[last()]/text()').extract_first('').replace('专家', '').replace(
'类', '科')
dept_name = re.sub(r'中医医师|中西医医师', '中医科',
dept_name1) if dept_name1 else dept_name2
doctor_level = custom_remove_tags(
''.join(doctor_level1)) if doctor_level1 else doctor_level2
doctor_intro = response.xpath(
'//div[@class="FrontProducts_detail02-'
'1482202997396_htmlbreak"]/p[2]').extract_first('')
loader.add_value('dept_name', dept_name,
MapCompose(custom_remove_tags, filter_info3))
loader.add_value('doctor_level', doctor_level,
MapCompose(filter_info4, custom_remove_tags))
loader.add_value('doctor_intro', doctor_intro,
MapCompose(remove_tags, custom_remove_tags))
loader.add_value('update_time', now_day())
doctor_item = loader.load_item()
yield doctor_item
def parse_dept_info(self, response):
dep_type = response.meta['dep_type']
self.logger.info('正在抓取[{}]科室信息'.format(custom_remove_tags(dep_type)))
all_dept_names = response.xpath('//div[@class="pic"]')
if all_dept_names:
# 一级科室有二级科室
for each_dept_name in all_dept_names:
dept_detail_link = each_dept_name.xpath(
'a/@href').extract_first('')
if dept_detail_link:
dept_detail_link = urljoin(self.host, dept_detail_link)
request = Request(dept_detail_link,
headers=self.headers,
callback=self.parse_dept_detail,
meta={'dep_type': dep_type})
request.meta['Referer'] = response.url
yield request
else:
# 一级科室没有二级科室
loader = PxfybjyLoader(item=HospitalDepItem(), response=response)
loader.add_value('dept_type', dep_type)
loader.add_value('hospital_name', self.hospital_name)
loader.add_value('update_time', now_day())
hospital_dep_item = loader.load_item()
yield hospital_dep_item
def parse_doctor_info_detail(self, response):
self.logger.info('>>>>>>正在抓取:医生详细信息>>>>>>')
try:
diagnosis_fee = response.meta.get('diagnosis_fee')
doctor_info = custom_remove_tags(
remove_tags(''.join(
response.xpath('//td[@class="bk '
'titletxt11"]').extract())))
doctor_intro1 = get_hospital_info(doctor_info, '个人简介：', '荣誉集锦：')
doctor_intro2 = get_hospital_info(doctor_info, '个人简介：', '出诊时间：')
doctor_intro = doctor_intro2 if doctor_intro2 else doctor_intro1
loader = CommonLoader2(item=DoctorInfoItem(), response=response)
loader.add_xpath(
'doctor_name',
'//table[@id="m_jkzs"]/tr/td[1]/a[last()]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'dept_name',
'//table[@id="m_jkzs"]/tr/td[1]/a[last()-1]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'hospital_name',
'//table[@id="m_jkzs"]/tr/td[1]/a[last()-2]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'doctor_level',
'//span[@class="selecttxt"][contains(text(),"医师") or contains(text(),"专家")]/text()',
MapCompose(custom_remove_tags))
loader.add_value('doctor_intro', doctor_intro,
MapCompose(custom_remove_tags))
loader.add_xpath(
'doctor_goodAt',
'//span[@class="titletxt11"]/b[contains(text(),"擅长")]/ancestor::span[1]/text()',
MapCompose(remove_tags, custom_remove_tags))
loader.add_value(
'diagnosis_amt', diagnosis_fee,
MapCompose(remove_tags, custom_remove_tags, get_number))
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('update_time', now_day())
doctor_item = loader.load_item()
yield doctor_item
except Exception as e:
self.logger.error('在抓取医生详细信息的过程中出错了,原因是：{}'.format(repr(e)))
def parse_doctor_info_detail(self, response):
hospital_name = response.meta.get('hospital_name')
dept_name = response.meta.get('dept_name')
self.logger.info('>>>>>>正在抓取[{}]医生详细信息>>>>>>'.format(hospital_name))
try:
# 获取医生信息
loader = CommonLoader2(item=DoctorInfoItem(), response=response)
loader.add_xpath(
'doctor_name', '//td/b[contains(text(),"姓名")]/ancestor::td[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_value('dept_name', dept_name,
MapCompose(custom_remove_tags))
loader.add_xpath(
'hospital_name',
'//div[@class="page_position"]/a[last()-1]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'sex', '//td/b[contains(text(),"性别")]/ancestor::td[1]',
MapCompose(remove_tags, custom_remove_tags, match_special,
clean_info2))
loader.add_xpath(
'doctor_level',
'//td/b[contains(text(),"职称")]/ancestor::td[1]',
MapCompose(remove_tags, custom_remove_tags, match_special,
clean_info2))
loader.add_xpath(
'doctor_intro',
'//td/b[contains(text(),"医生简介")]/ancestor::td[1]',
MapCompose(remove_tags, custom_remove_tags, clean_info2))
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('crawled_url', response.url)
loader.add_value('update_time', now_day())
doctor_item = loader.load_item()
yield doctor_item
# 获取医生排班信息
self.logger.info(
'>>>>>>正在抓取[{}]医生排班信息>>>>>>'.format(hospital_name))
has_doctor_scheduling = response.xpath(
'//td/div[@class="doctor-work"]')
if has_doctor_scheduling:
doctor_scheduling_tr = response.xpath(
'//table[@class="workTable"]/tbody/tr')
all_scheduling_date = response.xpath(
'//table[@class="workTable"]/thead/tr/td[position()>1]'
).extract()
scheduling_date_list = custom_remove_tags(
remove_tags(','.join(all_scheduling_date))).split(',')
for each_td in doctor_scheduling_tr:
scheduling_time = each_td.xpath(
'td[1]/text()').extract_first('')
scheduling_info = each_td.xpath('td[position()>1]')
for index, each_s_i in enumerate(scheduling_info):
has_scheduling = each_s_i.xpath('div')
if has_scheduling:
each_scheduling_date = scheduling_date_list[index][
0:3]
reg_info = '{0}{1}'.format(each_scheduling_date,
scheduling_time)
reg_loader = CommonLoader2(
item=DoctorRegInfoItem(), response=response)
reg_loader.add_xpath(
'doctor_name',
'//td/b[contains(text(),"姓名")]/ancestor::td[1]',
MapCompose(remove_tags, custom_remove_tags,
match_special))
reg_loader.add_value(
'dept_name', dept_name,
MapCompose(custom_remove_tags))
reg_loader.add_xpath(
'hospital_name',
'//div[@class="page_position"]/a[last()-1]/text()',
MapCompose(custom_remove_tags))
reg_loader.add_value('reg_info', reg_info)
reg_loader.add_value('dataSource_from',
self.data_source_from)
reg_loader.add_value('crawled_url', response.url)
reg_loader.add_value('update_time', now_day())
reg_item = reg_loader.load_item()
yield reg_item
except Exception as e:
self.logger.error('在抓取医生详细信息的过程中出错了,原因是：{}'.format(repr(e)))
def parse(self, response):
self.logger.info('>>>>>>正在抓取所有医院信息>>>>>>')
all_hospitals = response.xpath(
'//div[@class="serach-left-list"]/ul/li')
hospital_pro = response.xpath(
'//div[@id="yyk_header_location"]/strong/text()').extract_first('')
for each_hospital in all_hospitals:
each_hospital_link = each_hospital.xpath('a/@href').extract_first(
'')
each_hospital_name = each_hospital.xpath(
'div[1]/div[1]/a/text()').extract_first('')
if each_hospital_link:
hospital_id = re.search(r'.*/(.*?).html$', each_hospital_link)
hospital_link = urljoin(self.host, each_hospital_link)
if hospital_id:
hospital_id = hospital_id.group(1)
# 获取医院详细信息
hospital_detail_url = '{0}{1}'.format(
hospital_id, self.hospital_postfix)
hospital_intro_link = urljoin(self.hospital_url,
hospital_detail_url)
self.headers['Referer'] = hospital_link
yield Request(hospital_intro_link,
headers=self.headers,
callback=self.parse_hospital_info,
meta={
'hospital_pro':
hospital_pro,
'hospital_name':
custom_remove_tags(each_hospital_name)
},
dont_filter=True)
# 获取科室信息
dept_detail_url = '{0}{1}'.format(hospital_id,
self.dept_postfix)
dept_link = urljoin(self.hospital_url, dept_detail_url)
yield Request(urljoin(self.host, dept_link),
headers=self.headers,
callback=self.parse_hospital_dep,
dont_filter=True,
meta={'hospital_name': each_hospital_name})
# 获取医生信息,入口:医院列表页面[推荐专家],存在的问题:部分医生没有科室
# doctors_link = each_hospital.xpath('div[@class="as"]/a[contains(text(),'
# '"推荐专家")]/@href').extract_first('')
# if doctors_link:
# self.headers['Referer'] = response.url
# yield Request(urljoin(self.host, doctors_link),
# headers=self.headers,
# callback=self.parse_doctor_info,
# dont_filter=True,
# meta={'hospital_name': each_hospital_name})
# 翻页
has_next = response.xpath(
'//div[@class="next"]/a[contains(text(),"下一页")]/@href'
).extract_first('')
if has_next:
next_page_link = urljoin(self.host, has_next)
self.headers['Referer'] = response.url
yield Request(next_page_link,
headers=self.headers,
callback=self.parse)
def parse_hospital_info(self, response):
hospital_name = response.meta.get('hospital_name')
self.logger.info('>>>>>>正在抓取:[{}]医院详细信息>>>>>>'.format(hospital_name))
try:
# 获取医院等级与类别
l_a_c = response.xpath(
'//div[@class="l"]/h2/span/i/text()').extract()
l_a_c = custom_remove_tags(remove_tags('|'.join(l_a_c)))
h_l = h_c = m_t = None
if l_a_c:
# 等级
level = re.search(r'(.*等|.*级|.*甲)', l_a_c)
if level:
h_l = level.group(1).split('|')[-1]
# 类别
category = re.search(r'(.*?医院)', l_a_c.replace('医保定点医院', ''))
if category:
h_c = category.group(1).split('|')[-1]
# 医保类型
medical_type = re.search(r'(.*定点)', l_a_c)
if medical_type:
m_t = medical_type.group(1).split('|')[-1]
else:
h_l = h_c = None
# 获取省市信息
hospital_pro = response.meta.get('hospital_pro')
hospital_city = hospital_county = None
h_a = response.xpath(
'//dt[contains(text(),"地址")]/ancestor::dl[1]/dd').extract()
hospital_address = custom_remove_tags(
remove_tags(''.join(h_a).replace('查看地图', '')))
if hospital_pro and hospital_address:
if hospital_pro in MUNICIPALITY2:
hospital_city = hospital_pro
hospital_pro = ''
hos_c = hospital_city.replace('市', '')
useless_info = '{}{}|{}'.format(hos_c, '市', hos_c)
single_address = match_special2(
hospital_address.split(';')[0])
hospital_county = get_county2(useless_info, single_address)
else:
hos_p = hospital_pro
hospital_pro = '{0}{1}'.format(hospital_pro, '省')
single_address = match_special2(
hospital_address.split(';')[0])
hospital_city = get_city(hospital_pro, single_address)
if hospital_city:
hos_c = hospital_city.replace('市', '')
useless_info = '{}|{}|{}|{}'.format(
hospital_pro, hos_p, hospital_city, hos_c)
hospital_county = get_county2(useless_info,
single_address)
# 公立/私立
h_t = custom_remove_tags(
response.xpath(
'//li/b[contains(text(),"国营")]/text()').extract_first(''))
hospital_type = '公立' if h_t == '国营' else ''
# 医院信息item
loader = CommonLoader2(item=HospitalInfoItem(), response=response)
loader.add_xpath('hospital_name', '//div[@class="l"]/h2/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_level', h_l,
MapCompose(custom_remove_tags))
loader.add_value('hospital_type', hospital_type)
loader.add_value('hospital_category', h_c,
MapCompose(custom_remove_tags))
loader.add_value('hospital_addr', hospital_address,
MapCompose(custom_remove_tags))
loader.add_value('hospital_pro', hospital_pro,
MapCompose(custom_remove_tags))
loader.add_value('hospital_city', hospital_city,
MapCompose(custom_remove_tags))
loader.add_value('hospital_county', hospital_county,
MapCompose(custom_remove_tags))
loader.add_xpath('hospital_phone',
'//dt[contains(text(),"电话")]/ancestor::dl[1]/dd',
MapCompose(remove_tags, custom_remove_tags))
loader.add_xpath(
'hospital_intro',
'//dt/strong[contains(text(),"简介")]/ancestor::dl[1]/dd',
MapCompose(remove_tags, custom_remove_tags))
loader.add_value('medicare_type', m_t,
MapCompose(custom_remove_tags))
# loader.add_value('registered_channel', self.data_source_from)
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('hospital_url', response.url)
loader.add_value('update_time', now_day())
hospital_item = loader.load_item()
yield hospital_item
# 获取医院别名
hospital_alias = response.xpath(
'//div[@class="l"]/p/text()').extract_first('')
if hospital_alias:
h_s = custom_remove_tags(hospital_alias)
if h_s:
all_hospital_alias = h_s.split('，')
for each_alias in all_hospital_alias:
if each_alias != hospital_name:
alias_loader = CommonLoader2(
item=HospitalAliasItem(), response=response)
alias_loader.add_xpath(
'hospital_name', '//div[@class="l"]/h2/text()',
MapCompose(custom_remove_tags))
alias_loader.add_value(
'hospital_alias_name', each_alias,
MapCompose(custom_remove_tags, match_special))
alias_loader.add_value('dataSource_from',
self.data_source_from)
alias_loader.add_value('update_time', now_day())
alias_item = alias_loader.load_item()
yield alias_item
except Exception as e:
self.logger.error('在抓取医院详细信息过程中出错了,原因是：{}'.format(repr(e)))
def parse_hospital_info(self, response):
self.logger.info('>>>>>>正在抓取:医院详细信息和科室信息>>>>>>')
try:
# 获取医院信息
hospital_type = response.meta.get('hospital_type')
hospital_category = '{0}{1}'.format(
hospital_type, '医院') if hospital_type else None
hospital_info = custom_remove_tags(
remove_tags(''.join(
response.xpath('//td[@class='
'"title_yh14"]').extract())))
hospital_address = get_hospital_info(hospital_info, '地址：', '电话：')
hospital_address = hospital_address.replace(
'查看地图', '') if hospital_address else None
hospital_phone = get_hospital_info(hospital_info, '电话：', '官网')
hospital_intro = get_hospital_info(hospital_info, '简介：', '$')
hospital_intro = hospital_intro.replace(
'...更多>>', '') if hospital_intro else None
loader = CommonLoader2(item=HospitalInfoItem(), response=response)
loader.add_xpath('hospital_name', '//span[@class="title"]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath('hospital_level', '//span[@class="dj"]/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_category', hospital_category)
loader.add_value('hospital_addr', hospital_address,
MapCompose(custom_remove_tags))
loader.add_value('hospital_pro', '山西省')
loader.add_xpath(
'hospital_city',
'//td[contains(text(),"山西")]/ancestor::tr[1]/td[1]/a[1]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'hospital_county',
'//td[contains(text(),"山西")]/ancestor::tr[1]/td[1]/a[2]/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_phone', hospital_phone,
MapCompose(custom_remove_tags))
loader.add_value('hospital_intro', hospital_intro,
MapCompose(custom_remove_tags))
loader.add_value('registered_channel', self.data_source_from)
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('update_time', now_day())
hospital_info_item = loader.load_item()
yield hospital_info_item
# 获取科室信息
self.logger.info('>>>>>>正在抓取科室详细信息>>>>>>')
all_dept_links = response.xpath('//tr[@class="h_bottom"]')
for each_dept_link in all_dept_links:
dept_type = each_dept_link.xpath('td[1]/text()').extract_first(
'')
dept_name = each_dept_link.xpath(
'td[2]/table/tr/td/a/text()').extract()
for each_dept_name in dept_name:
dept_loader = CommonLoader2(item=HospitalDepItem(),
response=response)
dept_loader.add_value(
'dept_name', each_dept_name,
MapCompose(custom_remove_tags, match_special2))
dept_loader.add_value(
'dept_type', dept_type,
MapCompose(custom_remove_tags, match_special2))
dept_loader.add_xpath('hospital_name',
'//span[@class="title"]/text()',
MapCompose(custom_remove_tags))
dept_loader.add_value('dept_info', '')
dept_loader.add_value('dataSource_from',
self.data_source_from)
dept_loader.add_value('update_time', now_day())
dept_item = dept_loader.load_item()
yield dept_item
except Exception as e:
self.logger.error('在抓取医院详细信息和科室的过程中出错了,原因是：{}'.format(repr(e)))
def parse_hospital_info(self, response):
self.logger.info('>>>>>>正在抓取:医院详细信息>>>>>>')
try:
# 获取医院信息
# 获取医院等级与地区
hospital_info = response.xpath(
'//p[@class="yygh_box_top_p2"]').extract()
hospital_info2 = custom_remove_tags(
remove_tags(''.join(hospital_info)))
hospital_level = hospital_info2.split(':')[1].replace('区域', '')
hospital_county = hospital_info2.split(':')[2].replace('分类', '')
if hospital_level:
res = re.search(r'(.*等|.*级|.*合格|.*甲)(.*?)$', hospital_level)
if res:
h_l = res.group(1)
h_c = res.group(2)
if h_c:
h_c_2 = re.sub(r'合格|医院', '', h_c)
if h_c_2:
h_c = '{0}{1}'.format(h_c_2, '医院')
else:
h_l = h_c = None
else:
h_l = h_c = None
loader = CommonLoader2(item=HospitalInfoItem(), response=response)
loader.add_xpath('hospital_name',
'//p[@class="yygh_box_top_p"]/strong/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_level', h_l,
MapCompose(custom_remove_tags))
loader.add_value('hospital_category', h_c)
loader.add_xpath(
'hospital_addr',
'//span[@class="yygh_box_con_dl_span1"]/ancestor::dl[1]/dd[1]/p/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_pro', '')
loader.add_value('hospital_city', '北京市')
loader.add_value('hospital_county', hospital_county,
MapCompose(custom_remove_tags))
loader.add_xpath(
'hospital_phone',
'//span[@class="yygh_box_con_dl_span3"]/ancestor::dl[1]/dd[1]/p/text()',
MapCompose(custom_remove_tags))
loader.add_value('registered_channel', self.data_source_from)
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('hospital_url', response.url)
loader.add_value('update_time', now_day())
# 获取医院介绍
hospital_intro_link = response.xpath(
'//a[contains(text(),"医院介绍")]/@href').extract_first('')
if hospital_intro_link:
hospital_intro_link = urljoin(self.host, hospital_intro_link)
self.headers['Referer'] = response.url
yield Request(hospital_intro_link,
headers=self.headers,
callback=self.parse_hospital_detail_info,
meta={'loader': loader})
# 获取科室信息
all_dept_links = response.xpath('//div[@class="kfyuks_yyksbox"]')
for each_dept_link in all_dept_links:
dept_type = each_dept_link.xpath(
'div[1]/text()').extract_first('')
dept_info = each_dept_link.xpath(
'div[2]/div/ul/li/a/text()').extract()
for each_dept_info in dept_info:
dept_loader = CommonLoader2(item=HospitalDepItem(),
response=response)
dept_loader.add_value('dept_name', each_dept_info,
MapCompose(custom_remove_tags))
dept_loader.add_value('dept_type', dept_type,
MapCompose(custom_remove_tags))
dept_loader.add_xpath(
'hospital_name',
'//p[@class="yygh_box_top_p"]/strong/text()',
MapCompose(custom_remove_tags))
dept_loader.add_value('dept_info', '')
dept_loader.add_value('dataSource_from',
self.data_source_from)
dept_loader.add_value('update_time', now_day())
dept_item = dept_loader.load_item()
yield dept_item
except Exception as e:
self.logger.error('在抓取医院详细信息过程中出错了,原因是：{}'.format(repr(e)))
def parse_hospital_info(self, response):
hospital_name = response.meta.get('hospital_name')
self.logger.info('>>>>>>正在抓取[{}]医院信息和科室信息>>>>>>'.format(hospital_name))
hospital_city = response.xpath(
'//div[@class="jieshao_zi"]/p[4]/text()').extract()
if hospital_city:
hospital_address = custom_remove_tags(''.join(hospital_city))
hospital_city2 = get_city(hospital_address)
useless_info = '中国|湖南省|湖南|{}'.format(hospital_city2)
hospital_county = get_county2(useless_info, hospital_address)
else:
hospital_county = None
loader = CommonLoader2(item=HospitalInfoItem(), response=response)
loader.add_xpath('hospital_name',
'//div[@class="jieshao_zi"]/p/font/text()',
MapCompose(custom_remove_tags))
loader.add_xpath('hospital_level',
'//div[@class="jieshao_zi"]/p[2]/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_type', '公立')
loader.add_value('hospital_category', '')
loader.add_xpath('hospital_addr',
'//div[@class="jieshao_zi"]/p[4]/text()',
MapCompose(custom_remove_tags))
loader.add_value('hospital_pro', '湖南省')
loader.add_xpath('hospital_city',
'//div[@class="jieshao_zi"]/p[4]/text()',
MapCompose(custom_remove_tags, get_city))
loader.add_value('hospital_county', hospital_county)
loader.add_xpath('hospital_phone',
'//div[@class="jieshao_zi"]/p[3]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'hospital_intro', '//div[@id="starlist"]',
MapCompose(remove_tags, custom_remove_tags, clean_info))
loader.add_value('registered_channel', self.data_source_from)
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('update_time', now_day())
hospital_info_item = loader.load_item()
yield hospital_info_item
# 获取科室信息
self.logger.info('>>>>>>正在抓取[{}]科室信息>>>>>>'.format(hospital_name))
dept_links = response.xpath('//div[@class="xuanze_kslb"]')
if dept_links:
for each_dept_link in dept_links:
dept_type = each_dept_link.xpath(
'div[1]/ul/li/text()').extract_first('')
all_dept_links = each_dept_link.xpath('div[2]/ul/li/a')
for dept_link in all_dept_links:
# dept_name = dept_link.xpath('text()').extract_first('')
data_info = dept_link.xpath('@onclick').extract_first('')
if data_info:
data_info = ''.join(re.findall(r'\S+', data_info))
is_sp_time = re.search(r'isSpTime:\'(.*?)\'',
data_info)
pay_mode = re.search(r'paymode:\'(.*?)\'', data_info)
dept_id = re.search(r'platformDeptId:\'(.*?)\'',
data_info)
hos_id = re.search(r'platformHosId:\'(.*?)\'',
data_info, S)
dept_name = re.search(r'tempDeptName:\'(.*?)\'',
data_info, S)
org_name = re.search(r'orgname:\'(.*?)\'', data_info,
S)
if dept_id and hos_id and dept_name and org_name:
is_sp_time = is_sp_time.group(1)
pay_mode = pay_mode.group(1)
dept_id = dept_id.group(1)
hos_id = hos_id.group(1)
dept_name = dept_name.group(1)
org_name = org_name.group(1)
data = {
'isSpTime': str(is_sp_time),
'paymode': quote(pay_mode),
'doctorCollectResult': '',
'platformDeptId': str(dept_id),
'orgname': quote(org_name),
'tempDeptName': quote(dept_name),
'platformHosId': str(hos_id),
'platformDoctorId': ''
}
self.headers.update({
'Content-Type':
'application/x-www-form-urlencoded',
'Origin': 'http://www.hnyygh.com',
'Referer':
'http://www.hnyygh.com/searchDeptmentAction.action',
'Pragma': 'no-cache'
})
splash_args = {
'url': self.dept_detail_url,
'headers': self.headers,
'lua_source': self.dept_script,
'data': data
}
yield SplashRequest(
self.dept_detail_url,
endpoint='execute',
args=splash_args,
dont_filter=True,
headers=self.headers,
callback=self.parse_hospital_dep_detail,
meta={
'dept_type': dept_type,
'dept_name': dept_name,
'hospital_name': org_name
})
# 获取医生信息
data = {
'platformDeptId': dept_id,
'platformHosId': hos_id,
'platformDoctorId': '',
'nextNumInfo': '0'
}
self.headers.update({
'Content-Type':
'application/x-www-form-urlencoded',
'Origin':
'http://www.hnyygh.com',
'Referer':
'http://www.hnyygh.com/searchOrderNumInfoAction.action'
})
splash_args = {
'url': self.doctor_url,
'headers': self.headers,
'lua_source': self.dept_script,
'data': data
}
yield SplashRequest(
self.doctor_url,
endpoint='execute',
args=splash_args,
dont_filter=True,
headers=self.headers,
callback=self.parse_doctor_info,
meta={
'dept_type': dept_type,
'dept_name': dept_name,
'dept_id': dept_id,
'hospital_name': org_name
})
def parse_area(self, response):
hospital_city = response.meta.get('area_city', '默认城市')
self.logger.info('>>>>>>正在抓取[{}]医院列表……>>>>>>'.format(hospital_city))
# 获取省市县等信息
municipality = ['北京市', '上海市', '重庆市', '天津市']
pro_or_city = response.xpath(
'//table[@class="nav"]/tr/'
'td/a[3]/text()').extract_first('').replace('医院列表', '')
if pro_or_city:
if pro_or_city.strip() in municipality:
# 直辖市,包括市、区等信息
hos_prov = ''
hos_city = pro_or_city
hos_county = response.xpath('//h1[@id="firstHeading"]/text()'
).extract_first('').replace(
hos_city, '')
else:
# 非直辖市,包括省、市、县或区等信息
hos_prov = pro_or_city
hos_city = response.xpath('//h1[@id="firstHeading"]'
'/text()').extract_first('').replace(
'医院列表',
'').replace(hos_prov, '')
hos_county = ''
else:
hos_prov = hos_city = hos_county = None
# 有医院最终页的医院
# all_hospital_list = response.xpath('//div[@id="bodyContent"]/ul[3]/li/b/a/@href').extract()
all_hospital_list2 = response.xpath(
'//h2/span[contains(text(),"医院列表")]/'
'following::ul[1]/li/b/a[not(contains(@href,"index"))]')
special_hospital_list = response.xpath(
'//h2/span[contains(text(),"医院列表")]/'
'following::ul[1]/li/b/a[(contains(@href,"index"))]/ancestor::li[1]'
)
area_hos_cnt = len(all_hospital_list2) + len(special_hospital_list)
self.logger.info('>>>>>>[{}]总共有{}家医院……>>>>>>'.format(
hospital_city, str(area_hos_cnt)))
self.total_hospital_cnt += area_hos_cnt
self.crawler.signals.connect(self.output_statistics,
signals.spider_closed)
try:
# 有医院最终页的
for each_hospital in all_hospital_list2:
hospital_name = each_hospital.xpath('text()').extract_first('')
hospital_link = each_hospital.xpath('@href').extract_first('')
self.headers['Referer'] = response.url
yield Request(urljoin(self.host, hospital_link),
headers=self.headers,
callback=self.parse_hospital_detail,
meta={'hospital_name': hospital_name},
dont_filter=True)
# 没有医院最终页的
for each_special_hospital in special_hospital_list:
hospital_name = each_special_hospital.xpath(
'b/a/text()').extract_first('')
hospital_url = each_special_hospital.xpath(
'b/a/@href').extract_first('')
hospital_address = each_special_hospital.xpath(
'ul[1]/li/b[contains(text(),'
'"医院地址")]/ancestor::li[1]/text()').extract_first('')
hos_county = hos_county if hos_county else get_county(
hos_prov, hos_city, hospital_address)
loader = CommonLoader2(item=HospitalInfoTestItem(),
selector=each_special_hospital)
loader.add_value('hospital_name', hospital_name)
loader.add_xpath(
'hospital_level',
'ul[1]/li/b[contains(text(),"医院等级")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'hospital_category',
'ul[1]/li/b[contains(text(),"医院类型")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'hospital_addr',
'ul[1]/li/b[contains(text(),"医院地址")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_value('hospital_pro', hos_prov,
MapCompose(custom_remove_tags, match_special))
loader.add_value('hospital_city', hos_city,
MapCompose(custom_remove_tags, match_special))
loader.add_value('hospital_county', hos_county,
MapCompose(custom_remove_tags, match_special))
loader.add_xpath(
'hospital_phone',
'ul[1]/li/b[contains(text(),"联系电话")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_value('hospital_intro', '')
loader.add_xpath(
'hospital_postcode',
'ul[1]/li/b[contains(text(),"邮政编码")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'hospital_email',
'ul[1]/li/b[contains(text(),"电子邮箱")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'hospital_website',
'ul[1]/li/b[contains(text(),"医院网站")]/ancestor::li[1]/'
'a[not(contains(@href,"http://www.a-hospital.com"))]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'hospital_fax',
'ul[1]/li/b[contains(text(),"传真号码")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_xpath(
'operation_mode',
'ul[1]/li/b[contains(text(),"经营方式")]/ancestor::li[1]',
MapCompose(remove_tags, custom_remove_tags, match_special))
loader.add_value('hospital_url',
urljoin(self.host, hospital_url))
loader.add_value('dataSource_from', '医学百科')
loader.add_value('update_time', now_day())
hospital_info_item = loader.load_item()
yield hospital_info_item
# 科室信息
dept_info = each_special_hospital.xpath(
'ul[1]/li/b[contains(text(),"重点科室")]/ancestor::li[1]')
all_dept_info = match_special(
dept_info.xpath('string(.)').extract_first(''))
if all_dept_info:
for each_dept in all_dept_info.split('、'):
dept_loader = CommonLoader2(item=HospitalDepItem(),
response=response)
dept_loader.add_value('dept_name', each_dept,
MapCompose(custom_remove_tags))
dept_loader.add_value(
'hospital_name', hospital_name,
MapCompose(custom_remove_tags, match_special2))
dept_loader.add_value('update_time', now_day())
dept_item = dept_loader.load_item()
yield dept_item
# 医院别名信息
hospital_name2 = each_special_hospital.xpath(
'b/text()').extract_first('')
if hospital_name2 and '（' in hospital_name2:
alias_name = re.search(r'（(.*?)）',
custom_remove_tags(hospital_name2))
if alias_name:
for each_alias_name in alias_name.group(1).split('、'):
alias_loader = CommonLoader2(
item=HospitalAliasItem(), response=response)
alias_loader.add_value(
'hospital_name', hospital_name,
MapCompose(custom_remove_tags, match_special2))
alias_loader.add_value('hospital_alias_name',
each_alias_name)
alias_loader.add_value('update_time', now_day())
alias_item = alias_loader.load_item()
yield alias_item
except Exception as e:
self.logger.error('抓取[{}]医院列表的时候出错了,原因是:{}'.format(
hospital_city, repr(e)))
def parse_doctor_info_detail(self, response):
hospital_name = response.meta.get('hospital_name')
dept_name = response.meta.get('dept_name')
doctor_name = response.meta.get('doctor_name')
self.logger.info('>>>>>>正在抓取[{}]医院-[{}]医生详细信息>>>>>>'.format(
hospital_name, doctor_name))
try:
# 获取医生信息
doctor_photo_url = response.xpath(
'//div[@class="doctor_Img"]/img/@src').extract_first('')
loader = CommonLoader2(item=DoctorInfoItem(), response=response)
loader.add_value('doctor_name', doctor_name,
MapCompose(custom_remove_tags))
loader.add_value('dept_name', dept_name,
MapCompose(custom_remove_tags))
loader.add_value('hospital_name', hospital_name,
MapCompose(custom_remove_tags))
loader.add_xpath('sex', '//span[@class="doctor_grade"]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath('doctor_level',
'//span[@class="object_grade"]/text()',
MapCompose(custom_remove_tags))
loader.add_xpath(
'doctor_intro', '//div[@class="doctor_Text_Major"]',
MapCompose(remove_tags, custom_remove_tags, match_special2))
loader.add_value('dataSource_from', self.data_source_from)
loader.add_value('crawled_url', response.url)
loader.add_value('update_time', now_day())
loader.add_value('doctor_id', response.url,
MapCompose(match_special2))
loader.add_xpath(
'dept_id', '//div[@class="position_one"]/span/a[last()]/@href',
MapCompose(match_special2))
loader.add_xpath(
'hospital_id',
'//div[@class="position_one"]/span/a[last()-1]/@href',
MapCompose(match_special2))
loader.add_value('doctor_photo_url',
urljoin(self.host, doctor_photo_url))
loader.add_value('gmt_created', now_time())
loader.add_value('gmt_modified', now_time())
doctor_item = loader.load_item()
yield doctor_item
# 获取医生排班信息
self.logger.info(
'>>>>>>正在抓取[{}]医生排班信息>>>>>>'.format(hospital_name))
has_doctor_scheduling = response.xpath(
'//span[@class="yuyue"]/a[contains(text(),"预约")]')
if has_doctor_scheduling:
doctor_scheduling_list = response.xpath(
'//div[@class="whliesubscribe"]/ul/li[1]/div/'
'span/text()').extract()
doctor_scheduling_length = len(doctor_scheduling_list)
all_scheduling_date = response.xpath(
'//div[@class="datetable"]/ul/li[position()>1]/'
'span[1]/text()').extract()
scheduling_date_list = custom_remove_tags(
remove_tags(','.join(all_scheduling_date))).split(',')
for i in range(1, doctor_scheduling_length + 1):
scheduling_info = response.xpath(
'//div[@class="whliesubscribe"]/ul/li[position()>1]'
'/div[{}]'.format(str(i)))
scheduling_time = doctor_scheduling_list[i - 1]
for index, each_s_i in enumerate(scheduling_info):
has_scheduling = each_s_i.xpath('span/a')
if has_scheduling:
each_scheduling_date = scheduling_date_list[index]
reg_info = '{0}-{1}{2}'.format(
now_year(), each_scheduling_date,
scheduling_time)
reg_loader = CommonLoader2(
item=DoctorRegInfoItem(), response=response)
reg_loader.add_value(
'doctor_name', doctor_name,
MapCompose(custom_remove_tags))
reg_loader.add_value(
'dept_name', dept_name,
MapCompose(custom_remove_tags))
reg_loader.add_value(
'hospital_name', hospital_name,
MapCompose(custom_remove_tags))
reg_loader.add_value('reg_info', reg_info)
reg_loader.add_value('dataSource_from',
self.data_source_from)
reg_loader.add_value('crawled_url', response.url)
reg_loader.add_value('update_time', now_day())
reg_loader.add_value('doctor_id', response.url,
MapCompose(match_special2))
reg_loader.add_xpath(
'dept_id',
'//div[@class="position_one"]/span/a[last()]/@href',
MapCompose(match_special2))
reg_loader.add_xpath(
'hospital_id',
'//div[@class="position_one"]/span/a[last()-1]/@href',
MapCompose(match_special2))
reg_loader.add_value('gmt_created', now_time())
reg_loader.add_value('gmt_modified', now_time())
reg_item = reg_loader.load_item()
yield reg_item
except Exception as e:
self.logger.error('在抓取医生详细信息的过程中出错了,原因是：{}'.format(repr(e)))