def parse_job(self, response):
item_load = LagouJobItemLoader(item=LagouJobItem(), response=response)
item_load.add_value("url", response.url)
item_load.add_value("url_object_id", get_md5(response.url))
item_load.add_css("title", "div.job-name::attr(title)")
item_load.add_css("salary", ".salary::text")
item_load.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_load.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_load.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_load.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_load.add_css("pulish_time", ".publish_time::text")
item_load.add_xpath("tags",
"//*[@class='position-label clearfix']/li/text()")
item_load.add_xpath("job_advantage",
"//*[@class='job-advantage']/p/text()")
item_load.add_xpath("job_desc", "//*[@class='job_bt']/div")
item_load.add_xpath("job_addr", "//*[@class='work_addr']/a/text()")
item_load.add_xpath("company_url",
"//*[@class='c_feature']/li/a/@title")
item_load.add_css("company_name", ".job_company dt img::attr(alt)")
item_load.add_value("crawl_time", datetime.datetime.now())
item_load.add_value("crawl_update_time", datetime.datetime.now())
lagou_item = item_load.load_item()
#i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
#i['name'] = response.xpath('//div[@id="name"]').extract()
#i['description'] = response.xpath('//div[@id="description"]').extract()
return lagou_item
def parse_job(self, response):
# 解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_xpath("tags", '//li[@class="labels"]/text()')
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_value("post_url", response.url)
item_loader.add_css("title", "div .job-name::attr(title)")
item_loader.add_css("company", "div .company::text")
item_loader.add_css("min_salary", "span.salary::text")
item_loader.add_xpath("city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("min_work_years",
"//*[@class='job-request']/p/span[3]/text()")
item_loader.add_xpath("degree_req",
"//*[@class='job-request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job-request']/p/span[5]/text()")
item_loader.add_css("tags", ".position-label li::text")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("advantage", ".job-advantage p::text")
item_loader.add_css("description",
".job_bt div") # hot about .job_bt p
item_loader.add_css("addr", ".work_addr")
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
#解析拉勾网职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),response=response)
item_loader.add_css('title','.job-name::attr(title)')
item_loader.add_css('url', '')
item_loader.add_value('url_obj_id', '')
item_loader.add_css('salary', '')
item_loader.add_css('job_city', '')
item_loader.add_css('work_years', '')
item_loader.add_css('degree_need', '')
item_loader.add_css('job_type', '')
item_loader.add_css('publish_time', '')
item_loader.add_css('job_advantage', '')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '')
item_loader.add_css('company_url', '')
item_loader.add_css('tags', '')
item_loader.add_value('crawl_time', datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_item(self, response):
# item = {}
#item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
#item['name'] = response.xpath('//div[@id="name"]').get()
#item['description'] = response.xpath('//div[@id="description"]').get()
# return item
# if 'utrack/track' in response.url:
# # 本想解决302重定向问题，但此法无效
# num = re.match('.*2F(\d+).html.*', response.url)
# num = str(num.group(1))
# url = 'https://www.lagou.com/jobs/' + num + '.html'
# print(url)
# time.sleep(2)
# return scrapy.Request(url, dont_filter=True, headers=self.headers)
if response.status == 302:
self.redirect_url.append(response.url)
self.crawler.stats.inc_value("redirected_url")
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css('title', 'span.name:nth-child(2)::text')
item_loader.add_value('url', response.url)
item_loader.add_value('url_object_id', get_md5(response.url))
item_loader.add_css('salary', '.job_request .salary::text')
item_loader.add_xpath('job_city',
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath('work_years',
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree_need',
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_type',
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_css('tags', '.position-label li::text')
item_loader.add_css('publish_time', '.publish_time::text')
item_loader.add_css('job_advantage', '.job-advantage p::text')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '#job_company dt a img::attr(alt)')
item_loader.add_css('company_url', '#job_company dt a::attr(href)')
item_loader.add_value('crawl_time', datetime.datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
#解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css('title', '.job-name::attr(title)')
item_loader.add_value('url', response.url)
item_loader.add_value('url_object_id', get_md5(response.url))
item_loader.add_css('salary', '.salary::text')
item_loader.add_xpath('job_city',
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath('work_years',
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree_need',
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_type',
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_css('tags', '.position-label li::text')
item_loader.add_css('publish_time', '.publish_time::text')
item_loader.add_css('job_advantage', '.job-advantage p::text')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '#job_company dt a img::attr(alt)')
item_loader.add_css('company_url', '#job_company dt a::attr(href)')
item_loader.add_value('crawl_time', datetime.datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_item(self, response):
"""解析拉勾网的职位"""
# 实例化一个对象， LagouJobItem是拉勾的item
# LagouJobItemLoader是自定义的一个给lagou的item
# Lg = LagouJobItem()
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
# print(response.text)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']//span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']//span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']//span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']//span[5]/text()")
item_loader.add_css("tags", '.position-label li::text')
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css(
"job_addr", ".work_addr"
) # 获取的字段里面包含了一段html，所以不能用::text,我们在itemloader里去除不必要的html
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
"""
解析拉勾网的职位
:param response:
:return:
"""
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", ".job-name span::text")
item_loader.add_value("url", response.url)
item_loader.add_css("salary", ".salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_css("company_name", "#job_company dt a div h2::text")
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
# 解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", "")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text")
item_loader.add_css("work_years",
".job_request p span:nth-child(3)::text"
) # 这里使用css ，是为了在学习时，熟悉css选择器用法
item_loader.add_xpath("degree_need",
"//dd[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//dd[@class='job_request']/p/span[5]/text()")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("tags", ".position-label.clearfix li::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_value("crawl_time", datetime.datetime.now())
# item_loader.add_css("crawl_update_time", datetime.datetime.now())
job_item = item_loader.load_item(
) # 这里先赋值给一个变量，是考虑到便于调试以及代码可读性，而不是为了代码简洁而直接return
return job_item
def parse_job(self, response):
# 解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_css("tags", ".position-label li::text")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
# response_text = response.text
#i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
#i['name'] = response.xpath('//div[@id="name"]').extract()
#i['description'] = response.xpath('//div[@id="description"]').extract()
return job_item