def parse_job(self, response):
# 解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_xpath("tags", '//li[@class="labels"]/text()')
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
item_loader = LagouItemLoader(item=LagouJobItem(), response=response)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request p .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_css("tags", ".position-label li::text")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", ".c_feature li a::text ")
item_loader.add_css("company_name", "img.b2::attr(alt)")
item_loader.add_value("crawl_time", datetime.datetime.now())
lagoujob_item = item_loader.load_item()
yield lagoujob_item
def parse_job(self, response):
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_value("post_url", response.url)
item_loader.add_css("title", "div .job-name::attr(title)")
item_loader.add_css("company", "div .company::text")
item_loader.add_css("min_salary", "span.salary::text")
item_loader.add_xpath("city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("min_work_years",
"//*[@class='job-request']/p/span[3]/text()")
item_loader.add_xpath("degree_req",
"//*[@class='job-request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job-request']/p/span[5]/text()")
item_loader.add_css("tags", ".position-label li::text")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("advantage", ".job-advantage p::text")
item_loader.add_css("description",
".job_bt div") # hot about .job_bt p
item_loader.add_css("addr", ".work_addr")
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_xpath("title", '//div/span[@class="name"]/text()')
item_loader.add_xpath("url", response.url)
item_loader.add_xpath("url_object_id", get_md5(response.url))
item_loader.add_xpath("salary", '//dd/p/span[@class="salary"]/text()')
item_loader.add_xpath("job_city", '//dd/p/span[2]/text()')
item_loader.add_xpath("work_years", '//dd/p/span[3]/text()')
item_loader.add_xpath("degree_need", '//dd/p/span[4]/text()')
item_loader.add_xpath("job_type", '//dd/p/span[5]/text()')
item_loader.add_xpath("tags",
'//dd[@class="job_request"]/ul/li/text()')
item_loader.add_xpath("publish_time",
'//dd/p[@class="publish_time"]/text()')
item_loader.add_xpath("job_advantage", '//dl/dd/p/text()')
item_loader.add_xpath("job_desc", '//dd/div/p/text()')
item_loader.add_xpath("job_addr", '//dd/div[@class="work_addr"]')
item_loader.add_xpath("company_url", '//dl/dt/a/@href')
item_loader.add_xpath("company_name", '//dl/dt/a/img/@alt')
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
#解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css('title', '.job-name::attr(title)')
item_loader.add_value('url', response.url)
item_loader.add_value('url_object_id', get_md5(response.url))
item_loader.add_css('salary', '.salary::text')
item_loader.add_xpath('job_city',
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath('work_years',
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree_need',
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_type',
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_css('tags', '.position-label li::text')
item_loader.add_css('publish_time', '.publish_time::text')
item_loader.add_css('job_advantage', '.job-advantage p::text')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '#job_company dt a img::attr(alt)')
item_loader.add_css('company_url', '#job_company dt a::attr(href)')
item_loader.add_value('crawl_time', datetime.datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_item(self, response):
# i = {}
# #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
# #i['name'] = response.xpath('//div[@id="name"]').extract()
# #i['description'] = response.xpath('//div[@id="description"]').extract()
# return i
item_loader = LagouItemLoader(item=LagouJobItem(), response=response)
item_loader.add_css("title", '.job-name > .name::text')
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", '.job_request .salary::text')
item_loader.add_xpath("job_city",
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath("work_years",
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath("degree_need",
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath("job_type",
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_css("publish_time", '.publish_time::text')
item_loader.add_css("job_advantage", '.job-advantage p::text')
item_loader.add_css("job_desc", '.job_bt div::text')
item_loader.add_css("job_addr", '.work_addr')
item_loader.add_css("company_name", '#job_company dt a img::attr(alt)')
item_loader.add_css("company_url", '#job_company dt a::attr(href)')
item_loader.add_css("tags", '.position-label li::text ')
item_loader.add_value("crawl_time", datetime.datetime.now())
item = item_loader.load_item()
return item
def parse_job(self, response):
"""
解析拉勾网的职位
:param response:
:return:
"""
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", ".job-name span::text")
item_loader.add_value("url", response.url)
item_loader.add_css("salary", ".salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_css("company_name", "#job_company dt a div h2::text")
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
item_load = LagouJobItemLoader(item=LagouJobItem(), response=response)
item_load.add_value("url", response.url)
item_load.add_value("url_object_id", get_md5(response.url))
item_load.add_css("title", "div.job-name::attr(title)")
item_load.add_css("salary", ".salary::text")
item_load.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text()")
item_load.add_xpath("work_years",
"//*[@class='job_request']/p/span[3]/text()")
item_load.add_xpath("degree_need",
"//*[@class='job_request']/p/span[4]/text()")
item_load.add_xpath("job_type",
"//*[@class='job_request']/p/span[5]/text()")
item_load.add_css("pulish_time", ".publish_time::text")
item_load.add_xpath("tags",
"//*[@class='position-label clearfix']/li/text()")
item_load.add_xpath("job_advantage",
"//*[@class='job-advantage']/p/text()")
item_load.add_xpath("job_desc", "//*[@class='job_bt']/div")
item_load.add_xpath("job_addr", "//*[@class='work_addr']/a/text()")
item_load.add_xpath("company_url",
"//*[@class='c_feature']/li/a/@title")
item_load.add_css("company_name", ".job_company dt img::attr(alt)")
item_load.add_value("crawl_time", datetime.datetime.now())
item_load.add_value("crawl_update_time", datetime.datetime.now())
lagou_item = item_load.load_item()
#i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
#i['name'] = response.xpath('//div[@id="name"]').extract()
#i['description'] = response.xpath('//div[@id="description"]').extract()
return lagou_item
def parse_job(self, response):
i = {}
item_loader = LagouItemLoader(item=LagouJobItem(), response=response)
item_loader.add_xpath('title', '//div[@class="job-name"]/@title')
item_loader.add_value('url', response.url)
item_loader.add_value('url_object_id', get_md5(url=response.url))
item_loader.add_xpath('salary', '//span[@class="salary"]/text()')
item_loader.add_xpath('job_city',
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath('work_years',
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree_need',
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_type',
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_xpath('publish_time',
'//*[@class="publish_time"]/text()')
item_loader.add_css('tags', '.position-label li::text')
item_loader.add_xpath('job_advantage',
'//*[@class="job-advantage"]/p/text()')
item_loader.add_xpath('job_desc', '//*[@class="job_bt"]/div')
item_loader.add_xpath('company_url',
'//*[@class="job_company"]/dt/a/@href')
item_loader.add_xpath('company_name',
'//*[@class="job_company"]/dt/a/img/@alt')
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
# 解析拉勾网的职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css("title", "")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']/p/span[2]/text")
item_loader.add_css("work_years",
".job_request p span:nth-child(3)::text"
) # 这里使用css ，是为了在学习时，熟悉css选择器用法
item_loader.add_xpath("degree_need",
"//dd[@class='job_request']/p/span[4]/text()")
item_loader.add_xpath("job_type",
"//dd[@class='job_request']/p/span[5]/text()")
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("tags", ".position-label.clearfix li::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css("job_addr", ".work_addr")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_value("crawl_time", datetime.datetime.now())
# item_loader.add_css("crawl_update_time", datetime.datetime.now())
job_item = item_loader.load_item(
) # 这里先赋值给一个变量，是考虑到便于调试以及代码可读性，而不是为了代码简洁而直接return
return job_item
def parse_job(self, response):
job_loader = ItemLoader(item=LagouJobItem(), response=response)
job_loader.add_value('url', response.url)
job_loader.add_value('url_object_id', get_md5(response.url))
job_loader.add_css('title', '.job-name span.name::text')
job_loader.add_css('salary', '.job_request span.salary::text')
job_loader.add_xpath('job_city',
"//*[@class='job_request']/p/span[2]/text()")
job_loader.add_xpath('work_experience',
"//*[@class='job_request']/p/span[3]/text()")
job_loader.add_xpath('degree_needed',
"//*[@class='job_request']/p/span[4]/text()")
job_loader.add_xpath('job_type',
"//*[@class='job_request']/p/span[5]/text()")
job_loader.add_css('publish_time', '.publish_time::text')
job_loader.add_css('tags', '.position-label li.labels::text')
job_loader.add_css('job_advantage', 'dd.job-advantage p::text')
job_loader.add_css('job_desc', 'dd.job_bt div')
job_loader.add_css('job_addr', 'div.work_addr a::text')
job_loader.add_css('company_url', '#job_company dt a::attr(href)')
job_loader.add_css('company_name', '#job_company h2.fl::text')
job_loader.add_value(
'crawl_time',
datetime.datetime.now().strftime(SQL_DATETIME_FORMAT))
job_item = job_loader.load_item()
yield job_item
def parse_item(self, response):
# item = {}
#item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
#item['name'] = response.xpath('//div[@id="name"]').get()
#item['description'] = response.xpath('//div[@id="description"]').get()
# return item
# if 'utrack/track' in response.url:
# # 本想解决302重定向问题，但此法无效
# num = re.match('.*2F(\d+).html.*', response.url)
# num = str(num.group(1))
# url = 'https://www.lagou.com/jobs/' + num + '.html'
# print(url)
# time.sleep(2)
# return scrapy.Request(url, dont_filter=True, headers=self.headers)
if response.status == 302:
self.redirect_url.append(response.url)
self.crawler.stats.inc_value("redirected_url")
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
item_loader.add_css('title', 'span.name:nth-child(2)::text')
item_loader.add_value('url', response.url)
item_loader.add_value('url_object_id', get_md5(response.url))
item_loader.add_css('salary', '.job_request .salary::text')
item_loader.add_xpath('job_city',
'//*[@class="job_request"]/p/span[2]/text()')
item_loader.add_xpath('work_years',
'//*[@class="job_request"]/p/span[3]/text()')
item_loader.add_xpath('degree_need',
'//*[@class="job_request"]/p/span[4]/text()')
item_loader.add_xpath('job_type',
'//*[@class="job_request"]/p/span[5]/text()')
item_loader.add_css('tags', '.position-label li::text')
item_loader.add_css('publish_time', '.publish_time::text')
item_loader.add_css('job_advantage', '.job-advantage p::text')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '#job_company dt a img::attr(alt)')
item_loader.add_css('company_url', '#job_company dt a::attr(href)')
item_loader.add_value('crawl_time', datetime.datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_item(self, response):
# 解析拉勾网的职位
# 1.要想无界面启动selenium，先设置headless模式
chrome_options = Options() # 实例化这个Options(),要在webdriver.Chrome加上参数
chrome_options.add_argument("--headless") # 这个就是无界面启动selenium，一定要写的
chrome_options.add_argument("--disable-gpu") # 谷歌文档提到需要加上这个属性来规避bug
# 2.设置selenium不加载图片, blink-settings=imagesEnabled=false是固定的
# chrome_options.add_argument("blink-settings=imagesEnabled=false")
"""
1.启动chrome（启动之前确保所有的chrome实例己经关闭）
"""
browser = webdriver.Chrome(
executable_path=
"D:/DecomPression-File/chromedriver_win32 (2.45-70)/chromedriver.exe",
chrome_options=chrome_options)
# 把得到的url给这里
browser.get(response.url)
# page_source就是运行js完后的html网页
temp_xc = Selector(text=browser.page_source)
time.sleep(3)
# 实例化一个对象， LagouJobItem是拉勾的item
# Lg = LagouJobItem()
# item_loader = LagouJobItemLoader(item=LagouJobItem(), response=response)
# print(response.text)
# title = item_loader.add_css("title", ".position-head-wrap-name::text()")
title = temp_xc.css(".job-name::attr(title)").extract_first("")
print(type(title))
# item_loader.add_css("url", response.url)
# item_loader.add_css("url_object_id")
url_object_id = get_md5(response.url)
LG = LagouJobItem()
LG["title"] = title
LG["url"] = response.url
LG["url_object_id"] = url_object_id
return LG
def parse_job(self, response):
#解析拉勾网职位
item_loader = LagouJobItemLoader(item=LagouJobItem(),response=response)
item_loader.add_css('title','.job-name::attr(title)')
item_loader.add_css('url', '')
item_loader.add_value('url_obj_id', '')
item_loader.add_css('salary', '')
item_loader.add_css('job_city', '')
item_loader.add_css('work_years', '')
item_loader.add_css('degree_need', '')
item_loader.add_css('job_type', '')
item_loader.add_css('publish_time', '')
item_loader.add_css('job_advantage', '')
item_loader.add_css('job_desc', '.job_bt div')
item_loader.add_css('job_addr', '.work_addr')
item_loader.add_css('company_name', '')
item_loader.add_css('company_url', '')
item_loader.add_css('tags', '')
item_loader.add_value('crawl_time', datetime.now())
job_item = item_loader.load_item()
return job_item
def parse_job(self, response):
job_item = BaseItemLoader(item=LagouJobItem(), response=response)
job_item.add_css('title', '.job-name::attr(title)')
job_item.add_value('url', response.url)
job_item.add_value('url_object_id', get_md5(response.url))
job_item.add_xpath('salary', '//*[@class=job_request]/h3/span[1]/text()')
job_item.add_xpath('job_city', '//*[@class=job_request]/h3/span[2]/text()')
job_item.add_xpath('work_years', '//*[@class=job_request]/h3/span[3]/text()')
job_item.add_xpath('degree_need', '//*[@class=job_request]/h3/span[4]/text()')
job_item.add_xpath('job_type', '//*[@class=job_request]/h3/span[5]/text()')
job_item.add_css('publish_time', '.publish_time::text')
job_item.add_css('job_advantage', '.job-advantage p::text')
job_item.add_css('job_desc', '.job-detail::text')
job_item.add_css('job_addr', '.work_addr::text')
job_item.add_css('company_name', '.b2::attr(alt)')
job_item.add_css('company_url', 'a[data-lg-tj-track-code="jobs_logo"]::attr(href)')
job_item.add_css('tags', '.position-label li::text')
item = job_item.load_item()
return item
def parse_item(self, response):
"""解析拉勾网的职位"""
# 实例化一个对象， LagouJobItem是拉勾的item
# LagouJobItemLoader是自定义的一个给lagou的item
# Lg = LagouJobItem()
item_loader = LagouJobItemLoader(item=LagouJobItem(),
response=response)
# print(response.text)
item_loader.add_css("title", ".job-name::attr(title)")
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
item_loader.add_css("salary", ".job_request .salary::text")
item_loader.add_xpath("job_city",
"//*[@class='job_request']//span[2]/text()")
item_loader.add_xpath("work_years",
"//*[@class='job_request']//span[3]/text()")
item_loader.add_xpath("degree_need",
"//*[@class='job_request']//span[4]/text()")
item_loader.add_xpath("job_type",
"//*[@class='job_request']//span[5]/text()")
item_loader.add_css("tags", '.position-label li::text')
item_loader.add_css("publish_time", ".publish_time::text")
item_loader.add_css("job_advantage", ".job-advantage p::text")
item_loader.add_css("job_desc", ".job_bt div")
item_loader.add_css(
"job_addr", ".work_addr"
) # 获取的字段里面包含了一段html，所以不能用::text,我们在itemloader里去除不必要的html
item_loader.add_css("company_name", "#job_company dt a img::attr(alt)")
item_loader.add_css("company_url", "#job_company dt a::attr(href)")
item_loader.add_value("crawl_time", datetime.now())
job_item = item_loader.load_item()
return job_item