def crawl(self): min_time = self.data.get('updated', datetime.min) if self.data else datetime.min max_time = None time = None page = 1 while True: url = "http://www.265zy.com/list/?0-%s.html" % page hxs = load_html(url) for s in hxs.select("//body/.//tr[@class='row']"): try: href = s.select("td[1]/a/@href").extract()[0] source_id = re.findall("(\d+)\.html", href)[0] title = clean_title( s.select("td[1]/.//text()").extract()[0]) region = s.select("td[2]/.//text()").extract()[0].replace( u"地区", u"") category = s.select("td[3]/.//text()").extract()[0] time = s.select("td[4]/.//text()").extract()[0] time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") except: continue if not max_time: max_time = time if time < min_time: break data = { "title": title, "time": time, 'category': category, 'region': region, } lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get( "time", datetime.min) if lastdata else datetime.min Scheduler.schedule(type=AlbumCrawler.type, key=source_id, data=data, reset=data['time'] > lasttime) if time and time < min_time: break text = hxs.select("//div[@class='pages']/span/text()").extract()[0] page_count = int(re.findall(u"\d+/(\d+)页", text)[0]) if page >= 5: break page += 1 if max_time: if not self.data: self.data = {} self.data['updated'] = max_time
def crawl(self): min_time = self.data.get('updated', datetime.min) if self.data else datetime.min max_time = None time = None page = 1 while True: url = "http://zyqvod.com/?page=%s" % page hxs = load_html(url) for s in hxs.select("//table[@id='listTable']/tbody/tr"): try: source_id = re.findall("id=(\d+)", s.select("td[1]/a/@href").extract()[0])[0] title = clean_title(s.select("td[1]/.//text()").extract()[0]) category = s.select("td[2]/.//text()").extract()[0] region = s.select("td[3]/.//text()").extract()[0] completed = s.select("td[4]/.//text()").extract()[0] == u"完结" time = s.select("td[5]/.//text()").extract()[0] time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") except: continue if not max_time: max_time = time if time < min_time: break data = { "title" : title, "time" : time, 'category' : category, 'region' : region, 'completed' : completed, } lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get("time", datetime.min) if lastdata else datetime.min Scheduler.schedule(type = AlbumCrawler.type, key = source_id, data = data, reset = data['time'] > lasttime) if time and time < min_time: break text = hxs.select("//div[@class='page_num']/text()").extract()[0] page_count = int(re.findall(u"\d+/(\d+)页", text)[0]) if page >= page_count: break page += 1 if max_time: if not self.data: self.data = {} self.data['updated'] = max_time
def crawl(self): type = 3 channel_id = self.key channel = self.data['channel'] start = 0 num = 16 params = { "order": "times", "time": "today" } while 1: list_data = api_list(type, channel_id, start, num, params) if start == list_data['num']: return for item in list_data['data']: if channel in LONG_VIDEO_CHANNELS.values(): source_id = item['mid'] tags = [] time = item['public_time'] time = datetime.strptime(time, "%Y%m%d") lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get( "time", datetime.min) if lastdata else datetime.min reset = time > lasttime else: source_id = item['flvid'] tags = item.get("tags").split(",") time = datetime.utcnow() reset = False data = { "url": item.get("web_url"), "title": item.get("title"), "image": item.get("bpic"), "image2": item.get("mpic"), "description": item.get("introduce"), "duration": item.get("duration"), "tags": tags, "time": time, "channel": channel } Scheduler.schedule( AlbumCrawler.type, source_id, data, reset=reset ) start += 1
def crawl(self): min_time = self.data.get('updated', datetime.min) if self.data else datetime.min max_time = None time = None page = 1 while True: url = "http://www.265zy.com/list/?0-%s.html" % page hxs = load_html(url) for s in hxs.select("//body/.//tr[@class='row']"): try: href = s.select("td[1]/a/@href").extract()[0] source_id = re.findall("(\d+)\.html", href)[0] title = clean_title(s.select("td[1]/.//text()").extract()[0]) region = s.select("td[2]/.//text()").extract()[0].replace(u"地区", u"") category = s.select("td[3]/.//text()").extract()[0] time = s.select("td[4]/.//text()").extract()[0] time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S") except: continue if not max_time: max_time = time if time < min_time: break data = { "title" : title, "time" : time, 'category' : category, 'region' : region, } lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get("time", datetime.min) if lastdata else datetime.min Scheduler.schedule(type = AlbumCrawler.type, key = source_id, data = data, reset = data['time'] > lasttime) if time and time < min_time: break text = hxs.select("//div[@class='pages']/span/text()").extract()[0] page_count = int(re.findall(u"\d+/(\d+)页", text)[0]) if page >= 5: break page += 1 if max_time: if not self.data: self.data = {} self.data['updated'] = max_time
def crawl(self): type = 3 channel_id = self.key channel = self.data['channel'] start = 0 num = 16 params = {"order": "times", "time": "today"} while 1: list_data = api_list(type, channel_id, start, num, params) if start == list_data['num']: return for item in list_data['data']: if channel in LONG_VIDEO_CHANNELS.values(): source_id = item['mid'] tags = [] time = item['public_time'] time = datetime.strptime(time, "%Y%m%d") lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get( "time", datetime.min) if lastdata else datetime.min reset = time > lasttime else: source_id = item['flvid'] tags = item.get("tags").split(",") time = datetime.utcnow() reset = False data = { "url": item.get("web_url"), "title": item.get("title"), "image": item.get("bpic"), "image2": item.get("mpic"), "description": item.get("introduce"), "duration": item.get("duration"), "tags": tags, "time": time, "channel": channel } Scheduler.schedule(AlbumCrawler.type, source_id, data, reset=reset) start += 1
def crawl(self): min_time = self.data['updated'] #上次爬取到最新视频的更新时间, 为本次爬取的时间下界 max_time = None #本次抓取的最新视频的时间 page = 1 while True: url = "http://bdzy.cc/list/?0-%s.html" % page hxs = load_html(url) #读取网页html, 返回一个HtmlXPathSelector time = None for s in hxs.select("//body/.//tr[@class='row']"): #用xpath解析html try: href = s.select("td[1]/a/@href").extract()[0] source_id = re.findall("(\d+)\.html", href)[0] #源站ID title = clean_title(s.select("td[1]/.//text()").extract()[0]) region = s.select("td[2]/.//text()").extract()[0].replace(u"地区", u"") category = s.select("td[3]/.//text()").extract()[0] time = s.select("td[4]/.//text()").extract()[0] time = datetime.strptime(time, "%Y-%m-%d") if not max_time: #第一条是最新更新的 max_time = time if time < min_time: #已经爬取到上次最新的数据 break data = { #详情页爬虫任务的附加数据 "title" : title, "time" : time, "category" : category, "region" : region, } #获取对应详情页爬虫的附加数据,用time字段判断该内容是否已经更新,需要重新抓取. 如果第一次创建,则数据为空 lastdata = Scheduler.get_data(AlbumCrawler.type, source_id) lasttime = lastdata.get("time", datetime.min) if lastdata else datetime.min #创建相应的专辑爬虫,爬取相应的详情页. key为源站id Scheduler.schedule( AlbumCrawler.type, source_id, data, reset = data["time"] > lasttime #是否需要强制重新抓取 ) except: self.logger.warning(get_exception_info()) #纪录错误信息并继续 continue if time and time < min_time: #已经爬取到上次最新的数据 break #获取总页数 text = hxs.select("//div[@class='pages']/span/text()").extract()[0] page_count = int(re.findall(u"\d+/(\d+)页", text)[0]) #超过总页数 if page >= page_count: break page += 1 if max_time: self.data = {'updated' : max_time} #保存上次爬取到的最新的时间