def parse_kb(self, response): mib = None # need to perform some nasty segmentation because different firmware versions are not clearly separated # reverse order to get MIB before firmware items for entry in reversed(response.xpath( "//div[@id='support-article-downloads']/div/p")): for segment in reversed(entry.extract().split("<br><br>")): resp = HtmlResponse( url=response.url, body=segment, encoding=response.encoding) for href in resp.xpath("//a/@href").extract(): text = resp.xpath("//text()").extract() if "MIBs" in href: mib = href elif "firmware" in href: text = resp.xpath("//text()").extract() item = FirmwareLoader( item=FirmwareImage(), response=resp, date_fmt=["%m/%d/%Y"]) item.add_value("date", item.find_date(text)) item.add_xpath("url", "//a/@href") item.add_value("mib", mib) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) item.add_value( "version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_kb(self, response): # initial html tokenization to find regions segmented by e.g. "======" # or "------" filtered = response.xpath( "//div[@class='sfdc_richtext']").extract()[0].split("=-") for entry in [x and x.strip() for x in filtered]: resp = HtmlResponse(url=response.url, body=entry, encoding=response.encoding) for link in resp.xpath("//a"): href = link.xpath("@href").extract()[0] if "cache-www" in href: text = resp.xpath("//text()").extract() text_next = link.xpath("following::text()").extract() item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%b %d, %Y", "%B %d, %Y", "%m/%d/%Y"]) version = FirmwareLoader.find_version_period(text_next) if not version: version = FirmwareLoader.find_version_period(text) item.add_value("version", version) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_kb(self, response): mib = None # need to perform some nasty segmentation because different firmware versions are not clearly separated # reverse order to get MIB before firmware items for entry in reversed( response.xpath( "//div[@id='support-article-downloads']/div/p")): for segment in reversed(entry.extract().split("<br><br>")): resp = HtmlResponse(url=response.url, body=segment, encoding=response.encoding) for href in resp.xpath("//a/@href").extract(): text = resp.xpath("//text()").extract() if "MIBs" in href: mib = href elif "firmware" in href: text = resp.xpath("//text()").extract() item = FirmwareLoader(item=FirmwareImage(), response=resp, date_fmt=["%m/%d/%Y"]) item.add_value("date", item.find_date(text)) item.add_xpath("url", "//a/@href") item.add_value("mib", mib) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) item.add_value( "version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_url(self, response): for link in response.xpath("//a"): text = link.xpath("text()").extract()[0] href = link.xpath("@href").extract()[0] if ".." in href: continue elif href.endswith('/'): if "package/" not in text: product = "%s-%s" % ( response.meta["product"], text[0:-1] ) if "product" in response.meta else text[0:-1] yield Request(url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, meta={ "version": response.meta["version"], "product": product }, callback=self.parse_url) elif any( href.endswith(x) for x in [".bin", ".elf", ".fdt", ".imx", ".chk", ".trx"]): item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("version", response.meta["version"]) item.add_value("url", href) item.add_value( "date", item.find_date(link.xpath("following::text()").extract())) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product_sw_fw(self, response): product = response.meta['product'] fw_sect = None #inspect_response(response, self) col_selector_map = {} # Find the "Firmware" section. NOTE: whitespace in the class is intentional for section in response.css('div.docs-table__section '): for col in section.css('div.docs-table__column-name'): col_text = col.xpath('.//text()').extract_first().strip() if len(col_text) > 1: col_selector_map[col_text] = section try: fw_sect = col_selector_map[u'Firmware'] except KeyError: logging.debug("Did not find a 'Firmware' section in the downloads for %s", product) return # Iterate Firmware rows for fw_row in fw_sect.css('div.docs-table__row'): fw_version, fw_href, fw_date, fw_desc = self.extract_fw_info(fw_row, response) if fw_href is None: continue item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%m/%d/%y"]) item.add_value('product', product) item.add_value('vendor', self.name) item.add_value('url', fw_href) item.add_value('description', fw_desc) item.add_value('date', fw_date) yield item.load_item()
def parse_product(self, response): js = response.text if js.startswith("var commonInfo"): print response.url print js p_product = u"id:\"(?P<product>.*?)\"" p_description = u"title:\"(?P<description>.*?)\"" p_version = u"romVersions:\"(?P<version>.*?)\"" p_url = u"romUrl:\"(?P<url>.*?)\"" p_date = u"updateDate:\"(?P<date>.*?)\"" import re products = re.findall(p_version, js) descriptions = re.findall(p_description, js) versions = re.findall(p_version, js) urls = re.findall(p_url, js) dates = re.findall(p_date, js) for i in xrange(len(products)): product = products[i] url = urls[i] version = versions[i] description = descriptions[i] date = dates[i] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", url) item.add_value("product", product) item.add_value("description", description) item.add_value("date", date) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): #<a href="#Firmware"><span>Firmware</span></a> if not response.xpath("//a[@href=\"#Firmware\"]").extract(): yield None description = response.xpath( "//div[@class=\"product-name\"]//strong/text()").extract()[0] url = response.xpath( "//*[@id=\"content_Firmware\"]/table/tbody/tr[1]/th/a/@href" ).extract()[0] date = response.xpath( "//*[@id=\"content_Firmware\"]/table/tbody/tr[2]/td[1]/span[2]/text()" ).extract()[0] item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%d/%m/%y"]) item.add_value("url", url) item.add_value("date", item.find_date(date)) item.add_value("description", description) item.add_value("product", response.meta["product"]) item.add_value("version", response.meta["version"]) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_download(self, response): for entry in response.xpath("//div[@class='downloadtable']"): text = entry.xpath(".//text()").extract() if "firmware" in " ".join(text).lower(): text = entry.xpath( ".//li[@class='maindescription' and position() = 1]//text()" ).extract() date = entry.xpath( ".//li[@class='maindescription' and position() = 2]//text()" ).extract() href = entry.xpath( ".//li[@class='maindescription']//a/@onclick" ).extract()[0].split( '\'')[1] + "&button=Continue+with+Download&Continue=yes" item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%m/%d/%Y"]) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("date", item.find_date(date)) item.add_value("version", FirmwareLoader.find_version(text)) item.add_value("vendor", self.name) yield item.load_item()
def parse_url(self, response): for link in response.xpath("//a"): text = link.xpath("text()").extract()[0] href = link.xpath("@href").extract()[0] if ".." in href: continue elif href.endswith('/'): if "package/" not in text: product = "%s-%s" % (response.meta["product"], text[0: -1]) if "product" in response.meta else text[0: -1] yield Request( url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, meta={"version": response.meta[ "version"], "product": product}, callback=self.parse_url) elif any(href.endswith(x) for x in [".bin", ".elf", ".fdt", ".imx", ".chk", ".trx"]): item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("version", response.meta["version"]) item.add_value("url", href) item.add_value("date", item.find_date( link.xpath("following::text()").extract())) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): js = response.text if js.startswith("var commonInfo"): print response.url print js p_product = u"id:\"(?P<product>.*?)\"" p_description = u"title:\"(?P<description>.*?)\"" p_version = u"romVersions:\"(?P<version>.*?)\"" p_url = u"romUrl:\"(?P<url>.*?)\"" p_date = u"updateDate:\"(?P<date>.*?)\"" import re products = re.findall(p_version, js) descriptions = re.findall(p_description, js) versions = re.findall(p_version, js) urls = re.findall(p_url, js) dates = re.findall(p_date, js) for i in xrange(len(products)): product = products[i] url = urls[i] version = versions[i] description = descriptions[i] date = dates[i] item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value("url", url) item.add_value("product", product) item.add_value("description", description) item.add_value("date", date) item.add_value("vendor", self.name) yield item.load_item()
def parse_kb(self, response): # initial html tokenization to find regions segmented by e.g. "======" # or "------" filtered = response.xpath( "//div[@class='sfdc_richtext']").extract()[0].split("=-") for entry in [x and x.strip() for x in filtered]: resp = HtmlResponse(url=response.url, body=entry, encoding=response.encoding) for link in resp.xpath("//a"): href = link.xpath("@href").extract()[0] if "cache-www" in href: text = resp.xpath("//text()").extract() text_next = link.xpath("following::text()").extract() item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%b %d, %Y", "%B %d, %Y", "%m/%d/%Y"]) version = FirmwareLoader.find_version_period(text_next) if not version: version = FirmwareLoader.find_version_period(text) item.add_value("version", version) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def download_item(self, response): url = "https:" + str(response.xpath("//div[@class='downbtns']/a/@href").extract_first()) item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%Y-%m-%d"]) item.add_value("url", url) item.add_value("version", response.meta["version"]) item.add_value("date", response.meta["date"]) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for href in response.xpath("//a/@href").extract(): if href == ".." or href == "/": continue elif href.endswith(".bin") or href.endswith(".upg"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("vendor", self.name) yield item.load_item() elif "/" in href: yield Request(url=urllib.parse.urljoin(response.url, href), headers={"Referer": response.url}, callback=self.parse)
def parse_model_page(self, response): for entry in response.xpath( "//section[@id='topicsdownload']//div[@class='col topic']/" "section[@class='box articles']//div[@class='accordion-item']" ): name = entry.xpath( "./a[@class='accordion-title']/h1/text()").extract_first() url = entry.xpath( "./div[@class='accordion-content']//a/@href").extract_first() if '#confirm-download-' in url or 'http://kb.netgear.com/' in url: continue if 'Firmware' in name and not 'Upgrade' in name: name_split = name.split(" ") index = name_split.index('Version') # only continue if there is a version number if index: version = name_split[index + 1] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", version) item.add_value("url", url) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_download(self, response): for link in response.xpath("//div[@id='auto']//a"): href = link.xpath("./@href").extract()[0] text = link.xpath(".//text()").extract()[0] if ("downloads" in href or "firmware" in href) and \ not href.endswith(".html"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", response.meta["version"]) item.add_value("url", href) item.add_value("description", text) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_link(self, response): # some items will require captcha authentication and pass a cookie e.g. # DownloadAuthorizationToken = # 7CB8169BFC8848B097BB071118F9E067431714963E3A74A45C8883A70654999980D7F1412CB98B87C802403D74B6A2611122BB3CCEE0B2ACDEEAACA8054B8FFBC4AB2C2CC992649F733AFB2446AA3DC66131E62F0697E9267A374A9E965D1286EC3CFEA1142B5244D497974E5992A3F172581BE78559432DA3A64ECC940D3C43A3C91427EEC5FC712A4ADF64D2FC6C31D62BD8E4417964B31AC6E0B8344EADEA6E81DBB33F522979F3C4FE33ECA4240C188C2C88FAEBC3E0C27AEDF79558E9113F2E7BB2CA261666A26CDA82074F0DC777F2BDB28A5A2588F7F4F67E2A4F04C4DDEE6E3A2A78E2106D2F324986705580070A9016C96007E82332EA1F1D2E9688033F514754555CE186695284B05B24DE6C99F22CCF4F43A7CB5D8AD9053929E3EFDAD40FD20497F1D9ED45BAA4C7CF1C2207C751624D755EBF0C4FF98C9B2E41437E41674C836D80C83C902C4B8B8ADDA23D813D9FA5B3331C36B05CE3C1F479220B7A02 for link in response.xpath("//tbody[@class='etdownloaditems']//tr"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", link.xpath( ".//td[@class='column-version']//text()").extract()[0].strip()) item.add_value("url", link.xpath(".//th/a/@href").extract()[0]) item.add_value("description", link.xpath( ".//th/a//text()").extract()[0]) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_download(self, response): for link in response.xpath("//div[@id='auto']//a"): href = link.xpath("./@href").extract()[0] text = link.xpath(".//text()").extract()[0] if ("downloads" in href or "firmware" in href) and \ not href.endswith(".html"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", response.meta["version"]) item.add_value("url", href) item.add_value("description", text) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for product_group in response.css("div.item"): try: category = product_group.css("h2 span::text").get().strip() except: category = product_group.css("h2::text").get().strip() for product in product_group.css("a"): model = product.css("::text").get().strip() link = product.css("::attr(href)").get() if link[-1] == "/": yield response.follow(link, meta={ "category": category, "product": model }, callback=self.parse_product) else: item = FirmwareLoader(item=FirmwareImage(), date_fmt=["%y-%m-%d"]) item.add_value("vendor", self.vendor) item.add_value("url", link) item.add_value("product", model) item.add_value("category", category) yield item.load_item()
def parse(self, response): for href in response.xpath("//a/@href").extract(): if href == ".." or href == "/": continue elif href.endswith(".bin") or href.endswith(".upg"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("vendor", self.name) yield item.load_item() elif "/" in href: yield Request( url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, callback=self.parse)
def parse_product(self, response): for a in response.xpath("//div[@id='mainbox']//dd/a"): url = a.xpath("./@href").extract()[0] title = a.xpath("./text()").extract()[0] description = title items = title.split(' ') product = items[0] version = items[-1] #FH456V1.0 Firmware V10.1.1.1_EN #E101(V2.0) Firmware V1.10.0.1_EN #G3(V2.0) Firmware V2.0.0.1_EN #O3 Firmware V1.0.0.3_EN #i6 Firmware V1.0.0.9(3857)_EN import re p = r'^(?P<product>([a-uw-zA-UW-Z0-9])+)[\(\uff08]?(V\d\.0)?' try: ret = re.search(p, items[0].decode('utf-8')) if ret: product = ret.group('product') except: product = item[0] item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value( "version", version) item.add_value("url", url) item.add_value("product", product) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_product(self, response): for a in response.xpath("//div[@id='mainbox']//dd/a"): url = a.xpath("./@href").extract()[0] title = a.xpath("./text()").extract()[0] description = title items = title.split(' ') product = items[0] version = items[-1] #FH456V1.0 Firmware V10.1.1.1_EN #E101(V2.0) Firmware V1.10.0.1_EN #G3(V2.0) Firmware V2.0.0.1_EN #O3 Firmware V1.0.0.3_EN #i6 Firmware V1.0.0.9(3857)_EN import re p = ur'^(?P<product>([a-uw-zA-UW-Z0-9])+)[\(\uff08]?(V\d\.0)?' try: ret = re.search(p, items[0].decode('utf-8')) if ret: product = ret.group('product') except: product = item[0] item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value( "version", version) item.add_value("url", url) item.add_value("product", product) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_link(self, response): # some items will require captcha authentication and pass a cookie e.g. # DownloadAuthorizationToken = # 7CB8169BFC8848B097BB071118F9E067431714963E3A74A45C8883A70654999980D7F1412CB98B87C802403D74B6A2611122BB3CCEE0B2ACDEEAACA8054B8FFBC4AB2C2CC992649F733AFB2446AA3DC66131E62F0697E9267A374A9E965D1286EC3CFEA1142B5244D497974E5992A3F172581BE78559432DA3A64ECC940D3C43A3C91427EEC5FC712A4ADF64D2FC6C31D62BD8E4417964B31AC6E0B8344EADEA6E81DBB33F522979F3C4FE33ECA4240C188C2C88FAEBC3E0C27AEDF79558E9113F2E7BB2CA261666A26CDA82074F0DC777F2BDB28A5A2588F7F4F67E2A4F04C4DDEE6E3A2A78E2106D2F324986705580070A9016C96007E82332EA1F1D2E9688033F514754555CE186695284B05B24DE6C99F22CCF4F43A7CB5D8AD9053929E3EFDAD40FD20497F1D9ED45BAA4C7CF1C2207C751624D755EBF0C4FF98C9B2E41437E41674C836D80C83C902C4B8B8ADDA23D813D9FA5B3331C36B05CE3C1F479220B7A02 for link in response.xpath("//tbody[@class='etdownloaditems']//tr"): item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", link.xpath( ".//td[@class='column-version']//text()").extract()[0].strip()) item.add_value("url", link.xpath(".//th/a/@href").extract()[0]) item.add_value("description", link.xpath( ".//th/a//text()").extract()[0]) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): # bit ugly but it works :-) if "pid" not in response.meta: for pid in range(0, 1000): yield Request(url=urllib.parse.urljoin( response.url, "firmware_details.html?id=%s" % pid), meta={"pid": pid}, headers={ "Referer": response.url, "X-Requested-With": "XMLHttpRequest" }, callback=self.parse) else: for product in response.xpath( "//div[@class='download_list_icon']/span/text()").extract( ): prods = response.xpath("//table[@class='down_table']//tr") # print(prods) # skip the table header for p in [x for x in prods[1:]]: version = p.xpath('td[1]//text()').extract_first() # skip partial versions if '_p' in version: continue item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", version) item.add_value( "url", 'https://www.foscam.com' + p.xpath('td[6]//a/@href').extract_first()) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): for href in response.xpath("//a/@href").extract(): if href.endswith(".npk") or href.endswith(".lzb"): text = response.xpath("//text()").extract() basename = href.split("/")[-1] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%Y-%b-%d"]) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", basename[0: basename.rfind("-")]) item.add_value("vendor", self.name) item.add_value( "version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_product(self, response): for href in response.xpath("//a/@href").extract(): if href.endswith(".npk") or href.endswith(".lzb"): text = response.xpath("//text()").extract() basename = href.split("/")[-1] item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%Y-%b-%d"]) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", basename[0:basename.rfind("-")]) item.add_value("vendor", self.name) item.add_value("version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_product(self, response): url = response.xpath("//div[@class='thumbnail']//a/@href").extract()[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", response.meta['version']) item.add_value("url", url) item.add_value("product", response.meta['product']) item.add_value("vendor", self.vendor) yield item.load_item()
def parse(self, response): for link in response.xpath("//a"): text = link.xpath(".//text()").extract()[0] href = link.xpath(".//@href").extract()[0] if ".." in href: continue elif href.endswith('/'): yield Request(url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, callback=self.parse) elif href.endswith(".gz") and ".iso" not in href: # strip off multiple file extensions basename = os.path.splitext(text)[0] while ".img" in basename or ".iso" in basename: basename = os.path.splitext(basename)[0] basename = basename.split("-") version = FirmwareLoader.find_version_period(basename) # attempt to parse filename and generate product/version # strings remove = [version] if version else [] for i in range(0, len(basename)): if "BETA" in basename[i]: version += "-%s%s" % (basename[i], basename[i + 1]) remove.append(basename[i]) remove.append(basename[i + 1]) elif "RC" in basename[i]: version += "-%s" % (basename[i]) remove.append(basename[i]) elif "RELEASE" in basename[i]: remove.append(basename[i]) basename = [x for x in basename if x not in remove] item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("version", version) item.add_value("url", href) item.add_value( "date", item.find_date(link.xpath("following::text()").extract())) item.add_value("product", "-".join(basename)) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for link in response.xpath("//a"): text = link.xpath(".//text()").extract()[0] href = link.xpath(".//@href").extract()[0] if ".." in href: continue elif href.endswith('/'): yield Request( url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, callback=self.parse) elif href.endswith(".gz") and ".iso" not in href: # strip off multiple file extensions basename = os.path.splitext(text)[0] while ".img" in basename or ".iso" in basename: basename = os.path.splitext(basename)[0] basename = basename.split("-") version = FirmwareLoader.find_version_period(basename) # attempt to parse filename and generate product/version # strings remove = [version] if version else [] for i in range(0, len(basename)): if "BETA" in basename[i]: version += "-%s%s" % (basename[i], basename[i + 1]) remove.append(basename[i]) remove.append(basename[i + 1]) elif "RC" in basename[i]: version += "-%s" % (basename[i]) remove.append(basename[i]) elif "RELEASE" in basename[i]: remove.append(basename[i]) basename = [x for x in basename if x not in remove] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("version", version) item.add_value("url", href) item.add_value("date", item.find_date( link.xpath("following::text()").extract())) item.add_value("product", "-".join(basename)) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): url =self.firmware_url + response.xpath('//a[@id="downLoadHref"]/@href').extract()[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath("date", response.meta['date']) item.add_value("description", response.meta['description']) item.add_value("url", url) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for a in response.xpath("//table//tr//td[2]//a"): title = a.xpath('./@title').extract()[0] url = a.xpath('./@href').extract()[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", url) item.add_value("product", self.parse_product(title)) item.add_value("description", title) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): if "product" in response.meta: for entry in response.xpath("//div[@class='tab-content']//tr")[1:]: version = entry.xpath("./td[1]//a//text()").extract_first() url = entry.xpath("./td[2]//a/@href").extract_first() if version is None or url is None: continue # remove unnecessary files to_remove_list = [ "end user license agreement", "eula", "release notes", "mac os", "windows", "guide", "(pdf)", "sample", "client", "manager", "software", "virtual", "control_panel", "activexbypass" ] if any(x in url.lower() for x in to_remove_list) \ or any(x in version.lower() for x in to_remove_list) \ or any(url.endswith(x) for x in ["htm", "html", "pdf", "ova", ".plcm.vc"]): continue url = urllib.parse.urljoin(response.url, PolycomSpider.fix_url(url)), item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", version) item.add_value("url", url) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item() # all entries on the product overview pages elif response.xpath("//div[@class='product-listing']" ) and "product" not in response.meta: for entry in response.xpath("//div[@class='product-listing']//li"): if not entry.xpath("./a"): continue text = entry.xpath("./a//text()").extract_first() href = entry.xpath("./a/@href").extract_first().strip() # date = entry.xpath("./span//text()").extract() if any(x in text.lower() for x in ["advisories", "support", "notices", "features"]) \ or href.endswith(".pdf"): continue path = urllib.parse.urlparse(href).path if any(path.endswith(x) for x in [".htm", ".html"]) or "(html)" in text.lower(): yield Request(url=urllib.parse.urljoin( response.url, PolycomSpider.fix_url(href)), meta={"product": text}, headers={"Referer": response.url}, callback=self.parse)
def parse_product(self, response): url = response.xpath("//div[@class='thumbnail']//a/@href").extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value( "version", response.meta['version']) item.add_value("url", url) item.add_value("product", response.meta['product']) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_product(self, response): url = self.firmware_url + response.xpath( '//a[@id="downLoadHref"]/@href').extract()[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath("date", response.meta['date']) item.add_value("description", response.meta['description']) item.add_value("url", url) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): for i in range(0, len(response.xpath("//ul[@id='normaltab2']//a"))): if "firmware" in "".join(response.xpath( "//ul[@id='normaltab2']/li[%d]/a//text()" % (i + 1)).extract()).lower(): for entry in response.xpath( "//div[@id='normalcon2']/div[%d]//table/tr[1]" % (i + 1)): version = entry.xpath("./td[2]//text()").extract() date = entry.xpath("./td[4]//text()").extract() href = entry.xpath("./td[5]//a/@href").extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%Y-%m-%d"]) item.add_value( "version", FirmwareLoader.find_version_period(version)) item.add_value("url", href) item.add_value("date", item.find_date(date)) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_product(self, response): for i in range(0, len(response.xpath("//ul[@id='normaltab2']//a"))): if "firmware" in "".join(response.xpath( "//ul[@id='normaltab2']/li[%d]/a//text()" % (i + 1)).extract()).lower(): for entry in response.xpath( "//div[@id='normalcon2']/div[%d]//table/tr[1]" % (i + 1)): version = entry.xpath("./td[2]//text()").extract() date = entry.xpath("./td[4]//text()").extract() href = entry.xpath("./td[5]//a/@href").extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%Y-%m-%d"]) item.add_value( "version", FirmwareLoader.find_version_period(version)) item.add_value("url", href) item.add_value("date", item.find_date(date)) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.vendor) yield item.load_item()
def parse(self, response): for href in response.xpath("//a/@href").extract(): if href.endswith(".img"): basename = href.split("/")[-1].split("-") item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("product", self.name) item.add_value("vendor", self.name) item.add_value( "version", basename[-1][0: basename[-1].rfind(".img")]) yield item.load_item()
def parse(self, response): for a in response.xpath("//table//tr//td[2]//a"): title = a.xpath('./@title').extract()[0] url = a.xpath('./@href').extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value("url", url) item.add_value("product", self.parse_product(title)) item.add_value("description", title) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for href in response.xpath("//a/@href").extract(): if href.endswith(".npk") or href.endswith(".lzb"): if href.startswith("//"): href = "http:" + href text = response.xpath("//text()").extract() items = href.split('/') version = items[-2] basename = items[-1] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%Y-%b-%d"]) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", basename[0: basename.rfind("-")]) item.add_value("vendor", self.name) item.add_value( "version", version) yield item.load_item()
def parse_download(self, response): for entry in response.xpath("//div[@class='downloadtable']"): text = entry.xpath(".//text()").extract() if "firmware" in " ".join(text).lower(): text = entry.xpath( ".//li[@class='maindescription' and position() = 1]//text()").extract() date = entry.xpath( ".//li[@class='maindescription' and position() = 2]//text()").extract() href = entry.xpath(".//li[@class='maindescription']//a/@onclick").extract()[ 0].split('\'')[1] + "&button=Continue+with+Download&Continue=yes" item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%m/%d/%Y"]) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("date", item.find_date(date)) item.add_value("version", FirmwareLoader.find_version(text)) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for href in response.xpath("//a/@href").extract(): if href.endswith(".npk") or href.endswith(".lzb"): if href.startswith("//"): href = "http:" + href text = response.xpath("//text()").extract() items = href.split('/') version = items[-2] basename = items[-1] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%Y-%b-%d"]) item.add_value("date", item.find_date(text)) item.add_value("url", href) item.add_value("product", basename[0: basename.rfind("-")]) item.add_value("vendor", self.name) item.add_value( "version", version) yield item.load_item()
def parse_product(self, response): #<a href="#Firmware"><span>Firmware</span></a> if not response.xpath("//a[@href=\"#Firmware\"]").extract(): yield None description = response.xpath("//div[@class=\"product-name\"]//strong/text()").extract()[0] url = response.xpath("//*[@id=\"content_Firmware\"]/table/tbody/tr[1]/th/a/@href").extract()[0] date = response.xpath("//*[@id=\"content_Firmware\"]/table/tbody/tr[2]/td[1]/span[2]/text()").extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%d/%m/%y"]) item.add_value("url", url) item.add_value("date", item.find_date(date)) item.add_value("description", description) item.add_value("product", response.meta["product"]) item.add_value("version", response.meta["version"]) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_product(self, response): #<h3 class="firm">Firmware</h3> if response.xpath('//h3[@class="firm"]').extract(): for tr in response.xpath( '//*[@id="tab-downloads"]/table[1]/tbody/tr'): print tr.extract() url = tr.xpath("./td[2]/a/@href").extract()[0] date = tr.xpath("./td[4]/text()").extract()[0] version = tr.xpath("./td[5]/text()").extract()[0] description = tr.xpath("./td[7]/text()").extract()[0] product = url.split('-')[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", version) item.add_value("description", description) item.add_value("url", url) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): #<h3 class="firm">Firmware</h3> if response.xpath('//h3[@class="firm"]').extract(): for tr in response.xpath('//*[@id="tab-downloads"]/table[1]/tbody/tr'): print tr.extract() url = tr.xpath("./td[2]/a/@href").extract()[0] date = tr.xpath("./td[4]/text()").extract()[0] version = tr.xpath("./td[5]/text()").extract()[0] description = tr.xpath("./td[7]/text()").extract()[0] product = url.split('-')[0] item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", version) item.add_value("description", description) item.add_value("url", url) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item()
def parse_product_firmware(self, response): # Get product name product = response.meta["product"] # Get the product last updated date create_date = '' for li_elem in response.xpath('//li'): if li_elem.xpath('@class').re(r'(\[hide_empty:create_date\])'): create_date = li_elem.xpath( './/span[@class="badge"]/text()').extract_first() elif li_elem.xpath('@class').re(r'(\[hide_empty:update_date\])'): update_date = li_elem.xpath( './/span[@class="badge"]/text()').extract_first() break else: update_date = create_date # File list table of downloads file_table = response.xpath( '//table[@class="wpdm-filelist table table-hover"]') for dl_button in file_table.xpath( './/a[@class="inddl btn btn-primary btn-sm"]'): href = dl_button.xpath("@href") item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt="%B %d, %Y") item.add_value("product", product) item.add_value("vendor", self.name) item.add_value("date", update_date) item.add_value("url", href.extract_first()) yield item.load_item()
def parse(self, response): for url in self.firmware: item = FirmwareLoader(item=FirmwareImage()) item.add_value("url", url) item.add_value("product", url.split("/")[-1].split("_")[0]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): for image in response.xpath( "//div[@id='accordion-2']//tr[position() > 1]"): text = image.xpath("./td[2]//a[1]/text()").extract() if "firmware" in "".join(text).lower(): item = FirmwareLoader(item=FirmwareImage(), response=response, selector=image, date_fmt=["%Y-%m-%d"]) item.add_xpath("date", "td[1]//text()") item.add_value("description", text) item.add_xpath("url", "td[2]//a[1]/@href") item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) item.add_value( "version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_product(self, response): for image in response.xpath( "//div[@id='accordion-2']//tr[position() > 1]"): text = image.xpath("./td[2]//a[1]/text()").extract() if "firmware" in "".join(text).lower(): item = FirmwareLoader(item=FirmwareImage(), response=response, selector=image, date_fmt=["%Y-%m-%d"]) item.add_xpath("date", "td[1]//text()") item.add_value("description", text) item.add_xpath("url", "td[2]//a[1]/@href") item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) item.add_value( "version", FirmwareLoader.find_version_period(text)) yield item.load_item()
def parse_product(self, response): for section in response.xpath("//ul[@id='tab_conbox']/li"): if u"升级软件" in "".join(section.xpath("./h3//text()").extract()): for entry in section.xpath(".//dd/a"): text = entry.xpath(".//text()").extract() href = entry.xpath("./@href").extract()[0] desc = text[0] # reverse text because hw version can come before version # e.g. "FH330升级软件(V1.0) V1.0.0.24_CN" if len(text) == 1: text = text[0].split() text.reverse() item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("version", FirmwareLoader.find_version_period(text)) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("description", desc) item.add_value("vendor", self.vendor) yield item.load_item()
def parse_product(self, response): for section in response.xpath("//ul[@id='tab_conbox']/li"): if u"升级软件" in "".join(section.xpath("./h3//text()").extract()): for entry in section.xpath(".//dd/a"): text = entry.xpath(".//text()").extract() href = entry.xpath("./@href").extract()[0] desc = text[0] # reverse text because hw version can come before version # e.g. "FH330升级软件(V1.0) V1.0.0.24_CN" if len(text) == 1: text = text[0].split() text.reverse() item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value( "version", FirmwareLoader.find_version_period(text)) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("description", desc) item.add_value("vendor", self.vendor) yield item.load_item()
def parse(self, response): for entry in response.xpath( "//div[@class='main-container']//p|//div[@class='main-container']//ul"): text = entry.xpath(".//text()").extract() for href in entry.xpath(".//a/@href").extract(): if "Firmware" in href: item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value( "product", FirmwareLoader.find_product(text)) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for i in range(0, len(response.xpath("//div[@id='main_right']/span[1]/p")), 7): prods = response.xpath("//div[@id='main_right']/span[1]//p[%d]/text()" % (i + 2)).extract()[0].split("\r\n") for product in [x for x in prods]: item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath("version", "//div[@id='main_right']/span[1]//p[%d]/text()" % (i + 3)) item.add_xpath("url", "//div[@id='main_right']/span[1]//p[%d]/a/@href" % (i + 7)) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item() for i in range(0, len(response.xpath("//div[@id='main_right']/span[2]/p")), 5): prods = response.xpath("//div[@id='main_right']/span[2]//p[%d]/text()" % (i + 2)).extract()[0].split(",") for product in [x for x in prods]: item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath("version", "//div[@id='main_right']/span[2]//p[%d]/text()" % (i + 3)) item.add_xpath("url", "//div[@id='main_right']/span[2]//p[%d]/a/@href" % (i + 5)) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for entry in response.xpath( "//div[@class='main-container']//p|//div[@class='main-container']//ul" ): text = entry.xpath(".//text()").extract() for href in entry.xpath(".//a/@href").extract(): if "Firmware" in href: item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("product", FirmwareLoader.find_product(text)) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for i in range( 0, len(response.xpath("//div[@id='main_right']/span[1]/p")), 7): prods = response.xpath( "//div[@id='main_right']/span[1]//p[%d]/text()" % (i + 2)).extract()[0].split("\r\n") for product in [x for x in prods]: item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath( "version", "//div[@id='main_right']/span[1]//p[%d]/text()" % (i + 3)) item.add_xpath( "url", "//div[@id='main_right']/span[1]//p[%d]/a/@href" % (i + 7)) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item() for i in range( 0, len(response.xpath("//div[@id='main_right']/span[2]/p")), 5): prods = response.xpath( "//div[@id='main_right']/span[2]//p[%d]/text()" % (i + 2)).extract()[0].split(",") for product in [x for x in prods]: item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_xpath( "version", "//div[@id='main_right']/span[2]//p[%d]/text()" % (i + 3)) item.add_xpath( "url", "//div[@id='main_right']/span[2]//p[%d]/a/@href" % (i + 5)) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): for entry in response.xpath("//div[@class='content']//a"): text = entry.xpath(".//text()").extract() href = entry.xpath("./@href").extract()[0] idx = None for string in text: if "---" in string: idx = int(string.split("-")[0]) break if not idx: continue item = FirmwareLoader(item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("version", self.firmware[idx][1]) item.add_value("product", self.firmware[idx][0]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): for entry in response.xpath("//div[@class='view-content']//table"): if "firmware update" in " ".join(entry.xpath( "./caption//text()").extract()).lower(): for link in entry.xpath("./tbody/tr/td[1]/a"): if link.xpath(".//text()"): href = link.xpath("./@href").extract()[0] text = link.xpath(".//text()").extract()[0] product = response.xpath( "//div[@id='--2']/div[3]//div[@class='inside']//text()").extract()[2].upper().split() for category in ["RESOURCES", "FOR", "SERIES"]: if category in product: product.remove(category) item = FirmwareLoader( item=FirmwareImage(), response=response) item.add_value("url", href) item.add_value("product", " ".join(product)) item.add_value("description", text) item.add_value("vendor", self.name) yield item.load_item()
def parse(self, response): if response.xpath("//form[@name='UCagreement']"): for href in response.xpath( "//div[@id='productAndDoc']").extract()[0].split('"'): if "downloads.polycom.com" in href: item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%B %d, %Y"]) item.add_value("version", response.meta["version"]) item.add_value("url", href.encode("utf-8")) item.add_value("date", response.meta["date"]) item.add_value("description", response.meta["description"]) item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item() elif response.xpath("//div[@id='ContentChannel']"): for entry in response.xpath("//div[@id='ContentChannel']//li"): if not entry.xpath("./a"): continue text = entry.xpath("./a//text()").extract()[0] href = entry.xpath("./a/@href").extract()[0].strip() date = entry.xpath("./span//text()").extract() path = urlparse.urlparse(href).path if any(x in text.lower() for x in ["end user license agreement", "eula", "release notes", "mac os", "windows", "guide", "(pdf)", "sample"]) or href.endswith(".pdf"): continue elif any(path.endswith(x) for x in [".htm", ".html"]) or "(html)" in text.lower(): yield Request( url=urlparse.urljoin( response.url, PolycomSpider.fix_url(href)), meta={"product": response.meta["product"] if "product" in response.meta else text, "date": date, "version": FirmwareLoader.find_version_period([text]), "description": text}, headers={"Referer": response.url}, callback=self.parse) elif path: item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%B %d, %Y"]) item.add_value( "version", FirmwareLoader.find_version_period([text])) item.add_value("url", href.encode("utf-8")) item.add_value("date", item.find_date(date)) item.add_value("description", text) # item.add_value("product", response.meta["product"]) item.add_value("vendor", self.name) yield item.load_item()
def parse_product(self, response): if response.xpath("//dl[@id='dlDropDownBox']") and "build" not in response.meta: for entry in response.xpath("//dl[@id='dlDropDownBox']//li/a"): href = entry.xpath("./@href").extract()[0] text = entry.xpath(".//text()").extract()[0] yield Request( url=urlparse.urljoin(response.url, href), meta={"product": response.meta["product"], "build": text}, headers={"Referer": response.url}, callback=self.parse_product, ) else: sdk = None for href in reversed(response.xpath("//div[@id='content_gpl_code']//a/@href").extract()): sdk = href for entry in response.xpath("//div[@id='content_firmware']//table"): href = entry.xpath("./tbody/tr[1]/th[1]//a/@href").extract()[0] text = entry.xpath("./tbody/tr[1]/th[1]//a//text()").extract()[0] date = entry.xpath("./tbody/tr[1]/td[1]//text()").extract() item = FirmwareLoader(item=FirmwareImage(), response=response, date_fmt=["%d/%m/%y"]) item.add_value("url", href) item.add_value("date", item.find_date(date)) item.add_value("description", text) item.add_value("product", response.meta["product"]) item.add_value("build", response.meta["build"] if "build" in response.meta else None) item.add_value("vendor", self.vendor) item.add_value("sdk", sdk) yield item.load_item()
def parse_product(self, response): mib = None if not response.body: return for entry in reversed(response.xpath("//table/tbody/tr")): if entry.xpath("./td[contains(@class, 'versionTd')]/select"): for i in range( 0, len(entry.xpath("./td[contains(@class, 'versionTd')]/select/option"))): desc = entry.xpath( "./td[contains(@class, 'typeTd')]/span/text()").extract()[i].lower() if "firmware" in desc: date = entry.xpath( "./td[contains(@class, 'dateTd')]/span/text()").extract()[i] ver = entry.xpath( "./td[contains(@class, 'versionTd')]/select/option/text()").extract()[i] href = entry.xpath( "./td[contains(@class, 'downloadTd')]/div/a[1]/@data-filelink").extract()[i] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%m-%d-%Y"]) item.add_value("version", ver) item.add_value("date", date) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("mib", mib) item.add_value("vendor", self.name) yield item.load_item() else: desc = entry.xpath( "./td[contains(@class, 'typeTd')]//text()").extract()[1].lower() if "firmware" in desc: date = entry.xpath( "./td[contains(@class, 'dateTd')]//text()").extract() ver = entry.xpath( "./td[contains(@class, 'versionTd')]//text()").extract() href = entry.xpath( "./td[contains(@class, 'downloadTd')]//a/@data-filelink").extract()[0] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%m-%d-%Y"]) item.add_value("version", ver) item.add_value("date", date) item.add_value("url", href) item.add_value("product", response.meta["product"]) item.add_value("mib", mib) item.add_value("vendor", self.name) yield item.load_item() elif "mib" in desc: mib = entry.xpath( "./td[contains(@class, 'downloadTd')]//a/@href").extract()[0]
def parse(self, response): for entry in response.xpath("//table/tr[position() > 3]"): if not entry.xpath("./td[2]/a"): continue text = entry.xpath("./td[2]/a//text()").extract()[0] href = entry.xpath("./td[2]/a/@href").extract()[0] date = entry.xpath("./td[3]//text()").extract()[0] if "DSM" in response.url: if href.endswith('/'): build = None version = response.meta.get( "version", FirmwareLoader.find_version_period([text])) if not FirmwareLoader.find_version_period([text]): build = text[0: -1] yield Request( url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, meta={"build": build, "version": version}, callback=self.parse) elif all(not href.lower().endswith(x) for x in [".txt", ".md5", ".torrent"]): product = None basename = os.path.splitext(text)[0].split("_") if "DSM" in basename: if response.meta["build"] in basename: basename.remove(response.meta["build"]) basename.remove("DSM") product = " ".join(basename) else: product = basename[-2] item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("build", response.meta["build"]) item.add_value("version", response.meta["version"]) item.add_value( "mib", "http://dedl.synology.com/download/Document/MIBGuide/Synology_MIB_File.zip") item.add_value("url", href) item.add_value("date", date) item.add_value("product", product) item.add_value("vendor", self.name) yield item.load_item() elif "VSFirmware" in response.url: if href.endswith('/'): version, build = text[0: -1].split("-") yield Request( url=urlparse.urljoin(response.url, href), headers={"Referer": response.url}, meta={"build": build, "version": version}, callback=self.parse) elif all(not href.lower().endswith(x) for x in [".txt", ".md5", ".torrent"]): basename = os.path.splitext(text)[0].split("_") item = FirmwareLoader( item=FirmwareImage(), response=response, date_fmt=["%d-%b-%Y"]) item.add_value("build", response.meta["build"]) item.add_value("version", response.meta["version"]) item.add_value("url", href) item.add_value("date", date) item.add_value("product", basename[0]) item.add_value("vendor", self.name) yield item.load_item()