class SaraivaScrapper(QThread): notifySaraiva = pyqtSignal(object) def __init__(self, urlList, category, htmlTag, replaceTag): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.urlList = urlList self.category = category self.htmlTag = self.regex.replaceData('\r+', '', htmlTag) self.htmlTag = self.regex.replaceData('\n+', ' ', self.htmlTag) self.htmlTag = self.regex.replaceData('\s+', ' ', self.htmlTag) self.htmlTag = self.regex.replaceData(r'\"+', '\"', self.htmlTag) self.replaceTag = replaceTag self.csvWriter = Csv(category + '.csv') csvDataHeader = ['Link', 'Name', 'Subtitle', 'Price', 'Synopsis and Characteristics', 'Picture'] self.csvWriter.writeCsvRow(csvDataHeader) self.mainUrl = 'http://busca.livrariasaraiva.com.br' self.scrapUrl = None self.dbHelper = DbHelper('saraiva.db') self.dbHelper.createTable(category) self.total = self.dbHelper.getTotalProduct(category) def run(self, retry=0): try: if self.urlList is not None and len(self.urlList): for url in self.urlList: if len(url) > 0: url = self.regex.replaceData('(?i)\r', '', url) url = self.regex.replaceData('(?i)\n', '', url) self.notifySaraiva.emit('<font color=green><b>Saraiva Main URL: %s</b></font>' % url) paginationUrl, self.maxRecords = self.reformatUrl(url) self.notifySaraiva.emit( '<font color=black><b>Total Records: %s</b></font>' % str(self.maxRecords)) print 'Max records: ', self.maxRecords print 'URL: ' + str(paginationUrl) sortList = ['&isort=globalpop', '&isort=best', '&isort=title', '&isort=title+rev', '&isort=price+rev', '&isort=price', '&isort=date+rev'] for sort in sortList: self.scrapResults(paginationUrl, sort) self.notifySaraiva.emit('<font color=red><b>Saraiva Data Scraping finished.</b></font>') except Exception, x: print x.message self.logger.error('Exception at run: ', x.message) if retry < 5: self.run(retry + 1)
class GoogleFinanceScrapper: isFinished = False def __init__(self, filename): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.filename = filename self.url = 'https://www.google.com/finance?' self.main_url = 'https://www.google.com' self.csvWriter = Csv('google_finance.csv') csvDataHeader = ['Ticker Symbol', 'Quarter End', 'Revenue', 'Total Revenue', 'Date of Scrape'] self.csvWriter.writeCsvRow(csvDataHeader) def run(self): self.scrapData() self.csvWriter.closeWriter() def scrapData(self): try: file = open(self.filename, 'rb') for line in file.readlines(): if self.isFinished: return line = self.regex.replaceData('\r+', '', line) line = self.regex.reduceNewLine(line) line = self.regex.reduceBlankSpace(line) line = line.strip() params = urllib.urlencode({'q': line}) url = self.url + params self.scrapBykeyword(url, line) except Exception, x: print x self.logger.error('Error: ' + x.message)
class TopsyScrapper: isFinished = False def __init__(self, filename): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.filename = filename self.url = 'http://topsy.com/s?' self.csvWriter = Csv('topsy.csv') csvDataHeader = ['Keyword', 'Tweets in last 30 days', 'Topsy Sentiment Score', ' Date of scrape'] self.csvWriter.writeCsvRow(csvDataHeader) def run(self): self.scrapData() self.csvWriter.closeWriter() def scrapData(self): try: file = open(self.filename, 'rb') for line in file.readlines(): if self.isFinished: return line = self.regex.replaceData('\r+', '', line) line = self.regex.reduceNewLine(line) line = self.regex.reduceBlankSpace(line) line = line.strip() if len(line) > 0: params = urllib.urlencode({'q': line, 'window': 'm', 'type': 'tweet'}) url = self.url + params self.scrapBrowserData(url, line) except Exception, x: print x
class AmazonScrapper(QThread): notifyAmazon = pyqtSignal(object) def __init__(self, urlList, category): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.urlList = urlList self.category = category dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow(category + '.csv') self.csvWriter = Csv(category + '.csv') csvDataHeader = ['SKU', 'Title', 'Sub Title', 'Price', 'Shipping Weight', 'Image URL'] if csvDataHeader not in self.dupCsvRows: self.dupCsvRows.append(csvDataHeader) self.csvWriter.writeCsvRow(csvDataHeader) self.mainUrl = 'http://www.amazon.com' self.scrapUrl = None self.dbHelper = DbHelper('amazon.db') self.dbHelper.createTable(category) self.total = self.dbHelper.getTotalProduct(category) def run(self, retry=0): try: # self.scrapProductDetail( # 'http://www.amazon.com/Casio-MRW-S300H-8BVCF-Solar-Powered-Analog/dp/B00ELALKH2/ref=sr_1_544/184-7248556-2619812?s=watches&ie=UTF8&qid=1397580509&sr=1-544') # return if self.urlList is not None and len(self.urlList): for url in self.urlList: if len(url) > 0: url = self.regex.replaceData('(?i)\r', '', url) url = self.regex.replaceData('(?i)\n', '', url) self.notifyAmazon.emit('<font color=green><b>Amazon Main URL: %s</b></font>' % url) imUrl = None retry = 0 while imUrl is None and retry < 4: imUrl = self.reformatUrl(url) retry += 1 if imUrl is None: imUrl = url self.total = 0 print 'URL: ' + str(imUrl) sortList = ['relevance-fs-browse-rank', 'price', '-price', 'reviewrank_authority', 'date-desc-rank'] for sort in sortList: self.scrapReformatData(imUrl, sort) self.notifyAmazon.emit( '<font color=red><b>Finish data for Amazon Main URL: %s</b></font><br /><br />' % url) self.notifyAmazon.emit('<font color=red><b>Amazon Data Scraping finished.</b></font>') except Exception, x: print x.message self.logger.error('Exception at run: ', x.message) if retry < 5: self.run(retry + 1)
class PaodeacucarScrapper(QThread): notifyPaode = pyqtSignal(object) def __init__(self): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.mainUrl = 'http://www.paodeacucar.com.br/' self.url = 'http://www.paodeacucar.com.br/' dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow('paodeacucar.csv', 4) self.csvWriter = Csv('paodeacucar.csv') csvDataHeader = ['SKU', 'Category', 'Subcategory', 'Name', 'URL', 'URL Image', 'Details', 'Nutrients Table html code', 'Price from, 28/abr/14', '28/abr/14'] if 'URL' not in self.dupCsvRows: self.dupCsvRows.append(csvDataHeader) self.csvWriter.writeCsvRow(csvDataHeader) def run(self): self.scrapData() def scrapData(self): try: print 'Main URL: ', self.url self.notifyPaode.emit(('<font color=green><b>Main URL: %s</b></font>' % self.url)) data = self.spider.fetchData(self.url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) soup = BeautifulSoup(data) categories = soup.find('nav', class_='items-wrapper').find_all('li', class_=re.compile('\s*item\s*')) print 'Total Categories: ', len(categories) self.notifyPaode.emit(('<font color=black><b>Total Categories: %s</b></font>' % str(len(categories)))) for category in categories: if category.a is not None: submenu_target = self.regex.replaceData('#', '', category.a.get('data-target')) sub_categories = soup.find('ul', id=submenu_target).find_all('li', class_='item') print 'Total Sub Categories: ', len(sub_categories) self.notifyPaode.emit(('<font color=black><b>Total Subcategories: %s</b></font>' % str(len(sub_categories)))) for sub_category in sub_categories: sub_category_label = sub_category.find('span', class_='label').text sub_category_url = sub_category.a.get('href') if sub_category.a is not None else 'N/A' self.scrapItems(sub_category_url, category.text, sub_category_label) except Exception, x: self.logger.error(x.message) print x
class AmazonScrapper(): def __init__(self, url): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.url = url self.base_product_url = 'http://www.amazon.com/dp/' self.base_image_url = 'http://ecx.images-amazon.com/images/I/' self.csvWriter = Csv('amazon.csv') csvDataHeader = ['URL', 'HTML Path', 'Image URLS'] self.csvWriter.writeCsvRow(csvDataHeader) def scrapData(self): try: host = ('Host', 'www.amazon.com') data = self.spider.fetchData(self.url, host=host) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) searchParams = self.regex.getSearchedData('(?i)var searchParams = {([^\}]*)}', data) searchParams = searchParams.split(',') seller = '' marketPlaceId = '' useMYI = '' for searchParam in searchParams: searchParam = self.regex.reduceBlankSpace(searchParam) searchParam = self.regex.replaceData('\'', '', searchParam) if searchParam.startswith('seller'): seller = searchParam.split(':')[1].strip() seller = seller.decode('string-escape') if searchParam.startswith('marketplaceID'): marketPlaceId = searchParam.split(':')[1].strip() marketPlaceId = marketPlaceId.decode('string-escape') if searchParam.startswith('useMYI'): useMYI = searchParam.split(':')[1].strip() useMYI = useMYI.decode('string-escape') params = {'seller': seller, 'marketPlaceId': marketPlaceId, 'useMYI': useMYI} ajax_url = 'http://www.amazon.com/gp/aag/ajax/productWidget.html' self.scrapAjaxPage(ajax_url, params, host) except Exception, x: print x
class TopsyScrapper: isFinished = False def __init__(self, filename): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.filename = filename self.url = 'http://topsy.com/s?' self.csvWriter = Csv('topsy.csv') csvDataHeader = [ 'Keyword', 'Tweets in last 30 days', 'Topsy Sentiment Score', ' Date of scrape' ] self.csvWriter.writeCsvRow(csvDataHeader) def run(self): self.scrapData() self.csvWriter.closeWriter() def scrapData(self): try: file = open(self.filename, 'rb') for line in file.readlines(): if self.isFinished: return line = self.regex.replaceData('\r+', '', line) line = self.regex.reduceNewLine(line) line = self.regex.reduceBlankSpace(line) line = line.strip() if len(line) > 0: params = urllib.urlencode({ 'q': line, 'window': 'm', 'type': 'tweet' }) url = self.url + params self.scrapBrowserData(url, line) except Exception, x: print x
class GoogleFinanceScrapper: isFinished = False def __init__(self, filename): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() self.filename = filename self.url = 'https://www.google.com/finance?' self.main_url = 'https://www.google.com' self.csvWriter = Csv('google_finance.csv') csvDataHeader = [ 'Ticker Symbol', 'Quarter End', 'Revenue', 'Total Revenue', 'Date of Scrape' ] self.csvWriter.writeCsvRow(csvDataHeader) def run(self): self.scrapData() self.csvWriter.closeWriter() def scrapData(self): try: file = open(self.filename, 'rb') for line in file.readlines(): if self.isFinished: return line = self.regex.replaceData('\r+', '', line) line = self.regex.reduceNewLine(line) line = self.regex.reduceBlankSpace(line) line = line.strip() params = urllib.urlencode({'q': line}) url = self.url + params self.scrapBykeyword(url, line) except Exception, x: print x self.logger.error('Error: ' + x.message)
class Scrapper(QThread): notifyScrapper = pyqtSignal(object) isFinished = False def __init__(self, urllist): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() print urllist self.urllist = urllist self.csv = Csv('scrapper.csv') def run(self): self.scrapData() self.notifyScrapper.emit( '<font color=green><b>------------------ Finish! ------------------------- </b></font>') def scrapData(self): try: total = 0 csvHeader = ['URL', 'Title', 'Price', 'Brand', 'Features', 'Material', 'Measurements', 'Category', 'Size', 'Color', 'Design'] self.csv.writeCsvRow(csvHeader) if self.isFinished: return for url in self.urllist: if len(url) > 0: url = self.regex.replaceData('(?i)\r', '', url) url = self.regex.replaceData('(?i)\n', '', url) url = self.regex.getSearchedData('(?i)(http.*?)$', url) print 'URL: ', url self.notifyScrapper.emit(('<font color=green><b>URL: %s</b></font>' % url)) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) soup = BeautifulSoup(data) soup.prettify() title = '' price = '' size = '' brand = '' features = '' material = '' measurements = '' category = '' color = '' design = '' if soup.find('span', id='vi-lkhdr-itmTitl') is not None: title = soup.find('span', id='vi-lkhdr-itmTitl').text if soup.find('span', id='prcIsum'): price = soup.find('span', id='prcIsum').text if soup.find('div', class_='itemAttr'): specchunk = soup.find('div', class_='itemAttr') rows = specchunk.find_all('tr') for row in rows: cols = row.find_all('td') for i in range(0, len(cols), 2): # if self.regex.isFoundPattern('(?i)Condition:', cols[i].text.strip()): # conditionChunk = cols[i + 1] # conditionChunk = self.regex.replaceData(u'(?i)<span class="infoLink u-nowrap" id="readFull">.*?</span>', '', unicode(conditionChunk)) # conditionChunk = self.regex.replaceData(u'(?i)<b class="g-hdn">.*?</b>', '', conditionChunk) # condition = BeautifulSoup(conditionChunk).text # print condition if self.regex.isFoundPattern('(?i)Brand:', cols[i].text.strip()): brand = cols[i + 1].text if self.regex.isFoundPattern('(?i)Features:', cols[i].text.strip()): features = cols[i + 1].text if self.regex.isFoundPattern('(?i)Material:', cols[i].text.strip()): material = cols[i + 1].text if self.regex.isFoundPattern('(?i)Measurements:', cols[i].text.strip()): measurements = cols[i + 1].text if self.regex.isFoundPattern('(?i)Category:', cols[i].text.strip()): category = cols[i + 1].text if self.regex.isFoundPattern('(?i)Color:', cols[i].text.strip()): color = cols[i + 1].text if self.regex.isFoundPattern('(?i)Design:', cols[i].text.strip()): design = cols[i + 1].text if self.regex.isFoundPattern('(?i)Size:', cols[i].text.strip()): size = cols[i + 1].text self.notifyScrapper.emit('<font color=black><b>Writting data to csv file.</b></font>') csvData = [url, title, price, brand, features, material, measurements, category, size, color, design] self.notifyScrapper.emit('<font color=black><b>Data: %s</b></font>' % unicode(csvData)) self.csv.writeCsvRow(csvData) self.notifyScrapper.emit('<font color=black><b>Successfully Written data to csv file.</b></font>') total += 1 self.notifyScrapper.emit('<font color=green><b>Total Data scrapped: [%s]</b></font>' % str(total)) except Exception, x: self.notifyScrapper.emit('<font color=red><b>Error scrapping category: %s</b></font>' % x.message) self.logger.error(x.message) print x
class CsCat(QThread): notifyCategory = pyqtSignal(object) def __init__(self): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow('cs_cat.csv') self.csvWriter = Csv('cs_cat.csv') dupFilterCsvReader = Csv() self.dupFilterCsvRows = dupFilterCsvReader.readCsvRow('filter_cat' + '.csv') self.csvW = Csv('filter_cat' + '.csv') self.mainUrl = 'http://www.cs-catering-equipment.co.uk/' self.totalCategory = 0 def run(self): self.scrapCategories() self.notifyCategory.emit('<font color=red><b>Finished Scraping All Categories.</b></font>') def scrapCategories(self): # self.scrapFinalCategory('http://www.cs-catering-equipment.co.uk/kitchen-equipment/food-prep-machines/chocolate-fountains', '', '') # return self.notifyCategory.emit('<b>Start scraping Category.</b>') self.notifyCategory.emit('<font color=green><b>Main URL: %s</b></font>' % self.mainUrl) data = self.spider.fetchData(self.mainUrl) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) # <a href="http://www.cs-catering-equipment.co.uk/kitchen-equipment" class="level-top" title="Kitchen Equipment" categories = self.regex.getAllSearchedData('(?i)<a href="([^"]*)" class="level-top" title="([^"]*)"', data) if categories and len(categories) > 0: self.totalCategory += len(categories) self.notifyCategory.emit( '<font color=green><b>Total Category Found [%s]</b></font>' % unicode(self.totalCategory)) for category in categories: homeCategoryName = 'Home' categoryName = unicode(category[1]).strip() self.scrapCategory(str(category[0]).strip(), homeCategoryName, categoryName) def scrapCategory(self, url, rootCategoryName, categoryName): self.notifyCategory.emit('<font color=green><b>Start scraping URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: print 'category 1' data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) self.filterCategory(data, categoryName) categoryDesc = self.regex.getSearchedData('(?i)<div class="category-description std">([^<]*)</div>', data) if categoryDesc and len(categoryDesc) > 0: categoryDesc = unicode(categoryDesc).strip() csvData = [rootCategoryName, categoryName, categoryDesc] if csvData not in self.dupCsvRows: self.notifyCategory.emit('<b>Scraped Data: %s</b>' % unicode(csvData)) self.csvWriter.writeCsvRow(csvData) self.dupCsvRows.append(csvData) else: self.notifyCategory.emit('<font color=green><b>Already Exits Category [%s] in csv file. Skip it.</b></font>' % categoryName) subCategories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if subCategories and len(subCategories) > 0: self.totalCategory += len(subCategories) self.notifyCategory.emit( '<font color=green><b>Total Category Found [%s]</b></font>' % unicode(self.totalCategory)) for subCategory in subCategories: print subCategory self.scrapSubCategory(subCategory[0], categoryName, subCategory[1]) def scrapSubCategory(self, url, rootCategoryName, categoryName): self.notifyCategory.emit('<font color=green><b>Start scraping URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) self.filterCategory(data, categoryName) categoryDesc = self.regex.getSearchedData('(?i)<div class="category-description std">([^<]*)</div>', data) categoryDesc = unicode(categoryDesc).strip() csvData = [rootCategoryName, categoryName, categoryDesc] if csvData not in self.dupCsvRows: self.csvWriter.writeCsvRow(csvData) self.dupCsvRows.append(csvData) self.notifyCategory.emit('<b>Scraped Data: %s</b>' % unicode(csvData)) else: self.notifyCategory.emit('<font color=green><b>Already Exits Category [%s] in csv file. Skip it.</b></font>' % categoryName) subCategories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if subCategories and len(subCategories) > 0: self.totalCategory += len(subCategories) self.notifyCategory.emit( '<font color=green><b>Total Category Found [%s]</b></font>' % unicode(self.totalCategory)) for subCategory in subCategories: self.scrapFinalCategory(subCategory[0], categoryName, subCategory[1]) def scrapFinalCategory(self, url, rootCategoryName, categoryName): self.notifyCategory.emit('<font color=green><b>Start scraping URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) self.filterCategory(data, categoryName) categoryDesc = self.regex.getSearchedData(u'(?i)<div class="category-description std">([^<]*)</div>', data) if len(categoryDesc) > 0: categoryDesc = categoryDesc.strip() csvData = [rootCategoryName, categoryName, categoryDesc] if csvData not in self.dupCsvRows: self.csvWriter.writeCsvRow(csvData) self.dupCsvRows.append(csvData) self.notifyCategory.emit('<b>Scraped Data: %s</b>' % unicode(csvData)) else: self.notifyCategory.emit('<font color=green><b>Already Exits Category [%s] in csv file. Skip it.</b></font>' % categoryName) def filterCategory(self, data, categoryName): # self.csvW = Csv(category + '.csv') filterData = self.regex.getSearchedData('(?i)<h4>Filter your results</h4> <dl id="narrow-by-list">(.*?)</dl>', data) if filterData and len(filterData) > 0: self.notifyCategory.emit('<b>Filter Data found writing to csv</b>') allFilters = self.regex.getAllSearchedData('(?i)<dt>([^<]*)</dt> <dd>(.*?)</dd>', filterData) topData = [categoryName] childData = [] maxLen = 0 for allFilter in allFilters: topData.append(allFilter[0]) print 'Filter: ' + allFilter[0] filterName = self.regex.replaceData('(?i)<span class="price">', '', allFilter[1]) filterName = self.regex.replaceData('(?i)</span>', '', filterName) filters = self.regex.getAllSearchedData('(?i)<a href=[^>]*>([^<]*)</a>', filterName) if filters is not None and len(filters) > 0: childData.append(filters) if len(filters) > maxLen: maxLen = len(filters) if topData not in self.dupFilterCsvRows: self.csvW.writeCsvRow(topData) self.notifyCategory.emit( '<font color=green><b>Filters Found For Category [%s].</b></font> <br /><b>Filters are: %s</b>' % ( unicode(categoryName), unicode(topData[1:]))) else: self.notifyCategory.emit('<font color=green><b>Already scraped Filter For Category [%s]. Skip it.</b></font>' % categoryName) return for row in range(maxLen): rowData = [''] for columnData in childData: if len(columnData) > row: rowData.append(columnData[row]) else: rowData.append('') print rowData self.csvW.writeCsvRow(rowData) else: self.notifyCategory.emit( '<font color=green><b>No Filter Found For Category[%s].</b></font>' % categoryName)
class CsProduct(QThread): notifyProduct = pyqtSignal(object) def __init__(self): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() dupCsvReader = Csv() self.dupCsvRows0 = dupCsvReader.readCsvRow('cs_product.csv', 0) self.dupCsvRows = dupCsvReader.readCsvRow('cs_product.csv', 1) self.csvWriter = Csv('cs_product.csv') self.mainUrl = 'http://www.cs-catering-equipment.co.uk/' self.utils = Utils() if 'Product Code' not in self.dupCsvRows: self.csvWriter.writeCsvRow( ['URL', 'Product Code', 'Product Name', 'Manufacturer', 'List Price', 'Product Price', 'Discount', 'Product Short Description', 'Product Long Description', 'Product Technical Specifications', 'Warranty' , 'Delivery', 'Product Image', 'Category 1', 'Category 2', 'Category 3', 'Category 4', 'Brand Image']) self.totalProducts = len(self.dupCsvRows) def run(self): self.scrapProduct() self.notifyProduct.emit('<font color=red><b>Finished Scraping All products.</b></font>') def scrapProduct(self): # self.logger.debug('Main URL: ' + self.mainUrl) self.notifyProduct.emit('<font color=green><b>Main URL: %s</b></font>' % self.mainUrl) data = self.spider.fetchData(self.mainUrl) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) self.notifyProduct.emit('<b>Try to scrap all categories.</b>') categories = self.regex.getAllSearchedData('(?i)<a href="([^"]*)" class="level-top" title="([^"]*)"', data) if categories and len(categories) > 0: self.notifyProduct.emit('<b>Total Categories Found: %s</b>' % str(len(categories))) for category in categories: category1Name = unicode(category[1]).strip() self.scrapCategory1Data(str(category[0]).strip(), category1Name) def scrapCategory1Data(self, url, category1Name): # self.logger.debug('Category 1 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category1Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: self.notifyProduct.emit('<b>Total Categories Found: %s</b>' % str(len(categories))) for category in categories: self.scrapCategory2Data(category[0], category1Name, category[1]) def scrapCategory2Data(self, url, category1Name, category2Name): # self.logger.debug('Category 2 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category2Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: for category in categories: print 'category2: ' + category[0] self.scrapCategory3Data(category[0], category1Name, category2Name, category[1]) def scrapCategory3Data(self, url, category1Name, category2Name, category3Name): # self.logger.debug('Category 3 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category3Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: for category in categories: print [category1Name, category2Name, category3Name, category[1]] self.scrapProductsDetails(category[0], category1Name, category2Name, category3Name, category[1]) def scrapProductsDetails(self, url, category1Name, category2Name, category3Name, category4Name): self.logger.debug('Product Details URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all products under Category[%s]</b>' % category4Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) maxLimit = 25 maxLimitChunk = self.spider.fetchData(url + '?mode=list') if maxLimitChunk and len(maxLimitChunk): maxLimitChunk = self.regex.reduceNewLine(maxLimitChunk) maxLimitChunk = self.regex.reduceBlankSpace(maxLimitChunk) maxLimits = self.regex.getAllSearchedData('<option value="[^"]*limit=(\d+)[^"]*"', maxLimitChunk) # print maxLimits if maxLimits and len(maxLimits) > 0: maxLimit = max(map(int, maxLimits)) # print maxLimit # self.notifyProduct.emit('<font color=blue><b>Max Limit: %s</b></font>' % str(maxLimit)) data = self.spider.fetchData(url + '?limit=' + str(maxLimit) + '&mode=list') if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) products = self.regex.getAllSearchedData('(?i)<div class="listing-item[^"]*?">(.*?)</div>', data) if products and len(products) > 0: print len(products) # self.totalProducts += len(products) # self.logger.debug('Total Products for %s is [%s]' % (str(len(products)), self.totalProducts)) self.notifyProduct.emit('<font color=green><b>Total Products Found [%s] for category[%s]</b></font>' % ( str(len(products)), category4Name)) for product in products: productDetailUrl = self.regex.getSearchedData('(?i)<a href="([^"]*)"', product) if productDetailUrl not in self.dupCsvRows0: # self.totalProducts += 1 self.dupCsvRows0.append(productDetailUrl) self.scrapProductDetails(productDetailUrl, category1Name, category2Name, category3Name, category4Name) else: self.notifyProduct.emit( '<font color=green><b>Already Exists This Product Under Category[%s]. Skip It.</b></font>' % category4Name) self.notifyProduct.emit( '<font color=green><b>Total Products Scraped [%s].</b></font>' % str(self.totalProducts)) def scrapProductDetails(self, url, category1Name, category2Name, category3Name, category4Name): self.logger.debug('Product Detail URL: ' + url) self.notifyProduct.emit('<b>Try to scrap product details under Category[%s]</b>' % category4Name) self.notifyProduct.emit('<font color=green><b>Product Detail URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) manufacturer = self.regex.getSearchedData( '(?i)<span class="manufacturer-box-label">Manufacturer:</span>([^<]*)</p>', data) productCode = self.regex.getSearchedData( '(?i)<span class="manufacturer-box-label">Model No:</span>([^<]*)</p>', data) if productCode not in self.dupCsvRows: self.totalProducts += 1 self.dupCsvRows.append(productCode) else: self.notifyProduct.emit( '<font color=green><b>Already Exists This Product Under Category[%s]. Skip It.</b></font>' % category4Name) return productName = self.regex.getSearchedData('(?i)<div class="product-name"> <h1>([^<]*)</h1>', data) productTechnicalDesc = self.regex.getSearchedData('(?i)<div class="product-short-description">([^<]*)</div>' , data) productDescriptions = self.regex.getSearchedData('(?i)<div class="product-specs">(.*?)</div>', data) productShortDesc = '' productFullDesc = '' if productDescriptions and len(productDescriptions) > 0: print 'desc: ' + productDescriptions productShortDesc = self.regex.getSearchedData('(?i)<p>(.*?)</p>', productDescriptions) productFullDesc = '\n'.join( self.regex.getAllSearchedData('(?i)<li>([^<]*)</li>', productDescriptions)) listPriceChunk = self.regex.getSearchedData('(?i)<div class="rrp-price regular-price">(.*?)</div>', data) listPrice = '' if listPriceChunk and len(listPriceChunk) > 0: listPrice = self.regex.getSearchedData('(?i)([0-9,.]+)', listPriceChunk) savePriceChunk = self.regex.getSearchedData('(?i)<div class="regular-price saving-price">(.*?)</div>', data) savePrice = '' if savePriceChunk and len(savePriceChunk) > 0: savePrice = self.regex.getSearchedData('(?i)([0-9%]+)', savePriceChunk) priceChunk = self.regex.getSearchedData('(?i)<div class="[^"]*" id="product-price-\d+">(.*?)</div>', data) price = '' if priceChunk and len(priceChunk) > 0: price = self.regex.getSearchedData('(?i)([0-9,.]+)', priceChunk) deliveryChunk = self.regex.getSearchedData('(?i)<div class="delivery">(.*?)</div>', data) delivery = '' if deliveryChunk and len(deliveryChunk) > 0: delivery = self.regex.getSearchedData('(?i)<p>([^<]*)</p>', deliveryChunk) warrantyChunk = self.regex.getSearchedData('(?i)<div class="warranty">(.*?)</div>', data) warranty = '' if warrantyChunk and len(warrantyChunk) > 0: warranty = self.regex.getSearchedData('(?i)<p>([^<]*)</p>', warrantyChunk) ## Download and save product images productImageUrl = self.regex.getSearchedData( '(?i)src="(http://assets.cs-catering-equipment.co.uk/media/catalog/product/cache/1/image/256x/[^"]*)"', data) print productImageUrl productImage = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_.]*)$', productImageUrl) if productImage and len(productImage) > 0: print productImage self.notifyProduct.emit( '<font color=green><b>Downloading Product Image [%s]. Please wait...</b></font>' % productImage) self.downloadFile(productImageUrl, 'product_image/' + productImage) self.notifyProduct.emit('<font color=green><b>Downloaded Product Image [%s].</b></font>' % productImage) # self.utils.downloadFile(productImageUrl, 'product_image/' + productImage) ## Download and save brand images brandImageUrl = self.regex.getSearchedData( '(?i)<div class="manufacturer-box-left"><a href="[^"]*"[^>]*?><img src="([^"]*)"', data) brandImage = '' if brandImageUrl and len(brandImageUrl) > 0: brandImageUrl = self.regex.replaceData('(?i)logo/', '', brandImageUrl) brandImage = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_.]*)$', brandImageUrl) if brandImage and len(brandImage) > 0: self.notifyProduct.emit( '<font color=green><b>Downloading Brand Image [%s]. Please wait...</b></font>' % brandImage) # self.utils.downloadFile(brandImageUrl, 'brand_image/' + brandImage) self.downloadFile(brandImageUrl, 'brand_image/' + brandImage) self.notifyProduct.emit('<font color=green><b>Downloaded Brand Image [%s].</b></font>' % brandImage) csvData = [url, productCode, productName, manufacturer, listPrice, price, savePrice, productShortDesc, productFullDesc, productTechnicalDesc, warranty, delivery, productImage, category1Name, category2Name, category3Name, category4Name, brandImage] self.csvWriter.writeCsvRow(csvData) self.logger.debug(unicode(csvData)) self.notifyProduct.emit('<b>Product Details: %s</b>' % unicode(csvData)) def downloadFile(self, url, downloadPath, retry=0): print url self.notifyProduct.emit('<b>File URL: %s.</b>' % url) try: socket.setdefaulttimeout(10) opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0)) opener.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1')] urllib2.install_opener(opener) # resp = opener.open(url, timeout=30) # resp = urllib2.urlopen(url, timeout=30) resp = None try: # resp = urllib.urlopen(url) resp = opener.open(url, timeout=30) except Exception, x: print x if resp is None: return False print resp.info() print 'info.......' contentLength = resp.info()['Content-Length'] contentLength = self.regex.getSearchedData('(?i)^(\d+)', contentLength) totalSize = float(contentLength) directory = os.path.dirname(downloadPath) if not os.path.exists(directory): try: os.makedirs(directory) except Exception, x: print x dl_file = open(downloadPath, 'wb') currentSize = 0 CHUNK_SIZE = 32768 totalSizeKB = totalSize / 1024 if totalSize > 0 else totalSize print 'everything ok............' while True: data = None try: data = resp.read(CHUNK_SIZE) except Exception, x: print x if not data: break currentSize += len(data) dl_file.write(data) print('============> ' +\ str(round(float(currentSize * 100) / totalSize, 2)) +\ '% of ' + str(totalSize) + ' bytes') notifyDl = '===> Downloaded ' + str(round(float(currentSize * 100) / totalSize, 2)) + '% of ' + str( totalSizeKB) + ' KB.' self.notifyProduct.emit('<b>%s</b>' % notifyDl) if currentSize >= totalSize: dl_file.close() return True
# -*- coding: utf-8 -*- from utils.Regex import Regex __author__ = "Rabbi-Tuly" if __name__ == "__main__": text = """<table style="width: 187px;" align="center" background=" border="0" cellpadding="0" cellspacing="0"> <tbody> <tr> <td colspan="2"><img src="http://img.photobucket.com/albums/v402/WGT-32/1.png" /> <a href="http://eshops.mercadolivre.com.br/quackstore/" target="_blank"><img src="http://img.photobucket.com/albums/v402/WGT-32/2.png" border="0" /></a> <img src="http://img.photobucket.com/albums/v402/WGT-32/3.png" /> <table style="width: 100%;" border="0" cellpadding="0" cellspacing="0"> <tbody> <tr> <td style="padding-left: 60px;" background="http://img.photobucket.com/albums/v402/WGT-32/fundo.png"> <p> </p> <p>Você está convidado a juntar-se a um grupo que durante uma semana vai estudar com um dos maiores especialistas em liderança dos Estados Unidos. Leonard Hoffman, um famoso empresário que abandonou sua brilhante carreira para se tornar monge em um mosteiro beneditino, é o personagem central desta envolvente história criada por James C. Hunter para ensinar de forma clara e agradável os princÃpios fundamentais dos verdadeiros lÃderes. Se você tem dificuldade em fazer com que sua equipe dê o melhor de si no trabalho e gostaria de se relacionar melhor com sua famÃlia e seus amigos, vai encontrar neste livro personagens, idéias e discussões que vão abrir um novo horizonte em sua forma de lidar com os outros. É impossÃvel ler este livro sem sair transformado. "O Monge e o Executivo" é, sobretudo, uma lição sobre como se tornar uma pessoa melhor.<br /> <br /> <b>I.S.B.N.: </b>8575421026<br /> <br /> <b>Cód. Barras: </b>9788575421024<br /> <br /> <b>Reduzido: </b>149181<br /> <br /> <b>Altura: </b>21 cm.<br /><b>Largura: </b>14 cm.<br /><b>Acabamento : </b>Brochura<br /><b>Edição : </b>1 / 2004<br /><b>Idioma : </b>Português<br /><b>PaÃs de Origem : </b>Brasil<br /><b>Número de Paginas : </b>144<br /> <br /></p> <p> </p> <p style="text-align: center;"></p> </td> </tr> </tbody> </table> <img src="http://img.photobucket.com/albums/v402/WGT-32/5.png" /><br /> <img src="http://img.photobucket.com/albums/v402/WGT-32/6.png" /><br /> <img src="http://img.photobucket.com/albums/v402/WGT-32/7.png" /><br /> <img src="http://img.photobucket.com/albums/v402/WGT-32/8_usa_pf.png" /><br /> <img src="http://img.photobucket.com/albums/v402/WGT-32/10.png" /><br /> <img src="http://img.photobucket.com/albums/v402/WGT-32/11.png" /></td> </tr> <tr> <td valign="top" width="186"></td> <td valign="top" width="1"></td> </tr> <tr> <td colspan="2"></td> </tr> </tbody> </table>""" regex = Regex() d = regex.replaceData(r"(?i)(?:<br\s*/>\s*)+", "<br />", text) print d
class CsProduct(QThread): notifyProduct = pyqtSignal(object) def __init__(self): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow('cs_product.csv', 0) self.csvWriter = Csv('cs_product.csv') self.mainUrl = 'http://www.cs-catering-equipment.co.uk/' self.utils = Utils() self.csvWriter.writeCsvRow( ['URL', 'Product Code', 'Product Name', 'Manufacturer', 'List Price', 'Product Price', 'Discount', 'Product Short Description', 'Product Long Description', 'Product Technical Specifications', 'Warranty', 'Delivery', 'Product Image', 'Category 1', 'Category 2', 'Category 3', 'Category 4', 'Brand Image']) self.totalProducts = 0 def run(self): self.scrapProduct() self.notifyProduct.emit('<font color=red><b>Finished Scraping All products.</b></font>') def scrapProduct(self): self.logger.debug('Main URL: ' + self.mainUrl) self.notifyProduct.emit('<font color=green><b>Main URL: %s</b></font>' % self.mainUrl) data = self.spider.fetchData(self.mainUrl) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) self.notifyProduct.emit('<b>Try to scrap all categories.</b>') categories = self.regex.getAllSearchedData('(?i)<a href="([^"]*)" class="level-top" title="([^"]*)"', data) if categories and len(categories) > 0: self.notifyProduct.emit('<b>Total Categories Found: %s</b>' % str(len(categories))) for category in categories: category1Name = unicode(category[1]).strip() self.scrapCategory1Data(str(category[0]).strip(), category1Name) def scrapCategory1Data(self, url, category1Name): self.logger.debug('Category 1 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category1Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: self.notifyProduct.emit('<b>Total Categories Found: %s</b>' % str(len(categories))) for category in categories: self.scrapCategory2Data(category[0], category1Name, category[1]) def scrapCategory2Data(self, url, category1Name, category2Name): self.logger.debug('Category 2 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category2Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: for category in categories: print 'category2: ' + category[0] self.scrapCategory3Data(category[0], category1Name, category2Name, category[1]) def scrapCategory3Data(self, url, category1Name, category2Name, category3Name): self.logger.debug('Category 3 URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all categories under Category[%s]</b>' % category3Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categories = self.regex.getAllSearchedData( '(?i)<li> <a href="([^"]*)" title="([^"]*)"[^>]*?>[^<]*?</a> </li>', data) if categories and len(categories) > 0: for category in categories: print [category1Name, category2Name, category3Name, category[1]] self.scrapProductsDetails(category[0], category1Name, category2Name, category3Name, category[1]) def scrapProductsDetails(self, url, category1Name, category2Name, category3Name, category4Name): self.logger.debug('Product Details URL: ' + url) self.notifyProduct.emit('<b>Try to scrap all products under Category[%s]</b>' % category4Name) self.notifyProduct.emit('<font color=green><b>Category URL: %s</b></font>' % url) data = self.spider.fetchData(url + '?limit=10000&mode=list') if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) products = self.regex.getAllSearchedData('(?i)<div class="listing-item[^"]*?">(.*?)</div>', data) if products and len(products) > 0: self.totalProducts += len(products) self.notifyProduct.emit('<font color=green><b>Total Products Found [%s]</b></font>' % unicode(self.totalProducts)) for product in products: productDetailUrl = self.regex.getSearchedData('(?i)<a href="([^"]*)"', product) if productDetailUrl not in self.dupCsvRows: self.scrapProductDetails(productDetailUrl, category1Name, category2Name, category3Name, category4Name) else: self.notifyProduct.emit( '<font color=green><b>Already Exists This Product Under Category[%s]. Skip It.</b></font>' % category4Name) def scrapProductDetails(self, url, category1Name, category2Name, category3Name, category4Name): self.logger.debug('Product Detail URL: ' + url) self.notifyProduct.emit('<b>Try to scrap product details under Category[%s]</b>' % category4Name) self.notifyProduct.emit('<font color=green><b>Product Detail URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) manufacturer = self.regex.getSearchedData( '(?i)<span class="manufacturer-box-label">Manufacturer:</span>([^<]*)</p>', data) productCode = self.regex.getSearchedData( '(?i)<span class="manufacturer-box-label">Model No:</span>([^<]*)</p>', data) productName = self.regex.getSearchedData('(?i)<div class="product-name"> <h1>([^<]*)</h1>', data) productTechnicalDesc = self.regex.getSearchedData('(?i)<div class="product-short-description">([^<]*)</div>' , data) productDescriptions = self.regex.getSearchedData('(?i)<div class="product-specs">(.*?)</div>', data) productShortDesc = '' productFullDesc = '' if productDescriptions and len(productDescriptions) > 0: print 'desc: ' + productDescriptions productShortDesc = self.regex.getSearchedData('(?i)<p>(.*?)</p>', productDescriptions) productFullDesc = '\n'.join( self.regex.getAllSearchedData('(?i)<li>([^<]*)</li>', productDescriptions)) listPriceChunk = self.regex.getSearchedData('(?i)<div class="rrp-price regular-price">(.*?)</div>', data) listPrice = '' if listPriceChunk and len(listPriceChunk) > 0: listPrice = self.regex.getSearchedData('(?i)([0-9,.]+)', listPriceChunk) savePriceChunk = self.regex.getSearchedData('(?i)<div class="regular-price saving-price">(.*?)</div>', data) savePrice = '' if savePriceChunk and len(savePriceChunk) > 0: savePrice = self.regex.getSearchedData('(?i)([0-9%]+)', savePriceChunk) priceChunk = self.regex.getSearchedData('(?i)<div class="[^"]*" id="product-price-\d+">(.*?)</div>', data) price = '' if priceChunk and len(priceChunk) > 0: price = self.regex.getSearchedData('(?i)([0-9,.]+)', priceChunk) deliveryChunk = self.regex.getSearchedData('(?i)<div class="delivery">(.*?)</div>', data) delivery = '' if deliveryChunk and len(deliveryChunk) > 0: delivery = self.regex.getSearchedData('(?i)<p>([^<]*)</p>', deliveryChunk) warrantyChunk = self.regex.getSearchedData('(?i)<div class="warranty">(.*?)</div>', data) warranty = '' if warrantyChunk and len(warrantyChunk) > 0: warranty = self.regex.getSearchedData('(?i)<p>([^<]*)</p>', warrantyChunk) ## Download and save product images productImageUrl = self.regex.getSearchedData( '(?i)src="(http://assets.cs-catering-equipment.co.uk/media/catalog/product/cache/1/image/256x/[^"]*)"', data) print productImageUrl productImage = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_.]*)$', productImageUrl) if productImage and len(productImage) > 0: print productImage self.notifyProduct.emit('<b>Downloading Product Image [%s]. Please wait...</b>' % productImage) self.utils.downloadFile(productImageUrl, 'product_image/' + productImage) ## Download and save brand images brandImageUrl = self.regex.getSearchedData( '(?i)<div class="manufacturer-box-left"><a href="[^"]*"[^>]*?><img src="([^"]*)"', data) brandImage = '' if brandImageUrl and len(brandImageUrl) > 0: brandImageUrl = self.regex.replaceData('(?i)logo/', '', brandImageUrl) brandImage = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_.]*)$', brandImageUrl) if brandImage and len(brandImage) > 0: self.notifyProduct.emit('<b>Downloading Brand Image [%s]. Please wait...</b>' % brandImage) self.utils.downloadFile(brandImageUrl, 'brand_image/' + brandImage) csvData = [url, productCode, productName, manufacturer, listPrice, price, savePrice, productShortDesc, productFullDesc, productTechnicalDesc, warranty, delivery, productImage, category1Name, category2Name, category3Name, category4Name, brandImage] self.csvWriter.writeCsvRow(csvData) self.logger.debug(unicode(csvData)) self.notifyProduct.emit('<b>Product Details: %s</b>' % unicode(csvData))
class YoutubeScrapper(object): def __init__(self): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.utils = Utils() def scrapVideoDownloadUrl(self, url, filename=None): data = self.spider.fetchData(url) if data and len(data) > 0: title = self.scrapTitle(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) dlUrlChunk = self.regex.getSearchedData('(?i)"url_encoded_fmt_stream_map": "([^"]*)"', data) dlUrlChunk = self.regex.replaceData('(?i)\\\\u0026', ' ', dlUrlChunk) dlUrlParts = dlUrlChunk.split(',') sig = '' video = '' videoUrl = '' print dlUrlParts for dlUrlPart in dlUrlParts: dlUrlPart = urllib2.unquote(dlUrlPart) print dlUrlPart ## TODO if self.regex.isFoundPattern('(?i)itag=22', dlUrlPart) or self.regex.isFoundPattern('(?i)itag=18', dlUrlPart): urlPart = dlUrlPart.split(' ') for part in urlPart: print part if self.regex.isFoundPattern('(?i)sig=.*?', part): sig = self.regex.getSearchedData('(?i)sig=(.*?)$', part) if self.regex.isFoundPattern('(?i)url=.*?', part): video = self.regex.getSearchedData('(?i)url=(.*?)$', part) print video videoUrl = video + '&signature=' + sig self.downloadDir = './natok.mp4' print 'Video URL= ' + videoUrl print self.downloadDir break # dlPath = './natok.mp4' if filename is None else filename fname = self.regex.replaceData('\s+', '_', title) dlPath = './' + fname + '.mp4' if filename is None else filename print dlPath print '\n\n' if self.downloadFile(videoUrl, dlPath) is True: print 'Download complete' else: print 'No data found.' def scrapTitle(self, url): # https://www.youtube.com/oembed?url=http://www.youtube.com/watch?v=9bZkp7q19f0&format=xml xmlUrl = 'https://www.youtube.com/oembed?url=' + str(url) + '&format=xml' data = self.spider.fetchData(xmlUrl) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) print data return self.regex.getSearchedData('(?i)<title>([^<]*)</title>', data) def downloadFile(self, url, downloadPath, retry=0): try: opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0)) opener.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1'), ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Connection', 'keep-alive')] # resp = opener.open(url, timeout=10) resp = urllib2.urlopen(url, timeout=60) print 'ok' print resp.info() contentLength = resp.info()['Content-Length'] contentLength = self.regex.getSearchedData('(?i)^(\d+)', contentLength) totalSize = float(contentLength) directory = os.path.dirname(downloadPath) if not os.path.exists(directory): os.makedirs(directory) currentSize = 0 dl_file = open(downloadPath, 'ab') try: if os.path.getsize(downloadPath): start = os.path.getsize(downloadPath) currentSize = start opener.addheaders.append(('Range', 'bytes=%s-' % (start))) except Exception, x: print x res = opener.open(url, timeout=60) CHUNK_SIZE = 256 * 1024 while True: data = res.read(CHUNK_SIZE) # data = resp.read(CHUNK_SIZE) if not data: break currentSize += len(data) dl_file.write(data) print('============> ' + \ str(round(float(currentSize * 100) / totalSize, 2)) + \ '% of ' + str(totalSize / (1024 * 1024)) + ' Mega Bytes') notifyDl = '===> Downloaded ' + str(round(float(currentSize * 100) / totalSize, 2)) + '% of ' + str( totalSize) + ' KB.' if currentSize >= totalSize: dl_file.close() return True except Exception, x: error = 'Error downloading: ' print x if retry < 20: time.sleep(30) return self.downloadFile(url, downloadPath, retry + 1)
class Nisbets: def __init__(self): self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.csvWriter = Csv('nisbets.csv') self.mainUrl = 'http://www.nisbets.co.uk' csvHeaderList = ['Category', 'Product Image Url', 'Product Code', 'Product Name', 'Price'] self.csvWriter.writeCsvRow(csvHeaderList) def scrapData(self): self.logger.debug('===== URL [' + self.mainUrl + '] =====') data = self.spider.fetchData(self.mainUrl) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) data = self.regex.getSearchedData('(?i)<div class="cms-left-nav-category">(.*?)</ul>', data) if data: links = self.regex.getAllSearchedData('(?i)<a href="([^"]*)"', data) if links: for link in links: self.scrapLinkData(self.mainUrl + link) def scrapLinkData(self, link): self.logger.debug('== Link URL [' + link + '] ==') data = self.spider.fetchData(link) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) data = self.regex.getSearchedData('(?i)<h3>Brand</h3> <ul class="subCat02 clear-fix">(.*?)</ul>', data) if data: links = self.regex.getAllSearchedData('(?i)<a href="([^"]*)"', data) if links: for link in links: self.scrapInfo(self.mainUrl + link) def scrapInfo(self, link): self.logger.debug('= Info URL [' + link + '] =') data = self.spider.fetchData(link) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category = self.regex.getSearchedData('(?i)<li><h3>Category</h3></li> <li class="remCont"> <span class="block">([^<]*)</span>', data) allInfo = self.regex.getAllSearchedData('(?i)<div class="product-list-row clear-after">(.*?)</fieldset>', data) if allInfo: for info in allInfo: csvData = [] csvData.append(category) grpData = self.regex.getSearchedDataGroups('(?i)<img class="primaryImage" src="([^"]*)" alt="([^"]*)" />', info) if grpData.group(1): imageUrl = grpData.group(1) imageUrl = self.regex.replaceData('(?i)medium', 'xlarge', imageUrl) csvData.append(self.mainUrl + imageUrl) else: csvData.append('') csvData.append(grpData.group(2)) name = self.regex.getSearchedData('(?i)<h3 class="product-name"> <a href="[^"]*">([^<]*)</a>', info) csvData.append(name) price = self.regex.getSearchedData(u'(?i)<div class="reduced-price"> <span class="bold">([^<]*)</span>', info) csvData.append(price.strip()[1:]) self.logger.debug('Scraped Data ' + str(csvData)) self.csvWriter.writeCsvRow(csvData)
class NisbetProduct(QtCore.QThread): scrapProductData = QtCore.pyqtSignal(object) stopThread = QtCore.pyqtSignal(int) def __init__(self): QtCore.QThread.__init__(self) self.isExiting = False self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow("nisbets.csv", 0) self.csvWriter = Csv("nisbets.csv") self.mainUrl = "http://www.nisbets.co.uk" csvHeaderList = [ "URL", "Product Code", "Product Technical Specifications", "Product Name", "Brand", "Product Price", "Product Short Description", "Product Long Description", "Image File Name", "User Manual File Name", "Exploded View File Name", "Spares Code", "Accessories", "Product Status" "Category1", "Category2", "Category3", "Category4", ] if "URL" not in self.dupCsvRows: self.csvWriter.writeCsvRow(csvHeaderList) self.dupCsvRows.append(csvHeaderList[0]) self.utils = Utils() def run(self): self.scrapData() def stop(self): self.isExiting = True def scrapData(self): if self.isExiting: return self.scrapProductData.emit("<font color=green><b>Main URL: </b>%s</font>" % self.mainUrl) self.logger.debug("===== URL [" + self.mainUrl + "] =====") data = self.spider.fetchData(self.mainUrl) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category1Chunk = self.regex.getAllSearchedData('(?i)<li id="li-id-\d+">(.*?)</ul> </li>', data) if category1Chunk: for category1Data in category1Chunk: category1 = self.regex.getSearchedData('(?i)<a href="[^"]*">([^<]*)</a>', category1Data) category2Chunk = self.regex.getAllSearchedData( '(?i)<li><a href="([^"]*)">([^<]*)</a>', category1Data ) if category2Chunk: for category2Data in category2Chunk: self.scrapCategory2Data(self.mainUrl + category2Data[0], category1, category2Data[1]) self.scrapProductData.emit("<font color=red><b>Finish Scraping Product data from %s</b></font>" % self.mainUrl) def scrapCategory2Data(self, url, category1, category2): if self.isExiting: return self.scrapProductData.emit("<b>Category 2 URL: </b>%s" % url) self.logger.debug("== Category 2 URL [" + url + "] ==") data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category3Chunks = self.regex.getSearchedData('(?i)<ul class="topCat clear-fix">(.*?)</ul>', data) if category3Chunks: category3Chunk = self.regex.getAllSearchedData('(?i)<a href="([^"]*)">([^<]*)<', category3Chunks) if category3Chunk: for category3Data in category3Chunk: self.scrapCategory3Data(self.mainUrl + category3Data[0], category1, category2, category3Data[1]) def scrapCategory3Data(self, url, category1, category2, category3): if self.isExiting: return self.scrapProductData.emit("<b>Category 3 URL: </b>%s" % url) self.logger.debug("== Category 3 URL [" + url + "] ==") data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category4Chunks = self.regex.getSearchedData('(?i)<ul class="topCat clear-fix">(.*?)</ul>', data) if category4Chunks: category4Chunk = self.regex.getAllSearchedData('(?i)<a href="([^"]*)">([^<]*)<', category4Chunks) if category4Chunk: for category4Data in category4Chunk: category4Url = self.mainUrl + category4Data[0] self.scrapCategory4Data(category4Url, category1, category2, category3, category4Data[1]) def scrapCategory4Data(self, url, category1, category2, category3, category4): if self.isExiting: return self.scrapProductData.emit("<b>Category 4 URL: </b>%s" % url) self.logger.debug("== Category 4 URL [" + url + "] ==") data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categoryChunk = self.regex.getAllSearchedData( '(?i)<div class="product-list-row clear-after">(.*?)</fieldset>', data ) if categoryChunk: for categoryData in categoryChunk: if self.isExiting: return productInfo = self.regex.getSearchedDataGroups( '(?i)<h3 class="product-name"> <a href="([^"]*)"[^>]*?>([^<]*)</a>', categoryData ) productUrl = self.mainUrl + productInfo.group(1) productName = productInfo.group(2) if productUrl not in self.dupCsvRows: self.dupCsvRows.append(productUrl) else: self.scrapProductData.emit( "<font color=green><b>Already exists this item in csv Skip it</b></font>" ) self.logger.debug("========= Already exists this item Skip it ===========") return productImageInfo = self.regex.getSearchedDataGroups( '(?i)<img class="primaryImage" src="([^"]*)" alt="([^"]*)"', categoryData ) image = self.regex.replaceData("(?i)medium", "xlarge", str(productImageInfo.group(1))) productImageUrl = self.mainUrl + image productImage = self.regex.getSearchedData("(?i)/([a-zA-Z0-9-_.]*)$", image) self.utils.downloadFile(productImageUrl, "images/" + productImage) productCode = productImageInfo.group(2) productTechSpecs = self.regex.getSearchedData( '(?i)<p class="description">([^<]*)</p>', categoryData ) brandName = self.regex.getSearchedData( '(?i)<img class="brand-image" src="[^"]*" alt="([^"]*)"', categoryData ) price = self.regex.getSearchedData( '(?i)<div class="reduced-price"> <span class="[^"]*">([^<]*)</span>', categoryData ) if price: price = price.strip()[1:] productStatus = self.regex.getSearchedData( '(?i)<div class="availibility"> <img alt="([^"]*)"', categoryData ) productDesc = "" productLongDesc = "" spareCodes = "" accessoryCode = "" userManual = "" explodedView = "" self.scrapProductData.emit( "<br /><font color=green><b>Product Details URL: </b>%s</font>" % productUrl ) productChunk = self.spider.fetchData(productUrl) if productChunk: productChunk = self.regex.reduceNewLine(productChunk) productChunk = self.regex.reduceBlankSpace(productChunk) productDesc = self.regex.getSearchedData( '(?i)<div class="productDesc"> <h1 class="[^"]*"[^>]*?>[^<]*?</h1>.*?<p>([^<]*)</p>', productChunk, ) productLongDesc = self.regex.getSearchedData( '(?i)<div class="info-product[^>]*?>(.*?)</div>', productChunk ) otherUrl = self.regex.getSearchedData("(?i)(^.*?/)[a-zA-Z0-9._-]*?$", productUrl) self.logger.debug("== Common Product URL [" + otherUrl + "] ==") sparesUrl = otherUrl + "AjaxProductSpares.raction" self.logger.debug("== Spares URL [" + sparesUrl + "] ==") spares = self.spider.fetchData(sparesUrl) if spares: spares = self.regex.getAllSearchedData( '(?i)<p class="code"><span class="bold">Code:</span>([^<]*)</p>', spares ) if spares: spareCodes = ", ".join(spares) accessoriesUrl = otherUrl + "AjaxProductAccessories.raction" self.logger.debug("== Accessories URL [" + accessoriesUrl + "] ==") accessories = self.spider.fetchData(accessoriesUrl) if accessories: accessories = self.regex.getAllSearchedData( '(?i)<p class="code"><span class="bold">Code:</span>([^<]*)</p>', accessories ) if accessories: accessoryCode = ", ".join(accessories) docUrl = otherUrl + "AjaxProductDocuments.raction" self.logger.debug("== Document URL[" + docUrl + "] ==") userManuals = self.spider.fetchData(docUrl) if userManuals: userManual = self.regex.getSearchedData( '(?i)<a class="document-icon" href="([^"]*)"[^>]*?>Download User Manual</a>', userManuals, ) self.logger.debug("Manual URL: " + userManual) if userManual: userManualUrl = self.mainUrl + self.regex.replaceData(" ", "%20", userManual) self.logger.debug("User Manual URL: " + userManualUrl) self.scrapProductData.emit("<b>User Manual PDF URL: </b>%s" % userManualUrl) userManual = self.regex.getSearchedData("(?i)/([a-zA-Z0-9-_. ]*)$", userManual) userManual = self.regex.replaceData("\s+", "_", userManual.strip()) self.scrapProductData.emit( "<font color=green><b>Downloading User Manual: </b>%s <b>Please Wait...</b>" % userManual ) self.utils.downloadFile(userManualUrl, "user_manual/" + userManual) explodedView = self.regex.getSearchedData( '(?i)<a class="document-icon" href="([^"]*)"[^>]*?>Download Exploded Diagram</a>', userManuals, ) if explodedView: explodedViewUrl = self.mainUrl + self.regex.replaceData(" ", "%20", explodedView) self.scrapProductData.emit("<b>Exploded Diagram PDF URL: </b>%s" % explodedViewUrl) explodedView = self.regex.getSearchedData("(?i)/([a-zA-Z0-9-_. ]*)$", explodedView) explodedView = self.regex.replaceData("\s+", "_", explodedView.strip()) self.scrapProductData.emit( "<font color=green><b>Downloading Exploded Diagram: </b>%s <b>Please Wait...</b>" % explodedView ) self.utils.downloadFile(explodedViewUrl, "exploded_view/" + explodedView) csvData = [ productUrl, productCode, productTechSpecs, productName, brandName, price.strip(), productDesc, productLongDesc, productImage, userManual, explodedView, spareCodes, accessoryCode, productStatus, category1, category2, category3, category4, ] self.csvWriter.writeCsvRow(csvData) self.logger.debug("Scraped data " + str(csvData)) self.scrapProductData.emit("<div><b>Scraped Data: </b>%s<br /></div>" % str(csvData))
class BetrosProduct(QThread): notifyProduct = pyqtSignal(object) def __init__(self): QThread.__init__(self) self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() self.mainUrl = 'http://www.bertos.com' self.utils = Utils() self.csvHeader = ['Home Category', 'Sub Category', 'Category Description', 'Category Image', 'Code', 'Product Code', 'Product Name', 'Product Description', 'Product Image File', 'Technical Sheet File', 'Exploded View File'] self.totalProducts = 0 def run(self): self.scrapBertos() self.notifyProduct.emit('<font color=red><b>Finished Scraping All products.</b></font>') def scrapBertos(self, retry=0): # self.downloadFile('http://s900.bertos.it/download.php?file=editorcms/documentazione/schede/scheda_13722600.pdf', 'a.pdf') # self.scrapSubCategory('http://s900.bertos.it/en/', '', None, None) # self.scrapProducts('http://s900.bertos.it/en/pasta_cookers/', '', '', None, None) # return self.notifyProduct.emit('<font color=green><b>Try to get all language links.</b></font>') self.logger.debug(self.mainUrl) data = self.spider.fetchData(self.mainUrl) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) languages = self.regex.getAllSearchedData( '(?i)<div class="[^"]*"><a href="([^"]*)"\s*?class="boxalingua">([^<]*)</a>', data) if languages and len(languages) > 0: self.logger.debug('Total languages: %s' % str(len(languages))) self.notifyProduct.emit('<b>Total languages found[%s]</b>' % str(len(languages))) for language in languages: self.totalProducts = 0 url = language[0] # if str(language[1]).lower() != 'en': # continue urlChunk = self.spider.fetchData(url) if urlChunk and len(urlChunk) > 0: urlChunk = self.regex.reduceNewLine(urlChunk) urlChunk = self.regex.reduceBlankSpace(urlChunk) url = self.regex.getSearchedData('(?i)<a href="([^"]*)" onmouseover="vedi_po_cat\(2\)\s*?"', urlChunk) csvFile = str(language[1].strip()).lower() + '_' + 'bertos.csv' dupCsvReader = Csv() dupCsvRows = dupCsvReader.readCsvRow(csvFile) csvWriter = Csv(csvFile) if self.csvHeader not in dupCsvRows: dupCsvRows.append(self.csvHeader) csvWriter.writeCsvRow(self.csvHeader) self.notifyProduct.emit( '<font color=green><b>Try to get data for language [%s].</b></font>' % language[1]) self.scrapCategory(url, dupCsvRows, csvWriter) self.notifyProduct.emit( '<font color=red><b>===== Finish scraping data for [%s] =====</b></font><br /><br />' % language[1]) else: if retry < 5: return self.scrapBertos(retry + 1) def scrapCategory(self, mainUrl, dupCsvRows, csvWriter): url = mainUrl self.logger.debug('Main URL: ' + url) self.notifyProduct.emit('<font color=green><b>Main URL: %s</b></font>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) data = self.regex.reduceNbsp(data) self.notifyProduct.emit('<b>Try to scrap all categories.</b>') categoryChunk = self.regex.getSearchedData('(?i)<div id="contenuto1">(.*?)</div>\s*?</div>', data) if categoryChunk and len(categoryChunk) > 0: categories = self.regex.getAllSearchedData('(?i)<a href="([^"]*)"[^>]*?>([^<]*)</a>', categoryChunk) if categories and len(categories) > 0: self.notifyProduct.emit('<b>Total Categories Found: %s</b>' % str(len(categories))) for category in categories: categoryName = category[1].strip() self.scrapSubCategory(str(category[0]).strip(), categoryName, dupCsvRows, csvWriter) def scrapSubCategory(self, url, categoryName, dupCsvRows, csvWriter): self.logger.debug('Category URL: ' + url) self.notifyProduct.emit('<b>Try to scrap subcategories for: %s</b>' % categoryName) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) subCategories = self.regex.getAllSearchedData('(?i)<li\s*?><a href="([^"]*)" title="([^"]*)"', data) if subCategories and len(subCategories) > 0: self.notifyProduct.emit( '<font color=green><b>Total subcategories found %s.</b></font>' % str(len(subCategories))) for subCategory in subCategories: subCategoryName = subCategory[1].strip() self.scrapProducts(subCategory[0].strip(), categoryName, subCategoryName, dupCsvRows, csvWriter) def scrapProducts(self, url, categoryName, subCategoryName, dupCsvRows, csvWriter): self.logger.debug('Product URL: ' + url) self.notifyProduct.emit('<b>Product URL: %s.</b>' % url) data = self.spider.fetchData(url) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categoryDescription = self.regex.getSearchedData( '(?i)<td class="prodottidescrizione1">\s*?<h1>[^<]*?</h1>(.*?)</td>', data) categoryDescription = self.regex.replaceData('(?i)<!--.*?-->', '', categoryDescription) categoryDescription = self.regex.replaceData('(?i)<[^>]*>', '', categoryDescription) productUrl = self.regex.getSearchedData('(?i)^(http://.*?)/', url) categoryImage = self.regex.getSearchedData( '(?i)<div class="boximgcat" id="boximgcatid">\s*?<a rel="shadowbox" href="([^"]*)"', data) categoryImageName = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_. ]*)$', categoryImage) categoryImageName = self.regex.replaceData('\s+', '_', categoryImageName.strip()) if categoryImageName is not None and len(categoryImageName) > 0 and not os.path.exists( 'category_image/' + categoryImageName): self.notifyProduct.emit( '<font color=green><b>Downloading Category Image: </b>%s <b>Please Wait...</b></font>' % categoryImageName) self.downloadFile(productUrl + categoryImage, 'category_image/' + categoryImageName) # self.utils.downloadFile(categoryImage, 'category_image/' + categoryImageName) self.notifyProduct.emit( '<font color=green><b>Downloaded Category Image: %s.</b></font>' % categoryImageName) productChunks = self.regex.getSearchedData('(?i)<table.*?class="prodottiriga"[^>]*?>(.*?)</table>', data) if productChunks and len(productChunks) > 0: productChunk = self.regex.getAllSearchedData('(?i)<tr>(.*?</div>\s*?</td>)\s*?</tr>', productChunks) for products in productChunk: print 'url: ' + url code = self.regex.getSearchedData('(?i)Cod\. ([a-zA-Z0-9 /]+)', products).strip() for dup in dupCsvRows: if code == dup[4]: return model = self.regex.getSearchedData('(?i)Mod\. ([^<]*)<', products).strip() productName = self.regex.getSearchedData('(?i)<h1>([^<]*)</h1>', products).strip() self.notifyProduct.emit( '<font color=green><b>Product Name: %s.</b></font>' % productName) desc = self.regex.getSearchedData( '(?i)<div id="prtdescrizione\d+" style="display:none">(.*?)<div class="prodotticomando">', products).strip() productImage = productUrl + self.regex.getSearchedData('(?i)<img src="/tpl/\.\.([^"]*)"', products).strip() productImage = self.regex.replaceData('(?i)k_\d+_\d+', 'k_800_557', productImage) productImageName = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_. ]*)$', productImage) productImageName = self.regex.replaceData('\s+', '_', productImageName.strip()) if productImageName is not None and len(productImageName) > 0 and not os.path.exists( 'product_image/' + productImageName): self.notifyProduct.emit( '<font color=green><b>Downloading Product Image: </b>%s <b>Please Wait...</b></font>' % productImageName) self.downloadFile(productImage, 'product_image/' + productImageName) # self.utils.downloadFile(productImage, 'product_image/' + productImageName) self.notifyProduct.emit( '<font color=green><b>Downloaded Product Image: %s.</b></font>' % productImageName) techPdf = self.regex.getSearchedData( '(?i)<td class="prodottiriga\d+">\s*?<div class="scarica">\s*?<a href="([^"]*)"' , products).strip() techPdfName = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_. ]*)$', techPdf) techPdfName = self.regex.replaceData('\s+', '_', techPdfName.strip()) if techPdfName is not None and len(techPdfName) > 0 and not os.path.exists( 'tech_pdf/' + techPdfName): self.notifyProduct.emit( '<font color=green><b>Downloading Tech Pdf: </b>%s <b>Please Wait...</b></font>' % techPdfName) self.downloadFile(techPdf, 'tech_pdf/' + techPdfName) # self.utils.downloadFile(techPdf, 'tech_pdf/' + techPdfName) self.notifyProduct.emit( '<font color=green><b>Downloaded Tech Pdf: %s.</b></font>' % techPdfName) explodedViewPdf = self.regex.getSearchedData( '(?i)</a>\s*?</div>\s*?<div class="scarica">\s*?<a href="([^"]*\.pdf)">[^<]*?</a>' , products).strip() explodedViewPdfName = self.regex.getSearchedData('(?i)/([a-zA-Z0-9-_. ]*)$', explodedViewPdf) explodedViewPdfName = self.regex.replaceData('\s+', '_', explodedViewPdfName.strip()) if explodedViewPdfName is not None and len(explodedViewPdfName) > 0 and not os.path.exists( 'exploded_pdf/' + explodedViewPdfName): self.notifyProduct.emit( '<font color=green><b>Downloading Exploded View Pdf: </b>%s <b>Please Wait...</b></font>' % explodedViewPdfName) # self.utils.downloadFile(explodedViewPdf, 'exploded_pdf/' + explodedViewPdfName) self.downloadFile(explodedViewPdf, 'exploded_pdf/' + explodedViewPdfName) self.notifyProduct.emit( '<font color=green><b>Downloading Exploded View Pdf: %s.</b></font>' % explodedViewPdfName) csvData = [categoryName, subCategoryName, categoryDescription, categoryImageName, code, model, productName, desc, productImageName, techPdfName, explodedViewPdfName] print csvData if csvData not in dupCsvRows: csvWriter.writeCsvRow(csvData) dupCsvRows.append(csvData) self.notifyProduct.emit('<font color=green><b>Successfully write data to csv file.</b></font>') else: self.notifyProduct.emit('<font color=green><b>Already exists. Skip it.</b></font>') def downloadFile(self, url, downloadPath, retry=0): print url self.notifyProduct.emit('<b>File URL: %s.</b>' % url) try: socket.setdefaulttimeout(10) opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0)) opener.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1')] urllib2.install_opener(opener) # resp = opener.open(url, timeout=30) # resp = urllib2.urlopen(url, timeout=30) resp = None try: # resp = urllib.urlopen(url) resp = opener.open(url, timeout=30) except Exception, x: print x if resp is None: return False # if resp.info()['Connection'] == 'close' or resp.getcode() != 200: # if retry < 3: # self.notifyProduct.emit('<font color=red><b>Failed to download file. Retrying...</b></font>') # return self.downloadFile(url, downloadPath, retry + 1) # else: # self.notifyProduct.emit('<font color=red><b>Failed to download file after 3 retry.</b></font>') # return print resp.info() print 'info.......' contentLength = resp.info()['Content-Length'] contentLength = self.regex.getSearchedData('(?i)^(\d+)', contentLength) totalSize = float(contentLength) directory = os.path.dirname(downloadPath) if not os.path.exists(directory): try: os.makedirs(directory) except Exception, x: print x dl_file = open(downloadPath, 'wb') currentSize = 0 CHUNK_SIZE = 32768 totalSizeKB = totalSize / 1024 if totalSize > 0 else totalSize print 'everything ok............' while True: data = None try: data = resp.read(CHUNK_SIZE) except Exception, x: print x if not data: break currentSize += len(data) dl_file.write(data) print('============> ' +\ str(round(float(currentSize * 100) / totalSize, 2)) +\ '% of ' + str(totalSize) + ' bytes') notifyDl = '===> Downloaded ' + str(round(float(currentSize * 100) / totalSize, 2)) + '% of ' + str( totalSizeKB) + ' KB.' self.notifyProduct.emit('<b>%s</b>' % notifyDl) if currentSize >= totalSize: dl_file.close() return True
class WebPageToPdf(QObject): threadPdfStatusBar = QtCore.pyqtSignal(object) threadPdfWritingStatus = QtCore.pyqtSignal(object) threadPdfWritingDone = QtCore.pyqtSignal(int) def __init__(self): QObject.__init__(self) self.regex = Regex() self.title = '' self.webView = QWebView() self.webView.settings().setAttribute(QWebSettings.AutoLoadImages, True) self.webView.settings().setAttribute(QWebSettings.JavascriptEnabled, True) self.webView.settings().setAttribute(QWebSettings.PluginsEnabled, True) self.webView.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, True) self.pdfPrinter = QPrinter() self.webView.loadFinished.connect(self.convertToPdf) def setupDefaultPdfPrinter(self, fileName): self.pdfPrinter.setOrientation(QPrinter.Portrait) self.pdfPrinter.setPageSize(QPrinter.A4) self.pdfPrinter.setOutputFormat(QPrinter.PdfFormat) self.pdfPrinter.setOutputFileName(fileName) def printWebHtmlToPdf(self, url, filePath, fileName, groupType): self.tempPdfFile = filePath + 'out.pdf' self.filePath = filePath self.fileName = fileName self.url = url self.groupType = groupType self.setupDefaultPdfPrinter(self.tempPdfFile) self.threadPdfStatusBar.emit('Fetching Data From Web. Please Wait...') # self.threadPdfWritingStatus.emit( # '<font size=4 color=green><b>Method "%s": </b></font><font color=green><b>Fetching Data From Web for</b> %s<b>.<br />Please Wait...</b></font>' % ( # self.groupType, self.url)) self.threadPdfWritingStatus.emit( '<font color=green><b>Fetching Data From Web for</b> %s<b>.<br />Please Wait...</b></font>' % self.url) self.webView.load(QUrl(url)) self.title = self.webView.title() def convertToPdf(self): print 'Generating Pdf' # self.threadPdfWritingStatus.emit( # '<font size=4><b>Method "%s": </b></font><b>Generating Pdf for</b> %s<b>. Please Wait...</b>' % ( # self.groupType, self.url)) self.threadPdfWritingStatus.emit( '<b>Generating Pdf for</b> %s<b>. Please Wait...</b>' % self.url) self.threadPdfStatusBar.emit('Generating Pdf. Please Wait...') self.webView.print_(self.pdfPrinter) print 'Generated Pdf' # self.threadPdfWritingStatus.emit( # '<font size=4><b>Method "%s": </b></font><b>Generated Pdf for</b> %s<b>. Please Wait...</b>' % ( # self.groupType, self.url)) self.threadPdfWritingStatus.emit( '<b>Generated Pdf for</b> %s<b>. Please Wait...</b>' % self.url) self.threadPdfStatusBar.emit('Generated Pdf.') self.mergePdf() self.threadPdfWritingDone.emit(True) def mergePdf(self): # self.threadPdfWritingStatus.emit( # '<font size=4><b>Method "%s": </b></font><b>Setting Title for</b> %s<b>. Please Wait...</b><br />' % ( # self.groupType, self.url)) self.threadPdfWritingStatus.emit( '<b>Setting Title for</b> %s<b>. Please Wait...</b><br />' % self.url) packet = StringIO() # create a new PDF with Reportlab pdfCanvas = canvas.Canvas(packet, pagesize=A4) pdfCanvas.setFont('Helvetica', 8) if len(self.title) is 0: self.title = str(self.url).split('/')[-1] self.title = self.regex.getSearchedData('(?i)([a-zA-Z0-9-_ ]*?)\.[a-zA-Z0-9_]*$', self.title) self.title = self.regex.replaceData('(?i)_', ' ', self.title) title = unicode(self.title[:57] + '...') if (len(self.title) > 60) else unicode(self.title) url = self.url[:57] + '...' if (len(self.title) > 60) else self.url pdfCanvas.drawString(5, 830, title + ' ' + str(url).lower()) d = datetime.datetime.now() strDate = str(d.strftime("%Y-%m-%d %H-%M-%S %p")) pdfCanvas.drawString(420, 5, 'Created Date Time: ' + strDate) pdfCanvas.save() packet.seek(0) newPdf = PdfFileReader(packet) if not os.path.exists(self.tempPdfFile): return self.printWebHtmlToPdf(self.url, self.filePath, self.fileName) writer = PdfFileWriter() tmpPdfFile = file(self.tempPdfFile, 'rb') reader = PdfFileReader(tmpPdfFile) for i in range(0, (reader.getNumPages())): page = reader.getPage(i) page.mergePage(newPdf.getPage(0)) # page = newPdf.getPage(0) # page.mergePage(reader.getPage(i)) writer.addPage(page) print 'Filename: ' + self.fileName outputStream = file(self.filePath + self.fileName, "wb") writer.write(outputStream) outputStream.close() tmpPdfFile.close() os.remove(str(self.tempPdfFile))
class YtDownloadManager(object): def __init__(self): self.spider = Spider() self.regex = Regex() self.utils = Utils() def scrapVideoDownloadUrl(self, url): data = self.spider.fetchData(url) print data soup = BeautifulSoup(data) exit(1) if data and len(data) > 0: title = self.scrapTitle(url) data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) dlUrlChunk = self.regex.getSearchedData('(?i)"url_encoded_fmt_stream_map": "([^"]*)"', data) dlUrlChunk = self.regex.replaceData('(?i)\\\\u0026', ' ', dlUrlChunk) dlUrlParts = dlUrlChunk.split(',') sig = '' video = '' videoUrl = '' print dlUrlParts for dlUrlPart in dlUrlParts: dlUrlPart = urllib2.unquote(dlUrlPart) print dlUrlPart # if self.regex.isFoundPattern('(?i)itag=5', dlUrlPart): urlPart = dlUrlPart.split(' ') for part in urlPart: print part if self.regex.isFoundPattern('(?i)sig=.*?', part): sig = self.regex.getSearchedData('(?i)sig=(.*?)$', part) if self.regex.isFoundPattern('(?i)url=.*?', part): video = self.regex.getSearchedData('(?i)url=(.*?)$', part) print video videoUrl = video + '&signature=' + sig self.downloadDir = './test.flv' # print 'Video URL= ' + videoUrl # print self.downloadDir # dlPath = './test.flv' # print dlPath print '\n\n' # if self.downloadFile(videoUrl, dlPath) is True: # break def scrapTitle(self, url): # https://www.youtube.com/oembed?url=http://www.youtube.com/watch?v=9bZkp7q19f0&format=xml xmlUrl = 'https://www.youtube.com/oembed?url=' + str(url) + '&format=xml' data = self.spider.fetchData(xmlUrl) if data and len(data) > 0: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) return self.regex.getSearchedData('(?i)<title>([^<]*)</title>', data) def downloadFile(self, url, downloadPath, retry=0): try: opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(), urllib2.HTTPHandler(debuglevel=0), urllib2.HTTPSHandler(debuglevel=0)) opener.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1'), ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), ('Connection', 'keep-alive')] # resp = opener.open(url, timeout=10) resp = urllib2.urlopen(url, timeout=30) print resp.info() contentLength = resp.info()['Content-Length'] contentLength = self.regex.getSearchedData('(?i)^(\d+)', contentLength) totalSize = float(contentLength) directory = os.path.dirname(downloadPath) if not os.path.exists(directory): os.makedirs(directory) dl_file = open(downloadPath, 'wb') currentSize = 0 CHUNK_SIZE = 32768 while True: data = resp.read(CHUNK_SIZE) if not data: break currentSize += len(data) dl_file.write(data) print('============> ' + \ str(round(float(currentSize * 100) / totalSize, 2)) + \ '% of ' + str(totalSize) + ' bytes') notifyDl = '===> Downloaded ' + str(round(float(currentSize * 100) / totalSize, 2)) + '% of ' + str( totalSize) + ' KB.' if currentSize >= totalSize: dl_file.close() return True except Exception, x: error = 'Error downloading: ' + x return False
class MyLinkedInMembers(QThread): notifyLinkedIn = pyqtSignal(object) notifyMembers = pyqtSignal(object) cookieL = pyqtSignal(object) def __init__(self, spider, url, pageRange=None): QThread.__init__(self) # self.spider = Spider() self.spider = spider self.regex = Regex() self.url = url self.startPage = None self.endPage = None if self.regex.isFoundPattern('(?i)(\d+)-(\d+)', str(pageRange).strip()): pageRangeFormat = self.regex.getSearchedDataGroups('(?i)(\d+)-(\d+)', str(pageRange).strip()) self.startPage = int(pageRangeFormat.group(1)) self.endPage = int(pageRangeFormat.group(2)) elif self.regex.isFoundPattern('(?i)(\d+)', str(pageRange).strip()): pageRangeFormat = self.regex.getSearchedDataGroups('(?i)(\d+)', str(pageRange).strip()) self.startPage = int(pageRangeFormat.group(1)) self.endPage = self.startPage def run(self): self.getMembers(self.url) self.notifyLinkedIn.emit('<font color=red><b>Finish scraping members.<b></font>') def getMembers(self, url, pageNumber=0): print 'Members URL: ' + url self.notifyLinkedIn.emit('<font color=green><b>Start Scraping All Members.<b></font>') self.notifyLinkedIn.emit('<b>Wait For 15 seconds Break...<b>') time.sleep(15) self.notifyLinkedIn.emit('<b>15 seconds Break Finish.<b>') groupData = self.spider.fetchData(str(url).replace('&', '&')) groupData = self.regex.reduceNewLine(groupData) groupData = self.regex.reduceBlankSpace(groupData) print groupData print 'page number: ' + str(pageNumber) if pageNumber > 0: harvestedMembers = [] allMembers = self.regex.getAllSearchedData('(?i)<li class="member" id="member-[^"]*"[^>]*?>(.*?)</div>', groupData) for members in allMembers: memberId = self.regex.getSearchedData('(?i)data-li-memberId="([^"]*)"', members) memberName = self.regex.getSearchedData('(?i)data-li-fullName="([^"]*)"', members) memberTitle = self.regex.getSearchedData('(?i)<p class="headline">([^<]*?)</p>', members) memberTitle = self.regex.replaceData('(?i)&', '&', memberTitle) harvestedMembers.append((memberId, memberName, memberTitle)) self.notifyLinkedIn.emit('<b>Member ID: </b>%s <b>Member Name: </b>%s' % (memberId, memberName + ' (' + memberTitle + ')')) # members = self.regex.getAllSearchedData( # '(?i)class="send-message" data-li-memberId="([^"]*)" data-li-fullName="([^"]*)"', groupData) # print members self.notifyMembers.emit(harvestedMembers) # for member in members: # print member # self.notifyLinkedIn.emit('<b>Member Name: </b>%s <b>Member ID: </b>%s' % (member[1], member[0])) urlNext = self.regex.getSearchedData('(?i)<a href="([^"]*)"[^>]*?>\s*?<strong>\s*?next', groupData) if urlNext and len(urlNext) > 0: # nextP = int(self.regex.getSearchedData('(?i).*?(\d+)$', urlNext.strip())) urlNext = self.regex.replaceData('(?i)&', '&', urlNext) urlNext = self.regex.replaceData('(?i)split_page=\d+', 'split_page=', urlNext) pageNumber += 1 if self.startPage <= pageNumber <= self.endPage: self.notifyLinkedIn.emit('<b>Wait for 15 second break...</b>') time.sleep(15) print 'sleep 15 s' self.notifyLinkedIn.emit('<b>15 second break finish!!!</b>') self.getMembers('http://www.linkedin.com' + urlNext + str(pageNumber), pageNumber) elif pageNumber < self.startPage: pageNumber = self.startPage self.notifyLinkedIn.emit('<b>Wait for 15 second break...</b>') time.sleep(15) print 'page number less 0 sleep' self.notifyLinkedIn.emit('<b>15 second break finish!!!</b>') self.getMembers('http://www.linkedin.com' + urlNext + str(pageNumber), pageNumber) if self.startPage is None and self.endPage is None: pageNumber += 1 self.notifyLinkedIn.emit('<b>Wait for 15 second break...</b>') time.sleep(15) print 'page number less 0 sleep' self.notifyLinkedIn.emit('<b>15 second break finish!!!</b>') self.getMembers('http://www.linkedin.com' + urlNext + str(pageNumber), pageNumber)
class NisbetProduct(QtCore.QThread): scrapProductData = QtCore.pyqtSignal(object) stopThread = QtCore.pyqtSignal(int) def __init__(self): QtCore.QThread.__init__(self) self.isExiting = False self.logger = LogManager(__name__) self.spider = Spider() self.regex = Regex() dupCsvReader = Csv() self.dupCsvRows = dupCsvReader.readCsvRow('nisbets.csv', 0) self.csvWriter = Csv('nisbets.csv') self.mainUrl = 'http://www.nisbets.co.uk' csvHeaderList = [ 'URL', 'Product Code', 'Product Technical Specifications', 'Product Name', 'Brand', 'Product Price', 'Product Short Description', 'Product Long Description', 'Image File Name', 'User Manual File Name', 'Exploded View File Name', 'Spares Code', 'Accessories', 'Product Status' 'Category1', 'Category2', 'Category3', 'Category4' ] if 'URL' not in self.dupCsvRows: self.csvWriter.writeCsvRow(csvHeaderList) self.dupCsvRows.append(csvHeaderList[0]) self.utils = Utils() def run(self): self.scrapData() def stop(self): self.isExiting = True def scrapData(self): if self.isExiting: return self.scrapProductData.emit( '<font color=green><b>Main URL: </b>%s</font>' % self.mainUrl) self.logger.debug('===== URL [' + self.mainUrl + '] =====') data = self.spider.fetchData(self.mainUrl) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category1Chunk = self.regex.getAllSearchedData( '(?i)<li id="li-id-\d+">(.*?)</ul> </li>', data) if category1Chunk: for category1Data in category1Chunk: category1 = self.regex.getSearchedData( '(?i)<a href="[^"]*">([^<]*)</a>', category1Data) category2Chunk = self.regex.getAllSearchedData( '(?i)<li><a href="([^"]*)">([^<]*)</a>', category1Data) if category2Chunk: for category2Data in category2Chunk: self.scrapCategory2Data( self.mainUrl + category2Data[0], category1, category2Data[1]) self.scrapProductData.emit( '<font color=red><b>Finish Scraping Product data from %s</b></font>' % self.mainUrl) def scrapCategory2Data(self, url, category1, category2): if self.isExiting: return self.scrapProductData.emit('<b>Category 2 URL: </b>%s' % url) self.logger.debug('== Category 2 URL [' + url + '] ==') data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category3Chunks = self.regex.getSearchedData( '(?i)<ul class="topCat clear-fix">(.*?)</ul>', data) if category3Chunks: category3Chunk = self.regex.getAllSearchedData( '(?i)<a href="([^"]*)">([^<]*)<', category3Chunks) if category3Chunk: for category3Data in category3Chunk: self.scrapCategory3Data( self.mainUrl + category3Data[0], category1, category2, category3Data[1]) def scrapCategory3Data(self, url, category1, category2, category3): if self.isExiting: return self.scrapProductData.emit('<b>Category 3 URL: </b>%s' % url) self.logger.debug('== Category 3 URL [' + url + '] ==') data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) category4Chunks = self.regex.getSearchedData( '(?i)<ul class="topCat clear-fix">(.*?)</ul>', data) if category4Chunks: category4Chunk = self.regex.getAllSearchedData( '(?i)<a href="([^"]*)">([^<]*)<', category4Chunks) if category4Chunk: for category4Data in category4Chunk: category4Url = self.mainUrl + category4Data[0] self.scrapCategory4Data(category4Url, category1, category2, category3, category4Data[1]) def scrapCategory4Data(self, url, category1, category2, category3, category4): if self.isExiting: return self.scrapProductData.emit('<b>Category 4 URL: </b>%s' % url) self.logger.debug('== Category 4 URL [' + url + '] ==') data = self.spider.fetchData(url) if data: data = self.regex.reduceNewLine(data) data = self.regex.reduceBlankSpace(data) categoryChunk = self.regex.getAllSearchedData( '(?i)<div class="product-list-row clear-after">(.*?)</fieldset>', data) if categoryChunk: for categoryData in categoryChunk: if self.isExiting: return productInfo = self.regex.getSearchedDataGroups( '(?i)<h3 class="product-name"> <a href="([^"]*)"[^>]*?>([^<]*)</a>', categoryData) productUrl = self.mainUrl + productInfo.group(1) productName = productInfo.group(2) if productUrl not in self.dupCsvRows: self.dupCsvRows.append(productUrl) else: self.scrapProductData.emit( '<font color=green><b>Already exists this item in csv Skip it</b></font>' ) self.logger.debug( '========= Already exists this item Skip it ===========' ) return productImageInfo = self.regex.getSearchedDataGroups( '(?i)<img class="primaryImage" src="([^"]*)" alt="([^"]*)"', categoryData) image = self.regex.replaceData( '(?i)medium', 'xlarge', str(productImageInfo.group(1))) productImageUrl = self.mainUrl + image productImage = self.regex.getSearchedData( '(?i)/([a-zA-Z0-9-_.]*)$', image) self.utils.downloadFile(productImageUrl, 'images/' + productImage) productCode = productImageInfo.group(2) productTechSpecs = self.regex.getSearchedData( '(?i)<p class="description">([^<]*)</p>', categoryData) brandName = self.regex.getSearchedData( '(?i)<img class="brand-image" src="[^"]*" alt="([^"]*)"', categoryData) price = self.regex.getSearchedData( '(?i)<div class="reduced-price"> <span class="[^"]*">([^<]*)</span>', categoryData) if price: price = price.strip()[1:] productStatus = self.regex.getSearchedData( '(?i)<div class="availibility"> <img alt="([^"]*)"', categoryData) productDesc = '' productLongDesc = '' spareCodes = '' accessoryCode = '' userManual = '' explodedView = '' self.scrapProductData.emit( '<br /><font color=green><b>Product Details URL: </b>%s</font>' % productUrl) productChunk = self.spider.fetchData(productUrl) if productChunk: productChunk = self.regex.reduceNewLine(productChunk) productChunk = self.regex.reduceBlankSpace( productChunk) productDesc = self.regex.getSearchedData( '(?i)<div class="productDesc"> <h1 class="[^"]*"[^>]*?>[^<]*?</h1>.*?<p>([^<]*)</p>', productChunk) productLongDesc = self.regex.getSearchedData( '(?i)<div class="info-product[^>]*?>(.*?)</div>', productChunk) otherUrl = self.regex.getSearchedData( '(?i)(^.*?/)[a-zA-Z0-9._-]*?$', productUrl) self.logger.debug('== Common Product URL [' + otherUrl + '] ==') sparesUrl = otherUrl + "AjaxProductSpares.raction" self.logger.debug('== Spares URL [' + sparesUrl + '] ==') spares = self.spider.fetchData(sparesUrl) if spares: spares = self.regex.getAllSearchedData( '(?i)<p class="code"><span class="bold">Code:</span>([^<]*)</p>', spares) if spares: spareCodes = ', '.join(spares) accessoriesUrl = otherUrl + "AjaxProductAccessories.raction" self.logger.debug('== Accessories URL [' + accessoriesUrl + '] ==') accessories = self.spider.fetchData(accessoriesUrl) if accessories: accessories = self.regex.getAllSearchedData( '(?i)<p class="code"><span class="bold">Code:</span>([^<]*)</p>', accessories) if accessories: accessoryCode = ', '.join(accessories) docUrl = otherUrl + "AjaxProductDocuments.raction" self.logger.debug('== Document URL[' + docUrl + '] ==') userManuals = self.spider.fetchData(docUrl) if userManuals: userManual = self.regex.getSearchedData( '(?i)<a class="document-icon" href="([^"]*)"[^>]*?>Download User Manual</a>', userManuals) self.logger.debug('Manual URL: ' + userManual) if userManual: userManualUrl = self.mainUrl + self.regex.replaceData( ' ', '%20', userManual) self.logger.debug('User Manual URL: ' + userManualUrl) self.scrapProductData.emit( '<b>User Manual PDF URL: </b>%s' % userManualUrl) userManual = self.regex.getSearchedData( '(?i)/([a-zA-Z0-9-_. ]*)$', userManual) userManual = self.regex.replaceData( '\s+', '_', userManual.strip()) self.scrapProductData.emit( '<font color=green><b>Downloading User Manual: </b>%s <b>Please Wait...</b>' % userManual) self.utils.downloadFile( userManualUrl, 'user_manual/' + userManual) explodedView = self.regex.getSearchedData( '(?i)<a class="document-icon" href="([^"]*)"[^>]*?>Download Exploded Diagram</a>', userManuals) if explodedView: explodedViewUrl = self.mainUrl + self.regex.replaceData( ' ', '%20', explodedView) self.scrapProductData.emit( '<b>Exploded Diagram PDF URL: </b>%s' % explodedViewUrl) explodedView = self.regex.getSearchedData( '(?i)/([a-zA-Z0-9-_. ]*)$', explodedView) explodedView = self.regex.replaceData( '\s+', '_', explodedView.strip()) self.scrapProductData.emit( '<font color=green><b>Downloading Exploded Diagram: </b>%s <b>Please Wait...</b>' % explodedView) self.utils.downloadFile( explodedViewUrl, 'exploded_view/' + explodedView) csvData = [ productUrl, productCode, productTechSpecs, productName, brandName, price.strip(), productDesc, productLongDesc, productImage, userManual, explodedView, spareCodes, accessoryCode, productStatus, category1, category2, category3, category4 ] self.csvWriter.writeCsvRow(csvData) self.logger.debug('Scraped data ' + str(csvData)) self.scrapProductData.emit( '<div><b>Scraped Data: </b>%s<br /></div>' % str(csvData))