def __VenueParser(self, link): #link ='https://www.meilleur-garagiste.com/annuaire/garage-la-couronne.464207.html' print 'Scrapping: ' + link existing = [x for x in self.link_venues if link in x] if len(existing) > 0: print 'Len existing : ' + str(len(existing)) return None xmlBody = Util.getRequestsXML(link, '//div[@id="fiche-artisan"]') if xmlBody != None and len(xmlBody) > 0: ven = Venue() name_ = xmlBody.xpath('.//h1/parent::div') if len(name_) > 0: if name_ != None: name_h1 = name_[0].find('./h1') name_h2 = name_[0].find('.//h2') if name_h2 != None: ven.name = name_h2.text else: ven.name = name_h1.text else: return None xmldiv = xmlBody.find('.//div[@class="row nomargin"]/div') if xmldiv == None: return None span_ = xmldiv.xpath('./span') for i_ in span_: if i_.get('class') == 'street-address text-hide-mobile': ven.street = i_.text if ven.street != None: #ven.street = self.validateStreet(ven.street).replace('43442491700012', '') ven.street = self.validateStreet2(ven.street).replace( '43442491700012', '') if ven.street.strip() == '.': ven.street = None if i_.get('class') == 'postal-code': ven.zipcode = i_.text ven.zipcode = self.validateZipcode(ven.zipcode) if i_.get('class') == 'locality': ven.city = i_.text a = xmlBody.find( './/a[@class="col m12 s4 tel waves-effect waves-light btn center btn-fix bleu"]' ) if a != None: phone = a.get('href').replace('tel:', '').replace(' ', '') if phone.startswith('07') | phone.startswith('06'): ven.mobile_number = self.validatePhone__(phone, 'FR') else: ven.office_number = self.validatePhone__(phone, 'FR') logo = xmlBody.find('.//div[@class="center-align"]/img') if logo != None: ven.img_link = [self.__url__ + logo.get('src')] ven.scrape_page = link ven.pricelist_link = [link] listServices = xmlBody.xpath( '//li/div[@class="collapsible-body"]/div/a') sers = [] for ser in listServices: servic = Service() servic.service = ser.text sers.append(servic) self.services.append(servic) ven.services = sers if ven.city != None and ven.zipcode != None: if ven.street != None and len(ven.street) > 0: add_ = ven.street + ', ' + ven.city + ', ' + ven.zipcode else: add_ = ven.city + ', ' + ven.zipcode else: add_ = None (ven.latitude, ven.longitude) = self.getLatlng(add_, 'FR') if ven.latitude == None and ven.longitude == None: Util.log.coordinate_logger.error(ven.scrape_page + ' : Cannot get GEO code') self.link_venues.append(link) ven.country = 'fr' desc = xmlBody.find('.//p[@id="description"]') desc_ = '' if desc != None: desc_ = ''.join(desc.itertext()).strip().replace('\n', '|').replace( '\t', '') title = xmlBody.find('.//div[@class="container"]//h2') if title != None and desc != None: desc_ = title.text + ' | ' + desc_ img_link_arr = [] desc_ = self.replace__(desc_) desc_ = self.replaceSame(desc_, '||', '|').replace('|', ' | ') ven.description = desc_ img_link = xmlBody.find('.//div[@class="realisations"]/img') if img_link != None: temp_img = ven.img_link = self.__url__ + img_link.get('src') img_link_arr.append(temp_img) multi_img = xmlBody.xpath( '//div[@class="3photo realisations"]/div/img') for it in multi_img: temp_ml = self.__url__ + it.get('src') img_link_arr.append(temp_ml) if len(img_link_arr) > 0: ven.img_link = img_link_arr nr_reviewer = xmlBody.xpath('//div[@class="avisoperation row"]') if len(nr_reviewer) > 0: ven.hqdb_nr_reviews = str(len(nr_reviewer)) ven.is_get_by_address = True return ven
def __VenueParser(self,hqdb_type, linkItems,subcate,cate): #linkItems ='https://www.blauarbeit.de/p/modernisierung/_sanierung/berlin/daniel_kutscher/576667.htm' existing=[x for x in self.linkIn if linkItems in x] if len(existing)>0: print 'This venue exist in list' return None self.linkIn.append(linkItems) xmlPages = self.getRequest(linkItems) if xmlPages==None: return None xmlVen = xmlPages.xpath('//div[@class="page_move"]') cate__ = xmlPages.find('.//meta[@name="Description"]') if len(xmlVen)==0: return None name = xmlVen[0].xpath('.//h2') if len(name) <=0: name ='' else: name = name[0].text.strip() noneValues ={'ZERO','NULL'} if name.upper() in noneValues: return None ven = Venue() if cate__!=None: ven.category = cate__.get('content').split(',')[0] nameFromUrl = self.getNamefromUrl(linkItems) ven.name = nameFromUrl ven.hqdb_featured_ad_type = hqdb_type #ven.name =name ven.scrape_page = linkItems #ven.subcategory = subcate #ven.category= cate address_= '' #ven.formatted_address='' img_link= [] divInfo = xmlVen[0].find('.//div[@class="content_wrapper content_wrapper_main clearfix"]/div') if divInfo!=None: mainInfo = divInfo.xpath('./section') if len(mainInfo)>=2: leftInfo = mainInfo[0] rightInfo = mainInfo[1] tableInfo = leftInfo.find('./div/div[@class="profile_top_left"]/table') trinfo = tableInfo.xpath('./tr') for tr_ in trinfo: td =tr_.xpath('./td') if len(td)<2: continue key_ = ''.join(td[0].itertext()).strip() values_ = ' '.join(td[1].itertext()).strip().replace('keine Angabe','').replace('NULL','').replace('null','') if key_ =='Ansprechpartner:': if values_!=None and len(values_)>2: #values_ ='' ven.name_of_contact = values_ ven.name +=', '+ ven.name_of_contact if key_ =='Addresse:': address_ = values_ ven.formatted_address = self.validateFormat(address_) '''(ven.street,ven.city,ven.zipcode) = self.processAddress(address_) if ven.street!=None: ven.street = self.validateStreet2(ven.street) #ven.formatted_address = address_ if ven.city!=None: checkCity = ven.city.split() if len(checkCity)>0: if checkCity[0].isdigit(): if len(checkCity[0])==5: if ven.street!=None: ven.street+=' '+ ven.zipcode ven.zipcode = checkCity[0] ven.city = ven.city.replace(ven.zipcode,'') else: ven.city = None ven.street = None ven.zipcode= None ven.formatted_address = ' '.join(checkCity) if ven.zipcode!=None: if len(ven.zipcode)==5: ven.zipcode = ven.zipcode else: ven.zipcode = None''' if key_ =='Homepage:': a_ = td[1].find('./a') if a_ !=None: ven.business_website = a_.get('href') mobileCode =['015','016','017','+4915','+4916','+4917'] if key_ =='Tel:': values_ = values_.replace('/', '').replace(' ', '').replace('Tel', '') #values_ ='01735465435' for mCode in mobileCode: if values_.startswith(mCode): ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de') break if ven.mobile_number==None: ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de') '''if values_.startswith('01')| values_.startswith('+0041')| values_.startswith('0041'): ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de') else: ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de')''' img_ = leftInfo.find('./div/div[@class="profile_top_right"]/img') if img_!=None: img_ =img_.get('src') img_link.append(img_) rating = leftInfo.xpath('.//section[@id="ratings"]/div') if len(rating)>=2: rating1 = ''.join(rating[0].itertext()).strip().split()[1] rating2 = ''.join(rating[1].itertext()).strip() if len(rating2)>0: rating2 = rating2.split()[0] if rating2.find('/')!=-1: rating2 = rating2.split('/')[0].replace(',','.') try: float(rating2) except Exception,ex: rating2=None ven.hqdb_nr_reviews = rating1 ven.hqdb_review_score = rating2 if ven.hqdb_review_score==None: scoreIn = xmlVen[0].xpath('//div[@class="float_box"]//span[@class="txtLight"]/parent::div') if len(scoreIn)>0: core_ = scoreIn[0].text.replace(',','.') try: float(core_) except Exception,ex: core_ =None ven.hqdb_review_score = core_ script_ = xmlPages.xpath('./head/script') if ven.formatted_address.strip()=='' and ven.office_number==None and ven.office_number2 ==None and ven.mobile_number ==None and ven.mobile_number2 ==None: return None '''streetTemp = ven.street cityTemp =ven.city zipcodeTemp =ven.zipcode if streetTemp ==None: streetTemp ='' if ven.city ==None: cityTemp = '' if ven.zipcode ==None: zipcodeTemp ='' address_ = streetTemp+', '+cityTemp+', '+zipcodeTemp address_ = address_.strip().replace(', ,', ',').replace(',,', ',') if address_.startswith(','): address_ =address_[1:len(address_)] if address_.endswith(','): address_ = address_[0:len(address_)-1] if ven.formatted_address!=None: address_ = ven.formatted_address''' #if len(address_.strip())>5: # (ven.latitude,ven.longitude) = self.getLatlng(address_,'DE') #script_ zipFrom = self.findZipcode(ven.formatted_address) if zipFrom!=None: (ven.latitude,ven.longitude) = self.getLatlng(zipFrom, 'DE') if ven.latitude ==None and ven.longitude==None: Util.log.running_logger.info(ven.formatted_address+' : cannot get GEO code') redirecPhotos= rightInfo.find('./nav/div/ul/li[@class="tabOff tab_foto"]/a') if redirecPhotos!=None: linkPhotos = redirecPhotos.get('href') if linkPhotos.startswith('/'): linkPhotos = self.__url__+ linkPhotos #time.sleep(1) xpathPhotos = Util.getRequestsXML(linkPhotos, '//div[@class="portfolio thumbs"]/a') if xpathPhotos!=None: listImg = xpathPhotos.xpath('./a') for __img in listImg: img_link.append(__img.get('data-thumb')) desElement= rightInfo.find('./div/div[@id="cont_about"]') ''' pTag = desElement.xpath('//div[@class="overview"]/p') des = '' for desE in pTag : if ''.join(desE.itertext()).find('<xml>')>=0: continue des+=''.join(desE.itertext()) h5Tag = desElement.xpath('//div[@class="overview"]/h5') for desE_ in h5Tag: if ''.join(desE_.itertext()).find('<xml>')>=0: continue des += ''.join(desE_.itertext()) divTag =desElement.xpath('//div[@class="overview"]/h5') for div_ in divTag: if ''.join(div_.itertext()).find('<xml>')>=0: continue des+= ''.join(div_.itertext()) if len(pTag)==0 and len(h5Tag) ==0: if desElement.find('.//div[@class="overview"]')!=None: des = desElement.find('.//div[@class="overview"]').text ven.description = self.validateDes(des) ''' des ='' divTag = desElement.xpath('//div[@class="overview"]') for divDes in divTag: des+= ' '.join(divDes.itertext()) ven.description = self.validateDes(des) certi = rightInfo.find('.//div/div[@id="cont_certs"]') tablecerti = certi.find('./table') if tablecerti!=None: certi_ = ''.join(tablecerti.itertext()).replace('Geprüfte Zertifikate:','') ven.accreditations = certi_ ven.img_link = img_link ven.country ='de' ven.is_get_by_address = True return ven
def __VenueParser(self, url__, name__): print 'Scraping: ' + url__ existing = [x for x in self.venuesList if url__ in x] if len(existing) > 0: return None #url__ ='http://www.drivingschoolsfinder.co.uk/city-Accrington/1846198-driving-Terrys-School-of-Motoring.html' #name__ ='Terrys School of Motoring' city = url__.split('/')[3].replace('city-', '').replace('-', ' ') xmlDoc = Util.getRequestsXML(url__, '/html/body') if xmlDoc == None: return None else: ven = Venue() sers = [] ven.name = name__ ven.city = city ven.scrape_page = url__ td = xmlDoc.xpath('//td[@class="welcome-padding"]') iter__ = ''.join(td[0].itertext()) iter__ = iter__[iter__.find('Driving School:') + len('Driving School:'):iter__. find('[Edit Text]')].replace('\n', '|').replace( '\t', '') iter__ = iter__.replace('|||', ' | ') rep = '|' + name__ iter__ = iter__[0:iter__.find(rep)] rep = ' | |' iter__ = iter__[0:iter__.find(rep)] ven.description = iter__ div = td[0].xpath('./div') if len(div) < 5: return None else: # div info = position div gray-line[0]+1 div_info = 0 for div_ in div: if div_.find('./script') != None: div_info = 3 info = div[div_info] info_ = ''.join(info.itertext()) address = info_[0:info_.find('Phone')].replace( name__, '').replace(city, ',' + city).replace(',,', ',').replace( ', ,', ',').split(',') #street = ', '.join(address[0:len(address)-2]).replace(','+city,'') street = ', '.join(address[0:len(address)]) street = street[0:street.find(city) - 1] if street.endswith(','): street = street[0:len(street) - 1] zipcode = address[len(address) - 1] street__ = street.upper() if street__.find('PO BOX') == -1: ven.street = street.replace('n/a', '').replace( '***', '').replace('6 weldon place croy', '').replace( 'cumbernauld41 napier square bellshill ml4 1tb', '').replace('P.O. Box 1048', '') if ven.street == '-': ven.street = None ven.zipcode = self.validateZipcode(zipcode) phone = info_[info_.find('Phone:') + len('Phone:'):info_.find('Fax:')].replace( ' ', '') if phone.isdigit(): if phone.startswith('07') | phone.startswith('7'): ven.mobile_number = self.validatePhone(phone) ven.mobile_number = self.validatePhone__( ven.mobile_number, 'gb') else: ven.office_number = self.validatePhone(phone) ven.office_number = self.validatePhone__( ven.office_number, 'gb') services_ = info_[info_.find('Services Offered:') + len('Services Offered:'):info_. find('Areas Served:')].strip().replace( ';', ',') if services_ != 'None Listed - [Edit]': services_ = services_.replace('/', ',').replace(',,', ',').split(',') for s in services_: name = self.validateServices(s) if len(name) >= 5: name__ = name.split() for n in name__: name = self.validateNameServices(name) if len(name.strip()) >= 5: services = Service() services.service = name sers.append(services) #ven.description = ven.description +' | ' +services_ stringfind = 'No Website' if info_.find('No Website') == -1: stringfind = 'Website' area_coverd = info_[info_.find('Areas Served:') + len('Areas Served:'):info_. find(stringfind)].strip().replace( ';', ',') #area_coverd = area_coverd[0:area_coverd.find(stringfind)] if area_coverd != 'None Listed - [Edit]': ven.areas_covered = area_coverd ven.services = sers reviewer = len(xmlDoc.xpath('//td[@class="review-box"]')) if reviewer > 0: ven.hqdb_nr_reviews = str(reviewer) scoreInfo = div[div_info + 1] #http://www.drivingschoolsfinder.co.uk/halfstar.gif +0.5 #http://www.drivingschoolsfinder.co.uk/fullstar.gif +1 #http://www.drivingschoolsfinder.co.uk/emptystar.gif +0 tr = scoreInfo.xpath('./table/tr') tr = tr[1] img_core = tr.xpath('./td')[1] img_core = img_core.xpath('./table/tr/td/img') score__ = 0.0 for score in img_core: score_ = score.get('src') if score_ == 'http://www.drivingschoolsfinder.co.uk/halfstar.gif': score__ += 0.5 if score_ == 'http://www.drivingschoolsfinder.co.uk/fullstar.gif': score__ += 1 if score_ == 'http://www.drivingschoolsfinder.co.uk/emptystar.gif': score__ += 0 if score__ > 0: ven.hqdb_review_score = str(score__).replace('.0', '') ven.country = 'gb' emails_ = re.findall(r'[\w\.-]+@[\w\.-]+', info_) for email_ in emails_: ven.business_email = email_ # website_ = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', info_) # for web_ in website_: # ven.business_website = web_ if ven.business_email != None: if ven.business_email.startswith('http'): ven.business_email = None ven.business_email = None if info_.find('No Website') == -1: arrays__ = info_.split(' ') for i in range(0, len(arrays__)): if arrays__[i].find('Website') >= 0: web_ = arrays__[i + 1].replace('\t', ' ').replace( '\n', ' ').split()[0].replace('No', '') ven.business_website = self.formatWeb_(web_) print ven.business_website break address_ = '' if ven.street == None: address_ = ven.city + ', ' + ven.zipcode #ven.formatted_address = ven.city+', '+ven.zipcode else: if ven.zipcode != None: address_ = ven.street + ', ' + ven.city + ', ' + ven.zipcode else: address_ = ven.street + ', ' + ven.city ven.pricelist_link = [ven.scrape_page] ''' get lat -lng ''' if address_ != '': try: (ven.latitude, ven.longitude) = self.getLatlng(address_, 'UK') except Exception, ex: Util.log.running_logger.error(ven.scrape_page + ' : ' + ex) return None ven.is_get_by_address = True return ven
def __VenueParser(self, venueElement): try: img_link = [] ad_type = "none" if venueElement.find( './/span[@class="label label-success"]') != None: ad_type = "featured" divs = venueElement.xpath('./div') logo_ = divs[0].find('.//img') if logo_ != None: img_link.append(self.__url__ + logo_.get('src')) url__ = venueElement.xpath( './div[@class="col-xs-9 col-sm-9 col-md-9 listing-body"]//div[@class="h4 listing-heading"]/a' ) if url__ != None: url__ = url__[0].get('href') url__ = self.__url__ + url__ ''' files = open('D:\\test.txt','a') files.write(url__+'\r\n') files.close() ''' existing = [x for x in self.listLink if url__ in x] if len(existing) <= 0: self.listLink.append(url__) print 'Scraping' + ' : ' + url__ #if url__ =='http://www.garagesandrecovery.co.uk/business/dorset-auto-repirs': # print 'Debug' xmlDoc = Util.getRequestsXML( url__, '//body/div[@class="page-wrapper"]') ven = Venue() ven.name = xmlDoc.find( './/div[@class="page-heading"]//h1').text content = xmlDoc.find( './/div[@class="container page-content"]') if content != None: des_img = content.find('.//div[@class="article-body"]') if des_img != None: div_img = des_img.xpath('.//img/parent::div') if len(div_img) > 0: des_img.remove(div_img[0]) des = ' '.join(des_img.itertext()) ven.description = des ven.country = self._language ven.scrape_page = url__ ven.hqdb_featured_ad_type = ad_type offices_ = content.xpath( './/div[@id="offices"]/parent::div/div[@class="row"]' ) div_maps = offices_[0].find( './/div[@class="google-map"]') if div_maps != None: ven.latitude = div_maps.get('data-lat') ven.longitude = div_maps.get('data-lng') info_ = offices_[0].xpath( './div[@class="col-md-5 col-sm-6"]') info_ = info_[0] ul = info_.xpath('./ul') phones = [] for u in ul: phone_ = u.xpath('./li/a') for phone in phone_: if phone.get('title') == 'Phone Number': phone = phone.text.replace(' ', '') if phone.startswith('0800'): continue else: phones.append(phone) if len(ul) >= 2: ul_2 = ul[0] li__ = ul_2.xpath('./li') address = '' for li in li__: if li.get('class') != 'text-bold': address = '\n'.join(li.itertext()) addressArr = address.split('\n') if len(addressArr) >= 3: ven.street = addressArr[len(addressArr) - 3] ven.city = addressArr[len(addressArr) - 2].split(',')[0] ven.zipcode = addressArr[len(addressArr) - 1] if ven.zipcode != None: results = re.search(self.ukReg, ven.zipcode, flags=0) if ven.zipcode == 'Rotherham, South Yorkshire': ven.zipcode = '' ven.street = None if results == None: ven.zipcode = None (ven.office_number, ven.office_number2, ven.mobile_number, ven.mobile_number2) = self.processPhones(phones) # right sidebar : //div[@class="col-md-3 page-sidebar"]/div rightSidebar = xmlDoc.xpath( './/div[@class="col-md-3 page-sidebar"]/div[@class="section"]' ) for right in rightSidebar: website = right.xpath( './a[contains(text(),"Visit Our Website")]') if len(website) > 0: website = website[0].get('href') if website.find('facebook.com') == -1: ven.business_website = website else: ven.facebook = website reviews = right.xpath('./p/strong') if len(reviews) >= 3: ven.hqdb_nr_reviews = reviews[2].text ven.hqdb_review_score = reviews[1].text follows = right.xpath('./ul/li/a') for foll in follows: follow_link = foll.get('href') if follow_link.find('facebook.com') != -1: if ven.facebook == None: ven.facebook = self.addHTTP( follow_link) if follow_link.find('twitter.com') != -1: if ven.twitter == None: ven.twitter = self.addHTTP(follow_link) img_find = xmlDoc.xpath( '//div[@id="galleries"]/parent::div/div[@class="carousel slide equal-height"]//img' ) for ig in img_find: img_link.append(self.__url__ + ig.get('src')) if len(img_link) > 0: ven.img_link = img_link self.index += 1 ven.writeToFile(self.folder, self.index, ven.name, False) #img_link : //div[@id="galleries"]/parent::div/div[@class="carousel slide equal-height"]//img else: print '\nduplicate'.upper() print '*' * (len(url__) + 4) print '*' + ' ' * (len(url__) + 2) + '*' print '* ' + url__ + ' *' print '*' + ' ' * (len(url__) + 2) + '*' print '*' * (len(url__) + 4) + '\n' except Exception, ex: print ex