def __VenueParser(self, url, cateName): existing = [x for x in self.listlink if url in x] self.listlink.append(url) if len(existing) > 0: self.countduplicate += 1 print '[INFO] Duplicate count = ' + str(self.countduplicate) return try: print 'Scraping url: ' + url #url = 'http://www.uksecurity-directory.co.uk/the-directory/1905/ecpcco/' xmlDoc = Util.getRequestsXML(url, '//div[@class="gdl-page-content"]') xmlDoc = xmlDoc.xpath('//div[@class="gdl-page-content"]/div')[0] ven = Venue() imgs = [] ven.category = cateName ven.scrape_page = url ven.country = self._language ven.name = xmlDoc.find('./div/h2').text ven.hqdb_featured_ad_type = 'none' isFeatured = xmlDoc.find('./div[@class="stickytag"]/img') if isFeatured != None: if isFeatured.get('title') == 'Featured Listing': ven.hqdb_featured_ad_type = 'featured' divInfo = xmlDoc.xpath('./div[@class="listing-details cf"]/div') town_ = '' area_ = '' zipcode = '' listPhone_ = [] for div__ in divInfo: label = div__.find('./label') if label != None: label_ = label.text if label_ == 'Business Website Address:': website = div__.find('./span/a') if website != None: website = website.get('href') isFacebook = website.find('facebook.com') isTwiter = website.find('twiter.com') if isFacebook == -1 and isTwiter == -1: ven.business_website = website else: if isFacebook != -1: ven.facebook = website if ven.twitter != -1: ven.twitter = website if label_ == 'Security Services:': serviceStr = div__.xpath('./span/a') sers = [] for ser in serviceStr: serv = Service() serv.service = ser.text sers.append(serv) if len(sers) > 0: ven.services = sers ven.pricelist_link = [ven.scrape_page] if label_ == 'Long Business Description:': des = div__.find('./span') if des != None: des = ' '.join(des.itertext()) ven.description = des if label_ == 'Business Phone Number:': phone = div__.find('./span').text #phone = self.formatPhone(phone) findsplistPPhone = self.findSplitPhone(phone) if findsplistPPhone == None: listPhone_ = [phone] #(ven.office_number,ven.office_number2,ven.mobile_number,ven.mobile_number2) = self.processPhones([phone]) else: listPhone_ = phone.split(findsplistPPhone) (ven.office_number, ven.office_number2, ven.mobile_number, ven.mobile_number2) = self.processPhones(listPhone_) if label_ == 'Postcode:': zipcode = div__.find('./span').text if label_ == 'Town:': town_ = div__.find('./span').text if label_ == 'Area:': area_ = div__.find('./span').text zipcode = self.validateZipcode(zipcode) if url == 'http://www.uksecurity-directory.co.uk/the-directory/1981/s-comm-vehicle-surveillance-system': print if ven.office_number == 'NOT_GB' or ven.office_number2 == 'NOT_GB' or ven.mobile_number == 'NOT_GB' or ven.mobile_number2 == 'NOT_GB': return for p in listPhone_: if p == town_: town_ = '' break ven.zipcode = zipcode ven.formatted_address = ', '.join([area_, town_, zipcode]) ven.formatted_address = self.refixFormatAddress( ven.formatted_address.replace('0000000', '')) extraImg = xmlDoc.xpath('./div[@class="extra-images"]//a/img') listingThumbnail = xmlDoc.xpath( './div[@class="listing-thumbnail"]//a/img') for thumb in listingThumbnail: imgs.append(thumb.get('src')) for img in extraImg: imgs.append(img.get('src')) if len(imgs) > 0: ven.img_link = imgs self.index = self.index + 1 ven.writeToFile(self.folder, self.index, ven.name, False) except Exception, ex: print '[ERROR] ' + url + ': ' + str(ex)
def __VenueParser(self,hqdb_type, linkItems,subcate,cate): #linkItems ='https://www.blauarbeit.de/p/modernisierung/_sanierung/berlin/daniel_kutscher/576667.htm' existing=[x for x in self.linkIn if linkItems in x] if len(existing)>0: print 'This venue exist in list' return None self.linkIn.append(linkItems) xmlPages = self.getRequest(linkItems) if xmlPages==None: return None xmlVen = xmlPages.xpath('//div[@class="page_move"]') cate__ = xmlPages.find('.//meta[@name="Description"]') if len(xmlVen)==0: return None name = xmlVen[0].xpath('.//h2') if len(name) <=0: name ='' else: name = name[0].text.strip() noneValues ={'ZERO','NULL'} if name.upper() in noneValues: return None ven = Venue() if cate__!=None: ven.category = cate__.get('content').split(',')[0] nameFromUrl = self.getNamefromUrl(linkItems) ven.name = nameFromUrl ven.hqdb_featured_ad_type = hqdb_type #ven.name =name ven.scrape_page = linkItems #ven.subcategory = subcate #ven.category= cate address_= '' #ven.formatted_address='' img_link= [] divInfo = xmlVen[0].find('.//div[@class="content_wrapper content_wrapper_main clearfix"]/div') if divInfo!=None: mainInfo = divInfo.xpath('./section') if len(mainInfo)>=2: leftInfo = mainInfo[0] rightInfo = mainInfo[1] tableInfo = leftInfo.find('./div/div[@class="profile_top_left"]/table') trinfo = tableInfo.xpath('./tr') for tr_ in trinfo: td =tr_.xpath('./td') if len(td)<2: continue key_ = ''.join(td[0].itertext()).strip() values_ = ' '.join(td[1].itertext()).strip().replace('keine Angabe','').replace('NULL','').replace('null','') if key_ =='Ansprechpartner:': if values_!=None and len(values_)>2: #values_ ='' ven.name_of_contact = values_ ven.name +=', '+ ven.name_of_contact if key_ =='Addresse:': address_ = values_ ven.formatted_address = self.validateFormat(address_) '''(ven.street,ven.city,ven.zipcode) = self.processAddress(address_) if ven.street!=None: ven.street = self.validateStreet2(ven.street) #ven.formatted_address = address_ if ven.city!=None: checkCity = ven.city.split() if len(checkCity)>0: if checkCity[0].isdigit(): if len(checkCity[0])==5: if ven.street!=None: ven.street+=' '+ ven.zipcode ven.zipcode = checkCity[0] ven.city = ven.city.replace(ven.zipcode,'') else: ven.city = None ven.street = None ven.zipcode= None ven.formatted_address = ' '.join(checkCity) if ven.zipcode!=None: if len(ven.zipcode)==5: ven.zipcode = ven.zipcode else: ven.zipcode = None''' if key_ =='Homepage:': a_ = td[1].find('./a') if a_ !=None: ven.business_website = a_.get('href') mobileCode =['015','016','017','+4915','+4916','+4917'] if key_ =='Tel:': values_ = values_.replace('/', '').replace(' ', '').replace('Tel', '') #values_ ='01735465435' for mCode in mobileCode: if values_.startswith(mCode): ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de') break if ven.mobile_number==None: ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de') '''if values_.startswith('01')| values_.startswith('+0041')| values_.startswith('0041'): ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de') else: ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de')''' img_ = leftInfo.find('./div/div[@class="profile_top_right"]/img') if img_!=None: img_ =img_.get('src') img_link.append(img_) rating = leftInfo.xpath('.//section[@id="ratings"]/div') if len(rating)>=2: rating1 = ''.join(rating[0].itertext()).strip().split()[1] rating2 = ''.join(rating[1].itertext()).strip() if len(rating2)>0: rating2 = rating2.split()[0] if rating2.find('/')!=-1: rating2 = rating2.split('/')[0].replace(',','.') try: float(rating2) except Exception,ex: rating2=None ven.hqdb_nr_reviews = rating1 ven.hqdb_review_score = rating2 if ven.hqdb_review_score==None: scoreIn = xmlVen[0].xpath('//div[@class="float_box"]//span[@class="txtLight"]/parent::div') if len(scoreIn)>0: core_ = scoreIn[0].text.replace(',','.') try: float(core_) except Exception,ex: core_ =None ven.hqdb_review_score = core_ script_ = xmlPages.xpath('./head/script') if ven.formatted_address.strip()=='' and ven.office_number==None and ven.office_number2 ==None and ven.mobile_number ==None and ven.mobile_number2 ==None: return None '''streetTemp = ven.street cityTemp =ven.city zipcodeTemp =ven.zipcode if streetTemp ==None: streetTemp ='' if ven.city ==None: cityTemp = '' if ven.zipcode ==None: zipcodeTemp ='' address_ = streetTemp+', '+cityTemp+', '+zipcodeTemp address_ = address_.strip().replace(', ,', ',').replace(',,', ',') if address_.startswith(','): address_ =address_[1:len(address_)] if address_.endswith(','): address_ = address_[0:len(address_)-1] if ven.formatted_address!=None: address_ = ven.formatted_address''' #if len(address_.strip())>5: # (ven.latitude,ven.longitude) = self.getLatlng(address_,'DE') #script_ zipFrom = self.findZipcode(ven.formatted_address) if zipFrom!=None: (ven.latitude,ven.longitude) = self.getLatlng(zipFrom, 'DE') if ven.latitude ==None and ven.longitude==None: Util.log.running_logger.info(ven.formatted_address+' : cannot get GEO code') redirecPhotos= rightInfo.find('./nav/div/ul/li[@class="tabOff tab_foto"]/a') if redirecPhotos!=None: linkPhotos = redirecPhotos.get('href') if linkPhotos.startswith('/'): linkPhotos = self.__url__+ linkPhotos #time.sleep(1) xpathPhotos = Util.getRequestsXML(linkPhotos, '//div[@class="portfolio thumbs"]/a') if xpathPhotos!=None: listImg = xpathPhotos.xpath('./a') for __img in listImg: img_link.append(__img.get('data-thumb')) desElement= rightInfo.find('./div/div[@id="cont_about"]') ''' pTag = desElement.xpath('//div[@class="overview"]/p') des = '' for desE in pTag : if ''.join(desE.itertext()).find('<xml>')>=0: continue des+=''.join(desE.itertext()) h5Tag = desElement.xpath('//div[@class="overview"]/h5') for desE_ in h5Tag: if ''.join(desE_.itertext()).find('<xml>')>=0: continue des += ''.join(desE_.itertext()) divTag =desElement.xpath('//div[@class="overview"]/h5') for div_ in divTag: if ''.join(div_.itertext()).find('<xml>')>=0: continue des+= ''.join(div_.itertext()) if len(pTag)==0 and len(h5Tag) ==0: if desElement.find('.//div[@class="overview"]')!=None: des = desElement.find('.//div[@class="overview"]').text ven.description = self.validateDes(des) ''' des ='' divTag = desElement.xpath('//div[@class="overview"]') for divDes in divTag: des+= ' '.join(divDes.itertext()) ven.description = self.validateDes(des) certi = rightInfo.find('.//div/div[@id="cont_certs"]') tablecerti = certi.find('./table') if tablecerti!=None: certi_ = ''.join(tablecerti.itertext()).replace('Geprüfte Zertifikate:','') ven.accreditations = certi_ ven.img_link = img_link ven.country ='de' ven.is_get_by_address = True return ven
def __VenueParser(self, item, city): existing = [x for x in self.listLink if item in x] self.listLink.append(item) if len(existing) <= 0: try: xmlDoc = Util.getRequestsXML(item, '/html') ven = Venue() ven.scrape_page = item #ven.city = city ven.name = xmlDoc.xpath( '//div[@class="row top-buffer"]/h3')[0].text (ven.latitude, ven.longitude) = self.getLatlng(xmlDoc) xmlcontent = xmlDoc.find('.//div[@class="tab-content"]') services_schedule_info = xmlcontent.xpath( './div/div[@class="row top-buffer"]/h4/parent::div')[0] if services_schedule_info != None: services_schedule_info = ''.join( services_schedule_info.itertext()).split('\n') for it in services_schedule_info: if it.find('Style:') != -1: it = it[0:it.find('Schedule')] it = it.strip() ser_name = it[it.find('Style:') + len('Style:'):it. find('Ability level')] cost = len(it) cost_ = ['Cost:', 'Concession cost:'] char_cost = '' for c in cost_: if it.find(c) != -1: cost = it.find(c) char_cost = c break #cost = it.find('Cost:') if cost == -1: cost = len(it) ser_des = it[it.find('Ability level:') + len('Ability level:'):cost] ser_price = it[cost + len(char_cost):it.find('GBP') + len('GBP')] ser = Service() ser.service = ser_name ser.description = ser_des ser.price = ser_price.replace('-', '') ven.services = [ser] if it.find('a.m.') != -1 or it.find('p.m.') != -1: ven.opening_hours_raw = it.strip().replace( '.Monday', ' | Monday').replace( '.Tuesday', ' | Tuesday').replace( '.Wednesday', ' | Wednesday').replace( '.Thursday', ' | Thursday').replace( '.Friday', ' | Friday').replace( '.Saturday', ' | Saturday').replace( '.Sunday', ' | Sunday') ven.opening_hours_raw = self.formatOpenhour( ven.opening_hours_raw) address = xmlcontent.find('.//address') if address != None: #print ET.dump(address) address = ''.join(address.itertext()).replace( 'United Kingdom', '').strip() address = self.validateAddress(address) #address ='Ward Park Arras Pavilion,Gransha Road,Bangor,Northern Ireland,BT20 4TN' ven.country = 'gb' if address.upper().find('Ireland'.upper()) != -1: if address.upper().find( 'Northern Ireland'.upper()) != -1: ven.country = 'ie' if address.endswith(','): address = address[0:-1] ven.formatted_address = address posted = xmlcontent.find('./div/div[@class="row"]/p') imgs = xmlcontent.xpath('.//a/img') img_ = [] for img in imgs: img_.append(img.get('src')) ven.img_link = img_ if posted != None: ven.hqdb_ad_posted = posted.text.replace( 'Last updated', '') split_posted = ven.hqdb_ad_posted.split(',') if len(split_posted) >= 3: ven.hqdb_ad_posted = ', '.join( split_posted[0:len(split_posted) - 1]) ven.category = self.category #ven.country ='gb' des_info = xmlcontent.xpath( '//div[@class="row top-buffer"]/h3')[1] #print des_info.text des_info = des_info.getparent() des__ = des_info.xpath('./p') ven.pricelist_link = [ven.scrape_page] ven.hqdb_featured_ad_type = 'none' ven.description = '' for des in des__: ven.description += ''.join(des.itertext()) + ' ' des_info.remove(des) info = '____'.join(des_info.itertext()) a = des_info.find('./a') if a != None: a = a.get('href') if a.find('facebook.com') == -1: ven.business_website = a else: if a.startswith('http:'): a = a.replace('http:', 'https:') ven.facebook = a info = info.split('__') for inf in range(0, len(info)): if info[inf] == 'Qualifications:': ven.accreditations = info[inf + 2] if info[inf] == 'Phone:': phone = info[inf + 2].strip() pattren = '(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)' '''isEmail = re.search(pattren, phone, flags=0) if isEmail!=None: ven.business_email = isEmail.group(0) continue''' find_charSplit = self.findSplitPhone(phone) if find_charSplit == None: issMail = re.search(pattren, phone, flags=0) if issMail != None: ven.business_email = issMail.group(0) continue phone = phone.replace('Mobile:', '').replace( 'ext.225', '').replace('O7', '07').replace(' ', '') if phone.startswith('07') or phone.startswith( '447' ) or phone.startswith('+447') or phone.startswith( '00447') or phone.startswith( '+44(0)7') or phone.startswith( '44(0)7') or phone.startswith( '004407'): ven.mobile_number = self.validatePhone__( phone, ven.country) else: ven.office_number = self.validatePhone__( phone, ven.country) else: phone = phone.split(find_charSplit) for p in phone: issMail = re.search(pattren, p, flags=0) if issMail != None: ven.business_email = issMail.group(0) continue p = p.replace('Mobile', '').replace( 'ext225', '').replace('O7', '07').replace(' ', '') if p.startswith('07') or p.startswith( '447' ) or p.startswith('+447') or p.startswith( '00447' ) or p.startswith('+44(0)7') or p.startswith( '44(0)7') or p.startswith('004407'): if ven.mobile_number != None: ven.mobile_number2 = self.validatePhone__( p, ven.country) else: ven.mobile_number = self.validatePhone__( p, ven.country) else: if ven.office_number != None: ven.office_number2 = self.validatePhone__( p, ven.country) else: ven.office_number = self.validatePhone__( p, ven.country) isPhoneOverSea = self.checkPhoneOverSea([ ven.office_number, ven.office_number2, ven.mobile_number, ven.mobile_number2 ]) if isPhoneOverSea == False: index = self.addIndex() print str( index) + ' Scapping: ' + city + '---' + ven.scrape_page #ven.is_get_by_address =True ven.writeToFile(self.folder, index, ven.name, False) except Exception, ex: print ex return
def __VenueParser(self, jsonItems, hash): url = self.__url__ + 'profile/' + jsonItems.get( 'serviceSlug') + '/' + jsonItems.get( 'companySlug') + '-' + jsonItems.get('id') + '?hash=' + hash url__ = self.__url__ + 'profile/' + jsonItems.get( 'serviceSlug') + '/' + jsonItems.get( 'companySlug') + '-' + jsonItems.get('id') id_ = str(jsonItems.get('id')) existing = [x for x in self.list_url if url__ in x] if len(existing) > 0: print 'this venues existed in list' return None if len(existing) <= 0: print 'Scrapping: ' + url ven = Venue() services_v = [] ven.category = jsonItems.get('restriction').get('name') ven.adid = str(jsonItems.get('id')) ven.name = jsonItems.get('companyName') ven.latitude = jsonItems.get('coordinates').get('lat') ven.longitude = jsonItems.get('coordinates').get('long') ven.venue_images = jsonItems.get('logo') points_ = jsonItems.get('satisfaction_rating') if str(points_).find('.') >= 0: ven.hqdb_review_score = str(round(points_, 1)) else: ven.hqdb_review_score = str(points_) #ven.img_link = [url] #ven.description = jsonItems.get('salesPitch') ven.country = 'gb' ven.scrape_page = url #ven.pricelist_link = [url] self.list_url.append(url__) #url ='https://www.unbiased.co.uk/profile/financial-adviser/stiles-company-financial-services-petersfield-ltd-511274' xmlRequest = Util.getRequestsXML( url, '//div[@class="container-fluid"]') if xmlRequest != None: stringAddress = xmlRequest.find( './/span[@class="profile-meta__address"]').text.replace( ',,', ',') stringAddress = '1st and 2nd Floor Offices, 446 - 452 High street, Kingswinford, West Midlands,' ven.formatted_address = self.removeNameFromAdd( ven.name.strip(), stringAddress).replace('PO BOX', '').replace('PO Box', '').replace( 'Po Box', '') zipArr = stringAddress.split(',') ven.zipcode = zipArr[len(zipArr) - 1] ex_ = re.search( '([Gg][Ii][Rr]0[Aa]{2})|((([A-Za-z][0-9]{1,2})|(([A-Za-z][A-Ha-hJ-Yj-y][0-9]{1,2})|(([A-Za-z][0-9][A-Za-z])|([A-Za-z][A-Ha-hJ-Yj-y][0-9]?[A-Za-z]))))\s?[0-9][A-Za-z]{2})', stringAddress, flags=0) if ex_ != None: zip_c = ex_.group(0) #ven.zipcode = zip_c #ven.formatted_address = ven.formatted_address.replace(ven.zipcode,'').strip() if ven.zipcode != zip_c: poZip_c = stringAddress.find(zip_c) poZipcode = stringAddress.find(ven.zipcode) if len(ven.zipcode.strip()) > 1: if poZip_c > poZipcode: ven.zipcode = zip_c if ex_ == None: if ven.zipcode != None: ven.zipcode = None if ven.formatted_address.endswith(','): ven.formatted_address = ven.formatted_address[ 0:len(ven.formatted_address) - 2] phoneLabel = xmlRequest.xpath( './/span[@class="phone-label"]/parent::a') if len(phoneLabel) > 0: for phone_ in phoneLabel: phone = phone_.get('data-phone').replace('\n', '').replace( ' ', '') if phone.find('Shownumber') <= 0: phone = self.validatePhone(phone) for rePhone in self.listPhoneremove: if phone == rePhone: phone = None if phone != None: if phone.startswith('07'): ven.mobile_number = phone else: ven.office_number = phone break services = xmlRequest.find( './/ul[@class="advice-area__level-one"]') if services != None: list_ser = services.xpath('./li') for ser_name in list_ser: # feedback 3 : add category service cate = ser_name.find('./span').text.strip() list_services = ser_name.xpath('./ul/li') for service__ in list_services: service = Service() service.service_category = cate + ' advice' service.service = service__.text + ' advice' services_v.append(service) ven.services = services_v # append accreditations feedback 3 certi = [] cer = xmlRequest.xpath( './/div[@class="profile-team__skill-item collapsed"]') for c in cer: inCerti = [x_ for x_ in certi if c.text in x_] if len(inCerti) <= 0: certi.append(c.text) ven.accreditations = ', '.join(certi) # add follow : fb, twi, website feedback 3 follow = xmlRequest.xpath( '//div[@class="profile__follow"]/ul/li') for fol in follow: values_fol = fol.get('class') if values_fol == 'icon-soc-tw': ven.twitter = fol.find('./a').get('href') if values_fol == 'icon-soc-www': ven.business_website = fol.find('./a').get('href') if values_fol == 'icon-soc-fb': ven.facebook = fol.find('./a').get('href') # description feedback 3 des_1 = xmlRequest.find( './/div[@class="profile__text-block "]/p') if des_1 != None: ven.description = ''.join(des_1.itertext()).replace( '.\n', ' | ') des_2 = xmlRequest.find( './/div[@class="profile__text-block spacing-bottom-xs-0"]/p' ) if des_2 != None: ven.description += ' -Our services: ' + ''.join( des_2.itertext()).replace('.\n', ' | ') if ven.description != None: if ven.description.endswith(' | '): ven.description = ven.description[0:len(ven.description ) - 2] return ven else: return None