def __VenueParser_2(self, element):
try:
featured = 'none'
#print ET.dump(element)
self.urlmarks += 1
print '[COUNT]: ' + str(self.urlmarks)
ven = Venue()
ven.scrape_page = self._url_lstVenues + '#' + str(self.urlmarks)
td = element.find('./td')
div = td.find('./div')
if div != None:
a = div.find('./a').get('href')
ven.business_website = a
td.remove(div)
basicInfo = ''.join(td.itertext())
#basicInfo =td.text
if self.urlmarks == 881:
pass
street = ''
contactName = ''
name = ''
basicInfo_ = basicInfo.split('\n')
if basicInfo_[len(basicInfo_) - 1].find('Fax') != -1:
basicInfo_ = basicInfo_[0:-1]
positionInArr = -1
phoneNumber = basicInfo_[positionInArr].strip().replace(
'Tel.', '').replace(' ', '')
positionInArr -= 1
local = basicInfo_[positionInArr]
positionInArr -= 1
street = basicInfo_[positionInArr]
if self.isContactName(street) == True:
contactName = street
street = ''
positionInArr += 1
positionInArr -= 1
if contactName == '':
contactName = basicInfo_[positionInArr]
if self.isContactName(contactName) == False:
positionInArr += 1
contactName = ''
name = ' '.join(basicInfo_[0:positionInArr])
(ven.office_number, ven.office_number2, ven.mobile_number,
ven.mobile_number2) = self.processPhone([phoneNumber])
(ven.city, ven.zipcode) = self.processZipCity(local)
if ven.zipcode != None:
if self.validateZip(ven.zipcode) == None:
return
ven.street = street
ven.name_of_contact = contactName
ven.name = name
ven.country = self._language
ven.hqdb_featured_ad_type = featured
self.venues[ven.scrape_page] = ven
except Exception, ex:
print ex
def __VenueParser(self, element):
try:
self.urlmarks += 1
print '[COUNT]: ' + str(self.urlmarks)
featured = 'featured'
onclick = element.xpath('./td//a/@onclick')[0]
detailLink = onclick[onclick.find("MM_openBrWindow('") +
len("MM_openBrWindow('"):onclick.find("','','"
)]
detailLink = self.__url__ + '/' + detailLink
xmlDoc = Util.getRequestsXML(detailLink,
'//table[@cellspacing="3"]')
xmlDoc = xmlDoc[0]
ven = Venue()
ven.hqdb_featured_ad_type = featured
ven.country = self._language
ven.scrape_page = detailLink
detail_ = xmlDoc.xpath('./tr/td/table')
detail_1 = detail_[0]
detail_2 = detail_[2]
basicInfo = detail_1.find('./tr/td/table/tr/td[@class="text"]')
email_website = basicInfo.getparent().xpath(
'//table//div[@align="right"]/a')
for aTag in email_website:
link__ = aTag.get('href')
if link__.find('mailto') != -1:
ven.business_email = link__.replace('mailto:', '')
else:
if link__.find('http') != -1:
ven.business_website = link__
openxml = detail_1.xpath('./tr')
openxml = openxml[2].find('./td/table') # table openning hour
rows = openxml.xpath('./tr')
dayofweek = {
'Montag': 'Lunedì',
'Dienstag': 'Martedì',
'Mittwoch': 'Mercoledì',
'Donnerstag': 'Giovedì',
'Freitag': 'Venerdì',
'Samstag': 'Sabato',
'Sonntag': 'Domenica'
}
openning_hour_array = []
for row in rows:
tds = row.xpath('./td')
if len(tds) > 0:
if tds[0].text != None:
if dayofweek.get(tds[0].text, "NULL") != "NULL":
record = ''
count_ = 0
for td in tds:
if dayofweek.get(td.text, "NULL") != "NULL":
record += dayofweek.get(td.text) + ": "
else:
if td.text.strip() != '-':
record += td.text.replace('.',
':') + ", "
else:
count_ += 1
record = record.strip()
if record.endswith(','):
record = record[0:-1]
if count_ < 3:
openning_hour_array.append(record)
ven.opening_hours_raw = ' | '.join(openning_hour_array)
basicInfo_ = ''.join(basicInfo.itertext()).split('\n')
if basicInfo_[len(basicInfo_) - 1].find('Fax') != -1:
basicInfo_ = basicInfo_[0:-1]
phonenumber = basicInfo_[-1].strip().replace('Tel.',
'').replace(' ', '')
zip_ci = basicInfo_[-2]
street = basicInfo_[-3]
contactName = basicInfo_[-4]
name = ' '.join(basicInfo_[0:-4])
(ven.office_number, ven.office_number2, ven.mobile_number,
ven.mobile_number2) = self.processPhone([phonenumber])
(ven.city, ven.zipcode) = self.processZipCity(zip_ci)
if ven.zipcode != None:
if self.validateZip(ven.zipcode) == None:
return
ven.street = street
ven.name_of_contact = contactName
ven.name = name
services = detail_2.xpath('./tr[@valign="top"]/td')
if len(services) > 0:
service_ = []
services = services[0].text
ven.services = self.__ServicesParser(services)
self.index += 1
ven.writeToFile(self.folder, self.index, ven.name, False)
except Exception, ex:
print ex
def __VenueParser(self,hqdb_type, linkItems,subcate,cate):
#linkItems ='https://www.blauarbeit.de/p/modernisierung/_sanierung/berlin/daniel_kutscher/576667.htm'
existing=[x for x in self.linkIn if linkItems in x]
if len(existing)>0:
print 'This venue exist in list'
return None
self.linkIn.append(linkItems)
xmlPages = self.getRequest(linkItems)
if xmlPages==None:
return None
xmlVen = xmlPages.xpath('//div[@class="page_move"]')
cate__ = xmlPages.find('.//meta[@name="Description"]')
if len(xmlVen)==0:
return None
name = xmlVen[0].xpath('.//h2')
if len(name) <=0:
name =''
else:
name = name[0].text.strip()
noneValues ={'ZERO','NULL'}
if name.upper() in noneValues:
return None
ven = Venue()
if cate__!=None:
ven.category = cate__.get('content').split(',')[0]
nameFromUrl = self.getNamefromUrl(linkItems)
ven.name = nameFromUrl
ven.hqdb_featured_ad_type = hqdb_type
#ven.name =name
ven.scrape_page = linkItems
#ven.subcategory = subcate
#ven.category= cate
address_= ''
#ven.formatted_address=''
img_link= []
divInfo = xmlVen[0].find('.//div[@class="content_wrapper content_wrapper_main clearfix"]/div')
if divInfo!=None:
mainInfo = divInfo.xpath('./section')
if len(mainInfo)>=2:
leftInfo = mainInfo[0]
rightInfo = mainInfo[1]
tableInfo = leftInfo.find('./div/div[@class="profile_top_left"]/table')
trinfo = tableInfo.xpath('./tr')
for tr_ in trinfo:
td =tr_.xpath('./td')
if len(td)<2:
continue
key_ = ''.join(td[0].itertext()).strip()
values_ = ' '.join(td[1].itertext()).strip().replace('keine Angabe','').replace('NULL','').replace('null','')
if key_ =='Ansprechpartner:':
if values_!=None and len(values_)>2:
#values_ =''
ven.name_of_contact = values_
ven.name +=', '+ ven.name_of_contact
if key_ =='Addresse:':
address_ = values_
ven.formatted_address = self.validateFormat(address_)
'''(ven.street,ven.city,ven.zipcode) = self.processAddress(address_)
if ven.street!=None:
ven.street = self.validateStreet2(ven.street)
#ven.formatted_address = address_
if ven.city!=None:
checkCity = ven.city.split()
if len(checkCity)>0:
if checkCity[0].isdigit():
if len(checkCity[0])==5:
if ven.street!=None:
ven.street+=' '+ ven.zipcode
ven.zipcode = checkCity[0]
ven.city = ven.city.replace(ven.zipcode,'')
else:
ven.city = None
ven.street = None
ven.zipcode= None
ven.formatted_address = ' '.join(checkCity)
if ven.zipcode!=None:
if len(ven.zipcode)==5:
ven.zipcode = ven.zipcode
else:
ven.zipcode = None'''
if key_ =='Homepage:':
a_ = td[1].find('./a')
if a_ !=None:
ven.business_website = a_.get('href')
mobileCode =['015','016','017','+4915','+4916','+4917']
if key_ =='Tel:':
values_ = values_.replace('/', '').replace(' ', '').replace('Tel', '')
#values_ ='01735465435'
for mCode in mobileCode:
if values_.startswith(mCode):
ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de')
break
if ven.mobile_number==None:
ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de')
'''if values_.startswith('01')| values_.startswith('+0041')| values_.startswith('0041'):
ven.mobile_number = self.validatePhone__(self.validatePhone(values_), 'de')
else:
ven.office_number = self.validatePhone__(self.validatePhone(values_), 'de')'''
img_ = leftInfo.find('./div/div[@class="profile_top_right"]/img')
if img_!=None:
img_ =img_.get('src')
img_link.append(img_)
rating = leftInfo.xpath('.//section[@id="ratings"]/div')
if len(rating)>=2:
rating1 = ''.join(rating[0].itertext()).strip().split()[1]
rating2 = ''.join(rating[1].itertext()).strip()
if len(rating2)>0:
rating2 = rating2.split()[0]
if rating2.find('/')!=-1:
rating2 = rating2.split('/')[0].replace(',','.')
try:
float(rating2)
except Exception,ex:
rating2=None
ven.hqdb_nr_reviews = rating1
ven.hqdb_review_score = rating2
if ven.hqdb_review_score==None:
scoreIn = xmlVen[0].xpath('//div[@class="float_box"]//span[@class="txtLight"]/parent::div')
if len(scoreIn)>0:
core_ = scoreIn[0].text.replace(',','.')
try:
float(core_)
except Exception,ex:
core_ =None
ven.hqdb_review_score = core_
script_ = xmlPages.xpath('./head/script')
if ven.formatted_address.strip()=='' and ven.office_number==None and ven.office_number2 ==None and ven.mobile_number ==None and ven.mobile_number2 ==None:
return None
'''streetTemp = ven.street
cityTemp =ven.city
zipcodeTemp =ven.zipcode
if streetTemp ==None:
streetTemp =''
if ven.city ==None:
cityTemp = ''
if ven.zipcode ==None:
zipcodeTemp =''
address_ = streetTemp+', '+cityTemp+', '+zipcodeTemp
address_ = address_.strip().replace(', ,', ',').replace(',,', ',')
if address_.startswith(','):
address_ =address_[1:len(address_)]
if address_.endswith(','):
address_ = address_[0:len(address_)-1]
if ven.formatted_address!=None:
address_ = ven.formatted_address'''
#if len(address_.strip())>5:
# (ven.latitude,ven.longitude) = self.getLatlng(address_,'DE') #script_
zipFrom = self.findZipcode(ven.formatted_address)
if zipFrom!=None:
(ven.latitude,ven.longitude) = self.getLatlng(zipFrom, 'DE')
if ven.latitude ==None and ven.longitude==None:
Util.log.running_logger.info(ven.formatted_address+' : cannot get GEO code')
redirecPhotos= rightInfo.find('./nav/div/ul/li[@class="tabOff tab_foto"]/a')
if redirecPhotos!=None:
linkPhotos = redirecPhotos.get('href')
if linkPhotos.startswith('/'):
linkPhotos = self.__url__+ linkPhotos
#time.sleep(1)
xpathPhotos = Util.getRequestsXML(linkPhotos, '//div[@class="portfolio thumbs"]/a')
if xpathPhotos!=None:
listImg = xpathPhotos.xpath('./a')
for __img in listImg:
img_link.append(__img.get('data-thumb'))
desElement= rightInfo.find('./div/div[@id="cont_about"]')
'''
pTag = desElement.xpath('//div[@class="overview"]/p')
des = ''
for desE in pTag :
if ''.join(desE.itertext()).find('<xml>')>=0:
continue
des+=''.join(desE.itertext())
h5Tag = desElement.xpath('//div[@class="overview"]/h5')
for desE_ in h5Tag:
if ''.join(desE_.itertext()).find('<xml>')>=0:
continue
des += ''.join(desE_.itertext())
divTag =desElement.xpath('//div[@class="overview"]/h5')
for div_ in divTag:
if ''.join(div_.itertext()).find('<xml>')>=0:
continue
des+= ''.join(div_.itertext())
if len(pTag)==0 and len(h5Tag) ==0:
if desElement.find('.//div[@class="overview"]')!=None:
des = desElement.find('.//div[@class="overview"]').text
ven.description = self.validateDes(des)
'''
des =''
divTag = desElement.xpath('//div[@class="overview"]')
for divDes in divTag:
des+= ' '.join(divDes.itertext())
ven.description = self.validateDes(des)
certi = rightInfo.find('.//div/div[@id="cont_certs"]')
tablecerti = certi.find('./table')
if tablecerti!=None:
certi_ = ''.join(tablecerti.itertext()).replace('Geprüfte Zertifikate:','')
ven.accreditations = certi_
ven.img_link = img_link
ven.country ='de'
ven.is_get_by_address = True
return ven