def parseTeaserInfo(self, article, pos, pattern=teaserInfoPattern):
teaserInfoMatch = pattern.search(article, pos)
playable = False
duration = None
season = None
episode = None
genre = None
if teaserInfoMatch is not None:
teaserInfo = getTag('dd', article, teaserInfoMatch)
isTiviMatch = teaserInfoIsTiviPattern.search(teaserInfo)
if isTiviMatch is not None:
teaserInfo = teaserInfo[0:isTiviMatch.start(0)]
teaserInfo = cleanTags(teaserInfo)
sep = u'\xb7'.encode('utf-8')
parts = teaserInfo.split(sep)
for part in parts:
part = part.strip()
partMatch = teaserInfoDurationPattern.search(part)
if partMatch is not None:
duration = partMatch.group(1)
else:
partMatch = teaserInfoEpisodePattern.search(part)
if partMatch is not None:
season = partMatch.group(1)
episode = partMatch.group(2)
else:
genre = part
if duration is not None and duration.isdigit():
duration = int(duration) * 60
playable = True
pos = teaserInfoMatch.end(0)
if not self.playable and playable:
self.playable = playable
self.duration = duration
if self.genre is None:
self.genre = genre
self.season = season
self.episode = episode
return pos
def parse(self, string, pos=0, baseUrl=None, teaserMatch=None):
if teaserMatch is None:
teaserMatch = teaserPattern.search(string, pos)
if teaserMatch is None:
return -1
class_ = teaserMatch.group(1)
article = getTag('article', string, teaserMatch)
endPos = teaserMatch.start(0) + len(article)
if class_.find('m-hidden') != -1:
return endPos
pos = self.parseImage(article, pos)
pos = self.parseLabel(article, pos)
pos = self.parseCategory(article, pos)
pos = self.parseTitle(article, pos, baseUrl)
pos = self.parseText(article, pos)
pos = self.parseDate(article, pos)
return endPos
def parse(self, string, pos=0, baseUrl=None, teaserMatch=None):
if teaserMatch is None:
teaserMatch = teaserPattern.search(string, pos)
if teaserMatch is None:
return -1
class_ = teaserMatch.group(1)
article = getTag('article', string, teaserMatch)
endPos = teaserMatch.start(0) + len(article)
if class_.find('m-hidden') != -1:
return endPos
pos = self.parseImage(article, pos)
pos = self.parseCategory(article, pos)
pos = self.parseTitle(article, pos, baseUrl)
pos = self.parseText(article, pos)
pos = self.parseLabel(article, pos)
pos = self.parseFoot(article, pos)
return endPos
def parseLabel(self, article, pos):
labelMatch = labelPattern.search(article, pos)
label = None
type = None
if labelMatch is not None:
labelTags = getTag('div', article, labelMatch)
iconMatch = iconPattern.search(labelTags)
if iconMatch is not None:
type = iconMatch.group(1)
i = labelTags.find('>') + len('>')
j = labelTags.rfind('</div>')
pos = j + len('</div>')
label = labelTags[i:j]
label = stripTag('abbr', label)
label = cleanTags(label)
label = label.strip()
self.label = stripHtml(label)
self.type = type
return pos
def parse(self):
super(NavigationResource, self).parse()
leftNavMatch = leftNavPattern.search(self.content)
if leftNavMatch is None:
self.warn(
"can't find navigation in page '{}', no rubrics will be available ...",
self.url)
return
leftNav = getTag('ul', self.content, leftNavMatch)
pos = leftNavMatch.end(0)
dropdownLinksMatch = dropdownLinksPattern.search(self.content, pos)
self.rubrics = []
while dropdownLinksMatch is not None:
url = dropdownLinksMatch.group(1).strip()
title = stripHtml(dropdownLinksMatch.group(2))
rubric = Rubric(title, url)
self.rubrics.append(rubric)
pos = dropdownLinksMatch.end(0)
dropdownLinksMatch = dropdownLinksPattern.search(self.content, pos)
def parse(self):
super(NavigationResource, self).parse()
leftNavMatch = leftNavPattern.search(self.content)
if leftNavMatch is None:
self.warn("can't find navigation in page '{}', no rubrics will be available ...", self.url)
return
leftNav = getTag('ul', self.content, leftNavMatch)
pos = leftNavMatch.end(0)
dropdownLinksMatch = dropdownLinksPattern.search(self.content, pos)
self.rubrics = []
urls = Set([]);
while dropdownLinksMatch is not None:
url = self.parseUrl(dropdownLinksMatch.group(1))
if url not in urls:
urls.add(url)
title = stripHtml(dropdownLinksMatch.group(2))
rubric = Rubric(title, url)
self.rubrics.append(rubric)
pos = dropdownLinksMatch.end(0)
dropdownLinksMatch = dropdownLinksPattern.search(self.content, pos)
def parseLabel(self, article, pos):
labelMatch = labelPattern.search(article, pos)
label = None
type = None
if labelMatch is not None:
labelTags = getTag('div', article, labelMatch)
iconMatch = iconPattern.search(labelTags)
if iconMatch is not None:
type = iconMatch.group(1)
i = labelTags.find('</span>') + len('</span>')
j = labelTags.rfind('</div>')
pos = j + len('</div>')
label = labelTags[i:j]
label = label.replace('<strong>', '')
label = label.replace('</strong>', '')
label = stripTag('abbr', label)
label = stripTag('span', label)
label = label.strip()
self.label = stripHtml(label)
self.type = type
return pos