def drawBbcCoUkNew(url):
htmlContent = Downloader.getContentFromUrl(url=url)
title = Decoder.extract('<p class="story-body__introduction">', "</p><div", htmlContent)
if 'property="articleBody"' in htmlContent:
body = Decoder.extract(
'property="articleBody"',
" </div>",
htmlContent,
)
body = body.replace('<span class="off-screen">Image copyright</span>', "")
body = body.replace('<span class="story-image-copyright">AFP</span>', "")
body = body.replace('<span class="story-image-copyright">Reuters</span>', "")
body = body.replace('<span class="off-screen">Image caption</span>', "")
body = body.replace('<span class="off-screen">Media caption</span>', "")
while '<span class="media-caption__text">' in body:
line = Decoder.extractWithRegex('<span class="media-caption__text">', "</span>", body)
body = body.replace(line, "")
elif 'class="text-wrapper"' in htmlContent:
# special content
body = Decoder.extract('class="text-wrapper"', "</p>\n", htmlContent)
dates = Decoder.extractWithRegex('<div class="date', "</div>", body)
lastUpdate = Decoder.extractWithRegex('<p class="date ', "</p>", body)
body = body.replace(dates, "")
body = body.replace(lastUpdate, "")
elif '<figcaption class="sp-media-asset' in htmlContent:
body = Decoder.extract('<figcaption class="sp-media-asset', "</p><div ", htmlContent)
if ">" in body:
body = body[body.find(">") + 1 :]
body = Decoder.removeHTML(body).replace(".", ".\n").replace(">", "")
logger.debug("body is: " + body)
drawNew(textContent=(body))
def getChannels(page):
x = []
if page == '0':
url = Reuters.LAST_NEWS_RSS + str(time.time() * 1000)
logger.debug("news rss url is: "+url)
bruteResult = Reuters.getContentFromUrl(url=url,launchLocation=False,ajax=True)
logger.debug("brute ajax response: "+bruteResult)
results = json.loads(bruteResult)
i=0
for result in results["headlines"]:
if i>0:
element = {}
img = result["mainPicUrl"]
link = Reuters.MAIN_URL+result["url"]
title = result["formattedDate"]+" - "+result["headline"]
logger.debug("appending result: "+title+", url: "+link+", img: "+img)
element["title"] = title
element["link"] = link
element["thumbnail"] = img
x.append(element)
i+=1
else:
html = Reuters.getContentFromUrl(url=page)
startRegex = '<span id="article-text">'
if '<span id="article-text">' in html:
startRegex = '<span id="article-text">'
else:
startRegex = '<span id="articleText">'
body = Decoder.extract(startRegex,'<div class="linebreak"></div>',html)
body = Decoder.removeHTML(body)
if '|' in body:
body = body[body.find('|')+1:]
try:
lowerCaseIndex = int(re.search("[a-z]", body).start())
body = body[:lowerCaseIndex-1]+"\n"+body[lowerCaseIndex-1:]
except:
logger.error("No break for city was done. Something goes wrong")
pass
element = {}
element["link"] = page
element["title"] = body
element["thumbnail"] = ''
x.append(element)
return x