def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//ul/li/div/div/div[@class="horiz-offer-card__content"]/div[@class="horiz-offer-card__desc"]',
parent=True)
articleDataDict["images"] = parsers_common.xpath_to(
"list", pageTree,
'//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/img/@data-src'
)
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/@title')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/@href')
# remove unwanted content: titles
dictList = [
"radiaator",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
articleDataDict = parsers_common.dict_reverse_order(articleDataDict)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//article/ul/li/figure/div[@class="offer-thumb__content"]',
parent=True)
articleDataDict["images"] = parsers_common.xpath_to(
"list", pageTree, '//article/ul/li/figure/figure/a/@data-original')
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//article/ul/li/figure/div[@class="offer-thumb__content"]/h3/a/text()'
)
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//article/ul/li/figure/div[@class="offer-thumb__content"]/h3/a/@href')
# remove unwanted content: titles
dictList = [
"az ",
"jawa",
"kitarr",
"poolvääriskivi",
"tuulesuunaja",
"viiuli",
"xtrons",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//table[@class="views-table cols-5"]/tbody/tr',
parent=True)
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="views-table cols-5"]/tbody/tr/td[1]/text()')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="views-table cols-5"]/tbody/tr/td[5]/div[1]/a/@href')
# remove unwanted content: descriptions
dictList = [
"ametnik",
"hooldaja",
"jurist",
"koristaja",
"logopeed",
"pedagoog",
"psühholoog",
"raamatupidaja",
"sanitar",
"teenindaja",
"uurija",
"õpetaja",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list", pageTree, '//div[@id="t-content"]/table[1]/tr', parent=True)
articleDataDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree, '//div[@id="t-content"]/table[1]/tr/td[1]/text()')
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@id="t-content"]/table[1]/tr/td[@class="left"]/b/a/text()')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@id="t-content"]/table[1]/tr/td[@class="left"]/b/a/@href')
# remove unwanted content: descriptions
dictList = ["Tartu"]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "not in", "descriptions")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# pubDates magic from "29.08.19" to datetime()
curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i)
curArtPubDate = parsers_datetime.raw_to_datetime(
curArtPubDate, "%d.%m.%y")
articleDataDict["pubDates"] = parsers_common.list_add_or_assign(
articleDataDict["pubDates"], i, curArtPubDate)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["authors"] = parsers_common.xpath_to(
"list",
pageTree,
'//div[@class="message"]/div[@class="name"]',
parent=True)
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//div[@class="message"]/div[@class="content"]',
parent=True)
articleDataDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="message"]/div[@class="posttime"]/text()')
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="message"]/div[@class="title"]/a[3]/text()')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="message"]/div[@class="title"]/a[3]/@href')
# remove unwanted content: titles
dictList = [
"Hoiatus! Läbisõit 100% keritud",
"Kaebused",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# description
curArtDesc = parsers_common.get(articleDataDict["descriptions"], i)
curArtDesc = curArtDesc.split('<div class="userControls')[0]
articleDataDict["descriptions"] = parsers_common.list_add_or_assign(
articleDataDict["descriptions"], i, curArtDesc)
# title
curArtTitle = parsers_common.get(articleDataDict["titles"], i)
curArtTitle = parsers_common.str_lchop(curArtTitle, "Re: ")
curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add_or_assign(
articleDataDict["titles"], i, curArtTitle)
# pubDates magic from "20:22 01.09.2019" to Datetime()
curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i)
curArtPubDate = parsers_datetime.replace_string_with_timeformat(
curArtPubDate, "eile", "%d.%m.%Y", offsetDays=-1)
curArtPubDate = parsers_datetime.replace_string_with_timeformat(
curArtPubDate, "täna", "%d.%m.%Y", offsetDays=0)
curArtPubDate = parsers_datetime.add_missing_date_to_string(
curArtPubDate, "%H:%M %d.%m.%Y", " %d.%m.%Y")
curArtPubDate = parsers_datetime.raw_to_datetime(
curArtPubDate, "%H:%M %d.%m.%Y")
articleDataDict["pubDates"] = parsers_common.list_add_or_assign(
articleDataDict["pubDates"], i, curArtPubDate)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="box-news-block-title "]/a[@title]/@title')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="box-news-block-title "]/a[@title]/@href')
# remove unwanted content: urls
dictList = [
"https://sky.ee/rock-fmi-hommikuprogramm-igal-toopaeval-kell-7-10/"
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "urls")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
if parsers_common.should_get_article_body(i):
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(domain,
curArtUrl,
cache='cacheAll')
# author
curArtAuthor = parsers_common.xpath_to(
"single", pageTree,
'//div[@class="post-content"]/div[@class="article-page-author"]/p/text()'
)
if ": " in curArtAuthor:
curArtAuthor = curArtAuthor.split(": ")[1]
articleDataDict["authors"] = parsers_common.list_add_or_assign(
articleDataDict["authors"], i, curArtAuthor)
# description
curArtDesc1 = parsers_common.xpath_to(
"single", pageTree,
'//div[@class="posts"]/div[@class="two-side-content-container clearfix"][1]//div[@class="post-content"]/strong/p/text()'
)
curArtDesc2 = parsers_common.xpath_to(
"single",
pageTree,
'//div[@class="posts"]/div[@class="two-side-content-container clearfix"][2]//div[@class="post-content"]',
parent=True)
curArtDesc = curArtDesc1 + curArtDesc2
if '<p class="related-cta">LOE KA NEID UUDISEID!</p>' in curArtDesc:
curArtDesc = curArtDesc.split(
'<p class="related-cta">LOE KA NEID UUDISEID!</p>')[0]
articleDataDict[
"descriptions"] = parsers_common.list_add_or_assign(
articleDataDict["descriptions"], i, curArtDesc)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/p',
parent=True)
articleDataDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="right-content"]/div[@class="application-date"][1]/text()'
)
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/h2/a/text()'
)
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/h2/a/@href'
)
# remove unwanted content: descriptions
dictList = [
"jurist",
"logopeed",
"pedagoog",
"psühholoog",
"raamatupidaja",
"sotsiaal",
"õpetaja",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# pubDates magic from "Avaldatud: 12.12.2017" to datetime()
curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i)
curArtPubDate = curArtPubDate.split(': ')[1]
curArtPubDate = parsers_datetime.raw_to_datetime(
curArtPubDate, "%d.%m.%Y")
articleDataDict["pubDates"] = parsers_common.list_add_or_assign(
articleDataDict["pubDates"], i, curArtPubDate)
# url
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
curArtUrl = curArtUrl.split('?ref=')[0]
articleDataDict["urls"] = parsers_common.list_add_or_assign(
articleDataDict["urls"], i, curArtUrl)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 10)
maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX /
maxArticleBodies) # set 0 for all posts
parentPages = {}
parentPages["stamps"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="reply-count"]/span[@data-xf-init="tooltip"]/text()')
parentPages["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@qid="thread-item-parent"]/div[@qid="thread-item"]/div/div[@class="structItem-title"]/a[@qid="thread-item-title"]/text()'
)
parentPages["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@qid="thread-item-parent"]/div[@qid="thread-item"]/div/div[@class="structItem-title"]/a[@qid="thread-item-title"]/@href'
)
# remove unwanted content: titles
dictList = [
"$",
"*:::::::the official what did you do to you mkiv today thread::::::::*",
"??",
"Ask a Simple Question",
]
parentPages = parsers_common.article_data_dict_clean(
parentPages, dictList, "in", "titles")
# teemade läbivaatamine
for i in parsers_common.article_urls_range(parentPages["urls"]):
# teemalehe sisu hankimine
if parsers_common.should_get_article_body(i, maxArticleBodies):
curParentUrl = parsers_common.get(parentPages["urls"], i)
curParentUrl = curParentUrl + "page-1000"
parentPagesStamp = parsers_common.get(parentPages["stamps"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(
domain,
curParentUrl,
cache='cacheStamped',
pageStamp=parentPagesStamp)
articlePostsDict = {}
articlePostsDict["authors"] = parsers_common.xpath_to(
"list", pageTree, '//h4[@qid="message-username"]//text()')
articlePostsDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//article/div[@class="bbWrapper"]',
parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="message-attribution-main"]/a[@class="u-concealed"][2]/time/@datetime'
)
articlePostsDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="message-attribution-main"]/a[@class="u-concealed"][2]/@href'
)
# teema postituste läbivaatamine
for j in parsers_common.article_posts_range(
articlePostsDict["urls"], maxArticlePosts):
# author
articleDataDict["authors"] = parsers_common.list_add(
articleDataDict["authors"], j,
parsers_common.get(articlePostsDict["authors"], j))
# description
curArtDesc = parsers_common.get(
articlePostsDict["descriptions"], j)
articleDataDict["descriptions"] = parsers_common.list_add(
articleDataDict["descriptions"], j, curArtDesc)
# pubDates
curArtPubDate = parsers_common.get(
articlePostsDict["pubDates"], j)
curArtPubDate = parsers_datetime.raw_to_datetime(
curArtPubDate,
"%Y-%m-%dT%H:%M:%S%z") # 2021-01-28T16:15:42-0500
articleDataDict["pubDates"] = parsers_common.list_add(
articleDataDict["pubDates"], j, curArtPubDate)
# title
curArtTitle = parsers_common.get(parentPages["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(
curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add(
articleDataDict["titles"], j, curArtTitle)
# url
curArtUrl = parsers_common.get(articlePostsDict["urls"], j)
articleDataDict["urls"] = parsers_common.list_add(
articleDataDict["urls"], j, curArtUrl)
rss_print.print_debug(
__file__, "teema " + str(i + 1) + " postitus nr. " +
str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) +
") on " + articlePostsDict["urls"][j], 2)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]',
parent=True)
articleDataDict["images"] = parsers_common.xpath_to(
"list", pageTree,
'/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/p[@class="img"]/a/img/@src'
)
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]/h2/a/text()'
)
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]/h2/a/@href'
)
# video
# remove unwanted content: titles
dictList = [
"2000.ee:",
"AK filmikroonika 1958-1991:",
"Aktuaalne kaamera",
"Eesti Gloobus",
"Hommik Anuga:",
"Insight",
"Iseolemine:",
"Johannese lähetamine",
"Kes keda?",
"Kodukäijad",
"Koolitants",
"Lasteekraan",
"Lastetuba",
"Laulge kaasa!",
"Meie kõrval:",
"Mis teie kodus uudist?:",
"Mis? Kus? Millal?:",
"NOVA:",
"Noor meister:",
"Nädala intervjuu:",
"OP:",
"Oma tõde:",
"Ongi Koik",
"Pealtnägija",
"Peegel:",
"Plekktrumm",
"Prillitoos:",
"Püha päeva palvus:",
"Rahvas laulab:",
"Rakett 69:",
"Reibas hommik",
"Ringvaade",
"Suus sulav Eesti:",
"TECHnolik",
"TV 10 olümpiastarti",
"Taevavalvurid",
"Tarmo ja Aet liiguvad:",
"Terevisioon:",
"Tähendamisi",
"Tähtede lava",
"Välisilm",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
# remove unwanted content: description
dictList = [
"Johannes Tralla",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "descriptions")
# audio
# remove unwanted content: titles
dictList = [
"AIATARK.",
"DELTA.",
"GOGOL.",
"HOMMIKUMÕTISKLUS.",
"HUVITAJA.",
"KIHNUKEELSED UUDISED",
"KIRIKUELU.",
"KITARRIMUUSIKAST JA -MÄNGIJAIST.",
"KULDRANDEVUU.",
"LOETUD JA KIRJUTATUD.",
"LUULERUUM.",
"LUULETUS.",
"MELOTURNIIR.",
"MINITURNIIR.",
"MNEMOTURNIIR",
"MULGIKEELSED UUDISED",
"MUST KLAHV, VALGE KLAHV.",
"NAISTESAUN.",
"OLUKORRAST RIIGIS",
"PÄEVAKAJA",
"RAHVA OMA KAITSE",
"RAHVA TEENRID",
"REPORTERITUND.",
"SETUKEELSED UUDISED",
"SIILILEGI SELGE!",
"SPORDIPÜHAPÄEV.",
"TETRIS.",
"UUDISED.",
"VÕRUKEELSED UUDISED",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
return articleDataDict
parsers_common.dict_stats(curArticleDataDict)
# lisame viimased andmed
rss_print.print_debug(
__file__, "lisame andmed viimaselt alamlehelt: " + curDomainLong,
2)
articleDataDict = parsers_common.dict_add_dict(articleDataDict,
curArticleDataDict)
if not articleDataDict["urls"]:
rss_print.print_debug(__file__,
"ei leitud andmeid lehelt: " + curDomainShort, 0)
continue
# remove unwanted content: titles
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, rss_config.BAD_TITLES, "in", "titles")
# remove unwanted content: descriptions
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, rss_config.BAD_DESCRIPTIONS, "in", "descriptions")
# combine rss file
rss_print.print_debug(__file__, "asume koostame rss-i: " + curDomainLong,
3)
rssContent = rss_maker.rssmaker(articleDataDict, curTitle, curDomainShort,
curDomainLong, curDescription)
# make sure we have subfolder
OS_PATH = os.path.dirname(os.path.abspath(__file__))
LATEST_FEEDS_PATH = OS_PATH + "/" + "latest_feeds"
if not os.path.exists(LATEST_FEEDS_PATH):
def fill_article_dict(articleDataDict, pageTree, domain):
maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 5)
maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX /
maxArticleBodies) # set 0 for all posts
parentPages = {}
parentPages["stamps"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/h3[@class="adds-list-title"]/span[@class="adds-list-meta"]/text()'
)
parentPages["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/h3[@class="adds-list-title"]/text()'
)
parentPages["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/@href'
)
# muudame suuna sobivaks
parentPages = parsers_common.dict_reverse_order(parentPages)
# remove unwanted content: titles
dictList = [
"AMD",
"Apple",
"Assassin",
"Batman",
"Battlefield",
"Call of Duty",
"Cyberpunk",
"Diablo 2",
"Dying",
"Escape From Tarkov",
"Euro Truck",
"Evil",
"FIFA",
"Far Cry",
"Forza",
"Galaxy",
"Grand Theft",
"IPhon",
"Kindle",
"MMORPG",
"MSI",
"MacBook",
"MacOS",
"Mafia",
"Mass Effect",
"Meizu",
"Minecraft",
"Nintendo",
"Pixel",
"PlayStation",
"Steam",
"Tanks",
"Vidia",
"War Thunder",
"Watercool",
"Windows",
"Xbox",
"arvutikast",
"exile",
"foorumiga seotud",
"konsool",
"korpust",
"moderaatorite",
"seotud vead",
"siia lingid",
"toiteplok",
]
parentPages = parsers_common.article_data_dict_clean(
parentPages, dictList, "in", "titles")
# teemade läbivaatamine
for i in parsers_common.article_urls_range(parentPages["urls"]):
# teemalehe sisu hankimine
if parsers_common.should_get_article_body(i, maxArticleBodies):
parentPages["stamps"][i] = parentPages["stamps"][i].split("/")[0]
parentPages["urls"][i] = parentPages["urls"][i].split("&sid=")[0]
pageTree = parsers_common.get_article_tree(
domain,
parentPages["urls"][i],
cache='cacheStamped',
pageStamp=parentPages["stamps"][i])
articlePostsDict = {}
articlePostsDict["authors"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="forumline"]/tr/td[1]/span[@class="name"]/b/a/text()'
)
articlePostsDict["descriptions1"] = parsers_common.xpath_to(
"list",
pageTree,
'//table[@class="forumline"]/tr/td[2]/table/tr[3]/td/span[@class="postbody"][1]',
parent=True)
articlePostsDict["descriptions2"] = parsers_common.xpath_to(
"list",
pageTree,
'//table[@class="forumline"]/tr/td[2]/table/tr[3]/td/span[@class="postbody"][2]',
parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="forumline"]/tr/td[2]/table/tr[1]/td/span[@class="postdetails"]/span[@class="postdetails"][1]/text()[1]'
)
articlePostsDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="forumline"]/tr/td[2]/table/tr[1]/td/span[@class="postdetails"]/a/@href'
)
# teema postituste läbivaatamine
for j in parsers_common.article_posts_range(
articlePostsDict["urls"], maxArticlePosts):
# author
articleDataDict["authors"] = parsers_common.list_add(
articleDataDict["authors"], j,
parsers_common.get(articlePostsDict["authors"], j))
# description
curArtDesc = parsers_common.get(
articlePostsDict["descriptions1"], j)
if not curArtDesc:
curArtDesc = parsers_common.get(
articlePostsDict["descriptions2"], j)
curArtDesc = curArtDesc.replace(
'</div><div class="quotecontent">', '<br>')
curArtDesc = parsers_common.fix_quatation_tags(
curArtDesc, '<div class="quotetitle">', "</div>",
"<blockquote>", "</blockquote>")
articleDataDict["descriptions"] = parsers_common.list_add(
articleDataDict["descriptions"], j, curArtDesc)
# pubDates
curArtPubDate = parsers_common.get(
articlePostsDict["pubDates"], j)
curArtPubDate = curArtPubDate[0:16]
curArtPubDate = parsers_datetime.guess_datetime(
curArtPubDate) # 14.07.2020 07:59
articleDataDict["pubDates"] = parsers_common.list_add(
articleDataDict["pubDates"], j, curArtPubDate)
# title
curArtTitle = parsers_common.get(parentPages["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(
curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add(
articleDataDict["titles"], j, curArtTitle)
# url
curArtUrl = parsers_common.get(articlePostsDict["urls"], j)
curArtUrl = parsers_common.link_add_end(
curArtUrl, articlePostsDict["urls"][j])
articleDataDict["urls"] = parsers_common.list_add(
articleDataDict["urls"], j, curArtUrl)
rss_print.print_debug(
__file__, "teema " + str(i + 1) + " postitus nr. " +
str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) +
") on " + articlePostsDict["urls"][j], 2)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="history-item"]/span[@class="history-date ng-binding"]/text()'
)
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="history-item"]/p[@class="history-header"]/a[@class="ng-binding"]/text()[1]'
)
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="history-item"]/p[@class="history-header"]/a[@class="ng-binding"]/@href'
)
lastArtPubDate = ""
iMinus = 0
# remove unwanted content: titles
dictList = [
" reket",
" võidu",
"Aktuaalne kaamera",
"ERR-i teleuudised",
"ETV spordi",
"Ilm ",
"OTSE ",
"Päevakaja",
"Raadiouudised",
"Viipekeelsed uudised",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
i = i - iMinus
# pubDates magic from "11:34" to datetime()
curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i)
curArtPubDate = parsers_datetime.raw_to_datetime_guess_missing(
curArtPubDate, lastArtPubDate, "%Y %m %d ", "%H:%M", -1)
lastArtPubDate = curArtPubDate
articleDataDict["pubDates"] = parsers_common.list_add_or_assign(
articleDataDict["pubDates"], i, curArtPubDate)
# title
curArtTitle = parsers_common.get(articleDataDict["titles"], i)
if "|" in curArtTitle:
curArtTitle = curArtTitle.split("|")[-1]
articleDataDict["titles"] = parsers_common.list_add_or_assign(
articleDataDict["titles"], i, curArtTitle)
if parsers_common.should_get_article_body(i):
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(domain,
curArtUrl,
cache='cacheAll')
# author
curArtAuthor = parsers_common.xpath_to(
"single", pageTree,
'//article/div[@class="body"]/div/div[@class="meta"]/section/div[@class="byline"]/span/text()'
)
articleDataDict["authors"] = parsers_common.list_add_or_assign(
articleDataDict["authors"], i, curArtAuthor)
# description
curArtDesc1 = parsers_common.xpath_to(
"single",
pageTree,
'//article/div[@class="body"]/div[@class="lead"]',
parent=True)
curArtDesc2 = parsers_common.xpath_to(
"single",
pageTree,
'//article/div[@class="body"]/div[@class="text flex-row"]',
parent=True)
if not curArtDesc2:
rss_print.print_debug(
__file__, "tühja sisuga uudis, eemaldame rea " + str(i), 1)
articleDataDict = parsers_common.dict_del_article_index(
articleDataDict, i)
iMinus += 1
continue
curArtDesc = curArtDesc1 + "<br>" + curArtDesc2
articleDataDict[
"descriptions"] = parsers_common.list_add_or_assign(
articleDataDict["descriptions"], i, curArtDesc)
# image
curArtImg = parsers_common.xpath_to(
"single", pageTree,
'/html/head/meta[@property="og:image"]/@content')
if not curArtImg:
curArtImg = parsers_common.xpath_to(
"single", pageTree,
'/html/head/meta[@property="image"]/@content')
articleDataDict["images"] = parsers_common.list_add_or_assign(
articleDataDict["images"], i, curArtImg)
articleDataDict = parsers_common.dict_reverse_order(articleDataDict)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 8)
maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts
parentPages = {}
version = 0
if "kipper.ee" in domain or "militaar.net" in domain:
version = 1
parentPages["stamps"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/p[@class="topicdetails"]/text()')
parentPages["titles"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/a[@class="topictitle"]/text()')
parentPages["urls"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/a[@class="topictitle"]/@href')
else:
parentPages["stamps"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/dd[@class="posts"]/text()')
parentPages["titles"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/*/div/a[@class="topictitle"]/text()')
parentPages["urls"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/*/div/a[@class="topictitle"]/@href')
if not parentPages["urls"] and "arutelud.com" in domain:
rss_print.print_debug(__file__, "aktiivseid teemasid ei leitud, arutelude foorumis külastame mammutteemat", 1)
parentPages["stamps"] = parsers_common.list_add_or_assign(parentPages["stamps"], 0, "")
parentPages["titles"] = parsers_common.list_add_or_assign(parentPages["titles"], 0, "Arutelud")
parentPages["urls"] = parsers_common.list_add_or_assign(parentPages["urls"], 0, "https://arutelud.com/viewtopic.php?f=3&t=4&sd=d&sk=t&st=7")
# remove unwanted content: titles
dictList = [
"Race.Fi:",
"Write my",
"PÕLVAMAA, VÕRUMAA JA VALGAMAA CB JA HAM SIDE",
]
parentPages = parsers_common.article_data_dict_clean(parentPages, dictList, "in", "titles")
# teemade läbivaatamine
for i in parsers_common.article_urls_range(parentPages["urls"]):
# teemalehe sisu hankimine
if parsers_common.should_get_article_body(i, maxArticleBodies):
curParentUrl = parsers_common.get(parentPages["urls"], i)
curParentUrl = curParentUrl.split("&sid=")[0]
curParentUrl = curParentUrl + "&start=100000"
parentPagesStamp = parsers_common.get(parentPages["stamps"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(domain, curParentUrl, cache='cacheStamped', pageStamp=parentPagesStamp)
articlePostsDict = {}
if version in (1, 2):
rss_print.print_debug(__file__, "kasutame spetsiifilist hankimist, domain = " + domain, 2)
articlePostsDict["authors"] = parsers_common.xpath_to("list", pageTree, '//tr/td/b[@class="postauthor"]/text()')
articlePostsDict["descriptions"] = parsers_common.xpath_to("list", pageTree, '//tr/td/div[@class="postbody"][1]', parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//tr/td[@class="gensmall"]/div[@style="float: right;"]/text()')
articlePostsDict["urls"] = parsers_common.xpath_to("list", pageTree, '//tr/td[@class="gensmall"]/div[@style="float: right;"]/a/@href')
else:
rss_print.print_debug(__file__, "kasutame üldist hankimist, domain = " + domain, 3)
articlePostsDict["authors"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]//strong//text()')
articlePostsDict["descriptions"] = parsers_common.xpath_to("list", pageTree, '//div[@class="content"]', parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/time/@datetime')
articlePostsDict["urls"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/a/@href')
if not articlePostsDict["pubDates"]:
articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/time/text()')
if not articlePostsDict["pubDates"]:
articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/text()[1]')
if len(articlePostsDict["pubDates"][0]) < 5:
rss_print.print_debug(__file__, "hangitud aeg[0] liiga lühike: '" + articlePostsDict["pubDates"][0] + "', proovime alternatiivi...", 0)
articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/text()[2]')
if len(articlePostsDict["pubDates"][0]) < 5:
rss_print.print_debug(__file__, "hangitud aeg[0] liiga lühike: '" + articlePostsDict["pubDates"][0] + "'", 0)
else:
rss_print.print_debug(__file__, "hangitud aeg[0]: '" + articlePostsDict["pubDates"][0] + "'", 4)
if not articlePostsDict["pubDates"]:
rss_print.print_debug(__file__, "ei suutnud hankida ühtegi aega", 0)
# teema postituste läbivaatamine
for j in parsers_common.article_posts_range(articlePostsDict["urls"], maxArticlePosts):
# author
articleDataDict["authors"] = parsers_common.list_add(articleDataDict["authors"], j, parsers_common.get(articlePostsDict["authors"], j))
# description
curArtDesc = parsers_common.get(articlePostsDict["descriptions"], j)
curArtDesc = curArtDesc.replace('</div><div class="quotecontent">', '<br>')
curArtDesc = parsers_common.fix_quatation_tags(curArtDesc, '<div class="quotetitle">', "</div>", "<blockquote>", "</blockquote>")
articleDataDict["descriptions"] = parsers_common.list_add(articleDataDict["descriptions"], j, curArtDesc)
# pubDates
curArtPubDate = parsers_common.get(articlePostsDict["pubDates"], j)
curArtPubDate = parsers_datetime.months_to_int(curArtPubDate)
curArtPubDate = parsers_datetime.remove_weekday_strings(curArtPubDate)
curArtPubDate = parsers_datetime.replace_string_with_timeformat(curArtPubDate, "eile", "%d %m %Y", offsetDays=-1)
curArtPubDate = parsers_datetime.replace_string_with_timeformat(curArtPubDate, "täna", "%d %m %Y", offsetDays=0)
curArtPubDate = parsers_datetime.guess_datetime(curArtPubDate)
articleDataDict["pubDates"] = parsers_common.list_add(articleDataDict["pubDates"], j, curArtPubDate)
# title
curArtTitle = parsers_common.get(parentPages["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add(articleDataDict["titles"], j, curArtTitle)
# url
curArtUrl = parsers_common.get(articlePostsDict["urls"], j)
curArtUrl = parsers_common.link_add_end(curArtUrl, articlePostsDict["urls"][j])
articleDataDict["urls"] = parsers_common.list_add(articleDataDict["urls"], j, curArtUrl)
rss_print.print_debug(__file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//div[@class="article-content"]/div[@class="article-content__meta"]/span[@class="article-content__date-published"]/text()')
articleDataDict["titles"] = parsers_common.xpath_to("list", pageTree, '//div[@class="article-content"]/a[@class="article-content__headline"]/text()')
articleDataDict["urls"] = parsers_common.xpath_to("list", pageTree, '//div[@class="article-content"]/a[@class="article-content__headline"]/@href')
articleDataDictPubDatesDay = parsers_common.xpath_to("list", pageTree, '//div[@class="article-content"]/div[@class="article-content__meta"]/span[@class="article-content__date-published"]/span/text()')
# remove unwanted content: titles
dictList = [
"Sakala kuulutused",
"Tartu Börs,",
"positiivseid proove",
]
articleDataDict = parsers_common.article_data_dict_clean(articleDataDict, dictList, "in", "titles")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# pubDates magic from "24.12.2017 17:51" to datetime()
curArtPubDateDay = ""
if len(articleDataDictPubDatesDay) - 1 >= i:
curArtPubDateDay = parsers_common.get(articleDataDictPubDatesDay, i)
curArtPubDateDay = parsers_datetime.replace_string_with_timeformat(curArtPubDateDay, "Eile", "%d.%m.%Y", offsetDays=-1)
curArtPubDateDay = parsers_datetime.replace_string_with_timeformat(curArtPubDateDay, "Täna", "%d.%m.%Y", offsetDays=0)
curArtPubDate = articleDataDict["pubDates"][i]
curArtPubDate = parsers_datetime.raw_to_datetime(curArtPubDateDay + curArtPubDate, "%d.%m.%Y, %H:%M")
articleDataDict["pubDates"][i] = curArtPubDate
if parsers_common.should_get_article_body(i):
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
# load article into tree
pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll')
# author
curArtAuthor = parsers_common.xpath_to("single", pageTree, '//span[@class="article-authors__name"]/text()', multi=True)
articleDataDict["authors"] = parsers_common.list_add_or_assign(articleDataDict["authors"], i, curArtAuthor)
# description1 - enne pilti
curArtDesc1 = ""
if not curArtDesc1:
curArtDesc1 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--video"][1]', parent=True, count=True)
if not curArtDesc1:
curArtDesc1 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--articleBullets"]', parent=True, count=True)
# description2 - pildi ja kuulamise vahel
curArtDesc2 = ""
if not curArtDesc2:
curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//div[@itemprop="articleBody"]', parent=True, count=True)
if not curArtDesc2:
curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--htmlElement article-body__item--lead"]', parent=True, count=True, multi=True)
if not curArtDesc2:
curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//span[@class="figure__caption--title"][1]', parent=True, count=True)
if not curArtDesc2:
curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--htmlElement article-body--first-child article-body__item--lead"]', parent=True, count=True)
if not curArtDesc2:
curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//div[@itemprop="description"]', parent=True, count=True)
# description3 - pärast kuulamist
curArtDesc3 = ""
if not curArtDesc3:
curArtDesc3 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--htmlElement"]', parent=True, count=True, multi=True)
if not curArtDesc3:
curArtDesc3 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--premium-indicator"]', parent=True, count=True)
# description4 - hall väljajuhatus
curArtDesc4 = ""
if not curArtDesc4:
curArtDesc4 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--htmlElement article-body--teaser"]', parent=True, count=True)
if not curArtDesc4:
curArtDesc4 = parsers_common.xpath_to("single", pageTree, '//div[@class="article-body__item article-body__item--gallery"]', parent=True, count=True, multi=True)
# image
curArtImg = ""
if not curArtImg:
curArtImg = parsers_common.xpath_to("single", pageTree, '//div[@class="article-superheader article-superheader--figure"]/div[@class="article-superheader__background"]/@style', count=True)
curArtImg = curArtImg.split("url('")[-1].strip("');")
if not curArtImg:
curArtImg = parsers_common.xpath_to("single", pageTree, '//figure[@class="figure"]/img[@class="figure--has-fullscreen"]/@src', count=True)
if not curArtImg:
curArtImg = parsers_common.xpath_to("single", pageTree, '//meta[@property="og:image"]/@content', count=True)
# kontrollid
if "-kuulutused-" in curArtUrl:
rss_print.print_debug(__file__, "ei kontrolli plokke, kuna: kuulutused", 2)
elif "-karikatuur" in curArtUrl:
rss_print.print_debug(__file__, "ei kontrolli plokke, kuna: karikatuur", 2)
else:
if not curArtDesc1:
rss_print.print_debug(__file__, "1. plokk on tühi. (Pildieelne loendiplokk puudub?)", 2)
else:
rss_print.print_debug(__file__, "curArtDesc1 = " + curArtDesc1, 4)
if not curArtDesc2:
rss_print.print_debug(__file__, "2. plokk on tühi. (Pildi ja kuulamise vahe plokk puudub?) - " + curArtUrl, 0)
else:
rss_print.print_debug(__file__, "curArtDesc2 = " + curArtDesc2, 4)
if not curArtDesc3:
rss_print.print_debug(__file__, "3. plokk on tühi. (Pärast kuulamist plokk puudub?)", 0)
else:
rss_print.print_debug(__file__, "curArtDesc3 = " + curArtDesc3, 4)
if not curArtDesc4:
if "button--for-subscription" in curArtDesc3:
curArtDesc3 = curArtDesc3.split('<span class="button--for-subscription')[0]
rss_print.print_debug(__file__, "4. plokk on tühi. (Kolmandas plokis oli teemant)", 3)
else:
rss_print.print_debug(__file__, "4. plokk on tühi. (Hall fadeout plokk puudub?)", 2)
else:
rss_print.print_debug(__file__, "curArtDesc4 = " + curArtDesc4, 4)
if not curArtImg:
rss_print.print_debug(__file__, "pilti ei leitud.", 0)
else:
rss_print.print_debug(__file__, "curArtImg = " + curArtImg, 4)
if curArtDesc1 and curArtDesc1 == curArtDesc2:
rss_print.print_debug(__file__, "1. ja 2. plokk langevad kokku", 0)
rss_print.print_debug(__file__, "curArtDesc1 = " + curArtDesc1, 1)
rss_print.print_debug(__file__, "curArtDesc2 = " + curArtDesc2, 1)
if curArtDesc2 and curArtDesc2 == curArtDesc3:
rss_print.print_debug(__file__, "2. ja 3. plokk langevad kokku", 0)
rss_print.print_debug(__file__, "curArtDesc2 = " + curArtDesc2, 1)
rss_print.print_debug(__file__, "curArtDesc3 = " + curArtDesc3, 1)
if curArtDesc3 and curArtDesc3 == curArtDesc4:
rss_print.print_debug(__file__, "3. ja 4. plokk langevad kokku", 0)
rss_print.print_debug(__file__, "curArtDesc3 = " + curArtDesc3, 1)
rss_print.print_debug(__file__, "curArtDesc4 = " + curArtDesc4, 1)
if curArtDesc4 and curArtDesc4 == curArtDesc1:
rss_print.print_debug(__file__, "4. ja 1. plokk langevad kokku", 0)
rss_print.print_debug(__file__, "curArtDesc4 = " + curArtDesc3, 1)
rss_print.print_debug(__file__, "curArtDesc1 = " + curArtDesc4, 1)
curArtDesc = curArtDesc1 + ' ' + curArtDesc2 + ' ' + curArtDesc3 + ' ' + curArtDesc4
if "button--for-subscription" in curArtDesc:
curArtDesc = curArtDesc.replace(' tellijatele', '')
curArtDesc = curArtDesc.replace('<a href="https://minumeedia.postimees.ee/kampaania/" target="_blank" class="my-media-link">digipaketi</a>', '')
curArtDesc = curArtDesc.replace('<div class="article-body__item article-body__item--audio-teaser">', '<div>')
curArtDesc = curArtDesc.replace('<div class="audio-teaser">', '<div>')
curArtDesc = curArtDesc.replace('<img data-lazy-src="/v5/img/icons/diamond-black-on-yellow.svg" alt="Tellijale" src="/v5/img/icons/diamond-black-on-yellow.svg" width="30" height="30">', "")
curArtDesc = curArtDesc.replace('<img src="/v5/img/icons/diamond-black-on-yellow.svg" alt="Tellijale" width="30" height="30">', "")
curArtDesc = curArtDesc.replace('<span class="button--for-subscription">', "<span>")
curArtDesc = curArtDesc.replace('<span class="button--for-subscription__diamond diamond--ee">', "<span>")
curArtDesc = curArtDesc.replace('<span class="button--for-subscription__text"', "")
curArtDesc = curArtDesc.replace('Artikkel on kuulatav', '')
curArtDesc = curArtDesc.replace('Tellijale', '')
articleDataDict["descriptions"] = parsers_common.list_add_or_assign(articleDataDict["descriptions"], i, curArtDesc)
articleDataDict["images"] = parsers_common.list_add_or_assign(articleDataDict["images"], i, curArtImg)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 1)
maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX /
maxArticleBodies) # set 0 for all posts
parentPages = {}
parentPages["stamps"] = parsers_common.xpath_to(
"list", pageTree, '//tbody/tr/th[@class="col-1"]/text()')
parentPages["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//tbody/tr/th[@class="col-7 teemapealkiri"]/a/text()')
parentPages["urls"] = parsers_common.xpath_to(
"list", pageTree, '//tbody/tr/th[@class="col-4"]/a/@href')
# remove unwanted content: titles
dictList = [
"Lõvide perekonna uus teema",
"abort",
"beebi",
"ivf",
"lapse",
"rase ",
"rased",
"triibupüüdjad",
]
parentPages = parsers_common.article_data_dict_clean(
parentPages, dictList, "in", "titles")
# teemade läbivaatamine
for i in parsers_common.article_urls_range(parentPages["urls"]):
# teemalehe sisu hankimine
if parsers_common.should_get_article_body(i, maxArticleBodies):
curParentUrl = parsers_common.get(parentPages["urls"], i)
curParentUrl = curParentUrl.split("/#")[0]
pageTree = parsers_common.get_article_tree(
domain,
curParentUrl,
cache='cacheStamped',
pageStamp=parentPages["stamps"][i])
articlePostsDict = {}
articlePostsDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//div[@class="bbp-reply-content entry-content"]',
parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="post_date date updated"]/text()')
articlePostsDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="bbp-reply-header entry-title"]/@id')
# teema postituste läbivaatamine
for j in parsers_common.article_posts_range(
articlePostsDict["urls"], maxArticlePosts):
# description
curArtDesc = parsers_common.get(
articlePostsDict["descriptions"], j)
curArtDesc = curArtDesc.split(
'<div class="gdrts-rating-block ')[0]
curArtDesc = parsers_html.html_remove_single_parents(
curArtDesc)
articleDataDict["descriptions"] = parsers_common.list_add(
articleDataDict["descriptions"], j, curArtDesc)
# pubDates
curArtPubDate = parsers_common.get(
articlePostsDict["pubDates"], j)
curArtPubDate = parsers_datetime.guess_datetime(curArtPubDate)
articleDataDict["pubDates"] = parsers_common.list_add(
articleDataDict["pubDates"], j, curArtPubDate)
# title
curArtTitle = parsers_common.get(parentPages["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(
curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add(
articleDataDict["titles"], j, curArtTitle)
# url
curArtUrl = parsers_common.get(
parentPages["urls"],
i) + "/#" + articlePostsDict["urls"][j]
articleDataDict["urls"] = parsers_common.list_add(
articleDataDict["urls"], j, curArtUrl)
rss_print.print_debug(
__file__, "teema " + str(i + 1) + " postitus nr. " +
str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) +
") on " + articlePostsDict["urls"][j], 2)
# remove unwanted content: descriptions
dictList = [
" liba "
"Kommentaar eemaldatud.",
"Liba?",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "descriptions")
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//main/div[@class="content"]/div/div/div[1]/div/div',
parent=True)
articleDataDict["images"] = parsers_common.xpath_to(
"list", pageTree,
'//main/div[1]/div/div/div[1]/div/div/a/div/img/@src')
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//main/div[@class="content"]/div/div/div[1]/div/div/div/div[1]/div[2]/a/text()'
)
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//main/div[@class="content"]/div/div/div[1]/div/div/div/div[1]/div[2]/a/@href'
)
# remove unwanted content: titles
dictList = [
"(uus) raamat",
"abramova",
"akvavit",
"based broccoli",
"beats of no nation",
"bisweed",
"ekkm",
"error!",
"floorshow",
"gnoom",
"hard feeler",
"hillbilly picnic",
"ida jutud",
"ida räpp",
"intro",
"katus",
"keskkonnatund",
"kink konk",
"korrosioon",
"kräpp",
"let me juke",
"liin ",
"liin",
"lunchbreak lunchdate",
"meie igapäevane avalik ruum",
"milk",
"muster",
"myös",
"müürilehe hommik",
"n-lib"
"oleneb päevast!",
"oujee!",
"paneel",
"playa music",
"propel",
"puhkus",
"rets records",
"room_202",
"rõhk",
"saal raadio",
"soojad suhted",
"svet nureka",
"söökladisko",
"triinemets.",
"vitamiin k",
"zubrovka am",
"ära kaaguta!",
"öömaja",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
# remove unwanted content: descriptions
dictList = [
"#hip hop",
"#interview",
"#rap",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "descriptions")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# title
curArtTitle = parsers_common.get(articleDataDict["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add_or_assign(
articleDataDict["titles"], i, curArtTitle)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
articleDataDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//div[@class="list-item"]/div[@class="details"]',
parent=True)
articleDataDict["images"] = parsers_common.xpath_to(
"list", pageTree, '//div[@class="list-item"]/a[@class="image"]/@style')
articleDataDict["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="list-item"]/div[@class="details"]/a[1]/text()')
articleDataDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//div[@class="list-item"]/div[@class="details"]/a[1]/@href')
# remove unwanted content: titles
dictList = [
"edasi lükatud",
"ei toimu",
"jääb ära",
"lükkub edasi",
"tühistatud",
]
articleDataDict = parsers_common.article_data_dict_clean(
articleDataDict, dictList, "in", "titles")
for i in parsers_common.article_urls_range(articleDataDict["urls"]):
# image
curArtImage = parsers_common.get(articleDataDict["images"], i)
curArtImage = parsers_common.split_failsafe(curArtImage, "'", 1)
articleDataDict["images"] = parsers_common.list_add_or_assign(
articleDataDict["images"], i, curArtImage)
# url
curArtUrl = parsers_common.get(articleDataDict["urls"], i)
curArtUrl = curArtUrl.split("?event")[0]
articleDataDict["urls"] = parsers_common.list_add_or_assign(
articleDataDict["urls"], i, curArtUrl)
if parsers_common.should_get_article_body(i):
# load article into tree
pageTree = parsers_common.get_article_tree(domain,
curArtUrl,
cache='cacheAll')
# description
curArtDesc1 = parsers_common.get(articleDataDict["descriptions"],
i)
curArtDesc2 = parsers_common.xpath_to("single",
pageTree,
'//article',
parent=True)
curArtDesc2 = curArtDesc2.replace(
"<h5 class=\"sm-hide\">Galerii</h5>", "")
curArtDesc2 = curArtDesc2.replace(
"<h5 class=\"sm-hide\">Tutvustus</h5>", "")
curArtDesc2 = curArtDesc2.replace(
"<div class=\"link after-arrow_down sm-show\">Loe lähemalt</div>",
"")
curArtDesc = curArtDesc1 + curArtDesc2
articleDataDict[
"descriptions"] = parsers_common.list_add_or_assign(
articleDataDict["descriptions"], i, curArtDesc)
# muudame suuna sobivaks
articleDataDict = parsers_common.dict_reverse_order(articleDataDict)
return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain):
maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 5)
maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX /
maxArticleBodies) # set 0 for all posts
parentPages = {}
parentPages["stamps"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="grid zebra forum"]/tr/td[@class="meta"][4]/span/text()'
)
parentPages["titles"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="grid zebra forum"]/tr/td[@class="title"]/a/@title')
parentPages["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//table[@class="grid zebra forum"]/tr/td[@class="title"]/a/@href')
# remove unwanted content: titles
dictList = [
"Börsihai",
"Cleveroni aktsiate ost/müük/oksjon",
"Head uut aastat – prognoosid",
"Keegi malet soovib mängida",
"LHV Pank paremaks",
"Uurimis- ja lõputööde küsimustikud",
]
parentPages = parsers_common.article_data_dict_clean(
parentPages, dictList, "in", "titles")
# teemade läbivaatamine
for i in parsers_common.article_urls_range(parentPages["urls"]):
# teemalehe sisu hankimine
if parsers_common.should_get_article_body(i, maxArticleBodies):
# load article into tree
pageTree = parsers_common.get_article_tree(
domain,
parentPages["urls"][i] +
'?listEventId=jumpToPage&listEventParam=100&pagesOfMaxSize=true',
cache='cacheStamped',
pageStamp=parentPages["stamps"][i])
articlePostsDict = {}
articlePostsDict["authors"] = parsers_common.xpath_to(
"list", pageTree,
'//ul[@class="forum-topic"]/li/div[@class="col2"]/div[@class="forum-header clear"]/p[@class="author"]/strong/a/text()'
)
articlePostsDict["descriptions"] = parsers_common.xpath_to(
"list",
pageTree,
'//ul[@class="forum-topic"]/li/div[@class="col2"]/div[@class="forum-content temporary-class"]',
parent=True)
articlePostsDict["pubDates"] = parsers_common.xpath_to(
"list", pageTree,
'//ul[@class="forum-topic"]/li/div[@class="col2"]/div[@class="forum-header clear"]/div/p[@class="permalink"]/a/node()'
)
articlePostsDict["urls"] = parsers_common.xpath_to(
"list", pageTree,
'//ul[@class="forum-topic"]/li/div[@class="col2"]/div[@class="forum-header clear"]/div/p[@class="permalink"]/a/@href'
)
# teema postituste läbivaatamine
for j in parsers_common.article_posts_range(
articlePostsDict["urls"], maxArticlePosts):
# author
articleDataDict["authors"] = parsers_common.list_add(
articleDataDict["authors"], j,
parsers_common.get(articlePostsDict["authors"], j))
# description
articleDataDict["descriptions"] = parsers_common.list_add(
articleDataDict["descriptions"], j,
parsers_common.get(articlePostsDict["descriptions"], j))
# pubDates magic from "15.01.2012 23:49" to datetime()
curArtPubDate = parsers_common.get(
articlePostsDict["pubDates"], j)
curArtPubDate = parsers_datetime.replace_string_with_timeformat(
curArtPubDate, "Eile", "%d.%m.%Y", offsetDays=-1)
curArtPubDate = parsers_datetime.add_missing_date_to_string(
curArtPubDate, "%d.%m.%Y %H:%M", "%d.%m.%Y ")
curArtPubDate = parsers_datetime.raw_to_datetime(
curArtPubDate, "%d.%m.%Y %H:%M")
articleDataDict["pubDates"] = parsers_common.list_add(
articleDataDict["pubDates"], j, curArtPubDate)
# title
curArtTitle = parsers_common.get(parentPages["titles"], i)
curArtTitle = parsers_common.str_title_at_domain(
curArtTitle, domain)
articleDataDict["titles"] = parsers_common.list_add(
articleDataDict["titles"], j, curArtTitle)
# url
curArtUrl = parentPages["urls"][i] + articlePostsDict["urls"][j]
articleDataDict["urls"] = parsers_common.list_add(
articleDataDict["urls"], j, curArtUrl)
rss_print.print_debug(
__file__, "teema postitus nr. " + str(j + 1) + "/(" +
str(len(articlePostsDict["urls"])) + ") on " +
articlePostsDict["urls"][j], 2)
return articleDataDict