def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product"]/div[@class="image-cell"]/img/@src') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product"]/div[@class="description-cell"]/h2/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product"]/div[@class="description-cell"]/a/@href') articlePrices = parsers_common.xpath_to( "list", pageTree, '//div[@class="product"]/div[@class="price-cell"]', parent=True) articleDescriptions = parsers_common.xpath_to( "list", pageTree, '//div[@class="product"]/div[@class="description-cell"]', parent=True) for i in parsers_common.article_urls_range(articleDataDict["urls"]): # description curArtDesc = parsers_common.get(articlePrices, i) + parsers_common.get( articleDescriptions, i) articleDataDict["descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="news-list list"]/li/div[@class="inner"]/a[@class="news-list-link"]/div/p', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="news-list list"]/li/div[@class="inner"]/a[@class="news-list-link"]/p/img/@src' ) articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="news-list list"]/li/div[@class="inner"]/a[@class="news-list-link"]/@title' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="news-list list"]/li/div[@class="inner"]/a[@class="news-list-link"]/div/h3/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="news-list list"]/li/div[@class="inner"]/a[@class="news-list-link"]/@href' ) for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "03.01.2018 11:09.08 [Tanel]" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = curArtPubDate.split('[')[0] curArtPubDate = parsers_datetime.months_to_int(curArtPubDate) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d. %m %Y %H:%M:%S") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="views-table cols-5"]/tbody/tr', parent=True) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="views-table cols-5"]/tbody/tr/td[1]/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="views-table cols-5"]/tbody/tr/td[5]/div[1]/a/@href') # remove unwanted content: descriptions dictList = [ "ametnik", "hooldaja", "jurist", "koristaja", "logopeed", "pedagoog", "psühholoog", "raamatupidaja", "sanitar", "teenindaja", "uurija", "õpetaja", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//ul/li/div/div/div[@class="horiz-offer-card__content"]/div[@class="horiz-offer-card__desc"]', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/img/@data-src' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/@title') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//ul/li/div/div/div[@class="horiz-offer-card__image"]/a/@href') # remove unwanted content: titles dictList = [ "radiaator", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") articleDataDict = parsers_common.dict_reverse_order(articleDataDict) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@id="t-content"]/table[1]/tr', parent=True) articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@id="t-content"]/table[1]/tr/td[1]/text()') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@id="t-content"]/table[1]/tr/td[@class="left"]/b/a/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@id="t-content"]/table[1]/tr/td[@class="left"]/b/a/@href') # remove unwanted content: descriptions dictList = ["Tartu"] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "not in", "descriptions") for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "29.08.19" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d.%m.%y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//article/ul/li/figure/div[@class="offer-thumb__content"]', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//article/ul/li/figure/figure/a/@data-original') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//article/ul/li/figure/div[@class="offer-thumb__content"]/h3/a/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//article/ul/li/figure/div[@class="offer-thumb__content"]/h3/a/@href') # remove unwanted content: titles dictList = [ "az ", "jawa", "kitarr", "poolvääriskivi", "tuulesuunaja", "viiuli", "xtrons", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="results search-row list-unstyled"]/li/div[2]/div[@class="inner"]/div/@title' ) articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="results search-row list-unstyled"]/li/div[3]/div/div/div', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="results search-row list-unstyled"]/li/div[2]/div[@class="inner"]/a/img/@src' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="results search-row list-unstyled"]/li/div[3]/div/h2/a/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//ul[@class="results search-row list-unstyled"]/li/div[3]/div/h2/a/@href' ) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message"]/div[@class="name"]', parent=True) articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message"]/div[@class="content"]', parent=True) articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message"]/div[@class="posttime"]/text()') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message"]/div[@class="title"]/a[3]/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message"]/div[@class="title"]/a[3]/@href') # remove unwanted content: titles dictList = [ "Hoiatus! Läbisõit 100% keritud", "Kaebused", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") for i in parsers_common.article_urls_range(articleDataDict["urls"]): # description curArtDesc = parsers_common.get(articleDataDict["descriptions"], i) curArtDesc = curArtDesc.split('<div class="userControls')[0] articleDataDict["descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # title curArtTitle = parsers_common.get(articleDataDict["titles"], i) curArtTitle = parsers_common.str_lchop(curArtTitle, "Re: ") curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add_or_assign( articleDataDict["titles"], i, curArtTitle) # pubDates magic from "20:22 01.09.2019" to Datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "eile", "%d.%m.%Y", offsetDays=-1) curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "täna", "%d.%m.%Y", offsetDays=0) curArtPubDate = parsers_datetime.add_missing_date_to_string( curArtPubDate, "%H:%M %d.%m.%Y", " %d.%m.%Y") curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%H:%M %d.%m.%Y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="box-news-block-title "]/a[@title]/@title') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="box-news-block-title "]/a[@title]/@href') # remove unwanted content: urls dictList = [ "https://sky.ee/rock-fmi-hommikuprogramm-igal-toopaeval-kell-7-10/" ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "urls") for i in parsers_common.article_urls_range(articleDataDict["urls"]): if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # author curArtAuthor = parsers_common.xpath_to( "single", pageTree, '//div[@class="post-content"]/div[@class="article-page-author"]/p/text()' ) if ": " in curArtAuthor: curArtAuthor = curArtAuthor.split(": ")[1] articleDataDict["authors"] = parsers_common.list_add_or_assign( articleDataDict["authors"], i, curArtAuthor) # description curArtDesc1 = parsers_common.xpath_to( "single", pageTree, '//div[@class="posts"]/div[@class="two-side-content-container clearfix"][1]//div[@class="post-content"]/strong/p/text()' ) curArtDesc2 = parsers_common.xpath_to( "single", pageTree, '//div[@class="posts"]/div[@class="two-side-content-container clearfix"][2]//div[@class="post-content"]', parent=True) curArtDesc = curArtDesc1 + curArtDesc2 if '<p class="related-cta">LOE KA NEID UUDISEID!</p>' in curArtDesc: curArtDesc = curArtDesc.split( '<p class="related-cta">LOE KA NEID UUDISEID!</p>')[0] articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/p', parent=True) articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="right-content"]/div[@class="application-date"][1]/text()' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/h2/a/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="applied-jobs"]/div/div[@class="job-content"]/div[@class="left-content"]/h2/a/@href' ) # remove unwanted content: descriptions dictList = [ "jurist", "logopeed", "pedagoog", "psühholoog", "raamatupidaja", "sotsiaal", "õpetaja", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "Avaldatud: 12.12.2017" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = curArtPubDate.split(': ')[1] curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d.%m.%Y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) # url curArtUrl = parsers_common.get(articleDataDict["urls"], i) curArtUrl = curArtUrl.split('?ref=')[0] articleDataDict["urls"] = parsers_common.list_add_or_assign( articleDataDict["urls"], i, curArtUrl) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//body/div[2]/div[1]/main/div/div[3]/div/div/a/h2/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//body/div[2]/div[1]/main/div/div[3]/div/div/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # description curArtDesc = parsers_common.xpath_to("single", pageTree, '//div[@class="col-md-12"]', parent=True) if not curArtDesc: curArtDesc = parsers_common.xpath_to( "single", pageTree, '//div[@class="col-md-8"]', parent=True) if not curArtDesc: curArtDesc = parsers_common.xpath_to( "single", pageTree, '//div[@class="img-open-area basic-content pb-4"]', parent=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # image curArtImg = parsers_common.xpath_to( "single", pageTree, '//a[@data-lightbox="treimages"][1]/@href') articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImg) # pubDates magic from "7. märts 2021" to datetime() curArtPubDate = parsers_common.xpath_to( "single", pageTree, '//div[@class="col-sm-12 col-md-auto text-sm-center text-md-right"]/div[@class="mt-1 text-uppercase"]/small/text()' ) curArtPubDate = parsers_datetime.months_to_int(curArtPubDate) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d. %m %Y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="col-sm-6"]/div[@class="post-item"]/a/div/img/@src') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="col-sm-6"]/div[@class="post-item"]/a/h3/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="col-sm-6"]/div[@class="post-item"]/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # descriptions curArtDesc = parsers_common.xpath_to("single", pageTree, '//div[@class="col-sm-9"]/p', multi=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # timeformat magic from "Avaldatud: 14 detsember, 2017" to datetime() curArtPubDate = parsers_common.xpath_to( "single", pageTree, '//div[@class="col-sm-9"]/div[@class="page-header"]/em/text()') curArtPubDate = parsers_datetime.months_to_int( curArtPubDate.split(':')[1]) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d %m, %Y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div[3]/div/div[1]/div[@class="sb-article"]/a/div[@class="sb-article-cnt"]/div[@class="sb-article-prolog"]', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div[3]/div/div[1]/div[@class="sb-article"]/a/div[@class="sb-article-image"]/@style' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div[3]/div/div[1]/div[@class="sb-article"]/a/div[@class="sb-article-cnt"]/div[@class="sb-article-title"]/h3/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div[3]/div/div[1]/div[@class="sb-article"]/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # author curArtAuthor = parsers_common.xpath_to( "single", pageTree, '//div[@class="sg-article-details"]/div[@class="author"]/text()' ) articleDataDict["authors"] = parsers_common.list_add_or_assign( articleDataDict["authors"], i, curArtAuthor) # description curArtDesc = parsers_common.xpath_to( "single", pageTree, '/html/body/div[3]/div/div[@class="page-content"]/div[@class="sg-article"]/div[@class="sg-article-text"]', parent=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # pubDates magic from "18.08.2019 21:35" to datetime() curArtPubDate = parsers_common.xpath_to( "single", pageTree, '//div[@class="sg-article-details"]/div[@class="date"]/text()') curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d.%m.%Y %H:%M") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//article/a/div/div/img/@src') articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//article/a/div[@class="node__body"]/p[@class="node__date"]/span/@content' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//article/a/div[@class="node__body"]/h3/span/text()') articleDataDict["urls"] = parsers_common.xpath_to("list", pageTree, '//article/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "2021-03-23T12:35:36+00:00" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%Y-%m-%dt%H:%M:%S%z") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # description curArtDesc = parsers_common.xpath_to( "single", pageTree, '//div[@class="node__content"]/div/div[@class="field__item"]', parent=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="js-newsline-container"]/span[1]/text()') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="js-newsline-container"]/div/a/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="js-newsline-container"]/div/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "14 dets 2017 11:34" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.months_to_int(curArtPubDate) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d %m %Y %H:%M") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # description curArtDesc = parsers_common.xpath_to( "single", pageTree, '//div[@class="news-preview"]/div/text()') if not curArtDesc: curArtDesc = parsers_common.xpath_to( "single", pageTree, '//div[@class="content_item"]', parent=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): author = parsers_common.xpath_to("single", pageTree, '//p[@id="band-name-location"]/span[@class="title"]/text()') description = parsers_common.xpath_to("single", pageTree, '//meta[@name="description"]/@content') domain = parsers_common.xpath_to("single", pageTree, '//meta[@property="og:url"]/@content') articleDataDict["images"] = parsers_common.xpath_to("list", pageTree, '//div[@class="leftMiddleColumns"]/ol/li/a/div/img/@src') articleDataDict["titles"] = parsers_common.xpath_to("list", pageTree, '//div[@class="leftMiddleColumns"]/ol/li/a/p/text()') articleDataDict["urls"] = parsers_common.xpath_to("list", pageTree, '//div[@class="leftMiddleColumns"]/ol/li/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # author articleDataDict["authors"] = parsers_common.list_add_or_assign(articleDataDict["authors"], i, author) # description articleDataDict["descriptions"] = parsers_common.list_add_or_assign(articleDataDict["descriptions"], i, description) # title curArtTitle = parsers_common.get(articleDataDict["titles"], i) curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add_or_assign(articleDataDict["titles"], i, curArtTitle) # url curArtUrl = parsers_common.get(articleDataDict["urls"], i) articleDataDict["urls"] = parsers_common.list_add_or_assign(articleDataDict["urls"], i, curArtUrl) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product_camp_box w"]/a/div[@class="product_camp"]/div[@class="leftC"]/div/img/@data-src' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product_camp_box w"]/a/div[@class="product_camp"]/div[@class="leftC"]/h3/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="product_camp_box w"]/a/@href') articlePrices = parsers_common.xpath_to( "list", pageTree, '//div[@class="product_camp_box w"]/div[@class="priceCont"]', parent=True) articleDescriptions = parsers_common.xpath_to( "list", pageTree, '//div[@class="product_camp_box w"]/a/div[@class="product_camp"]/div[@class="leftC"]', parent=True) for i in parsers_common.article_urls_range(articleDataDict["urls"]): # description curArtDesc = articlePrices[i] + articleDescriptions[i] curArtDesc = curArtDesc.replace( '<img src="https://www.stokker.ee/gfx/blank.gif">', "") articleDataDict["descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # image curArtImg = parsers_common.get(articleDataDict["images"], i) curArtImg = curArtImg.split("?")[0] curArtImg = curArtImg.replace("%26", "&") articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImg) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//section/div/div/div/div/div/article/div/div/div/div[2]/div[2]/div[1]/div/div/div[1]/div[1]/a/div/div[1]/div[1]/span/span/text()' ) articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//section/div/div/div/div/div/article/div/div/div/div[2]/div[2]/div[2]/div[1]/div/span[1]', parent=True) articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//section/div/div/div/div/div/article/div/div/div/div[2]/div[2]/div[1]/div/div/div[1]/a/time/@datetime' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//section/div/div/div/div/div/article/div/div/div/div[2]/div[2]/div[1]/div/div/div[1]/div[1]/a/div/div[1]/div[1]/span/span/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//section/div/div/div/div/div/article/div/div/div/div[2]/div[2]/div[1]/div/div/div[1]/a/@href' ) for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates 2021-02-12T16:08:02.000Z curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%Y-%m-%dT%H:%M:%S.000Z") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) # title curArtTitle = parsers_common.get(articleDataDict["titles"], i) curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add_or_assign( articleDataDict["titles"], i, curArtTitle) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//li[@class="b-posts__list-item"]/p[@class="b-posts__list-item-summary"]/text()') articleDataDict["titles"] = parsers_common.xpath_to("list", pageTree, '//li[@class="b-posts__list-item"]/h2[@class="b-posts__list-item-title"]/a/text()') articleDataDict["urls"] = parsers_common.xpath_to("list", pageTree, '//li[@class="b-posts__list-item"]/h2[@class="b-posts__list-item-title"]/a/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "30.01.2021" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime(curArtPubDate, "%d.%m.%Y") articleDataDict["pubDates"] = parsers_common.list_add_or_assign(articleDataDict["pubDates"], i, curArtPubDate) if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # description curArtDesc = parsers_common.xpath_to("single", pageTree, '//div[@class="b-article"]', parent=True) articleDataDict["descriptions"] = parsers_common.list_add_or_assign(articleDataDict["descriptions"], i, curArtDesc) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//section/article/h2/a[2]/text()') articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//section/article/time/text()') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//section/article/h2/a[@itemprop="url"]/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//section/article/h2/a[@itemprop="url"]/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # pubDates magic from "2021-01-06T10:11:04Z" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%Y-%m-%dt%H:%M:%S%z") articleDataDict["pubDates"][i] = curArtPubDate if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache="cacheAll") # description curArtDesc = parsers_common.xpath_to("single", pageTree, '//body/div//article', parent=True) articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # image curArtImg = parsers_common.xpath_to( "single", pageTree, '//body/div//article/p/img/@src') articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImg) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="grid-main--item "]/div/div[2]/div[1]/h6/a[1]/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="grid-main--item "]/div/div[2]/div[1]/h6/a[1]/@href') for i in parsers_common.article_urls_range(articleDataDict["urls"]): # titles curArtTitle = parsers_common.get(articleDataDict["titles"], i) curArtTitle = parsers_common.str_remove_clickbait(curArtTitle) articleDataDict["titles"] = parsers_common.list_add_or_assign( articleDataDict["titles"], i, curArtTitle) if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # author curArtAuthor = parsers_common.xpath_to( "single", pageTree, '//span[@class="author"]/text()') articleDataDict["authors"] = parsers_common.list_add_or_assign( articleDataDict["authors"], i, curArtAuthor) # description curArtDesc1 = parsers_common.xpath_to( "single", pageTree, '//div[@class="page-layout--block"][1]/div[@class="page-layout--content"]/div[@class="page-layout--inner"]/div[@class="article-main--content article-main--excerpt formatted--content"]', parent=True) if not curArtDesc1: curArtDesc1 = parsers_common.xpath_to( "single", pageTree, '//div[@class="page-layout--content"]/div[@class="page-layout--inner"]/div[@class="article-main--content article-main--excerpt formatted--content"]', parent=True) curArtDesc2 = parsers_common.xpath_to( "single", pageTree, '//div[@class="page-layout--block"][2]/div[@class="page-layout--content"]/div[@class="page-layout--inner"]', parent=True, multi=True) curArtDesc = curArtDesc1 + curArtDesc2 curArtDesc = curArtDesc.split("Edasi lugemiseks")[0] curArtDesc = curArtDesc.split("Jaga:")[0] curArtDesc = curArtDesc.split("Samal teemal")[0] articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # image curArtImg = parsers_common.xpath_to( "single", pageTree, '//div[@class="page-layout--block"][1]//div[@class="image-gallery-image first-in-gallery"][1]/picture[1]/img[@class="article-image"]/@src' ) if not curArtImg: curArtImg = parsers_common.xpath_to( "single", pageTree, '//div[@class="part"][1]/div/p/img/@src') articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImg) # pubDates from "täna 16:53" to datetime() curArtPubDate = parsers_common.xpath_to( "single", pageTree, '//div[@class="details--inner"]/text()') curArtPubDate = curArtPubDate.split(",")[-1] curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "eile", "%d. %m %Y ", offsetDays=-1) curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "täna", "%d. %m %Y ", offsetDays=0) curArtPubDate = parsers_datetime.months_to_int(curArtPubDate) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%d. %m %Y %H:%M") articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/p[@class="img"]/a/img/@src' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]/h2/a/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '/html/body/div/div/div/div[@id="page"]/ul/li[@class="clear "]/div[@class="content"]/h2/a/@href' ) # video # remove unwanted content: titles dictList = [ "2000.ee:", "AK filmikroonika 1958-1991:", "Aktuaalne kaamera", "Eesti Gloobus", "Hommik Anuga:", "Insight", "Iseolemine:", "Johannese lähetamine", "Kes keda?", "Kodukäijad", "Koolitants", "Lasteekraan", "Lastetuba", "Laulge kaasa!", "Meie kõrval:", "Mis teie kodus uudist?:", "Mis? Kus? Millal?:", "NOVA:", "Noor meister:", "Nädala intervjuu:", "OP:", "Oma tõde:", "Ongi Koik", "Pealtnägija", "Peegel:", "Plekktrumm", "Prillitoos:", "Püha päeva palvus:", "Rahvas laulab:", "Rakett 69:", "Reibas hommik", "Ringvaade", "Suus sulav Eesti:", "TECHnolik", "TV 10 olümpiastarti", "Taevavalvurid", "Tarmo ja Aet liiguvad:", "Terevisioon:", "Tähendamisi", "Tähtede lava", "Välisilm", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") # remove unwanted content: description dictList = [ "Johannes Tralla", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "descriptions") # audio # remove unwanted content: titles dictList = [ "AIATARK.", "DELTA.", "GOGOL.", "HOMMIKUMÕTISKLUS.", "HUVITAJA.", "KIHNUKEELSED UUDISED", "KIRIKUELU.", "KITARRIMUUSIKAST JA -MÄNGIJAIST.", "KULDRANDEVUU.", "LOETUD JA KIRJUTATUD.", "LUULERUUM.", "LUULETUS.", "MELOTURNIIR.", "MINITURNIIR.", "MNEMOTURNIIR", "MULGIKEELSED UUDISED", "MUST KLAHV, VALGE KLAHV.", "NAISTESAUN.", "OLUKORRAST RIIGIS", "PÄEVAKAJA", "RAHVA OMA KAITSE", "RAHVA TEENRID", "REPORTERITUND.", "SETUKEELSED UUDISED", "SIILILEGI SELGE!", "SPORDIPÜHAPÄEV.", "TETRIS.", "UUDISED.", "VÕRUKEELSED UUDISED", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="list-item"]/div[@class="details"]', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="list-item"]/a[@class="image"]/@style') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="list-item"]/div[@class="details"]/a[1]/text()') articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="list-item"]/div[@class="details"]/a[1]/@href') # remove unwanted content: titles dictList = [ "edasi lükatud", "ei toimu", "jääb ära", "lükkub edasi", "tühistatud", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") for i in parsers_common.article_urls_range(articleDataDict["urls"]): # image curArtImage = parsers_common.get(articleDataDict["images"], i) curArtImage = parsers_common.split_failsafe(curArtImage, "'", 1) articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImage) # url curArtUrl = parsers_common.get(articleDataDict["urls"], i) curArtUrl = curArtUrl.split("?event")[0] articleDataDict["urls"] = parsers_common.list_add_or_assign( articleDataDict["urls"], i, curArtUrl) if parsers_common.should_get_article_body(i): # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # description curArtDesc1 = parsers_common.get(articleDataDict["descriptions"], i) curArtDesc2 = parsers_common.xpath_to("single", pageTree, '//article', parent=True) curArtDesc2 = curArtDesc2.replace( "<h5 class=\"sm-hide\">Galerii</h5>", "") curArtDesc2 = curArtDesc2.replace( "<h5 class=\"sm-hide\">Tutvustus</h5>", "") curArtDesc2 = curArtDesc2.replace( "<div class=\"link after-arrow_down sm-show\">Loe lähemalt</div>", "") curArtDesc = curArtDesc1 + curArtDesc2 articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # muudame suuna sobivaks articleDataDict = parsers_common.dict_reverse_order(articleDataDict) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 5) maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts parentPages = {} parentPages["stamps"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/h3[@class="adds-list-title"]/span[@class="adds-list-meta"]/text()' ) parentPages["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/h3[@class="adds-list-title"]/text()' ) parentPages["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="tabs-content tab-active shadow"]/ul[@class="adds-list list"]/li[@class="adds-list-item"]/a[h3/@class="adds-list-title"]/@href' ) # muudame suuna sobivaks parentPages = parsers_common.dict_reverse_order(parentPages) # remove unwanted content: titles dictList = [ "AMD", "Apple", "Assassin", "Batman", "Battlefield", "Call of Duty", "Cyberpunk", "Diablo 2", "Dying", "Escape From Tarkov", "Euro Truck", "Evil", "FIFA", "Far Cry", "Forza", "Galaxy", "Grand Theft", "IPhon", "Kindle", "MMORPG", "MSI", "MacBook", "MacOS", "Mafia", "Mass Effect", "Meizu", "Minecraft", "Nintendo", "Pixel", "PlayStation", "Steam", "Tanks", "Vidia", "War Thunder", "Watercool", "Windows", "Xbox", "arvutikast", "exile", "foorumiga seotud", "konsool", "korpust", "moderaatorite", "seotud vead", "siia lingid", "toiteplok", ] parentPages = parsers_common.article_data_dict_clean( parentPages, dictList, "in", "titles") # teemade läbivaatamine for i in parsers_common.article_urls_range(parentPages["urls"]): # teemalehe sisu hankimine if parsers_common.should_get_article_body(i, maxArticleBodies): parentPages["stamps"][i] = parentPages["stamps"][i].split("/")[0] parentPages["urls"][i] = parentPages["urls"][i].split("&sid=")[0] pageTree = parsers_common.get_article_tree( domain, parentPages["urls"][i], cache='cacheStamped', pageStamp=parentPages["stamps"][i]) articlePostsDict = {} articlePostsDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="forumline"]/tr/td[1]/span[@class="name"]/b/a/text()' ) articlePostsDict["descriptions1"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="forumline"]/tr/td[2]/table/tr[3]/td/span[@class="postbody"][1]', parent=True) articlePostsDict["descriptions2"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="forumline"]/tr/td[2]/table/tr[3]/td/span[@class="postbody"][2]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="forumline"]/tr/td[2]/table/tr[1]/td/span[@class="postdetails"]/span[@class="postdetails"][1]/text()[1]' ) articlePostsDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//table[@class="forumline"]/tr/td[2]/table/tr[1]/td/span[@class="postdetails"]/a/@href' ) # teema postituste läbivaatamine for j in parsers_common.article_posts_range( articlePostsDict["urls"], maxArticlePosts): # author articleDataDict["authors"] = parsers_common.list_add( articleDataDict["authors"], j, parsers_common.get(articlePostsDict["authors"], j)) # description curArtDesc = parsers_common.get( articlePostsDict["descriptions1"], j) if not curArtDesc: curArtDesc = parsers_common.get( articlePostsDict["descriptions2"], j) curArtDesc = curArtDesc.replace( '</div><div class="quotecontent">', '<br>') curArtDesc = parsers_common.fix_quatation_tags( curArtDesc, '<div class="quotetitle">', "</div>", "<blockquote>", "</blockquote>") articleDataDict["descriptions"] = parsers_common.list_add( articleDataDict["descriptions"], j, curArtDesc) # pubDates curArtPubDate = parsers_common.get( articlePostsDict["pubDates"], j) curArtPubDate = curArtPubDate[0:16] curArtPubDate = parsers_datetime.guess_datetime( curArtPubDate) # 14.07.2020 07:59 articleDataDict["pubDates"] = parsers_common.list_add( articleDataDict["pubDates"], j, curArtPubDate) # title curArtTitle = parsers_common.get(parentPages["titles"], i) curArtTitle = parsers_common.str_title_at_domain( curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add( articleDataDict["titles"], j, curArtTitle) # url curArtUrl = parsers_common.get(articlePostsDict["urls"], j) curArtUrl = parsers_common.link_add_end( curArtUrl, articlePostsDict["urls"][j]) articleDataDict["urls"] = parsers_common.list_add( articleDataDict["urls"], j, curArtUrl) rss_print.print_debug( __file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="history-item"]/span[@class="history-date ng-binding"]/text()' ) articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="history-item"]/p[@class="history-header"]/a[@class="ng-binding"]/text()[1]' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="history-item"]/p[@class="history-header"]/a[@class="ng-binding"]/@href' ) lastArtPubDate = "" iMinus = 0 # remove unwanted content: titles dictList = [ " reket", " võidu", "Aktuaalne kaamera", "ERR-i teleuudised", "ETV spordi", "Ilm ", "OTSE ", "Päevakaja", "Raadiouudised", "Viipekeelsed uudised", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") for i in parsers_common.article_urls_range(articleDataDict["urls"]): i = i - iMinus # pubDates magic from "11:34" to datetime() curArtPubDate = parsers_common.get(articleDataDict["pubDates"], i) curArtPubDate = parsers_datetime.raw_to_datetime_guess_missing( curArtPubDate, lastArtPubDate, "%Y %m %d ", "%H:%M", -1) lastArtPubDate = curArtPubDate articleDataDict["pubDates"] = parsers_common.list_add_or_assign( articleDataDict["pubDates"], i, curArtPubDate) # title curArtTitle = parsers_common.get(articleDataDict["titles"], i) if "|" in curArtTitle: curArtTitle = curArtTitle.split("|")[-1] articleDataDict["titles"] = parsers_common.list_add_or_assign( articleDataDict["titles"], i, curArtTitle) if parsers_common.should_get_article_body(i): curArtUrl = parsers_common.get(articleDataDict["urls"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curArtUrl, cache='cacheAll') # author curArtAuthor = parsers_common.xpath_to( "single", pageTree, '//article/div[@class="body"]/div/div[@class="meta"]/section/div[@class="byline"]/span/text()' ) articleDataDict["authors"] = parsers_common.list_add_or_assign( articleDataDict["authors"], i, curArtAuthor) # description curArtDesc1 = parsers_common.xpath_to( "single", pageTree, '//article/div[@class="body"]/div[@class="lead"]', parent=True) curArtDesc2 = parsers_common.xpath_to( "single", pageTree, '//article/div[@class="body"]/div[@class="text flex-row"]', parent=True) if not curArtDesc2: rss_print.print_debug( __file__, "tühja sisuga uudis, eemaldame rea " + str(i), 1) articleDataDict = parsers_common.dict_del_article_index( articleDataDict, i) iMinus += 1 continue curArtDesc = curArtDesc1 + "<br>" + curArtDesc2 articleDataDict[ "descriptions"] = parsers_common.list_add_or_assign( articleDataDict["descriptions"], i, curArtDesc) # image curArtImg = parsers_common.xpath_to( "single", pageTree, '/html/head/meta[@property="og:image"]/@content') if not curArtImg: curArtImg = parsers_common.xpath_to( "single", pageTree, '/html/head/meta[@property="image"]/@content') articleDataDict["images"] = parsers_common.list_add_or_assign( articleDataDict["images"], i, curArtImg) articleDataDict = parsers_common.dict_reverse_order(articleDataDict) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 5) maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts parentPages = {} parentPages["stamps"] = parsers_common.xpath_to( "list", pageTree, '//div[@data-lang="Vastuseid"]/a/text()') parentPages["titles"] = parsers_common.xpath_to( "list", pageTree, '//span[@class=" subject_old"]/a/text()') parentPages["urls"] = parsers_common.xpath_to( "list", pageTree, '//span[@class=" subject_old"]/a/@href') # remove unwanted content: titles #dictList = [] #parentPages = parsers_common.article_data_dict_clean(parentPages, dictList, "in", "titles") # teemade läbivaatamine for i in parsers_common.article_urls_range(parentPages["urls"]): # teemalehe sisu hankimine if parsers_common.should_get_article_body(i, maxArticleBodies): curParentUrl = parsers_common.get(parentPages["urls"], i) curParentUrl = curParentUrl.split("&sid=")[0] curParentUrl = curParentUrl + "&page=-1" curParentUrl = parsers_common.str_domain_url(domain, curParentUrl) parentPagesStamp = parsers_common.get(parentPages["stamps"], i) # load article into tree pageTree = parsers_common.get_article_tree( domain, curParentUrl, cache='cacheStamped', pageStamp=parentPagesStamp) articlePostsDict = {} articlePostsDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="author_information"]/strong/span[@class="largetext"]/a/text()' ) articlePostsDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="post_body scaleimages"]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="post_head"]/span[@class="post_date"]', parent=True) articlePostsDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="post_content"]/div[@class="post_head"]/div/strong/a/@href' ) # teema postituste läbivaatamine for j in parsers_common.article_posts_range( articlePostsDict["urls"], maxArticlePosts): # author articleDataDict["authors"] = parsers_common.list_add( articleDataDict["authors"], j, parsers_common.get(articlePostsDict["authors"], j)) # description curArtDesc = parsers_common.get( articlePostsDict["descriptions"], j) curArtDesc = curArtDesc.replace( '</div><div class="quotecontent">', '<br>') curArtDesc = parsers_common.fix_quatation_tags( curArtDesc, '<div class="quotetitle">', "</div>", "<blockquote>", "</blockquote>") articleDataDict["descriptions"] = parsers_common.list_add( articleDataDict["descriptions"], j, curArtDesc) # pubDates curArtPubDate = parsers_common.get( articlePostsDict["pubDates"], j) curArtPubDate = parsers_common.str_lchop( curArtPubDate, '<span title="') curArtPubDate = curArtPubDate.split(" <span class")[0] if "Eile" in curArtPubDate or "Täna" in curArtPubDate: curArtPubDate = curArtPubDate.split('">')[1] curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "Eile</span>", "%d-%m-%Y", offsetDays=-1) curArtPubDate = parsers_datetime.replace_string_with_timeformat( curArtPubDate, "Täna</span>", "%d-%m-%Y", offsetDays=0) else: curArtPubDate = curArtPubDate.split('">')[0] curArtPubDate = parsers_datetime.guess_datetime(curArtPubDate) articleDataDict["pubDates"] = parsers_common.list_add( articleDataDict["pubDates"], j, curArtPubDate) # title curArtTitle = parsers_common.get(parentPages["titles"], i) curArtTitle = parsers_common.str_title_at_domain( curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add( articleDataDict["titles"], j, curArtTitle) # url curArtUrl = parsers_common.get(articlePostsDict["urls"], j) curArtUrl = parsers_common.link_add_end( curArtUrl, articlePostsDict["urls"][j]) articleDataDict["urls"] = parsers_common.list_add( articleDataDict["urls"], j, curArtUrl) rss_print.print_debug( __file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 8) maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts parentPages = {} version = 0 if "kipper.ee" in domain or "militaar.net" in domain: version = 1 parentPages["stamps"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/p[@class="topicdetails"]/text()') parentPages["titles"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/a[@class="topictitle"]/text()') parentPages["urls"] = parsers_common.xpath_to("list", pageTree, '//table[@class="tablebg"]//tr/td/a[@class="topictitle"]/@href') else: parentPages["stamps"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/dd[@class="posts"]/text()') parentPages["titles"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/*/div/a[@class="topictitle"]/text()') parentPages["urls"] = parsers_common.xpath_to("list", pageTree, '//ul[2]/li/dl/*/div/a[@class="topictitle"]/@href') if not parentPages["urls"] and "arutelud.com" in domain: rss_print.print_debug(__file__, "aktiivseid teemasid ei leitud, arutelude foorumis külastame mammutteemat", 1) parentPages["stamps"] = parsers_common.list_add_or_assign(parentPages["stamps"], 0, "") parentPages["titles"] = parsers_common.list_add_or_assign(parentPages["titles"], 0, "Arutelud") parentPages["urls"] = parsers_common.list_add_or_assign(parentPages["urls"], 0, "https://arutelud.com/viewtopic.php?f=3&t=4&sd=d&sk=t&st=7") # remove unwanted content: titles dictList = [ "Race.Fi:", "Write my", "PÕLVAMAA, VÕRUMAA JA VALGAMAA CB JA HAM SIDE", ] parentPages = parsers_common.article_data_dict_clean(parentPages, dictList, "in", "titles") # teemade läbivaatamine for i in parsers_common.article_urls_range(parentPages["urls"]): # teemalehe sisu hankimine if parsers_common.should_get_article_body(i, maxArticleBodies): curParentUrl = parsers_common.get(parentPages["urls"], i) curParentUrl = curParentUrl.split("&sid=")[0] curParentUrl = curParentUrl + "&start=100000" parentPagesStamp = parsers_common.get(parentPages["stamps"], i) # load article into tree pageTree = parsers_common.get_article_tree(domain, curParentUrl, cache='cacheStamped', pageStamp=parentPagesStamp) articlePostsDict = {} if version in (1, 2): rss_print.print_debug(__file__, "kasutame spetsiifilist hankimist, domain = " + domain, 2) articlePostsDict["authors"] = parsers_common.xpath_to("list", pageTree, '//tr/td/b[@class="postauthor"]/text()') articlePostsDict["descriptions"] = parsers_common.xpath_to("list", pageTree, '//tr/td/div[@class="postbody"][1]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//tr/td[@class="gensmall"]/div[@style="float: right;"]/text()') articlePostsDict["urls"] = parsers_common.xpath_to("list", pageTree, '//tr/td[@class="gensmall"]/div[@style="float: right;"]/a/@href') else: rss_print.print_debug(__file__, "kasutame üldist hankimist, domain = " + domain, 3) articlePostsDict["authors"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]//strong//text()') articlePostsDict["descriptions"] = parsers_common.xpath_to("list", pageTree, '//div[@class="content"]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/time/@datetime') articlePostsDict["urls"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/a/@href') if not articlePostsDict["pubDates"]: articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/time/text()') if not articlePostsDict["pubDates"]: articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/text()[1]') if len(articlePostsDict["pubDates"][0]) < 5: rss_print.print_debug(__file__, "hangitud aeg[0] liiga lühike: '" + articlePostsDict["pubDates"][0] + "', proovime alternatiivi...", 0) articlePostsDict["pubDates"] = parsers_common.xpath_to("list", pageTree, '//p[@class="author"]/text()[2]') if len(articlePostsDict["pubDates"][0]) < 5: rss_print.print_debug(__file__, "hangitud aeg[0] liiga lühike: '" + articlePostsDict["pubDates"][0] + "'", 0) else: rss_print.print_debug(__file__, "hangitud aeg[0]: '" + articlePostsDict["pubDates"][0] + "'", 4) if not articlePostsDict["pubDates"]: rss_print.print_debug(__file__, "ei suutnud hankida ühtegi aega", 0) # teema postituste läbivaatamine for j in parsers_common.article_posts_range(articlePostsDict["urls"], maxArticlePosts): # author articleDataDict["authors"] = parsers_common.list_add(articleDataDict["authors"], j, parsers_common.get(articlePostsDict["authors"], j)) # description curArtDesc = parsers_common.get(articlePostsDict["descriptions"], j) curArtDesc = curArtDesc.replace('</div><div class="quotecontent">', '<br>') curArtDesc = parsers_common.fix_quatation_tags(curArtDesc, '<div class="quotetitle">', "</div>", "<blockquote>", "</blockquote>") articleDataDict["descriptions"] = parsers_common.list_add(articleDataDict["descriptions"], j, curArtDesc) # pubDates curArtPubDate = parsers_common.get(articlePostsDict["pubDates"], j) curArtPubDate = parsers_datetime.months_to_int(curArtPubDate) curArtPubDate = parsers_datetime.remove_weekday_strings(curArtPubDate) curArtPubDate = parsers_datetime.replace_string_with_timeformat(curArtPubDate, "eile", "%d %m %Y", offsetDays=-1) curArtPubDate = parsers_datetime.replace_string_with_timeformat(curArtPubDate, "täna", "%d %m %Y", offsetDays=0) curArtPubDate = parsers_datetime.guess_datetime(curArtPubDate) articleDataDict["pubDates"] = parsers_common.list_add(articleDataDict["pubDates"], j, curArtPubDate) # title curArtTitle = parsers_common.get(parentPages["titles"], i) curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add(articleDataDict["titles"], j, curArtTitle) # url curArtUrl = parsers_common.get(articlePostsDict["urls"], j) curArtUrl = parsers_common.link_add_end(curArtUrl, articlePostsDict["urls"][j]) articleDataDict["urls"] = parsers_common.list_add(articleDataDict["urls"], j, curArtUrl) rss_print.print_debug(__file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): articleDataDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//main/div[@class="content"]/div/div/div[1]/div/div', parent=True) articleDataDict["images"] = parsers_common.xpath_to( "list", pageTree, '//main/div[1]/div/div/div[1]/div/div/a/div/img/@src') articleDataDict["titles"] = parsers_common.xpath_to( "list", pageTree, '//main/div[@class="content"]/div/div/div[1]/div/div/div/div[1]/div[2]/a/text()' ) articleDataDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//main/div[@class="content"]/div/div/div[1]/div/div/div/div[1]/div[2]/a/@href' ) # remove unwanted content: titles dictList = [ "(uus) raamat", "abramova", "akvavit", "based broccoli", "beats of no nation", "bisweed", "ekkm", "error!", "floorshow", "gnoom", "hard feeler", "hillbilly picnic", "ida jutud", "ida räpp", "intro", "katus", "keskkonnatund", "kink konk", "korrosioon", "kräpp", "let me juke", "liin ", "liin", "lunchbreak lunchdate", "meie igapäevane avalik ruum", "milk", "muster", "myös", "müürilehe hommik", "n-lib" "oleneb päevast!", "oujee!", "paneel", "playa music", "propel", "puhkus", "rets records", "room_202", "rõhk", "saal raadio", "soojad suhted", "svet nureka", "söökladisko", "triinemets.", "vitamiin k", "zubrovka am", "ära kaaguta!", "öömaja", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "titles") # remove unwanted content: descriptions dictList = [ "#hip hop", "#interview", "#rap", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "descriptions") for i in parsers_common.article_urls_range(articleDataDict["urls"]): # title curArtTitle = parsers_common.get(articleDataDict["titles"], i) curArtTitle = parsers_common.str_title_at_domain(curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add_or_assign( articleDataDict["titles"], i, curArtTitle) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 10) maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts parentPages = {} parentPages["stamps"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="reply-count"]/span[@data-xf-init="tooltip"]/text()') parentPages["titles"] = parsers_common.xpath_to( "list", pageTree, '//div[@qid="thread-item-parent"]/div[@qid="thread-item"]/div/div[@class="structItem-title"]/a[@qid="thread-item-title"]/text()' ) parentPages["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@qid="thread-item-parent"]/div[@qid="thread-item"]/div/div[@class="structItem-title"]/a[@qid="thread-item-title"]/@href' ) # remove unwanted content: titles dictList = [ "$", "*:::::::the official what did you do to you mkiv today thread::::::::*", "??", "Ask a Simple Question", ] parentPages = parsers_common.article_data_dict_clean( parentPages, dictList, "in", "titles") # teemade läbivaatamine for i in parsers_common.article_urls_range(parentPages["urls"]): # teemalehe sisu hankimine if parsers_common.should_get_article_body(i, maxArticleBodies): curParentUrl = parsers_common.get(parentPages["urls"], i) curParentUrl = curParentUrl + "page-1000" parentPagesStamp = parsers_common.get(parentPages["stamps"], i) # load article into tree pageTree = parsers_common.get_article_tree( domain, curParentUrl, cache='cacheStamped', pageStamp=parentPagesStamp) articlePostsDict = {} articlePostsDict["authors"] = parsers_common.xpath_to( "list", pageTree, '//h4[@qid="message-username"]//text()') articlePostsDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//article/div[@class="bbWrapper"]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message-attribution-main"]/a[@class="u-concealed"][2]/time/@datetime' ) articlePostsDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="message-attribution-main"]/a[@class="u-concealed"][2]/@href' ) # teema postituste läbivaatamine for j in parsers_common.article_posts_range( articlePostsDict["urls"], maxArticlePosts): # author articleDataDict["authors"] = parsers_common.list_add( articleDataDict["authors"], j, parsers_common.get(articlePostsDict["authors"], j)) # description curArtDesc = parsers_common.get( articlePostsDict["descriptions"], j) articleDataDict["descriptions"] = parsers_common.list_add( articleDataDict["descriptions"], j, curArtDesc) # pubDates curArtPubDate = parsers_common.get( articlePostsDict["pubDates"], j) curArtPubDate = parsers_datetime.raw_to_datetime( curArtPubDate, "%Y-%m-%dT%H:%M:%S%z") # 2021-01-28T16:15:42-0500 articleDataDict["pubDates"] = parsers_common.list_add( articleDataDict["pubDates"], j, curArtPubDate) # title curArtTitle = parsers_common.get(parentPages["titles"], i) curArtTitle = parsers_common.str_title_at_domain( curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add( articleDataDict["titles"], j, curArtTitle) # url curArtUrl = parsers_common.get(articlePostsDict["urls"], j) articleDataDict["urls"] = parsers_common.list_add( articleDataDict["urls"], j, curArtUrl) rss_print.print_debug( __file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2) return articleDataDict
def fill_article_dict(articleDataDict, pageTree, domain): maxArticleBodies = min(rss_config.REQUEST_ARTICLE_BODIES_MAX, 1) maxArticlePosts = round(rss_config.REQUEST_ARTICLE_POSTS_MAX / maxArticleBodies) # set 0 for all posts parentPages = {} parentPages["stamps"] = parsers_common.xpath_to( "list", pageTree, '//tbody/tr/th[@class="col-1"]/text()') parentPages["titles"] = parsers_common.xpath_to( "list", pageTree, '//tbody/tr/th[@class="col-7 teemapealkiri"]/a/text()') parentPages["urls"] = parsers_common.xpath_to( "list", pageTree, '//tbody/tr/th[@class="col-4"]/a/@href') # remove unwanted content: titles dictList = [ "Lõvide perekonna uus teema", "abort", "beebi", "ivf", "lapse", "rase ", "rased", "triibupüüdjad", ] parentPages = parsers_common.article_data_dict_clean( parentPages, dictList, "in", "titles") # teemade läbivaatamine for i in parsers_common.article_urls_range(parentPages["urls"]): # teemalehe sisu hankimine if parsers_common.should_get_article_body(i, maxArticleBodies): curParentUrl = parsers_common.get(parentPages["urls"], i) curParentUrl = curParentUrl.split("/#")[0] pageTree = parsers_common.get_article_tree( domain, curParentUrl, cache='cacheStamped', pageStamp=parentPages["stamps"][i]) articlePostsDict = {} articlePostsDict["descriptions"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="bbp-reply-content entry-content"]', parent=True) articlePostsDict["pubDates"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="post_date date updated"]/text()') articlePostsDict["urls"] = parsers_common.xpath_to( "list", pageTree, '//div[@class="bbp-reply-header entry-title"]/@id') # teema postituste läbivaatamine for j in parsers_common.article_posts_range( articlePostsDict["urls"], maxArticlePosts): # description curArtDesc = parsers_common.get( articlePostsDict["descriptions"], j) curArtDesc = curArtDesc.split( '<div class="gdrts-rating-block ')[0] curArtDesc = parsers_html.html_remove_single_parents( curArtDesc) articleDataDict["descriptions"] = parsers_common.list_add( articleDataDict["descriptions"], j, curArtDesc) # pubDates curArtPubDate = parsers_common.get( articlePostsDict["pubDates"], j) curArtPubDate = parsers_datetime.guess_datetime(curArtPubDate) articleDataDict["pubDates"] = parsers_common.list_add( articleDataDict["pubDates"], j, curArtPubDate) # title curArtTitle = parsers_common.get(parentPages["titles"], i) curArtTitle = parsers_common.str_title_at_domain( curArtTitle, domain) articleDataDict["titles"] = parsers_common.list_add( articleDataDict["titles"], j, curArtTitle) # url curArtUrl = parsers_common.get( parentPages["urls"], i) + "/#" + articlePostsDict["urls"][j] articleDataDict["urls"] = parsers_common.list_add( articleDataDict["urls"], j, curArtUrl) rss_print.print_debug( __file__, "teema " + str(i + 1) + " postitus nr. " + str(j + 1) + "/(" + str(len(articlePostsDict["urls"])) + ") on " + articlePostsDict["urls"][j], 2) # remove unwanted content: descriptions dictList = [ " liba " "Kommentaar eemaldatud.", "Liba?", ] articleDataDict = parsers_common.article_data_dict_clean( articleDataDict, dictList, "in", "descriptions") return articleDataDict