def __init__(self, search_type): user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:24.0) Gecko/20100101 Firefox/24.0' self.config = Config() self.config.browser_user_agent = user_agent self.config.request_timeout = 120 self.search_type = search_type self.prepare_input()
def home(request): main_url = "http://newsapi.org/v2/top-headlines?country=in&pageSize=17&apiKey=813ecab326254348a7d1e8ef73d2838f" # fetching data in json format open_news_page = requests.get(main_url).json() # getting all articles in a string article article = open_news_page["articles"] results = [] user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36' config = Config() config.browser_user_agent = user_agent for ar in article: article = Articles() intl = [] intl.append(ar["title"]) intl.append(ar["description"]) intl.append(ar["urlToImage"]) intl.append(ar["url"]) toi_article = Article(intl[3], language="en", config=config) toi_article.download() #To parse the article toi_article.parse() #To perform natural language processing ie..nlp toi_article.nlp() summ = toi_article.summary intl.append(summ) results.append(intl) content = {'article': results} return render(request, 'summnews/index.html', content)
def handle(task, progress): url = task.url progress.set_status("Requesting page...") resp = http_downloader.page_text(url, json=False) if not resp: return False config = Config() config.memoize_articles = False config.verbose = False article = Article(url='', config=config) article.download() article.set_html(resp) article.parse() if not article.top_image: return None src = article.top_image if 'http' not in src: if 'https' in url: src = 'https://' + src.lstrip('/ ').strip() else: src = 'http://' + src.lstrip('/ ').strip() progress.set_status("Downloading image...") return http_downloader.download_binary(src, task.file, prog=progress, handler_id=tag)
def _config(): config = Config() config.fetch_images = False config.memoize_articles = False config.request_timeout = 10 config.language = 'en' return config
def __init__(self, url): self.url = url self.config = Config() self.config.memoize_articles = False #self.config.fetch_images = False self.config.browser_user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.35 Safari/537.36' self.config.request_timeout = 30 #logger.info('url is %s', url) self.article = Article(url, config=self.config) self._import_article() self.results = {} self.results['sourceurl'] = url try: self.results['title'] = self.article.title self.results['authors'] = self.article.authors self.results['canonical_link'] = self.article.canonical_link self.results['url'] = self.article.url self.results['top_image'] = self.article.top_image self.results['images'] = list(self.article.images) self.results['movies'] = self.article.movies self.results['description'] = self.article.meta_description #self.results['meta_favicon'] = self.article.meta_favicon self.results['keywords'] = self.article.meta_keywords self.results['lang'] = self.article.meta_lang self.results['summary'] = self.article.summary self.results['tags'] = list(self.article.tags) self.results['text'] = self.article.text except Exception: logger.fatal('Unable to make results for:%s, %s', url, results) pass
def get_content_dates(link): global keywords global dates config = Config() config.browser_user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' config.memoize_articles = True config.fetch_images = False config.https_success_only = False article = Article(url=link, config=config) sleep(0.1) article.download() if article.download_state == 1: #has the download failed? eg. due to 403 print(link) print("\033[101m\033[1m\033[!] Download failed\033[0m") return None else: article.parse() article.nlp() if (str(article.publish_date).split(" "))[0] not in dates: return None else: if relevancy_check(link, article.text, article.keywords) == False: return None else: if len(article.text) < 200: #too short? return None else: print(link) print(article.title) print(article.publish_date) print("=" * 80) keywords = keywords + article.keywords return (article.text + "\n=====DATE:" + str(article.publish_date))
def retrieveUrlText(url): try: config = Config() config.request_timeout = 1000 config.memoize_articles = False config.fetch_images = False config.browser_user_agent = 'Mozilla/5.0' article = Article(url, config) article.download(recursion_counter=5) if article.download_state != 2: return '' article.parse() articleText = article.text.replace('\n', ' ') except KeyboardInterrupt: raise except Exception: return '' punkt_param = PunktParameters() punkt_param.abbrev_types = set([ 'dr', 'vs', 'mr', 'mrs', 'prof', 'inc', 'et', 'al', 'fig', 'figs', 'chem', 'ph' ]) sentence_splitter = PunktSentenceTokenizer(punkt_param) articleSentences = validateSentences( sentence_splitter.tokenize(articleText)) return articleSentences
def get_text(url): """Given URL of the article extract text from it Arguments: url {str} -- url of the article Returns: str -- Text of thr article """ global CNT global NAN if (CNT % 100 == 0): print(CNT) config = Config() config.keep_article_html = True try: article = Article(url, config=config) article.download() article.parse() return article.text except Exception as e: print(e) NAN += 1 return np.nan finally: CNT += 1
def fetch_by_newspaper(url): is_linkedin_url = url.startswith(linkedinUrl) if is_linkedin_url: config = Config() config.MAX_TITLE = 1000 article = get_article(url, config) article = replace_title_text_from_title_url(article) else: article = get_article(url) return json.dumps({ "authors": article.authors, "html": article.html, "images:": list(article.images), "movies": article.movies, "publish_date": article.publish_date.strftime("%s") if article.publish_date else None, "text": article.text, "title": article.title, "topimage": article.top_image }), 200, { 'Content-Type': 'application/json' }
def filterNews(url, keyword): """ 对新闻进行筛选,通过关键词,以新闻标题为对象筛查 通过user_agent和timeout来解决访问超时问题 https://stackoverflow.com/questions/63061172/newspaper3k-api-article-download-failed-with-httpsconnectionpool-port-443-read :param url: :param keyword: :return: """ user_agent = r"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " r"Chrome/87.0.4280.67 Safari/537.36 Edg/87.0.664.47 " config = Config() config.browser_user_agent = user_agent config.request_timeout = 30 # Try-Except-Continue will skip to the next article in the For loop if there is an exception # https://github.com/codelucas/newspaper/issues/444 solve the bugs for not found url try: news = newspaper.Article(url, config=config) news.download() news.parse() for word in keyword: if word in news.title: # print(word) return True return False except ArticleException: return False
def cache_art(url): headers = { 'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0", 'From': '*****@*****.**' } r = requests.get(url, headers=headers) from newspaper import Config from newspaper import Article config = Config() config.browser_user_agent = headers['User-Agent'] article = Article(url, keep_article_html=True, config=config) article.download() # article = Article(url='') # article.set_html(str(soup)) article.parse() struct = { 'html': article.article_html, 'author': article.authors, 'pubdate': article.publish_date, 'text': article.text, 'image': article.top_image, 'rawhtml': str(r.content), 'buzotext': scrape(r.content, 'html5lib') } return struct
def get_top_news(news_information, todays_date): load_dotenv(find_dotenv(r"path to env variable")) news_api_key = os.getenv("news_api") params = ( ('country', 'us'), ('apiKey', f'{news_api_key}'), ) response = requests.get( """https://newsapi.org/v2/top-headlines""", params=params) news_articles = response.json()['articles'] for i in news_articles: user_agent = """Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:78.0) Gecko/20100101 Firefox/78.0""" config = Config() config.browser_user_agent = user_agent config.request_timeout = 10 article = Article(i['url'], config=config) print(i["url"]) article.download() article.parse() article.nlp() summary = article.summary # noqa news_information[i["source"]["name"]] = {"title": i["title"], "summary": summary, "url": i["url"], "date": f"{todays_date}"} # noqa
def get_article_text(url: str) -> str: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.54' } text = '' try: config = Config() config.browser_user_agent = headers['User-Agent'] article = Article(url, keep_article_html=True, fetch_images=False, config=config) article.download() article.parse() # article.nlp() text = article.text meta = { "source": "web_article", "source_url": article.source_url, "article_html": article.article_html, "title": article.title, "top_image": article.top_image, "images": article.images, "videos": article.movies, "meta_language": article.meta_lang, "meta_keywords": article.meta_keywords, "authors": article.authors, "publish_date": article.publish_date, } text = clean(text) except Exception as e: msg = f"Error using newspaper3k extractor: {str(e)}" print(msg) return text
def download_article(url): config = Config() config.follow_meta_refresh = True config.keep_article_html = True article = Article(url, config=config) article.download() return article
def getPolarity(uniName): from GoogleNews import GoogleNews from newspaper import Article from newspaper import Config import pandas as pd from textblob import TextBlob uniName = uniName + ' Coronavirus' user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36' config = Config() config.browser_user_agent = user_agent googlenews = GoogleNews(start='08/01/2020', end='09/26/2020') googlenews.search(uniName) result = googlenews.result() for i in range(0, 5): googlenews.getpage(i) result = googlenews.result() df = pd.DataFrame(result) sum = 0 counter = 1 for ind in df.index: try: article = Article(df['link'][ind], config=config) article.download() article.parse() article.nlp() testimonial = TextBlob(article.summary) counter += 1 sum += testimonial.sentiment.polarity except: pass return sum / counter
def cache_art(url): headers = { 'User-Agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0", 'From': '*****@*****.**' } from newspaper import Config from newspaper import Article config = Config() config.browser_user_agent = headers['User-Agent'] article = Article(url, keep_article_html=True, config=config) article.download() article.parse() struct = { 'html': article.article_html, 'link': url, 'author': article.authors, 'pubdate': article.publish_date, 'text': article.text, 'title': article.title, 'image': article.top_image, 'description': article.meta_description } return struct
def __init__(self, output_dir, dataframe, num_rows=None, user_agent=default_user_agent) -> None: self.output_dir = output_dir temp_df = dataframe if num_rows: self.df = temp_df.iloc[:num_rows] else: self.df = temp_df self.user_agent = user_agent self.config = Config() self.config.browser_user_agent = user_agent self.download_logs_dict = { "link": [], "download_state": [], "download_exception_msg": [] } self.number_of_downloaded = 0 self.preprocess_df()
def extract(self, request: ExtractorRequest) -> ExtractorResponse: try: user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" config = Config() config.browser_user_agent = user_agent article = Article( request.url, keep_article_html=True, fetch_images=False, config=config ) article.download() article.parse() # article.nlp() text = article.text meta = { "source": "web_article", "source_url": article.source_url, "article_html": article.article_html, "title": article.title, "top_image": article.top_image, "images": article.images, "videos": article.movies, "meta_language": article.meta_lang, "meta_keywords": article.meta_keywords, "authors": article.authors, "publish_date": article.publish_date, } # construct response response = ExtractorResponse(meta=meta, text=text or "") except Exception as e: msg = f"Error using newspaper3k extractor: {str(e)}" log.error(msg) response = ExtractorResponse(error=msg) return response
def process_link(self, link=None, nlp=False): """processes the linksobtain by get_links() method, extracts both the text and a summary of the article with newspaper package Args: link :: str URL of links stored in the dictionary returned by get_links() nlp :: bool Whether or not to perform nlp on the text of the link. This extracts a summary of the text, but is a somewhat expensive operation. Returns: article :: 'article' object object that contains parsed properties for the link, such as summary, text and date. """ #parameters for the processing config = Config() config.fetch_images = False #no need for images config.memoize_articles = False #no need for article caching try: article = Article(link, language="en", config=config) article.download() article.parse() if nlp: article.nlp() #extract summary as per the newspaper API except: return None return article
def subjectivity(): """ Graphs the posts based on the relationship between subjectivity and popularity :return: a graph with the x axis being the subjectivity value, and y axis being upvotes """ scores = [] for index, row in topics_data.iterrows(): if index in actual_list: scores.append(row['score']) subs = [] for index, row in topics_data.iterrows(): if index in actual_list: url = row['url'] if 'newsweek' or 'democracynow' in url: user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36' config = Config() config.browser_user_agent = user_agent article = Article(url, config=config) else: article = Article(url) article.download() article.parse() article.nlp() text = article.summary obj = TextBlob(text) subjectivity = obj.sentiment.subjectivity subs.append(subjectivity) plt.figure(figsize=(50, 10)) plt.scatter(subs, scores) plt.xlabel('Subjectivity') plt.ylabel('Score') plt.title('Posts in r/politics') plt.show()
def fetch_and_format(url, fetch_img=True): cfg = Config() cfg.keep_article_html = True cfg.browser_user_agent = USERAGENT cfg.drop_text_node = lambda x: x in ('', 'Advertisement', 'advertisement') def transform_img(i): try: i.attrib['src'] = fetch_image_to_b64(i.attrib['src']) except Exception as e: # print(e) pass if fetch_img: cfg.element_transformers['img'] = transform_img def transform_picture(i): try: img = i.find('img') if img is not None: i.tag = 'img' for k, v in img.items(): i.attrib[k] = v if 'src' in i.attrib: i.attrib['src'] = fetch_image_to_b64(i.attrib['src']) except Exception as e: # print(e) pass if fetch_img: cfg.element_transformers['picture'] = transform_picture art = Article(url, config=cfg) art.download() art.parse() title = decode_to_entity(art.title) # TODO: Newspaper3k parses the same author multiple times. Fix! author = art.authors[0] if art.authors else '' publish_date = art.publish_date.strftime( '%B %d %Y') if art.publish_date else '' source = tldextract.extract( url).registered_domain # Is there a better source for this? text = art.article_html doc = f'''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"><html> <head> <title>{title}</title> <META http-equiv="Content-Type" content="text/html; charset=utf-8"> <META name="author" content="{author}"> <META name="title" content="{title}"> </head> <body><div> <h1>{title}</h1> <h3>{author}</h3> <h4><a href='{url}'>{source}</a></h4> <h4>{publish_date}</h4> {text} </body> </html>''' return title, doc
def GetArticle(FeedLink, request): user_agent = request.META['HTTP_USER_AGENT'] config = Config() config.browser_user_agent = user_agent newarticle = Article(FeedLink, config=config) newarticle.download() newarticle.parse() return newarticle
def _getArticleObj(self): config = Config() config.keep_article_html = True article = Article(self.url, config=config) article.download() article.parse() article.nlp() return article
def make_news_table(db, articles, logger): class News(db.Model): uuid = db.Column(UUIDType(), primary_key=True) url = db.Column(db.Text, nullable=False) title = db.Column(db.Text, nullable=False) picture = db.Column(URLType, nullable=True) icon = db.Column(URLType, nullable=True) body = db.Column(db.Text, nullable=False) publish_time = db.Column(db.DateTime, nullable=False) __tablename__ = 'News' def __repr__(self): return f'<News: {self.uuid}>' db.create_all() db.session.commit() urls = tuple( set(articles) - set(a[0] for a in News.query.with_entities(News.url).all())) if len(urls) > 0: from newspaper import Article, Config from newspaper.mthreading import NewsPool from uuid import uuid4 config = Config() config.memoize_articles = False config.fetch_images = True config.request_timeout = 20 config.thread_request_timeout = 20 config.follow_meta_refresh = True config.MIN_WORD_COUNT = 150 config.MIN_SENT_COUNT = 4 config.browser_user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' news_pool = NewsPool(config=config) articles = tuple(Article(url, config=config) for url in urls) news_pool.set(articles, override_threads=4) logger.info(f"Articles being downloaded: {urls}") news_pool.join() for article in articles: logger.debug(f"Processing: {article.url}") if article.download_state == 1: logger.warn(f'Failed to download: {article.url}') continue article.parse() try: db.session.add( News(uuid=uuid4(), url=article.url, title=article.title, picture=article.top_image, icon=article.meta_favicon, body=article.text, publish_time=article.publish_date)) except Exception as e: logger.warn(e) db.session.commit() return News
def parse_submission(url): ''' Input: HTTP URL Ouput: List of keywords compiled from names found in the article title and keywords found in the article text ''' user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36' config = Config() config.browser_user_agent = user_agent a = Article(url, config=config) try: a.download() a.parse() except Exception as e: logging.error(e) return a.nlp() keywords = a.keywords title = a.title content = a.text thumbnail_image = get_thumbnail_image(a.html) #clean up content content = to_lower(content) content = strip_numbers(content) #clean up title title = to_lower(title) title = strip_numbers(title) #get names and identities from content text content_names = get_names(''.join(content)) top_keywords = [] for name in content_names: if name in title: #if the name is in the title, it is significant top_keywords.append(name) #Find other keywords that are not names and are found in the title for keyword in keywords: if keyword in title and keyword not in stop_words and not keyword.isdigit( ): top_keywords.append(keyword) if not top_keywords: top_keywords = title.split(" ") else: #remove duplicates top_keywords = list(dict.fromkeys(top_keywords)) top_keywords = remove_duplicates(top_keywords) return { 'keywords': top_keywords, 'title': a.title, 'thumbnail_url': thumbnail_image }
def load_and_parse_full_article_text_and_image(url: str) -> Article: config = Config() config.MAX_SUMMARY_SENT = 8 article = Article(url, config=config) article.set_html(load_page_safe(url)) # safer than article.download() article.parse() return article
def handle(url, data, log): try: log.out(0, 'Downloading article...') resp = requests.get(url, headers={'User-Agent': data['user_agent']}) if resp.status_code != 200: return False #!cover config = Config() config.memoize_articles = False config.verbose = False article = Article(url='', config=config) log.out(0, 'Parsing article...') article.download() article.set_html(resp.text) article.parse() if article.top_image: src = article.top_image if 'http' not in src: #!cover if 'https' in url: src = 'https://' + src.lstrip('/ ').strip() else: src = 'http://' + src.lstrip('/ ').strip() log.out(0, 'Newspaper located image: %s' % src) r = requests.get(src, headers={'User-Agent': data['user_agent']}, stream=True) if r.status_code == 200: content_type = r.headers['content-type'] ext = mimetypes.guess_extension(content_type) if not ext or ext == '': #!cover log.out( 1, 'NewsPaper Error locating file MIME Type: %s' % url) return False if '.jp' in ext: ext = '.jpg' #!cover path = data['single_file'] % ext if not os.path.isfile(path): if not os.path.isdir(data['parent_dir']): #!cover log.out(1, ("+Building dir: %s" % data['parent_dir'])) os.makedirs( data['parent_dir'] ) # Parent dir for the full filepath is supplied already. with open(path, 'wb') as f: r.raw.decode_content = True shutil.copyfileobj(r.raw, f) return path else: #!cover log.out( 0, ('\t\tError Reading Image: %s responded with code %i!' % (url, r.status_code))) return False except Exception as e: log.out(0, ('"Newspaper" Generic handler failed. ' + (str(e).strip()))) return False #!cover
def get_article(url): config = Config() config.request_timeout = 1.3 article = Article(url, language='zh', config=config) try: article.download() article.parse() # article.nlp() except Exception: return return article.text
def getConf(language, parser): ''' get newspaper Config object :return: ''' conf = Config() conf.fetch_images = False conf.memoize_articles = False conf.set_language(language) conf.parser_class = parser return conf
def parse(self, response): newsItem = CNNItem() newsItem['title'] = uniToAscii( response.selector.xpath( '//meta[@property="og:title"]/@content').extract_first()) newsItem['author'] = uniToAscii( response.selector.xpath( '//meta[@name="author"]/@content').extract_first()) newsItem['link'] = response.url print '^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', newsItem[ 'title'] text = "" pars = response.selector.xpath( '//*[@class="zn-body__paragraph"]//text()').extract() for par in pars: par_text = uniToAscii(par) if par_text[-1:] != '.': par_text += '.' text = text + par_text + " " newsItem['text'] = text keywords = uniToAscii( response.xpath( "//meta[@name='keywords']/@content").extract_first()) keywords_list = keywords.split(',') newsItem['keywords'] = [word.strip() for word in keywords_list] print '^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', newsItem[ 'title'], ' ', len(pars), ' ', newsItem['keywords'] if newsItem['text']: print newsItem['text'][:100] yield newsItem config = Config() config.fetch_images = False config.memoize_articles = False url_build = newspaper.build(url=response.url, config=config) print response.url storyLinks = [article_url for article_url in url_build.article_urls()] print storyLinks for link in storyLinks: if re.match('.*/2017/03/0.*\.html', str(link)): print 'Fetching from ', str(link) yield scrapy.Request(str(link))