def info():
api = NewsApiClient(api_key='0c121bea533546759ad2551c94d28118')
api.get_sources()
api.get_everything(sources='bbc-news', q='women AND safety')
apiKey_list = [
'20d469827dbb4eb384d22490ea5df888', '75d16a33351a44969f3a5ac41eb7cf20',
'6496b9cb73c34054a8b58a3dee86c672', '5e6b9203fe4247369e70351f0ab2b1b3',
'3907a8165aec4be89b2e12f3a5ad541a', '79bbb20ec53e4d1b85c2caca76402488',
'388eff313e1a4d399d55ebb19d4db4cd', 'a0936894b7904a03a4c35ca6627ebc33',
'c15a4b03480c4081bd3d184bc8559f23', 'c05beec776fa4b1fbcc46bdad8efa951',
'be98dcb51dd64998ad08a6dd2c5f9e80', '376c9dfc704748279df3e6f30a751a1e',
'46660f56bd6e45f986fea91dc87b1fc1', '5dbf8944da394e4ca003b7fea5b736c5',
'31ba8f79f57d41c8b03d3334760154b3', '723db6ceb2e8465daffa882be629d6fb',
'1a88fcc99b0b41de902fcdbc45bd4a97', '06d039549c914c78a46d2c0c137b7f7c',
'1b47a4f26fc949c4ad280f9bfb81cd5d'
] # List of different API Keys
a, b, c = fetch_keyword_news(
[
"women", "women empowerment", "menstural cycle", "periods",
"girl power", "women in business", "health"
],
datetime.now(pytz.timezone('US/Eastern')).date(),
apiKey_list[4]) # List which will contain all the latest news inputs
print(a, b, c)
arr = [a[0:3], b[0:3], c[0:3]]
print(arr)
# session['final'] = final
return render_template('info.html', final=arr)
def getSourcesForCategory():
newsapi = NewsApiClient(api_key='546efd0c61864b23969558f0dd4cd2ed')
news_sources={}
if request.args.get('category'):
category=request.args.get('category')
news_sources=newsapi.get_sources(category=category, language="en", country="us")
else:
news_sources=newsapi.get_sources(language="en", country="us")
return jsonify(news_sources)
def index(request):
newsapi = NewsApiClient(api_key='2fdb18e9297a437ab91a7ec7669bed35')
try:
sources = newsapi.get_sources()
except Exception as e:
return HttpResponse("Error: " + str(e))
sources = newsapi.get_sources()
print(sources)
return JsonResponse(sources)
def fetch_source(category):
# init api client
newsapi = NewsApiClient(api_key='cf6de82e0a034204b073376d55103619')
# get corresponding sources based on category
if 'all' not in category:
sources = newsapi.get_sources(category=category, language='en', country = 'us')
else:
sources = newsapi.get_sources(language='en', country = 'us')
return sources
def get_sources(category):
newsapi = NewsApiClient(api_key="9a9be6bb99934ba68825728cba40674b")
if category == "all":
sources = newsapi.get_sources(language="en", country="us")
else:
sources = newsapi.get_sources(category=category,
language="en",
country="us")
if (len(sources["sources"]) < 10):
return jsonify(sources["sources"])
else:
return jsonify(sources["sources"][:10])
def api_client(api_key: str):
"""
Initializes a new api client and returns it, or None.
To detect the error, it performs a test query. If query is successful, returns client,
if query fails, returns None.
Query might fail because the key expired, or is invalid.
"""
newsapi = NewsApiClient(api_key=api_key)
# test query
try:
newsapi.get_sources()
return newsapi
except NewsAPIException:
return None
def get_sources():
category_now = request.args.get('category')
newsapi = NewsApiClient(api_key="34a75cc37de44563acd6cfd166285520")
all_sources = newsapi.get_sources(category=category_now, country='us', language='en')
return jsonify(all_sources)
def retrieve_articles_newsapi():
newsapi = NewsApiClient(api_key='2050df7a6a014501a04c5f42fa6eef54')
# /v2/top-headlines
top_headlines = newsapi.get_top_headlines(
q=
'sector OR big OR corporate OR product OR investor OR pointed OR gavekal OR sovereign OR vincent OR louis',
sources='bbc-news,the-verge',
language='en')
# /v2/everything
all_articles = newsapi.get_everything(
q=
'reality OR long OR central OR capital OR political OR dollars OR trading OR algorithmic OR banks OR released',
sources=
'bbc-news, the-verge, the-wall-street-journal, the-washington-post, the-hill',
domains=
'bbc.co.uk, techcrunch.com, ft.com, economist.com, wsj.com, thewashingtonpost.com',
from_param='2019-07-18',
to='2019-08-12',
language='en',
sort_by='relevancy')
# /v2/sources
sources = newsapi.get_sources()
for article in all_articles['articles']:
print(article)
print('\n')
def index(request):
newsapi = NewsApiClient(api_key=settings.API_KEY)
top_headlines = newsapi.get_top_headlines(
sources='bbc-news,the-verge,techcrunch')
pprint(top_headlines)
all_articles = newsapi.get_everything(
sources='bbc-news,the-verge,techcrunch')
# pprint(all_articles)
# /v2/sources
sources = newsapi.get_sources()
# pprint(sources)
# pprint(dir(newsapi))
l = top_headlines['articles']
desc = []
news = []
img = []
url = []
for i in range(len(l)):
f = l[i]
news.append(f['title'])
desc.append(f['description'])
img.append(f['urlToImage'])
url.append(f['url'])
mylist = zip(news, desc, img, url)
return render(request, 'index.html', context={"mylist": mylist})
class Crawler:
def __init__(self):
logging.basicConfig(level=logging.INFO)
self.logger = logging.getLogger()
f = open('server\\crawler\\key.txt', 'r')
key = eval(f.read())
self.newsapi = NewsApiClient(api_key=key['NewsAPIKey'])
def news_top_headlines(self, country): # HIDE API KEY.
try:
top_headlines = self.newsapi.get_top_headlines(country=country)
return top_headlines
except Exception as e:
self.logger.error('news_top_headlines failure: ' + str(e))
# outfile = open('data//topHeadlines//' + country + '.json', 'w') // PUT IN DB
# json.dump(data, outfile)
def news_categories(self, country, category):
try:
category_url = self.newsapi.get_top_headlines(country=country,
category=category)
return category_url
except Exception as e:
self.logger.error('news_categories failure: ' + str(e))
# INSERT IN DB
def news_sources(self, source, country):
try:
news_source_url = self.newsapi.get_top_headlines(country=country,
sources=source)
return news_source_url
# outfile = open(location + '//' + source + '.json', 'w') //INSERT in DB or cache
# json.dump(data, outfile)
except Exception as e:
self.logger.error('news_sources failure: ' + str(e))
def lang_sources(self, language, country):
try:
lang_url = self.newsapi.get_sources(
country=country,
language=language) # Gives all sources for the params.
return lang_url
except Exception as e:
self.logger.error('lang_sources failure: ' + str(e))
def keyword_based(self, keyword, language='en'):
try:
all_articles = self.newsapi.get_everything(q=keyword,
sort_by='relevancy',
language=language)
return all_articles
except Exception as e:
self.logger.error('keyword_based failure: ' + str(e))
def news_source_headlines(self, source):
try:
src_articles = self.newsapi.get_top_headlines(sources=source)
return src_articles
except Exception as e:
self.logger.error('news_source_headlines failure: ' + str(e))
def newsapi_client(stock, company):
"""
Using NewsApiClient object -
Note that NewsApiClient uses underscores in parameter names:
exclude_domains, sort_by, page_size, api_key
"""
# Init
newsapi = NewsApiClient(api_key=API_KEY)
# /v2/top-headlines
top_headlines = newsapi.get_top_headlines(q=stock,
sources='bbc-news,the-verge',
language='en')
save_data.save_json(top_headlines, "top_headlines")
# /v2/everything
everything = newsapi.get_everything(q=company,
from_param='2021-01-01',
to='2021-01-07',
language='en',
sort_by='relevancy',
page_size=3,
page=1)
save_data.save_json(everything, "everything")
# /v2/sources
sources = newsapi.get_sources(language="en")
save_data.save_json(sources, "sources")
def source():
'''
View source page function and returns the news source page and its data
'''
title = 'News Source'
newsapi = NewsApiClient(api_key='d967ec379de94378b37409a2aa549775')
sources = newsapi.get_sources()
sources = sources['sources']
name = []
description = []
url = []
for i in range(len(sources)):
mysources = sources[i]
name.append(mysources['name'])
description.append(mysources['description'])
url.append(mysources['url'])
mylist = zip(name, description, url)
return render_template('source.html', title = title, context = mylist)
class NewsAPI(Scraper):
"""docstring for NewsAPI."""
def __init__(self):
"""Initializer."""
self.newsapi = NewsApiClient(api_key=os.environ['NEWSAPI_KEY'])
def get_top_articles(self, n):
"""Return the top articles from the source."""
sources = self.newsapi.get_sources(country='us', category='general')
sources = sources['sources']
sources = [
'abc-news', 'al-jazeera-english', 'associated-press',
'breitbart-news', 'cbs-news', 'cnn', 'fox-news', 'msnbc'
',nbc-news', 'newsweek', 'politico', 'reuters',
'the-huffington-post', 'the-new-york-times', 'the-washington-post',
'the-washington-times', 'time', 'usa-today', 'vice-news'
]
# sources = [x['id'] for x in sources]
sources = str(reduce(lambda x, y: x + ',' + y, sources))
articles = self.newsapi.get_everything(sources=sources,
language='en',
page_size=n,
sort_by='popularity')
articles = articles['articles']
for a in articles:
a['categories'] = ['US']
a['source'] = a['source']['name']
return articles
def display_exam():
Subject = request.form['Subject']
if (str(Subject).isspace() or Subject == ''):
Subject = 'America'
print(Subject)
# Init
newsapi = NewsApiClient(api_key='a9288315ab104f7aa9564ca6d08b8ce4')
# /v2/top-headlines
top_headlines = newsapi.get_top_headlines(q=str(Subject), language='en')
# /v2/everything
all_articles = newsapi.get_everything(q=str(Subject), language='en')
# /v2/sources
sources = newsapi.get_sources()
#print(sources)
print(type(all_articles))
#print(top_headlines)
#print(newsapi)
return render_template('home.html',
sources=sources,
all_articles=json.dumps(all_articles),
top_headlines=top_headlines,
newsapi=newsapi)
def home():
from newsapi import NewsApiClient
newsapi = NewsApiClient(api_key='e442517dd5d3486b8e2fa0b9674edfd4')
articlelists,srcs,sourceid=[],[],[]
print('main')
today = str(dt.today())
sources = newsapi.get_sources()
sources = sources['sources']
for i in range(len(sources)):
srcs.append(sources[i]['name'])
sourceid.append(sources[i]['id'])
sources = zip(srcs,sourceid)
sourceids = ','.join(sourceid)
print(sourceids)
# for i in sourceid:
# print(i)
if request.method == "POST":
print("hmmm")
if request.form['search']:
searchterm = request.form['search']
articlelists = getnews(sourceids,today,today,searchterm = searchterm)
else:
articlelists = getnews(sourceids,today,today)
# for j in articlelist:
# articlelists.append(j)
# print(len(articlelists))
return render_template('news.html',sources=sources,articlelist=articlelists)
class FNSCrawler:
def __init__(self):
self.api_client = NewsApiClient(api_key=get_api_key())
def crawl_articles(self):
results = self.api_client.get_everything(q="deep learning",
language="en",
page_size=100)
print(results["totalResults"])
articles = results["articles"]
print(articles[0])
titles = [article["title"] for article in articles]
descriptions = [
article["description"] for article in articles
if article["description"] is not None
]
titles_text = "\n".join(titles)
descriptions_text = "\n".join(descriptions)
titles_text_model = markovify.Text(titles_text)
descriptions_text_model = markovify.Text(descriptions_text)
for _ in range(5):
print(
titles_text_model.make_short_sentence(max_chars=120,
min_chars=60))
print(descriptions_text_model.make_sentence())
def categorize_sources(self):
results = self.api_client.get_sources(category="technology",
language="en")
print(results["sources"])
def dashboard(request, *args, **kwargs):
newsapi = NewsApiClient(api_key='8d3f042c3715480e8737a046721b64db')
if request.method == 'GET':
snippets = newsapi.get_sources()
serializer = SnippetSerializer(snippets, many=True)
return Response(serializer.data)
# return render(request, "tech_sprint/index.html",sources)
def fetch_from_api():
newsapi = NewsApiClient(api_key=API_KEY)
response = newsapi.get_sources()
if response['status'] != 'ok':
raise Exception('Error querying news api', response)
return list(
map(
lambda x: Source(x['id'], x['name'], x['description'], x[
'url'], x['language'], x['country']), response['sources']))
def get_all_srcs(*args):
client = NewsApiClient(api_key=config["newsapikey"])
sources = defaultdict(list)
for arg in args:
sources[arg] = [
src['id'] for src in client.get_sources(
country=Newschannel.COUNTRY_CODE.get(arg, "us"))["sources"]
]
return sources
def details(self):
client = NewsApiClient(api_key=config["newsapikey"])
det = next(
filter(lambda x: x["id"] == self.id,
client.get_sources()["sources"]))
if len(det):
return det
return {}
class News:
def __init__(self):
self.newsapi = NewsApiClient(api_key=key)
def get_sources(self, addi):
l = self.newsapi.get_sources()['sources']
ret = []
for i in l:
if i["language"] == "en" and i["id"] not in addi:
ret.append(i["id"])
return ret
def write_sources(self):
l = self.newsapi.get_sources()['sources']
ret = []
for i in l:
if i["language"] == "en":
ret.append(i["id"])
r = open('../../sources', 'w')
for i in ret:
r.write(str(i) + '*')
def get_articles(self, sources):
while '' in sources:
sources.remove('')
if len(sources) == 0:
return []
all_articles = self.newsapi.get_everything(
sources=','.join(sources),
language='en',
sort_by='relevancy',
pageSize=20,
)
arts = []
print("NUMBER ARTICLES : " + str(len(all_articles['articles'])))
count = 1
for i in range(len(all_articles['articles'])):
print("count : " + str(count))
a = Article(all_articles['articles'][i])
arts.append(a)
count += 1
print("ARTS LEN :" + str(len(arts)))
return arts
def requestSource():
newsapi = NewsApiClient(api_key='c2174dfb402940cab6d1829c14225861')
#print("--------------------------")
#print(request.form['category'])
#print("--------------------------")
if request.form['category'] == "all":
sources = newsapi.get_sources(country='us', language='en')
else:
sources = newsapi.get_sources(category=request.form['category'],
country='us',
language='en')
result = []
for i in range(len(sources['sources'])):
result.append(sources['sources'][i]['id'])
return json.dumps(result)
def getSources():
newsAPI = NewsApiClient(api_key='c2db8aff33aa4821aa7dc71a66e8a4cd')
category = request.args.get('category')
if category is None or category.lower() == 'all':
sources = newsAPI.get_sources(language='en', country='us')
src = sources['sources'][:10]
source = {'status': 'ok', 'sources': src}
return Response(json.dumps(source),
status=200,
mimetype='application/json')
try:
sources = newsAPI.get_sources(category=category.lower(),
language='en',
country='us')
return Response(json.dumps(sources),
status=200,
mimetype='application/json')
except:
return Response(None, status=500)
class GoogleNewsStreamConnector(IStreamConnector):
def __init__(self, secrets_manager: GoogleNewsSecretsManager):
secrets = secrets_manager.get_secrets()
self._api_client = NewsApiClient(
api_key=secrets["GOOGLE_NEWS_API_KEY"])
self._REQUEST_INTERVAL = 60 * 5
self._PAGE_SIZE = 100
self._all_news_sources = None
self._last_download_time = None
self._last_urls: Set[str] = set()
def _retrieve_news_sources(self) -> str:
response = self._api_client.get_sources(language="en")
assert response["status"] == "ok"
all_news_sources = ",".join([s["id"] for s in response["sources"]])
return all_news_sources
def _get_article_time(self, article: Dict) -> datetime:
return datetime.strptime(article["publishedAt"], "%Y-%m-%dT%H:%M:%SZ")
def _search_top_stories(self):
if self._all_news_sources is None:
self._all_news_sources = self._retrieve_news_sources()
response = self._api_client.get_top_headlines(
sources=self._all_news_sources, page_size=self._PAGE_SIZE)
assert response["status"] == "ok"
self._last_download_time = datetime.utcnow()
new_articles = [
x for x in response["articles"] if x["url"] not in self._last_urls
]
self._last_urls = set([x["url"] for x in response["articles"]])
return new_articles
def _listen_top_stories(self):
# Free users can only retrieve a max of 100 results
# in that case we can download just the first page
# and increase the max size to 100. We do not have
# to iterate through pages.
while True:
if (self._last_download_time is not None and
(datetime.utcnow() - self._last_download_time).total_seconds()
< self._REQUEST_INTERVAL):
time.sleep(self._REQUEST_INTERVAL)
for article in self._search_top_stories():
yield article
def stream_comments(self) -> Iterator[Mention]:
for article in self._listen_top_stories():
yield create_gn_mention(article)
def get_news_sources():
if request.method == 'GET':
keys = ['id', 'name']
newsapi = NewsApiClient(api_key=NEWS_API_KEY)
request_object = {'language': 'en', 'country': 'us'}
if 'category' in request.args:
category = request.args['category']
request_object['category'] = category
if 'category' in request_object and request_object['category'] != 'all':
get_sources = newsapi.get_sources(
category=category,
language=request_object['language'],
country=request_object['country'])
else:
get_sources = newsapi.get_sources(
language=request_object['language'],
country=request_object['country'])
get_sources_filtered = transform_data(get_sources['sources'], keys)
return json.dumps({"data" : get_sources_filtered})
def sourcenews():
newsapi = NewsApiClient(api_key="10f35268a4d94987b67e28acc18319a2")
fcat = request.args['formcategory'].lower()
if(fcat == "all"):
top_headlines = newsapi.get_sources(language='en',country='us')
else:
top_headlines = newsapi.get_sources(category=fcat,language='en',country='us')
#top_headlines = newsapi.get_top_headlines(category=fcat,language='en',country='us')
articles = top_headlines['sources']
cid = []
cname = []
datadic = []
for i in range(len(articles)):
sourcec = articles[i]
cid.append(sourcec['id'])
cname.append(sourcec['name'])
for j in range(len(articles)):
if(cid[j]!=None and cid[j]!="" and cid[j]!="null" and cname[j]!=None and cname[j]!="" and cname[j]!="null"):
Dict = dict({'id':cid[j],'name':cname[j]})
datadic.append(Dict)
return jsonify(datadic[0:10])
def sports_news(request):
#logger.log("Entered sports news")
newsapi = NewsApiClient(api_key=NEWS_API_TOKEN)
sources = newsapi.get_sources()
top_headlines = newsapi.get_top_headlines(category='sports',language='en',country='in',)
news_status = top_headlines['status']
news_count = top_headlines['totalResults']
news_list = top_headlines['articles']
for i in range(len(news_list)):
news_list[i]['publishedAt'] = news_list[i]['publishedAt'][:10]+' at '+news_list[i]['publishedAt'][12:-1]
#logger.log("Rendering sports news")
return render(request, 'inapp/sports.html', {'news_list':news_list, 'news_count':news_count, 'news_status':news_status})
def test_connect_to_api():
'''Confirm connection to the NewsAPI.
Returns
-------
True if API sources retrieved, else raises a ConnectionError
'''
news_api = NewsApiClient(NEWS_API_KEY)
print('Getting all NewsAPI sources.')
response = news_api.get_sources() # noqa: F841
assert response['status'] == 'ok'
def get_sources(Category):
'''
Function that gets the json response to our url request
'''
newsapi = NewsApiClient(api_key=key)
news_sources_response = newsapi.get_sources(category=Category,
language='en')
if news_sources_response['sources']:
news_results_list = news_sources_response['sources']
news_results = process_results(news_results_list)
return news_results
def get_all_category():
# def list_of_filters(params, **filters):
api_key = 'c6b4dff59361415dada7968f05685325'
newsapi = NewsApiClient(api_key)
all_sources = newsapi.get_sources()
sources = all_sources['sources']
selections = set()
for news_number in range(0, len(all_sources['sources'])):
selections.add(sources[news_number]['category'])
print(str(list(selections)))
# for news_number in range(0, len(selections)):
# print(str(news_number+1)+'. '+list(selections)[news_number])
print('0. Все')
return list(selections)
