def listOfSources():
s = Sources(API_KEY="40e40820d389493abb369f099605fec3")
a = s.all()
sourceList = []
for source in a['sources']:
sourceList.append(source['id'])
return sourceList
def news():
#news = 'this is awesome'
from newsapi.sources import Sources
s = Sources(API_KEY="6e8a402cd42f454c80e1377a86cada80")
print s.search('cnn')
#analysis = TextBlob(news)
#print s.search('Obama')
from pws import Bing
URL = 'https://www.google.com/search?pz=1&cf=all&ned=us&hl=en&tbm=nws&gl=us&as_q={query}'
def __init__(self, *args):
super(ReporterModule, self).__init__(*args)
self.API_KEY = self.get_configuration("newsapi.org_key")
self.threshold = int(self.get_configuration("news_limit"))
if self.API_KEY:
self.articles = Articles(self.API_KEY)
self.sources = Sources(self.API_KEY)
else:
print(_("error.news.configuration"))
return False
self.sources_url = {}
self.sources.information()
class Reporter:
def __init__(self, API_KEY):
self.articles = Articles(API_KEY)
self.sources = Sources(API_KEY)
self.sources.information()
def get_all_categories(self):
categories = list(self.sources.all_categories())
return categories
def get_all_categories(self):
categories = list(self.sources.all_categories())
return categories
def __init__(self, *args):
super(ReporterModule, self).__init__(*args)
self.API_KEY = self.get_configuration("newsapi.org_key")
self.threshold = int(self.get_configuration("news_limit"))
if self.API_KEY:
self.articles = Articles(self.API_KEY)
self.sources = Sources(self.API_KEY)
else:
print(
"Kindly look back at the documentation to configure news module properly especially the API keys."
)
return False
self.sources_url = {}
self.sources.information()
class Scraper:
# example code
# -----------------------
# x = Scraper(api_key='xyz')
# print(x.scrape_all_articles(language='en'))
articles = None
sources = None
api_key = None
def __init__(self, api_key) -> None:
super().__init__()
self.api_key = api_key
self.articles = Articles(API_KEY=self.api_key)
self.sources = Sources(API_KEY=self.api_key)
def scrape_articles_for_sources(self, sources):
'''
Accepts the list of source names and returns all articles downloaded from the given sources
:param sources: List of source id's
:return: List of article json objects, containing:
'author', 'title', 'description', 'url', 'urlToImage', 'publishedAt'
'''
articles = []
for source in sources:
try:
# list of json objects
# author, title, description, url, urlToImage, publishedAt
articles_for_source = self.articles.get(source=source).articles
except BaseException: # if the server does not respond
continue
for article in articles_for_source:
articles.append(article)
return articles
def scrape_sources(self, categories=[], language=None):
'''
Gets the newsapi sources associated with the given category (optional) and language (optional)
:param categories: List of categories (optional)
:param language: Language (optional)
:return: List of source id's
'''
sources_dict = []
for category in categories:
sources_dict += self.sources.get(category, language).sources
sources = set([source['id'] for source in sources_dict])
return sources
def scrape_all_articles(self, categories=[], language=None):
'''
Scrapes and returns all articles for the given category and language (parameters are optional)
:param categories: list of categories (optional)
:param language: language (optional)
:return: List of article json objects, containing:
'author', 'title', 'description', 'url', 'urlToImage', 'publishedAt'
'''
return self.scrape_articles_for_sources(
self.scrape_sources(categories, language))
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 03 10:57:46 2017
@author: Sarang
"""
from newsapi.articles import Articles
from newsapi.sources import Sources
f = open("API_KEY.txt")
api_key = f.read()
a = Articles(api_key)
s = Sources(api_key)
print a
#print s.get(category='technology', language='en', country='uk')
import requests
r = requests.get(
'https://newsapi.org/v1/articles?source=the-next-web&sortBy=latest&apiKey=153cffe401b84aa8ab8f19d01a354747'
)
print r.text
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import unicodedata
import math
import h5py
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.python.framework import ops
key = '96af62a035db45bda517a9ca62a25ac3'
a = Articles(API_KEY=key)
s = Sources(API_KEY=key)
class APIKeyException(Exception):
def __init__(self, message):
self.message = message
class InvalidQueryException(Exception):
def __init__(self, message):
self.message = message
class ArchiveAPI(object):
def __init__(self, key=None):
self.key = key
import json
import os
import re
import requests
from newsapi.articles import Articles
from newsapi.sources import Sources
API_KEY = "f044f5b63a7c4139858611a1ae6dc5f0"
s = Sources(API_KEY=API_KEY)
a = Articles(API_KEY=API_KEY)
# print(s.information().all_categories())
# print(s.get_by_category("general"))
def get_country_news(country):
country_id = country + "&"
url = ('https://newsapi.org/v2/top-headlines?'
'country=' + country_id + 'apiKey=' + API_KEY)
response = requests.get(url)
response = response.json()
path = os.path.join(os.getcwd(), "posts")
path = os.path.join(path, "regional_news")
path = os.path.join(path, country)
for main_key in response.items():
if main_key[0] == "articles":
from newsapi.articles import Articles
from newsapi.sources import Sources
key = '96af62a035db45bda517a9ca62a25ac3'
a, s = Articles(API_KEY=key), Sources(API_KEY=key)
s.all() # get all sources offered by newsapi
a.get(source='the-new-york-times')
s.get(category='technology', language='en', country='US')
from newsapi import NewsAPI
key = '96af62a035db45bda517a9ca62a25ac3'
params = {}
api = NewsAPI(key)
sources = api.sources(params)
articles = api.articles(sources[0]['id'], params)
################ NY Times API #############################################
import sys, csv, json
reload(sys)
sys.setdefaultencoding('utf8')
"""
About:
Python wrapper for the New York Times Archive API
https://developer.nytimes.com/article_search_v2.json
"""
class APIKeyException(Exception):
def __init__(self, message):
def __init__(self, api_key=keys.news['api_key']):
self.api_key = api_key
self.article = Articles(self.api_key)
self.source = Sources(self.api_key)
self.base_url = keys.news['base_everything_url']
self.logger = logutils.get_logger('News Data Ingestion')
from flask import Flask, jsonify, render_template, request, session, flash, redirect,abort
from newsapi.articles import Articles
from newsapi.sources import Sources
#from flask.ext.socketio import SocketIO, emit
from sqlalchemy.orm import sessionmaker
import os
from tabledef import *
engine = create_engine('sqlite:///database.db', echo=True)
app = Flask(__name__)
a = Articles(API_KEY="867af1dffb80450b9770b4bcc10c8e14")
s = Sources(API_KEY="867af1dffb80450b9770b4bcc10c8e14")
"""app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app)
@socketio.on('my event') # Decorator to catch an event called "my event":
def test_message(message): # test_message() is the event callback function.
emit('my response', {'data': 'got it!'}) # Trigger a new event called "my response"
""" # that can be caught by another callback later in the program.
@app.route("/")
def home():
if session.get('logged_in'):
return render_template("Welcome.html")
else:
return render_template("login.html")
@app.route('/login',methods=["POST"])
def do_admin_login():
POST_USERNAME = str(request.form['username'])
import newsapi
import requests
import json
import os
from newsapi.articles import Articles
from newsapi.sources import Sources
a = Articles(API_KEY="537b165a4f314fedae8cb39788d4d713")
s = Sources(API_KEY="537b165a4f314fedae8cb39788d4d713")
res = a.get(source="daily-mail")['articles']
bbc = a.get(source="bbc-news")['articles']
telegraph = a.get(source="the-telegraph")['articles']
guardian = a.get(source="the-guardian-uk")['articles']
independent = a.get(source="independent")['articles']
sports = a.get(source="the-sport-bible")['articles']
# results = s.get_by_country("gb").sources
# # s.get_by_category("politics")
#resultsString = ''.join(str(e) for e in results)
# filename = 'news_stream.py'
# with open(filename, 'a') as file:
# for result in independent:
# print(result['title'])
# # If you want other things from the tweet object you can specify it here
# file.write(result['title'] + os.linesep)
class ReporterModule(BaseModule):
AFFIRMATIVE = ["YES", "YEAH", "SURE", "YAH", "YA"]
NEGATIVE = ["NO", "NEGATIVE", "NAH", "NA", "NOPE"]
def __init__(self, *args):
super(ReporterModule, self).__init__(*args)
self.API_KEY = self.get_configuration("newsapi.org_key")
self.threshold = int(self.get_configuration("news_limit"))
if self.API_KEY:
self.articles = Articles(self.API_KEY)
self.sources = Sources(self.API_KEY)
else:
print(
"Kindly look back at the documentation to configure news module properly especially the API keys."
)
return False
self.sources_url = {}
self.sources.information()
def get_all_categories(self):
return list(self.sources.all_categories())
def get_by_category(self, category):
srcs = self.sources.get_by_category(category).sources
self.sources_url = {}
for src in srcs:
self.sources_url[src['name']] = src['url']
return self.sources_url
def get_sort_bys_of_source(self, source_name):
return self.sources.search(source_name)[0]['sortBysAvailable']
def all_sources(self):
self.sources_url = self.sources.all_names()
return self.sources_url
def get_news(self):
self.assistant.say(
"Would you prefer any specific category? If yes then what would it be?"
)
category_status = self.assistant.listen().decipher()
if category_status.upper() in self.NEGATIVE:
category = False
else:
categories = self.get_all_categories()
category = self.search(categories, category_status)
self.assistant.say(
"Any preference you would like to have about source of your news? like CNN"
"or Time magazine or maybe The hindu?")
source_status = self.assistant.listen().decipher()
if source_status.upper() in self.NEGATIVE:
source = False
else:
if category:
sources_available = self.get_by_category(category)
response = "Out of all the sources as follows"
for source_name, source_url in sources_available.items():
response += " %s," % source_name
response += ", which one would you like to pick?"
self.assistant.say(response)
source_command = self.assistant.listen().decipher()
source = self.search(list(sources_available), source_command)
else:
self.assistant.say(
"So would you want me to list all the sources around 70 which to be"
"honest would be a hefty task, so if not, then just let me know of"
"your source name and I would let you know if it's available or not."
)
all_sources_status = self.assistant.listen().decipher()
sources_available = self.all_sources()
if all_sources_status.upper() in self.AFFIRMATIVE:
response = "Good job, lazy ass, so here are all the available sources as follows "
sources_available_list = list(sources_available)
for source_name in sources_available_list:
response += " %s," % source_name
response += ", which one would you like to pick?"
self.assistant.say(response)
source_command = self.assistant.listen().decipher()
all_sources_status = source_command
source_found = self.search(list(sources_available),
all_sources_status)
source = source_found
if source:
sort_bys_available = self.get_sort_bys_of_source(source)
if len(sort_bys_available) == 1:
sort_by = sort_bys_available[0]
else:
if len(sort_bys_available) == 2:
response = "And what kind of news sort would you like? " \
"%s or %s?" % (sort_bys_available[0], sort_bys_available[1])
else:
response = "And what kind of news sort would you like? " \
"%s or %s, or maybe %s?" % (sort_bys_available[0],
sort_bys_available[1],
sort_bys_available[2])
self.assistant.say(response)
sort_by_command = self.assistant.listen().decipher()
sort_by = self.search(sort_bys_available, sort_by_command)
else:
self.assistant.say("And what kind of news sort would you like?"
"latest or maybe top ones shown in front page?")
sort_status_command = self.assistant.listen().decipher()
sort_by = self.search(['top', 'popular'
'latest'], sort_status_command)
if not source:
if sort_by.lower() == "top":
source = "google-news"
elif sort_by.lower() == "latest":
source = "the-telegraph"
else:
source = "time"
response = self.get_response(source, sort_by)
return response
def handle(self):
source = self.get_configuration("news_source")
response = self.get_response(source)
return response
def get_response(self, source, sort_by=None, threshold=5):
if self.threshold:
threshold = self.threshold
source = source.lower().replace(" ", "-")
articles = self.articles.get(source, sort_by=sort_by).articles
articles = articles[:threshold]
response = "So the %s news from %s news source are as follows " % (
sort_by, source)
for article in articles:
if article['title']:
response += "%s, " % article['title']
if article['description']:
response += "%s, " % article['description']
if article['author']:
response += "was reported by %s." % article['author']
response += "and in the other news. "
return response
@staticmethod
def search(dataset, query):
values = [0 for _ in range(0, len(dataset))]
search = query.lower().split()
upper_threshold = len(search)
for index, data in enumerate(dataset):
search_array = data.split()
for index2, text in enumerate(search_array):
if index2 >= upper_threshold:
break
threshold = len(search[index2])
for i in range(0, len(text)):
if i >= threshold - 1:
break
if text[i] == search[index2][i]:
values[index] += 1
max_value = max(values)
max_index = values.index(max_value)
return dataset[max_index]
class ReporterModule(BaseModule):
AFFIRMATIVE = ["YES", "YEAH", "SURE", "YAH", "YA"]
NEGATIVE = ["NO", "NEGATIVE", "NAH", "NA", "NOPE"]
def __init__(self, *args):
super(ReporterModule, self).__init__(*args)
self.API_KEY = self.get_configuration("newsapi.org_key")
self.threshold = int(self.get_configuration("news_limit"))
if self.API_KEY:
self.articles = Articles(self.API_KEY)
self.sources = Sources(self.API_KEY)
else:
print(_("error.news.configuration"))
return False
self.sources_url = {}
self.sources.information()
def get_all_categories(self):
return list(self.sources.all_categories())
def get_by_category(self, category):
srcs = self.sources.get_by_category(category).sources
self.sources_url = {}
for src in srcs:
self.sources_url[src['name']] = src['url']
return self.sources_url
def get_sort_bys_of_source(self, source_name):
return self.sources.search(source_name)[0]['sortBysAvailable']
def all_sources(self):
self.sources_url = self.sources.all_names()
return self.sources_url
def get_news(self):
self.assistant.say(_("news.category.ask"))
category_status = self.assistant.listen().decipher()
if category_status.upper() in self.NEGATIVE:
category = False
else:
categories = self.get_all_categories()
category = self.search(categories, category_status)
self.assistant.say(_("news.sources.ask"))
source_status = self.assistant.listen().decipher()
if source_status.upper() in self.NEGATIVE:
source = False
else:
if category:
sources_available = self.get_by_category(category)
response = _("news.sources.list")
for source_name, source_url in sources_available.items():
response += " %s," % source_name
response += _("news.sources.select")
self.assistant.say(response)
source_command = self.assistant.listen().decipher()
source = self.search(list(sources_available), source_command)
else:
self.assistant.say(_("news.sources.all.ask"))
all_sources_status = self.assistant.listen().decipher()
sources_available = self.all_sources()
if all_sources_status.upper() in self.AFFIRMATIVE:
response = _("news.sources.all")
sources_available_list = list(sources_available)
for source_name in sources_available_list:
response += " %s," % source_name
response += _("news.sources.select")
self.assistant.say(response)
source_command = self.assistant.listen().decipher()
all_sources_status = source_command
source_found = self.search(list(sources_available), all_sources_status)
source = source_found
if source:
sort_bys_available = self.get_sort_bys_of_source(source)
if len(sort_bys_available) == 1:
sort_by = sort_bys_available[0]
else:
if len(sort_bys_available) == 2:
response = _("news.sort.two_options").format(sort_bys_available[0], sort_bys_available[1])
else:
response = _("news.sort.three_options").format(
sort_bys_available[0],
sort_bys_available[1],
sort_bys_available[2],
)
self.assistant.say(response)
sort_by_command = self.assistant.listen().decipher()
sort_by = self.search(sort_bys_available, sort_by_command)
else:
self.assistant.say(_("news.sort.described_options"))
sort_status_command = self.assistant.listen().decipher()
sort_by = self.search(['top', 'popular' 'latest'], sort_status_command)
if not source:
if sort_by.lower() == "top":
source = "google-news"
elif sort_by.lower() == "latest":
source = "the-telegraph"
else:
source = "time"
response = self.get_response(source, sort_by)
return response
def handle(self):
source = self.get_configuration("news_source")
response = self.get_response(source)
return response
def get_response(self, source, sort_by=None, threshold=5):
if self.threshold:
threshold = self.threshold
source = source.lower().replace(" ", "-")
articles = self.articles.get(source, sort_by=sort_by).articles
articles = articles[:threshold]
response = _("news.report").format(sort_by, source)
for article in articles:
if article['title']:
response += "%s, " % article['title']
if article['description']:
response += "%s, " % article['description']
if article['author']:
response += _("news.report.by").format(article['author'])
response += _("news.report.continue")
return response
@staticmethod
def search(dataset, query):
values = [0 for _ in range(0, len(dataset))]
search = query.lower().split()
upper_threshold = len(search)
for index, data in enumerate(dataset):
search_array = data.split()
for index2, text in enumerate(search_array):
if index2 >= upper_threshold:
break
threshold = len(search[index2])
for i in range(0, len(text)):
if i >= threshold - 1:
break
if text[i] == search[index2][i]:
values[index] += 1
max_value = max(values)
max_index = values.index(max_value)
return dataset[max_index]
def __init__(self, api_key=keys.news['api_key']):
self.api_key = api_key
self.article = Articles(self.api_key)
self.source = Sources(self.api_key)
self.base_url = keys.news['base_everything_url']
def __init__(self, API_KEY):
self.articles = Articles(API_KEY)
self.sources = Sources(API_KEY)
self.sources.information()
from newsapi.articles import Articles from newsapi.sources import Sources import secrets a = Articles(API_KEY=secrets.newapi_api) s = Sources(API_KEY=secrets.newapi_api) print(s.informaion())
def __init__(self, api_key) -> None:
super().__init__()
self.api_key = api_key
self.articles = Articles(API_KEY=self.api_key)
self.sources = Sources(API_KEY=self.api_key)