def article_view(request, slug=None):
nobj = news.objects.get(slug=slug)
aobj = article.objects.get(title=nobj.title)
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = 'b143cf6a'
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = '89f2ead95da87d1dc9a27bbd5989d6c4'
api_instance = aylien_news_api.DefaultApi()
opts = {
'cluster': False,
'cluster_algorithm': 'stc',
'_return': ['title', 'links'],
'story_url': nobj.url,
#'story_title': obj.title,
#'story_body': obj.text,
'boost_by': 'popularity',
'story_language': 'auto'
}
rel_list = None
rel_title = []
rel_links = []
length = 0
try:
# List related stories
api_response = api_instance.list_related_stories(**opts)
filtered_list = api_response.related_stories #[i]
length = len(filtered_list)
rel_list = [[] for i in range(length)]
#rel_title = []
#rel_links = []
for i in range(0, len(filtered_list)):
rel_list[0].append(filtered_list[i].title)
rel_list[1].append(filtered_list[i].links.permalink)
#rel_title += filtered_list[i].title
#rel_links += filtered_list[i].links.permalink
print rel_title
print rel_links
print(api_response)
except ApiException as e:
print "api error"
#pprint("Exception when calling DefaultApi->list_related_stories: %s\n" % e)
#aobj = article.objects.get(slug=slug)
#nobj = news.objects.get(title=aobj.title)
context = {
'aobj': aobj,
'nobj': nobj,
#'rel_title':rel_title,
#'rel_links':rel_links,
'rel_list': rel_list,
'length': length,
}
return render(request, "article.html", context)
def generate_articles():
"""
Function to generate article models to put into the database.
Args:
none
Returns:
none
"""
apikey = "087e404043fb6f7df4c4ed55e72f3f7f"
appid = "dc8dc66d"
configuration = aylien_news_api.Configuration()
configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = appid
configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = apikey
client = aylien_news_api.ApiClient(configuration)
api_instance = aylien_news_api.DefaultApi(client)
# 1. Fetch articles
eligible_publishers = [
"theguardian.com", "nytimes.com", "ft.com", "bloomberg.com",
"reuters.com", "apnews.com", "thetimes.co.uk", "washingtonpost.com",
"afp.com", "abcnews.go.com", "time.com", "wsj.com", "economist.com",
"politico.com", "bbc.com", "pbs.com", "thehill.com", "usatoday.com",
"npr.org", "cbsnews.com", "axios.com", "huffpost.com", "newyorker.com",
"nationalreview.com", "slate.com", "theatlantic.com", "theweek.com",
"vanityfair.com", "msnbc.com", "cnn.com",
"theamericanconservative.com", "vox.com", "mic.com",
"independent.co.uk", "thesun.co.uk", "metro.co.uk", "dailymail.co.uk",
"telegraph.co.uk", "latimes.com", "cnet.com", "engadget.com",
"theverge.com", "vice.com", "hollywoodreporter.com", "newsweek.com",
"forbes.com", "sciencemag.org", "rte.com", "natgeo.com",
"wanderlust.co.uk", "skysports.com", "espn.com", "theathletic.co.uk",
"phys.org", "physicsworld.com", "sky.com", "techradar.com",
"entertainmentdaily.co.uk", "digitalspy.com", "inews.co.uk", "ign.com",
"france24.com", "dw.com", "euronews.com", "thelocal.it", "elpais.com",
"cbc.ca", "globalnews.ca", "nationalpost.com", "msn.com",
"nbcnews.com", "abc.net.au", "scmp.com", "seattletimes.com",
"independent.ie", "standard.co.uk", "wired.co.uk", "fortune.com",
"techcrunch.com", "usnews.com"
]
for publisher in eligible_publishers:
api_response = fetch_articles(api_instance, publisher)
if api_response:
stories = api_response.stories
# TBD error handling
# 2. Get the full text for every article
# 2.1 Analyse sentiment
for story in stories:
save_from_story(story)
def __init__(self):
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = 'XXXXXX'
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = 'XXXXXXXXXXXXXXXXXX'
self.api_instance = aylien_news_api.DefaultApi()
self.opts = {
'sort_by': 'relevance',
'language': ['en'],
'per_page': 100
}
def __init__(self):
# ÷ Configure API key authorization: app_id
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = 'af9d3b70'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = '3eeceb408f60f77d7719509bf902c09e'
self.api_instance = aylien_news_api.DefaultApi()
self.opts = {
'text': '',
'sort_by': 'relevance',
'language': ['en'],
'published_at_start': '',
'published_at_end': '',
'_return': ['id', 'title', 'body', 'summary', 'sentiment', 'links']
}
def SearchNews():
# Retrieving and storing user-input (News to be searched and type of news)
searchFor = searchnews.get()
newsType = s.get()
# Storing aylien_news_api license keys
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = "YOUR ID"
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = "YOUR KEY"
# Creating an instance of API Class
api_instance = aylien_news_api.DefaultApi()
# Parameters for fetching the news
# Title, Language, Type of news(Sentiment), Published Time and
# many more parameters can be specifed as well
opts = {
'title': searchFor,
'language': ['en'],
'sentiment_title_polarity' : newsType,
'published_at_start': 'NOW-7DAYS',
'published_at_end': 'NOW',
}
# Trying to Search and Fetch the News(stories)
try:
# List stories
count = 1
api_response = api_instance.list_stories(**opts)
# Looping over the fetched News (stories)
for news in api_response.stories:
storyTitle = news.title
storySource = news.source.name
storyBody = news.body
storyDate = news.published_at
displayNews = "\n\n"+str(count)+". "+storyTitle+"\n\nSOURCE : "+storySource+\
"| PUBLISHED ON : "+str(storyDate)+"\n\n"+storyBody
# Printing the fetched news
root.newsResults.insert('1.0', displayNews)
count = count + 1
# Catching the API Exception if it occurs
except ApiException as e:
print("Exception when calling DefaultApi->list_stories: %sn" % e)
def SearchNews():
searchFor = searchNews.get()
newsType = s.get()
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = "b2adff8a"
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = "6034baed6a768a178e3013128ce47838"
api_instance = aylien_news_api.DefaultApi()
otps = {
'title': searchFor,
'language': ['en'],
'sentiment_title_polarity': newsType,
'published_at_start': 'NOW-7DAYS',
'published_at_end': 'NOW',
}
try:
count = 1
api_response = api_instance.list_stories(**otps)
for news in api_response.stories:
storyTitle = news.title
storySource = news.source.name
storyBody = news.body
storyDate = news.published_at
displayNews = "\n\n"+str(count)+". "+storyTitle+"\n\nSOURCE : "+storySource+\
"| PUBLISHED ON : "+str(storyDate)+"\n\n"+storyBody
root.newsResults.insert('1.0', displayNews)
count = count + 1
except ApiException as e:
print("Exception when calling DefaultApi->list_stories: $sn" % e)
import aylien_news_api
from aylien_news_api.rest import ApiException
# Configure API key authorization: app_id
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = ' d31b61f1'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = ' 88aa8cda7ec8c9e65611e7c463ca82f3'
# create an instance of the API class
api_instance = aylien_news_api.DefaultApi()
opts = {
'title':
'trump',
'sort_by':
'social_shares_count.facebook',
'language': ['en'],
'published_at_start':
'NOW-7DAYS',
'published_at_end':
'NOW',
'entities_body_links_dbpedia': [
'http://dbpedia.org/resource/Donald_Trump',
'http://dbpedia.org/resource/Hillary_Rodham_Clinton'
]
}
try:
# List stories
def ay_lookup(keyword, subject='', entities=[]):
# Configure API key authorization: app_id
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = 'e6ddf398'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = 'db5180ad10ac1324aadd57451fded0a8'
# create an instance of the API class
api_instance = aylien_news_api.DefaultApi()
opts = {
'title': keyword,
'body': keyword,
'sort_by': 'relevance',
'language': ['en'],
'published_at_start': 'NOW-2DAYS',
'published_at_end': 'NOW',
'entities_body_text': entities
}
if subject != '':
opts['text'] = subject
try:
api_response = api_instance.list_stories(**opts)
api_response.stories
for i in range(0, len(api_response.stories)):
story = api_response.stories[i]
if "Sign up for one of our email newsletters." not in story.body:
try:
body_clauses = story.body.replace("\n", " ").split('. ')
except:
body_clauses = []
lst = body_clauses
try:
summary = story.summary['sentences']
except:
summary = []
st = ''
counter1 = 0
counter2 = 0
while counter2 < len(body_clauses) and st == '':
try:
st = summary[counter1]
intitial = counter1
counter1 += 1
initial = counter1
except:
st = body_clauses[counter2] + "."
initial = counter2
counter2 += 1
top = initial + 1
st = st.replace('\n', ' ')
if len(st) > 40:
st = st.split('. ')[0] + "."
if initial != counter2:
lst = summary
while bad_string(st) and top < len(lst):
if lst[initial:top] != []:
st = lst[initial:top].join('. ') + "."
top += 1
if 'photo' in st.lower() or 'file' in st.lower(
) or st[0] == st[0].lower() or '(' in st or ')' in st:
st = ''
st = st.replace('..', '.')
if len(st) > 10:
return story, st
except ApiException as e:
print("Exception when calling DefaultApi->list_stories: %s\n" % e)
return {}, ''
def config(self, Aylien_ID, Aylien_key):
configuration = aylien_news_api.Configuration()
client = aylien_news_api.ApiClient(configuration)
configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = Aylien_ID
configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = Aylien_key
self.api_instance = aylien_news_api.DefaultApi(client)
def updateData():
############################################### Update aylien_news.csv ###############################################
df = pd.read_csv("data/aylien_news.csv")
df['date'] = pd.to_datetime(df.date, format='%Y/%m/%d')
df.sort_values(by='date', inplace=True, ascending=False)
configuration = aylien_news_api.Configuration()
configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = '97c45927'
configuration = aylien_news_api.Configuration()
configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = '8e5a95f7bafa835d340a1be85a90588a'
configuration.host = "https://api.aylien.com/news"
api_instance = aylien_news_api.DefaultApi(aylien_news_api.ApiClient(configuration))
date = list(df["date"])[0]
e = str(datetime.date.today())
s = str((date + datetime.timedelta(days=1)).date())
from_date = s
to_date = str((datetime.datetime.strptime(str(from_date), "%Y-%m-%d") + datetime.timedelta(days=5)).date())
rows = []
crimes = {'burglary': 0, 'robbery': 0, 'murder': 0, 'kidnapping': 0, 'rape': 0}
total_news_count = 0
while dateutil.parser.parse(from_date).date() < dateutil.parser.parse(e).date():
if dateutil.parser.parse(to_date).date() > dateutil.parser.parse(e).date():
to_date = e
for crime in crimes.keys():
opts = {
'title': crime,
'sort_by': 'social_shares_count.facebook',
'language': ['en'],
'published_at_start': from_date + 'T00:00:00Z',
'published_at_end': to_date + 'T00:00:00Z',
'per_page': 100,
'source_locations_country': ['IN'],
'source_locations_state': ['Delhi'],
'source_locations_city': ['Delhi'],
}
headlines = api_instance.list_stories_with_http_info(**opts)
for i in headlines[0].stories:
date = i.published_at.date()
source = i.source.name
title = i.title
url = i.links.permalink
content = i.body
rows.append([date, source, title, crime, url, content])
total_news_count += 1
crimes[crime] += 1
from_date = str((datetime.datetime.strptime(str(to_date), "%Y-%m-%d") + datetime.timedelta(days=1)).date())
to_date = str((datetime.datetime.strptime(str(from_date), "%Y-%m-%d") + datetime.timedelta(days=5)).date())
with open('data/aylien_news.csv', 'a') as f:
writer = csv.writer(f)
writer.writerows(rows)
############################################### Update news_data.csv ###############################################
contents = ""
for i in range(len(rows)):
contents = contents + rows[i][5] + " "
locations = findLocations(contents)
f = open('data/loc.txt', 'r')
s = f.read()
loc = s.split('\n')
delhi_locs = []
for i in loc:
delhi_locs.append(i.lower())
all_locations = list(set(delhi_locs).intersection(locations))
if len(all_locations) > 0:
df1 = pd.read_csv("data/news_data.csv")
d = {}
for i in range(len(df1)):
lst = list(df1.iloc[i])
location = lst[0]
dates_lst = lst[1].split("\t")
dates = []
for j in dates_lst:
dates.append(str(j))
srcs = lst[2].split("\t")
headlines = lst[3].split("\t")
types = lst[4].split("\t")
articles = lst[5].split("\t")
ages_lst = lst[6].split("\t")
ages = []
for i in ages_lst:
ages.append(int(i))
businessmans_lst = lst[7].split("\t")
businessmans = []
for i in businessmans_lst:
businessmans.append(int(i))
urls = lst[8].split("\t")
d[location] = [dates, headlines, types, articles, ages, businessmans, urls, srcs]
business = ['businessman', 'jeweller', 'jeweler', 'shop owner', 'property dealer']
c = 0
for i in range(len(rows)):
article = rows[i][-1] # [date, source, title, crime, url, content]
date = rows[i][0]
headline = rows[i][2]
crime_type = rows[i][3]
news_url = rows[i][-2]
if str(news_url) == "nan":
news_url = "na"
news_src = rows[i][1]
businessman = 0
for j in business:
if j in article:
businessman = 1
break
age = findAge(article)
for i in all_locations:
if i in article.lower():
c += 1
flag = 1
for j in loc:
if i == j.lower() or i == j:
loc_name = j
break
lst = d.get(loc_name, [[], [], [], [], [], [], [], []])
lst[0].append(str(date))
lst[1].append(headline)
lst[2].append(crime_type)
lst[3].append(article)
lst[4].append(age)
lst[5].append(businessman)
lst[6].append(news_url)
lst[7].append(news_src)
d[i] = lst
crime_counts = {}
max_count = 0
for i in d.keys():
crime_counts[i] = len(d[i][0])
if len(d[i][0]) > max_count:
max_count = len(d[i][0])
percentile = {}
for i in crime_counts.keys():
p = 100 - int((crime_counts[i] / max_count) * 100)
percentile[i] = p
rows = []
header = ['location', 'date', 'source', 'headline', 'crime_type', 'article', 'age', 'businessman', 'url',
'percentile']
rows.append(header)
for i in d.keys():
for j in loc:
if i == j.lower() or i == j:
loc_name = j
break
row = [loc_name]
lst = d[i]
row.append('\t'.join(lst[0]))
row.append('\t'.join(lst[7]))
row.append('\t'.join(lst[1]))
row.append('\t'.join(lst[2]))
row.append('\t'.join(lst[3]))
row.append('\t'.join(repr(n) for n in lst[4]))
row.append('\t'.join(repr(n) for n in lst[5]))
row.append('\t'.join(lst[6]))
row.append(percentile[i])
rows.append(row)
data = open('data/news_data.csv', 'w')
writer = csv.writer(data)
writer.writerows(rows)
data.close()
df1 = pd.read_csv("data/news_data.csv")
df2 = pd.read_csv("data/area.csv")
crime_locations = list(df1["location"])
rows = [list(df2.columns)]
for i in range(len(df2)):
loc = df2.iloc[i, 0]
if loc == "AIIMS":
continue
if loc not in crime_locations:
rows.append(list(df2.iloc[i]))
else:
dt = list(df1[df1["location"] == loc].iloc[0])
age_lst = dt[-4].split("\t")
crimes = dt[4].split("\t")
businessman_count = list(dt[-3].split("\t")).count('1')
crime_count = len(age_lst)
age_crimes = {"0-21": 0, "22-50": 0, "50+": 0}
crime_type = {'burglary': 0, 'robbery': 0, 'murder': 0, 'kidnapping': 0, 'rape': 0}
for j in age_lst:
age = int(j)
if age > 0 and age <= 21:
age_crimes["0-21"] += 1
elif age > 21 and age <= 50:
age_crimes["22-50"] += 1
elif age > 50:
age_crimes["50+"] += 1
for j in crimes:
crime_type[j] += 1
lst = list(df2.iloc[i])
lst[1] = crime_count
lst[2] = age_crimes["0-21"]
lst[3] = age_crimes["22-50"]
lst[4] = age_crimes["50+"]
lst[5] = crime_type["murder"]
lst[6] = crime_type["burglary"]
lst[7] = crime_type["robbery"]
lst[8] = crime_type["kidnapping"]
lst[9] = crime_type["rape"]
lst[10] = 100 - dt[-1]
lst[-1] = businessman_count
rows.append(lst)
with open('data/area.csv', 'w') as f:
writer = csv.writer(f)
writer.writerows(rows)
def ay_lookup(keyword, entities = [], timeframe = '0,2', league=''):
# Configure API key authorization: app_id
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = '115c381f'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = '876fb6a2163fec8db0b725304363eb78'
# create an instance of the API class
api_instance = aylien_news_api.DefaultApi()
codes = {}
codes['NFL'] = ['15003000']
codes['MLB'] = ['15007000']
codes['NBA'] = ['15008000']
codes['NHL'] = ['15031000']
codes[''] = []
categories = ['15000000','15073000']
categories += codes[league]
opts = {
'title': keyword,
'body': keyword,
'sort_by': 'relevance',
'language': ['en'],
'entities_body_text' : entities,
'categories_taxonomy' : 'iptc-subjectcode',
'categories_id' : categories
}
first = timeframe.split(',')[0]
second = timeframe.split(',')[1]
if first == '0':
first = 'NOW'
else:
first = 'NOW-'+first+'DAYS'
second = 'NOW-'+second+'DAYS'
opts['published_at_start'] = second
opts['published_at_end'] = first
counter = 0
try:
api_response = api_instance.list_stories(**opts)
for i in range(0, len(api_response.stories)):
if counter < 10:
counter += 1
story = api_response.stories[i]
if "Sign up for" not in story.body and "newsletter" not in story.body:
try:
body_clauses = clean_st(story.body).split('.')
while '' in body_clauses:
body_clauses.remove('')
body_clauses = fix_clauses(body_clauses)
except:
body_clauses = []
try:
summary = story.summary['sentences']
except:
summary = []
summarycounter = 0
bodycounter = 0
st = ''
while '' in body_clauses:
body_clauses.remove('')
while bodycounter < len(body_clauses) and st == '':
try:
st = summary[summarycounter]
initial = summarycounter
summarycounter += 1
lst = body_clauses
except:
st = body_clauses[bodycounter]+"."
initial = bodycounter
bodycounter += 1
lst = summary
top = initial + 1
st = clean_st(st)
while bad_string(st) and top < len(lst):
if lst[initial:top] != []:
st = lst[initial:top].join('.') + '.'
top += 1
if 'photo' in st.lower() or 'file' in st.lower() or st[0] == st[0].lower() or '(' in st or ')' in st:
st = ''
st = clean_st(st)
if len(st) > 10 and bad_string(st) != True:
return story, st
else:
return {}, ''
except ApiException as e:
print("Exception when calling DefaultApi->list_stories: %s\n" % e)
def ay_lookup(keyword, subject='', entities=[], timeframe='0,2'):
# Configure API key authorization: app_id
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = 'e6ddf398'
# Configure API key authorization: app_key
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = 'db5180ad10ac1324aadd57451fded0a8'
# create an instance of the API class
api_instance = aylien_news_api.DefaultApi()
opts = {
'title': keyword,
'body': keyword,
'sort_by': 'relevance',
'language': ['en'],
'entities_body_text': entities
}
if subject != '':
opts['text'] = subject
first = timeframe.split(',')[0]
second = timeframe.split(',')[1]
if first == '0':
first = 'NOW'
else:
first = 'NOW-' + first + 'DAYS'
second = 'NOW-' + second + 'DAYS'
opts['published_at_start'] = second
opts['published_at_end'] = first
try:
api_response = api_instance.list_stories(**opts)
api_response.stories
for i in range(0, len(api_response.stories)):
story = api_response.stories[i]
if "Sign up for one of our email newsletters." not in story.body:
try:
body_clauses = clean_st(story.body.replace("\n",
" ")).split('.')
while '' in body_clauses:
body_clauses.remove('')
except:
body_clauses = []
try:
summary = story.summary['sentences']
except:
summary = []
summarycounter = 0
bodycounter = 0
st = ''
body_clauses = fix_clauses(body_clauses)
while '' in body_clauses:
body_clauses.remove('')
while bodycounter < len(body_clauses) and st == '':
try:
st = summary[summarycounter]
initial = summarycounter
summarycounter += 1
lst = body_clauses
except:
st = body_clauses[bodycounter] + "."
initial = bodycounter
bodycounter += 1
lst = summary
top = initial + 1
st = clean_st(st)
while bad_string(st) and top < len(lst):
if lst[initial:top] != []:
st = lst[initial:top].join('.') + '.'
top += 1
if 'photo' in st.lower() or 'file' in st.lower(
) or st[0] == st[0].lower() or '(' in st or ')' in st:
st = ''
st = clean_st(st)
if len(st) > 10 and bad_string(st) != True:
return story, st
except ApiException as e:
print("Exception when calling DefaultApi->list_stories: %s\n" % e)
return {}, ''
def index(request):
all_keys = Apipasswords.objects.all()
#auth = OAuthHandler(consumerKey, consumerSecret)
#auth.set_access_token(accessToken, accessSecret)
#twitterStream = Stream(auth, FetchTweets())
tweets = []
for line in open(tweets_filename):
try:
tweets.append(json.loads(line))
except:
pass
texts = ""
for tweet in tweets:
try:
texts += tweet['text']
except:
pass
tokenizedSentences = nltk.sent_tokenize(texts)
data = []
for sent in tokenizedSentences:
data = data + nltk.pos_tag(nltk.word_tokenize(sent))
entity_names = []
namedEnt = nltk.ne_chunk(data, binary=True)
for tree in namedEnt:
entity_names.extend(extract_entity_names(tree))
lowerFinalEntity = []
for eachEntity in final_entity_names:
lowerFinalEntity.append(eachEntity.lower())
all_words = nltk.FreqDist(lowerFinalEntity)
finalResult = all_words.most_common(5)
api_instance = aylien_news_api.DefaultApi()
f = open("new_queryKeywords2.txt", "w+")
data_dict = {}
i = 0
for query_data in finalResult:
try:
data = query_data
text = data
language = ['en']
since = 'NOW-10DAYS'
until = 'NOW'
api_response = api_instance.list_stories(text=text, language=language, published_at_start=since,
published_at_end=until)
story = api_response.stories
pnews = [data for data in story if data.sentiment.body.polarity == 'positive']
nnews = [data for data in story if data.sentiment.body.polarity == 'negative']
neutralnews = [data for data in story if data.sentiment.body.polarity == 'neutral']
positive_news_sentiment = format(100 * len(pnews) / len(story))
negetive_news_sentiment = format(100 * len(nnews) / len(story))
neutral_news_sentiment = format(100 * len(neutralnews) / len(story))
api = TwitterClient()
tweets = api.get_tweets(query=data, count=200)
ptweets = [tweet for tweet in tweets if tweet['sentiment'] == 'positive']
ntweets = [tweet for tweet in tweets if tweet['sentiment'] == 'negative']
neutraltweets = [tweet for tweet in tweets if tweet['sentiment'] == 'neutral']
positive_twitter_sentiment = format(100 * len(ptweets) / len(tweets))
negetive_twitter_sentiment = format(100 * len(ntweets) / len(tweets))
neutral_twitter_sentiment = format(100 * len(neutraltweets) / len(tweets))
data_dict[i] = {'query_val': query_data[0], 'positive_news_sentiment': positive_news_sentiment,
'negetive_news_sentiment':negetive_news_sentiment, 'neutral_news_sentiment':neutral_news_sentiment,
'positive_twitter_sentiment':positive_twitter_sentiment, 'negetive_twitter_sentiment':negetive_twitter_sentiment,
'neutral_twitter_sentiment':neutral_twitter_sentiment}
i = i+1
except:
pass
f.close()
context = {
'max_tweets': max_tweets,
'tweets': tweets,
'finalResult': finalResult,
'data_dict': data_dict,
}
return render(request, 'sentiAnalyze/index.html', context)
def get_news(request):
response1 = unirest.get("https://newsapi.org/v1/articles/",
headers={"Accept": "application/json"},
params={
"apiKey": "7a233aaececb44a3a0dd8576350c680e",
"sortBy": "top",
"source": "google-news"
})
articles = response1.body['articles']
client = textapi.Client("faf883f4", "057f1bdde3dba11b14cf9abc9abbf986")
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-ID'] = 'b143cf6a'
aylien_news_api.configuration.api_key[
'X-AYLIEN-NewsAPI-Application-Key'] = '89f2ead95da87d1dc9a27bbd5989d6c4'
api_instance = aylien_news_api.DefaultApi()
for i in articles:
try:
data = news()
data.title = i['title']
data.save()
data.description = i['description']
data.url = i['url']
data.urlToImage = i['urlToImage']
data.publishedAt = i['publishedAt']
data.save()
extract = client.Extract({"url": data.url, "best_image": True})
atext = extract['article']
#print extract
"""
#try:
response = unirest.post("https://api.aylien.com/api/v1/extract", headers={ "Accept": "application/json" }, params={ "text": data.url, "language": "en", "tab":"extract" })
atext = response.body
"""
#print atext.rstrip()
obj = article()
obj.title = i['title']
obj.text = atext.rstrip()
emotion = unirest.post("http://apidemo.theysay.io/api/v1/emotion",
headers={"Accept": "application/json"},
params={
"text": obj.text,
"level": "sentence"
})
sentiment = unirest.post(
"http://apidemo.theysay.io/api/v1/sentiment",
headers={"Accept": "application/json"},
params={
"text": obj.text,
"level": "sentence"
})
try:
'''
obj.calm = emotion.body[0]['emotions'][1]['score']
obj.fear = emotion.body[0]['emotions'][2]['score']
obj.happy = emotion.body[0]['emotions'][3]['score']
obj.like = emotion.body[0]['emotions'][4]['score']
obj.shame = emotion.body[0]['emotions'][5]['score']
obj.sure = emotion.body[0]['emotions'][6]['score']
obj.surprise = emotion.body[0]['emotions'][7]['score']
topic = unirest.post("http://apidemo.theysay.io/api/v1/topic", headers={ "Accept": "application/json" }, params={ "text": obj.text, "level": "sentence" })
top_3 = []
try:
top_3[0] = topic.body[0]['scores'][0]['label']
top_3[1] = topic.body[0]['scores'][1]['label']
top_3[2] = topic.body[0]['scores'][2]['label']
except:
pass
print top_3
obj.negative = sentiment.body[0]['sentiment']['negative']
obj.positive = sentiment.body[0]['sentiment']['positive']
obj.neutral = sentiment.body[0]['sentiment']['neutral']
obj.label = sentiment.body[0]['sentiment']['label']
obj.anger = emotion.body[0]['emotions'][0]['score']
obj.save()
'''
topic = unirest.post("http://apidemo.theysay.io/api/v1/topic",
headers={"Accept": "application/json"},
params={
"text": obj.text,
"level": "sentence"
})
top_3 = []
try:
top_3[0] = topic.body[0]['scores'][0]['label']
top_3[1] = topic.body[0]['scores'][1]['label']
top_3[2] = topic.body[0]['scores'][2]['label']
except:
pass
#print top_3
data.positive = sentiment.body[0]['sentiment']['positive']
data.label = sentiment.body[0]['sentiment']['label']
data.negative = sentiment.body[0]['sentiment']['negative']
data.neutral = sentiment.body[0]['sentiment']['neutral']
data.calm = emotion.body[0]['emotions'][1]['score']
data.fear = emotion.body[0]['emotions'][2]['score']
data.happy = emotion.body[0]['emotions'][3]['score']
data.like = emotion.body[0]['emotions'][4]['score']
data.shame = emotion.body[0]['emotions'][5]['score']
data.sure = emotion.body[0]['emotions'][6]['score']
data.surprise = emotion.body[0]['emotions'][7]['score']
data.anger = emotion.body[0]['emotions'][0]['score']
obj.save()
data.save()
'''
opts = {
'cluster': False,
'cluster_algorithm': 'stc',
'_return': ['title', 'links'],
'story_url': data.url,
#'story_title': obj.title,
#'story_body': obj.text,
'boost_by': 'popularity',
'story_language': 'auto'
}
try:
# List related stories
api_response = api_instance.list_related_stories(**opts)
print(api_response)
except ApiException as e:
print "api error"
#pprint("Exception when calling DefaultApi->list_related_stories: %s\n" % e)
'''
except:
print "error a"
except:
print "error b"
return HttpResponse("DONE")
import aylien_news_api
from aylien_news_api.rest import ApiException
from pprint import pprint as pp
import pandas as pd
import time
# import json
## Configure your connection to the API
configuration = aylien_news_api.Configuration()
configuration.api_key['X-AYLIEN-NewsAPI-Application-ID'] = 'xxx'
configuration.api_key['X-AYLIEN-NewsAPI-Application-Key'] = 'xxx'
configuration.host = "https://api.aylien.com/news"
api_instance = aylien_news_api.DefaultApi(aylien_news_api.ApiClient(configuration))
## List our parameters as search operators
opts_energy= {
# 'title': '"energy transition" OR "climate change"',
'title': '"energy transition',
'body': '"oil company" OR "oil companies"',
'language': ['en'],
'published_at_start': 'NOW-30DAYS',
'published_at_end': 'NOW',
'per_page': 100,
'sort_by': 'relevance'
}
opts_digital_twin= {
# 'title': '"energy transition" OR "climate change"',
'title': '"digital twin',
'body': 'upstream E&P',
'language': ['en'],
def __init__(self):
self.api_instance = aylien_news_api.DefaultApi()
