global article_handler

global source_handler

source_handler = Sources(API_KEY="f9f1206fc1e54d5ba63a09f34cc63af0")

article_handler = Articles(API_KEY="f9f1206fc1e54d5ba63a09f34cc63af0")

if datum is None:

todays_date = asking_date()

else:

todays_date = datum

all_relevant_articles = {}

for each in paper_id:

all_output = article_handler.get(source=each)

all_relevant_articles[each] = all_output['articles']

#pp.pprint(all_relevant_articles['mashable'])

print("Summary Count of Articles collected:")

source_url_dict = {}

for key, value in all_relevant_articles.items():

print("Source: ", key, "has relevant articles", len(value))

source_url_dict[key] = [x['url'] for x in value]

NER_classifier = "/Users/aparnaghosh87/Downloads/stanford-ner-2014-06-16/classifiers/english.all.3class.distsim.crf.ser.gz"

os.environ['CLASSPATH'] = "/Users/aparnaghosh87/Downloads/stanford-ner-2014-06-16"

st = StanfordNERTagger(NER_classifier, encoding='utf-8')

tokenized_text = word_tokenize(content)

classified_text = st.tag(tokenized_text)

# output looks like this:

# [('While', 'O'), ('in', 'O'), ('France', 'LOCATION'), (',', 'O'), ('Christine', 'PERSON'), ('Lagarde', 'PERSON'), ('discussed', 'O'), ('short-term', 'O'), ('stimulus', 'O'), ('efforts', 'O'), ('in', 'O'), ('a', 'O'), ('recent', 'O'),

# ('interview', 'O'), ('with', 'O'), ('the', 'O'), ('Wall', 'O'), ('Street', 'O'), ('Journal', 'O'), ('.', 'O')]

# TODO improve this with more tags for metadata

content = {}

body_text = ""

page = requests.get(url)

soup = BeautifulSoup(page.content, 'html.parser')

if soup.title is not None:

heading = soup.title.string

else:

heading = url.split("/")[-1].replace("-"," ")

content["heading"] = heading

for script in soup.find_all('script'):

script.extract()

paragraphs = soup.find_all('p')

for paragraph in paragraphs:

body_text = body_text + " " + (paragraph.get_text(strip=True))

content["body_text"] = body_text

# Check finders here:

## http: // www.nltk.org / howto / collocations.html

tokens = nltk.wordpunct_tokenize(all_content)

bigram_list = nltk.bigrams(tokens)

fdist = nltk.FreqDist(bigram_list)

#TODO use the inbuild nltk functions to filter and cross check only for top

#frequently occurring bigrms / trigrams

tokens = nltk.wordpunct_tokenize(all_content)

trigram_list = nltk.trigrams(tokens)

fdist = nltk.FreqDist(trigram_list)

# havent done NER on headings for both person and org

src_url_list = get_relevant_articles(src)

persons = [word_tokenize(x) for x in poi]

organizations = [word_tokenize(x) for x in orgs]

source_scores = {}

for src in src_url_list:

article_scores = {}

print("Debug: Working on ", src)

for each_url in src_url_list[src]:

print("Debug: Working on url: ", each_url)

score = 0

all_content = getContent(each_url)

article_content = all_content["body_text"]

article_heading = all_content["heading"]

# Rewrite this bit to only check the contents that are tagged with NP/PP etc.

for each in persons:

for term in each:

if term in article_content:

score = score + 1

if term in article_heading:

score = score + 3

for each in persons:

if (" ".join(each)) in article_content:

score = score + 2

if (" ".join(each)) in article_heading:

score = score + 4

NER_tagged = get_NER_Tagger(article_content)

only_personalities = [x[0] for x in NER_tagged if x[1] == 'PERSON']

for each in persons:

if each[0] in only_personalities:

score = score + 3

if each[1] in only_personalities:

score = score + 3

score_for_only_personalities = score

# Organizations

only_organizations = [x[0] for x in NER_tagged if x[1] == 'O']

for each in organizations:

for term in each:

if term in article_content:

score = score + 3

if term in article_heading:

score = score + 3

for each in organizations:

if (" ".join(each)) in article_content:

score = score + 2

if (" ".join(each)) in article_heading:

score = score + 4

# as the output of NER for organization doesnt always make sense, lets only check for the whole org name!!!

for each in organizations:

if (" ".join(each)) in only_organizations:

score = score + 3

score_for_only_organizations = score - score_for_only_personalities

article_scores[each_url] = score

print("Debug: score for article", "each_url", score)

# after testing this, go back and TODO: count the number of occurrences!!

source_scores[src] = article_scores

source_scores[src] = sorted(source_scores[src].items(), key=lambda x: x[1], reverse=True)

print(source_scores)