# Module: summarize.py

# Usage: python summarize.py [token] -d [database yml] -r [redis yml]

# ---------------------------------------------------------------------------

# It takes the narrative texts that satisfy the query and summarizes

# the text into a summary that is 1/4 of the length of the original text

# or has no more sentences than the number of paragraphs in the original text.

# It summarizes the French portion and the English portion of the text

# separately.

import nltk

import csv

from collections import *

from nltk.corpus import stopwords

import string

import networkx as nx

import pydot

import warnings

import mysql.connector

import random

import yaml

import redis

import sys

import argparse

import json

warnings.filterwarnings("ignore")

text_eng = ''

text_fre = ''

stopWordLang = dict()

stopWordLang['english'] = set(stopwords.words('english'))

stopWordLang['french'] = set(stopwords.words('french'))

sent_list = []

sent_score = defaultdict(int)

# Method: detectLanguages

# Usage: if detectLanguages(tokens) == 'english': ...

# ----------------------------------------------------

# It takes a tokenized text and determines if the text is

# in French or English.

def detectLanguages(tokens):

return 'french'

# Method: deletePunc

# Usage: tokens = deletePunc(tokens)

# ---------------------------------------------------

# It takes a tokenized text and returns the text with all

# punctuations stripped.

def deletePunc(tokens):

return [token for token in tokens if token not in set(string.punctuation)]

# Method: inter_score

# Usage: score = inter_score(sent1, sent2)

# ---------------------------------------------------

# It takes two sentences and returns the intersection

# score of the sentences.

def inter_score(sent1, sent2):

return float(2 * len([x for x in tok1 if x in tok2]))/(len(tok1) + len(tok2))

# Method: checkValidSent

# Usage: if checkValidSent(sent)

# --------------------------------------

# It takes a sentence and determines if

# the sentence has non-punctuation tokens

# in it.

def checkValidSent(sent):

tok = nltk.word_tokenize(sent)

tok = deletePunc(tok)

return tok != []

# Method: buildGraph

# Usage: graph = buildGraph(sentList)

# -------------------------------------------

# It takes a list of sentences and builds a graph

# where each node is a sentence and the weight of an edge

# is the intersection score of its two endpoints.

def buildGraph(sentList):

return gr

# Method: make_query_string

# Usage: querySt = make_query_string()

# ----------------------------------------------

# With the lists of conditions, it makes the condition

# part of the SQL query string to be used.

def make_query_string():

return condition_str

# Method: query_database

# Usage: query_database(dbyml)

# -----------------------------------------------

# It takes the address of the yml file and queries

# the database to get the texts needed for the particular

# query.

def query_database(dbadd = 'config/database.yml'):

database.close()

# Method: return_summary

# Usage: summary = return_summary(text)

# ----------------------------------------------

# It takes a text and summarizes it using TextRank

# algorithm. If the text is too long (> 150 sentences),

# then it randomly picks 150 sentences before running TextRank.

def return_summary(text):

return summary

if __name__ == "__main__":

parser = argparse.ArgumentParser(description = 'Summarizer!')

parser.add_argument('token', type = str, help = 'token for redis')

parser.add_argument('-d', '--database', help= 'database yml file', required = True)

parser.add_argument('-r', '--redis', help= 'redis yml file')

args = parser.parse_args()

if args.redis:

# set redis up from server

print "redis"

else:

r_server = redis.Redis()

# json_str = r_server.get(token)

json_str = '{"output_id": [], "ppg_id": ["LRZQ", "LTFL", "LP61"], "problem_objective_id": ["c70f5d80-a7cd-4d68-a085-aa04702c0fea", "acb439ae-0d06-463c-b3c9-030f9bc889f8"], "goal_id": ["EM"], "report_type": "Year-End Report", "year": 2013, "operation_id": ["7VR", "7VC"]}'

list_dict = json.loads(json_str)

oper_list = list_dict["operation_id"]

ppg_list = list_dict["ppg_id"]

goal_list = list_dict["goal_id"]

po_list = list_dict["problem_objective_id"]

output_list = list_dict["output_id"]

report_type = list_dict["report_type"]

year = list_dict["year"]

query_database(args.database)

print "Summary: "

print return_summary(text_eng).encode('utf-8')

print return_summary(text_fre).encode('utf-8')

print "Text: "

print text_eng.encode('utf-8')