import sys

# Add path for textrank

sys.path.append("/root/textrank/")

from textrank import textrank

import os

from os import listdir

from os.path import isfile, join

import traceback

import datetime

from datetime import datetime

import time

import string

import pymongo

from pymongo import MongoClient

#import base64

import hashlib

import lxml.html, lxml.html.clean

from lxml import etree

import urllib2

import httplib

import re

import random

from boilerpipe.extract import Extractor

import simplejson

import StringIO

from dateutil.parser import *

import requests

from functools import wraps

import deliciousapi

import cld

import codecs

import datetime

from datetime import datetime, date, time, timedelta

from dateutil.relativedelta import relativedelta

# Add path for summarize

#sys.path.append("/root/summarize/")

import summarize

import local_passwords

TOPSY_API_KEY = local_passwords.TOPSY_API_KEY

AAFTER_URL = local_passwords.AAFTER_URL

DELICIOUS_USERNAME = local_passwords.DELICIOUS_USERNAME

AAFTER_NEW_API = local_passwords.AAFTER_NEW_API

def guess_date(text):

return None

class SleepAfter(object): # pylint: disable-msg=R0903

return self.function(*args, **kwargs)

@SleepAfter

def get_article_features(articleText, articleURL):

return features

def map_todays_inserted_domains_from_articles(domain_analysis_outfile):

outfile.close()

def reduce_relevant_domains(domain_analysis_outfile):

outfile.close()

def download_article_file(articleURL, articleFileDirectory, code):

return None

def parse_news_articles(php_directory, download_directory, file_name, query):

print r

def parse_webpages(php_directory, term, option, excludes):

# TODO: Insert webpage into ElasticSearch?

def validate_url(url):

return True

def clean_text(text):

text = text.replace('’', "'").replace('’', "'").replace('\n', '\\n').replace('&', '&').replace('\r','\\r').replace('"','\\"').replace('\t','\\t')

return text

def mk_es_json(page_id, text, url, title, abst, pd):

return json_str

def post_to_elastic_search(es_url, index, index_type, index_id, json_str):

return r.json()

def parse_Subhankar_API_v2(sub_api_v2_path):

print "ERROR: IndexError at: " + str(i)

def download_articles_from_url(api_url, download_directory):

print r

if __name__ == "__main__":

php_directory = "/data/search-results/"

download_directory = "/data/article-files/"

# TODO: Find more excludes

webpage_excludes = '-meetup.com -buy'

if len(sys.argv) != 2:

print "Usage: python parse-and-download.py <output-directory>"

print "INFO: Output directory not specified. Defaulting to " + php_directory

else:

php_directory = sys.argv[1]

# Instantiate Delicious API

dapi = deliciousapi.DeliciousAPI()

username = DELICIOUS_USERNAME

# Open MongoDB connection

connection = MongoClient()

db = connection.db

perm = connection.perm

query_term_options_db = db.queries

tech_blogs = perm.tech_blogs

articles = db.articles

webpages = db.webpages

cursor = db.all_terms.find({'_id': 1})

all_terms = cursor[0]['terms']

query_term_options = query_term_options_db.find_one()

#for qto in query_term_options['q']:

# query = qto[0]

# term = qto[1]

# option = qto[2]

# # For each keyword and option string, parse the URLs

# try:

# # Run the parsers by sending the queries (term + option) -- i.e. "MongoDB Healthcare"

# print "Parsing: " + term + "-" + option

# file_name = (term + "-" + option).replace(" ", "_").replace('/','_')

# #parse_webpages(php_directory, term, option, webpage_excludes)

# #parse_news_articles(php_directory, download_directory, file_name, query)

# except OSError:

# print "ERROR: Directory for " + file_name + " doesn't exist! Content wasn't downloaded?"

# Subhankar's new article API

api_url = AAFTER_NEW_API

now = datetime.now()

process_date = now.strftime("%Y-%m-%d")

#download_articles_from_url(api_url, download_directory, process_date)

# TEST: Using Subhankar's API v1 to download individual terms

for term in all_terms[0]:

try:

# Run the parsers by sending the query term -- i.e. "MongoDB"

print "Parsing: " + term

file_name = term.replace(" ", "_").replace('/','_')

#parse_webpages(php_directory, term, option, webpage_excludes)

parse_news_articles(php_directory, download_directory, file_name, term)

except OSError:

print "ERROR: Directory for " + file_name + " doesn't exist! Content wasn't downloaded?"

domain_analysis_outfile = "/root/biffle-prototype/domains_inserted_into_MongoDB"

# TODO: Over what timeframe should we be analyzing domains? This deletes the file every time

# Delete previous domain analysis file

if (os.path.isfile(domain_analysis_outfile)):

os.unlink(domain_analysis_outfile)

else:

print "INFO: Existing domain_analysis_outfile not found for deletion"

# TODO: Count domains inserted for webpages or downloaded articles from url

map_todays_inserted_domains_from_articles(domain_analysis_outfile)

# Print summary statistics

print "SUMMARY STATISTICS:"

count_total_articles = articles.find().count()

print "Total articles: ", count_total_articles

count_fb_share = articles.find({"f.facebook_shares":{"$exists": True, "$ne": 0}}).count()

print "Total with Facebook share data: ", count_fb_share, " ({0:.2%})".format(float(count_fb_share)/float(count_total_articles))

count_retweet_all = articles.find({"f.retweet_count_all":{"$exists": True, "$ne": 0}}).count()

print "Total retweets: ", count_retweet_all, " ({0:.2%})".format(float(count_retweet_all)/float(count_total_articles))

#print "Total retweets by influentials: " + str(articles.find({"f.retweet_count_influential":{"$exists": True, "$ne": 0}}).count())

count_technorati = articles.find({"f.technorati_auth_score":{"$exists": True, "$ne": 0}}).count()

print "Total with Technorati auth score: ", count_technorati, " ({0:.2%})".format(float(count_technorati)/float(count_total_articles))

count_publish_date = articles.find({"pubd":{"$exists": True, "$ne": None}}).count()

print "Total with publish date (guessed): ", count_publish_date, " ({0:.2%})".format(float(count_publish_date)/float(count_total_articles))

count_processed_date = articles.find({"procd":{"$exists": True, "$ne": None}}).count()

print "Total with processed date: ", count_processed_date, " ({0:.2%})".format(float(count_processed_date)/float(count_total_articles))

count_reddit = articles.find({"f.reddit_info.data.modhash":{"$exists": True, "$ne": None}}).count()

print "Total that has Reddit data: ", count_reddit, " ({0:.2%})".format(float(count_reddit)/float(count_total_articles))

count_delicious = articles.find({"f.delicious_bookmarks":{"$exists": True, "$ne": None}}).count()

print "Total that has Delicious data: ", count_delicious, " ({0:.2%})".format(float(count_delicious)/float(count_total_articles))