def __init__(self, inputfile="dummyInput", outputfile="dummyOutput"):
     self.regex = re.compile(
         '"((Sun|Mon|Tue|Thu|Wed|Fri|Sat) [A-Za-z0-9\s:\+]*)".*"text":"(.*)"'
     )
     self.hashtag_regex = re.compile("#(\w+)")
     self.format_string = "%a %b %d %H:%M:%S +0000 %Y"
     self.time_key = {}
     self.adjacency_list = {}
     self.input_filepath = inputfile
     self.output_filepath = outputfile
     self.clean_tweet_object = CleanTweet()
	def test_unicode_tweet_count(self):
		# Should have mocked. But did not.
		cleaned_tweet_list = ["Spark Summit East this week! #Spark #Apache (timestamp: Thu Oct 29 17:51:01 +0000 2015)", "Just saw a great post on Insight Data Engineering #Apache #Hadoop #Storm (timestamp: Thu Oct 29 17:51:30 +0000 2015)", "Doing great work #Apache (timestamp: Thu Oct 29 17:51:55 +0000 2015)", "Excellent post on #Flink and #Spark (timestamp: Thu Oct 29 17:51:56 +0000 2015)", "New and improved #HBase connector for #Spark (timestamp: Thu Oct 29 17:51:59 +0000 2015)", "New 2.7.1 version update for #Hadoop #Apache (timestamp: Thu Oct 29 17:52:05 +0000 2015)"]
		self.clean_tweet = CleanTweet(self.base_dir+"tweet_input/sample_tweets.txt", self.base_dir+"tweet_output/test_output.txt")
		cleaned_tweets = self.clean_tweet.process_tweets()
		self.failUnlessEqual(set(cleaned_tweet_list), set(cleaned_tweets))
		self.failUnlessEqual(4, self.clean_tweet.get_count_of_unicode_tweets())
class TestCase(unittest.TestCase):
	def setUp(self):
		self.clean_tweet = CleanTweet()
		self.avg_deg = AverageDegree()
		self.base_dir = "/home/sesha/Interests/insightDataFella/coding-challenge/"

	def test_if_unicode_removed(self):
		tweet = "Spark \u009FSummit East this week! #Spark #Apache"
		clean_tweet = "Spark Summit East this week! #Spark #Apache"
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_if_unicode_spared(self):
		# Spare unicode between 0000 and 007F
		tweet = "Spark \u003cSummit East this week! #Spark #Apache"
		clean_tweet = "Spark <Summit East this week! #Spark #Apache"
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_if_double_backslash_handled(self):
		tweet = "This concert \\\\m/. Am enjoying!!"
		clean_tweet = "This concert \\m/. Am enjoying!!"
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_if_escape_url_handled(self):
		tweet = "PB https:\/\/t.co\/HOl34REL1a hello"
		clean_tweet = "PB https://t.co/HOl34REL1a hello"
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_if_escape_sequences_handled(self):
		tweet = "Should \n\nclean\t\rall of \"this\""
		clean_tweet = 'Should   clean  all of "this"'
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_if_empty(self):
		tweet = "\u3084\u3070\u3044\u7709\u6d88\u3048\u305f\uff01\u30d6\u30ea\u30fc\u30c1\u6642\u9593\u30df\u30b9\u3063\u305f\u308f\u2026"
		clean_tweet = ""
		self.failUnlessEqual(clean_tweet, self.clean_tweet.get_clean_tweet(tweet))

	def test_unicode_tweet_count(self):
		# Should have mocked. But did not.
		cleaned_tweet_list = ["Spark Summit East this week! #Spark #Apache (timestamp: Thu Oct 29 17:51:01 +0000 2015)", "Just saw a great post on Insight Data Engineering #Apache #Hadoop #Storm (timestamp: Thu Oct 29 17:51:30 +0000 2015)", "Doing great work #Apache (timestamp: Thu Oct 29 17:51:55 +0000 2015)", "Excellent post on #Flink and #Spark (timestamp: Thu Oct 29 17:51:56 +0000 2015)", "New and improved #HBase connector for #Spark (timestamp: Thu Oct 29 17:51:59 +0000 2015)", "New 2.7.1 version update for #Hadoop #Apache (timestamp: Thu Oct 29 17:52:05 +0000 2015)"]
		self.clean_tweet = CleanTweet(self.base_dir+"tweet_input/sample_tweets.txt", self.base_dir+"tweet_output/test_output.txt")
		cleaned_tweets = self.clean_tweet.process_tweets()
		self.failUnlessEqual(set(cleaned_tweet_list), set(cleaned_tweets))
		self.failUnlessEqual(4, self.clean_tweet.get_count_of_unicode_tweets())

	def test_average_degree_changes(self):
		self.avg_deg = AverageDegree(self.base_dir+"tweet_input/half_sample_tweets.txt", self.base_dir+"tweet_output/test_output.txt")
		self.avg_deg.process_tweets()
		self.failUnlessEqual(2.0, self.avg_deg.calculate_avg_degree())
		self.avg_deg = AverageDegree(self.base_dir+"tweet_input/sample_tweets.txt", self.base_dir+"tweet_output/test_output.txt")
		self.avg_deg.process_tweets()
		self.failUnlessEqual(1.67, self.avg_deg.calculate_avg_degree())

	def tearDown(self):
		self.clean_tweet = CleanTweet()
		self.avg_deg = AverageDegree()
 def __init__(self, inputfile="dummyInput", outputfile="dummyOutput"):
     self.regex = re.compile('"((Sun|Mon|Tue|Thu|Wed|Fri|Sat) [A-Za-z0-9\s:\+]*)".*"text":"(.*)"')
     self.hashtag_regex = re.compile("#(\w+)")
     self.format_string = "%a %b %d %H:%M:%S +0000 %Y"
     self.time_key = {}
     self.adjacency_list = {}
     self.input_filepath = inputfile
     self.output_filepath = outputfile
     self.clean_tweet_object = CleanTweet()
	def tearDown(self):
		self.clean_tweet = CleanTweet()
		self.avg_deg = AverageDegree()
	def setUp(self):
		self.clean_tweet = CleanTweet()
		self.avg_deg = AverageDegree()
		self.base_dir = "/home/sesha/Interests/insightDataFella/coding-challenge/"
class AverageDegree(object):
    def __init__(self, inputfile="dummyInput", outputfile="dummyOutput"):
        self.regex = re.compile(
            '"((Sun|Mon|Tue|Thu|Wed|Fri|Sat) [A-Za-z0-9\s:\+]*)".*"text":"(.*)"'
        )
        self.hashtag_regex = re.compile("#(\w+)")
        self.format_string = "%a %b %d %H:%M:%S +0000 %Y"
        self.time_key = {}
        self.adjacency_list = {}
        self.input_filepath = inputfile
        self.output_filepath = outputfile
        self.clean_tweet_object = CleanTweet()

    def remove_older_hashtags(self, timestamp):
        one_minute = datetime.timedelta(minutes=1)
        current_time = datetime.datetime.strptime(timestamp,
                                                  self.format_string)
        timestamps = self.time_key.keys()
        ts_to_remove = filter(
            lambda old_ts: (current_time - datetime.datetime.strptime(
                old_ts, self.format_string)) > one_minute, timestamps)
        hashtags = []
        for ts in ts_to_remove:
            hashtags += self.time_key.pop(ts)
        unique_hashtags = set(hashtags)
        for hashtag in unique_hashtags:
            self.adjacency_list[hashtag] -= unique_hashtags

    def add_nodes(self, tweet, timestamp):
        raw_hashtags = self.hashtag_regex.findall(tweet)
        hashtags = map(lambda tag: tag.lower().strip(), raw_hashtags)

        self.remove_older_hashtags(timestamp)
        if timestamp not in self.time_key:
            self.time_key[timestamp] = []

        unique_hashtags = set(hashtags)

        if len(unique_hashtags) > 1:
            self.time_key[timestamp] += list(unique_hashtags)
            temp_hashtags = unique_hashtags.copy()
            for hashtag in unique_hashtags:
                if hashtag not in self.adjacency_list:
                    self.adjacency_list[hashtag] = set([])
                temp_hashtags.remove(hashtag)
                self.adjacency_list[hashtag].update(temp_hashtags)
                temp_hashtags.add(hashtag)

    def calculate_avg_degree(self):
        avg_degree = 0
        no_of_nodes = len(self.adjacency_list.keys())
        if no_of_nodes != 0:
            for links in self.adjacency_list.values():
                avg_degree += len(links)

            avg_degree = round(avg_degree * 1. / no_of_nodes, 2)
        return avg_degree

    def process_tweets(self):
        output = open(self.output_filepath, "a+")

        with open(self.input_filepath) as tweets:
            for raw_tweet in tweets:
                if "source" in raw_tweet:
                    source_word_idx = raw_tweet.index(',"source"')
                    trimmed_tweet = raw_tweet[0:source_word_idx]
                    matches = self.regex.search(trimmed_tweet)

                    if matches != None:
                        timestamp = matches.group(1)
                        content = str(matches.group(3))

                        clean_tweet = self.clean_tweet_object.get_clean_tweet(
                            content)
                        self.add_nodes(clean_tweet, timestamp)

                output.write(str(self.calculate_avg_degree()))
                output.write("\n")
        output.close()
class AverageDegree(object):
    def __init__(self, inputfile="dummyInput", outputfile="dummyOutput"):
        self.regex = re.compile('"((Sun|Mon|Tue|Thu|Wed|Fri|Sat) [A-Za-z0-9\s:\+]*)".*"text":"(.*)"')
        self.hashtag_regex = re.compile("#(\w+)")
        self.format_string = "%a %b %d %H:%M:%S +0000 %Y"
        self.time_key = {}
        self.adjacency_list = {}
        self.input_filepath = inputfile
        self.output_filepath = outputfile
        self.clean_tweet_object = CleanTweet()

    def remove_older_hashtags(self, timestamp):
        one_minute = datetime.timedelta(minutes=1)
        current_time = datetime.datetime.strptime(timestamp, self.format_string)
        timestamps = self.time_key.keys()
        ts_to_remove = filter(
            lambda old_ts: (current_time - datetime.datetime.strptime(old_ts, self.format_string)) > one_minute,
            timestamps,
        )
        hashtags = []
        for ts in ts_to_remove:
            hashtags += self.time_key.pop(ts)
        unique_hashtags = set(hashtags)
        for hashtag in unique_hashtags:
            self.adjacency_list[hashtag] -= unique_hashtags

    def add_nodes(self, tweet, timestamp):
        raw_hashtags = self.hashtag_regex.findall(tweet)
        hashtags = map(lambda tag: tag.lower().strip(), raw_hashtags)

        self.remove_older_hashtags(timestamp)
        if timestamp not in self.time_key:
            self.time_key[timestamp] = []

        unique_hashtags = set(hashtags)

        if len(unique_hashtags) > 1:
            self.time_key[timestamp] += list(unique_hashtags)
            temp_hashtags = unique_hashtags.copy()
            for hashtag in unique_hashtags:
                if hashtag not in self.adjacency_list:
                    self.adjacency_list[hashtag] = set([])
                temp_hashtags.remove(hashtag)
                self.adjacency_list[hashtag].update(temp_hashtags)
                temp_hashtags.add(hashtag)

    def calculate_avg_degree(self):
        avg_degree = 0
        no_of_nodes = len(self.adjacency_list.keys())
        if no_of_nodes != 0:
            for links in self.adjacency_list.values():
                avg_degree += len(links)

            avg_degree = round(avg_degree * 1.0 / no_of_nodes, 2)
        return avg_degree

    def process_tweets(self):
        output = open(self.output_filepath, "a+")

        with open(self.input_filepath) as tweets:
            for raw_tweet in tweets:
                if "source" in raw_tweet:
                    source_word_idx = raw_tweet.index(',"source"')
                    trimmed_tweet = raw_tweet[0:source_word_idx]
                    matches = self.regex.search(trimmed_tweet)

                    if matches != None:
                        timestamp = matches.group(1)
                        content = str(matches.group(3))

                        clean_tweet = self.clean_tweet_object.get_clean_tweet(content)
                        self.add_nodes(clean_tweet, timestamp)

                output.write(str(self.calculate_avg_degree()))
                output.write("\n")
        output.close()