def on_data(self, data):

        global positive_count
        global negative_count
        global start_time
        global current_time
        global start_date_time
        global current_date_time
        global f

        tweets = json.loads(data)
        if 'text' in tweets and tweets:
            s = tweets['text']
            if not s.isspace():
                language = tweets['lang']
                if language == 'en':
                    s=s.encode('ascii','ignore')
                    print "original Tweet           : "+ s
                    s= re.sub( '\s+', ' ', s).strip()
                    if s:
                        s= normalize_sentence(s)
                        s = remove.remove(s)
                        s = re.sub(url,'',s)
                        s = re.sub(username,'',s)
                        s = re.sub(hashtags,'',s)
                        tokenized = tok.tokenize(s)   
                        s=''
                        for token in tokenized:
                            token = remove.remove(token)
                            s=s+token+' '
                        s= re.sub( '\s+', ' ', s).strip()
                        s = remove.remove(s)
                        print "normalised Tweet         : " + s
    ##                    table = string.maketrans("","")
    ##                    s=s.translate(table, string.punctuation)
                        if not s.isspace():
                            f.write(s+'\n')
                            current_time=time.time();
                            elapsed_time = current_time-start_time
                            if elapsed_time>3600:
                                f.close()
                                start_date_time=datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')
                                filename='live_tweets_' + str(start_date_time)+'.txt'
                                f = codecs.open(filename,'w','utf-8')               
                                start_time=time.time();           
        return True
    def on_data(self, data):
        global i
        i=i+1
        print i
        if i==100000:
            exit(i)
        tweets = json.loads(data)
        s = tweets['text']
        language = tweets['lang']
        if language == 'en':
            s=s.encode('ascii','ignore')
##          print "======================================================================"
            print "original Tweet           : " + s
            s= re.sub( '\s+', ' ', s).strip()
            s= normalize_sentence(s)
##            print "normalized Tweet         : " + s
            s = remove.remove(s)
##            print "stop words removed Tweet : " + s
            
            s = re.sub(url,'',s)
#            s = re.sub(hashtags,'',s)
            s = re.sub(username,'',s)
##            print "url  hashtags usernames removed Tweet : " + s
            
            tokenized = tok.tokenize(s)
##            print "Tokens in Tweet : "
##            print "\n".join(tokenized)    
            s=''
            for token in tokenized:
                token = remove.remove(token)
                s=s+token+' '
            s= re.sub( '\s+', ' ', s).strip()
##            f.write(s+'\n');
            s = remove.remove(s)
##            print "Final Tweet : " + s
##            if 'retweeted_status' in tweets:
##                retweet_count = tweets['retweeted_status']['retweet_count']   
##            else:
##                retweet_count=0

            ##writer.writerow((s,retweet_count))
##            print Stored_classifier.classify(extract_features(s.split()))

            return True
Stored_classifier = pickle.load(pkl_file)
##f1 = open('classified_tweets.txt', 'w')
pkl_file = open('word_features.pkl', 'rb')
word_features= pickle.load(pkl_file)
print "classifier loaded"
f1 = codecs.open('classified_tweets.txt','w','utf-8')
writer = csv.writer(f1, delimiter = '\t')
print "file opened for writing"
with open('200tweets.txt', 'rb') as csvfile:
    spamreader = csv.reader(csvfile, dialect="excel-tab")
    for (s, labelled_sentiment) in spamreader:
        print i
        i=i+1
##        s=s.decode("utf8")
        s= re.sub( '\s+', ' ', s).strip()
        s= normalize_sentence(s)
        s = remove.remove(s)
        s = re.sub(url,'',s)
##        s = re.sub(hashtags,'',s)
        s = re.sub(username,'',s)
        tokenized = tok.tokenize(s)
        s=''
        for token in tokenized:
            token = remove.remove(token)
            s=s+token+' '
        s= re.sub( '\s+', ' ', s).strip()
        s = remove.remove(s)
        classified_sentiment =  Stored_classifier.classify(extract_features(s.split()))
        print s+'\n'
        print classified_sentiment
        writer.writerow((s,classified_sentiment))