def on_data(self, data): #global model try: with open(self.outfile, 'a') as f: #if 'korea' in data: f.write(data) #print(data) all_data=json.loads(data) tweet=all_data["text"].encode("utf-8") #username=all_data["user"]["screen_name"] tweet=" ".join(re.findall("[a-zA-Z]+", tweet)) print("TEXT:::", tweet) #model=lm.load_model() #X_newinput=[[1, 40445, 55824, 52346, 67196, 72368, 76113, 49609, 45222, 77017, 18685, 43795, 44566, 20744, 18336, 71466, 16945, 42630, 23024, 56334, 58251, 53247, 67560]] #X_newinput = sequence.pad_sequences(X_newinput, 100) X_newinput = lm.load_X_newinput(tweet) print(X_newinput) print("ANSWER IS: ", model.predict_classes(X_newinput)) # t0 = time.clock() # X_newinput= aidr2.load_and_numberize_data2(input_tweet,path=data_dir, nb_words=max_features, init_type=init_type, embfile=emb_file, validate_train_merge=0, #map_labels_to_five_class=0) # print ("Thoi gian map to vector ", time.clock() - t0) # print ( X_newinput) # X_newinput = sequence.pad_sequences(X_newinput, maxlen)#Quan them vao-------------------- return True except BaseException as e: print("Error on_data: %s" % str(e)) time.sleep(5) return True
def on_data(self, data): #global model try: with open("streamming_quan.json", 'a') as f: f.write(data) #print(data) all_data = json.loads(data) tweet = all_data["text"].encode("utf-8") #username=all_data["user"]["screen_name"] tweet = " ".join(re.findall("[a-zA-Z]+", tweet)) print("TEXT:::", tweet) X_newinput = lm.load_X_newinput(tweet) Y_pred = str(model.predict_classes(X_newinput)[0][0]) print(X_newinput) print("ANSWER IS: ", Y_pred) #self.listWidget.addItem(tweet) if Y_pred == '0': self.listWidget.addItem(tweet) else: self.listWidget_2.addItem(tweet) return tweet #True except BaseException as e: print("Error on_data: %s" % str(e)) time.sleep(5) return True
def on_data(self, data): global keywords, limits, lang try: with open("streamming_quan.json", 'a') as f: if 'created_at' in data and '"lang":"en"' in data and ( check_limit(data.lower(), limits)): # ('' in data.lower() or '' in data.lower()) : t = int(calctime(initime)) # f.write(data) # print(data) send_tweets_to_spark(data, conn) all_data = json.loads(data) tweet1 = all_data["text"].encode("utf-8") created_at = all_data['created_at'].encode("utf-8") timestamp_ms = all_data['timestamp_ms'].encode("utf-8") # username=all_data["user"]["screen_name"] # tweet=" ".join(re.findall("[a-zA-Z]+", tweet)) tweet = pre_process(tweet1) print("TEXT:::", tweet) X_newinput = lm.load_X_newinput(tweet) # Y_pred='0' Y_pred = str(model.predict(X_newinput)[0][0]) # Y_pred2=str(model2.predict_classes(X_newinput)[0][0]) # print(X_newinput) print("ANSWER IS: ", Y_pred) # self.listWidget.addItem(tweet) # myQCustomQWidget = QCustomQWidget() # myQCustomQWidget.setIcon('earthquake.png') # self.delegate = ItemDelegate() username = all_data["user"]["screen_name"] self.win_draw.pushButtonPlot.setText(username) # self.win_draw.matplotlibWidget.axis.clear() # self.win_draw.matplotlibWidget.axis.plot(random.sample(range(0, 20),10)) # x=[[3, 3 ,3,3 ,3 ,3 ,3 ,3 ,3]] # x = np.array(x) # modelRNN=lr.load_modelRNN() # print ("ANSERRRRRRRRR", lr.RNN_predict(modelRNN,x)[0,0]) # self.win_draw.matplotlibWidget.canvas.draw() # self.win_draw.on_pushButtonPlot_clicked(5,10) if Y_pred == '0': # self.listWidget.addItem(created_at + ": "+tweet) self.win_draw.Accumulate(timestamp_ms) self.addnew_Item_listWidget(created_at + ": " + tweet) self.listWidget.scrollToBottom() # self.win_draw.RealtimePlot() else: # self.listWidget_2.addItem(created_at + ": "+tweet) self.addnew_Item_listWidget_2(created_at + ": " + tweet) self.listWidget_2.scrollToBottom() # self.win_draw.Accumulate(timestamp_ms) # self.win_draw.RealtimePlot() # plt.axis([ 0, 70, -20,20]) # plt.xlabel('Time') # plt.ylabel('Sentiment') # plt.plot([t],20,'go',[t] ,30,'ro') # plt.show() return tweet # True except BaseException as e: print("Error on_data: %s" % str(e)) time.sleep(5) return True
def run(self): global keywords, limits, lang,conn # print(check_limit("i love uou", limits)) tweet_count = 0 file_in = 'stream_earthquake8_ngay21thag9.json' # keyword_list=['quanap5'] print("RUN DEMO:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::RUN DEMO") f = open(file_in, 'r') line = f.readline() while line and self.win_draw.check_stopDemo == 0: if 'created_at' in line and '"lang":"en"' in line and ( check_limit(line.lower(), limits)): # ('korea' in line.lower() or 'kor' in line.lower()) : # t=int(calctime(initime)) # f.write(data) # print(data) tweet_count += 1 send_tweets_to_spark(line, conn) print (conn) print ("AlREADY read: ", tweet_count) all_data = json.loads(line) tweet = all_data["text"].encode("utf-8") created_at = all_data['created_at'].encode("utf-8") timestamp_ms = all_data['timestamp_ms'].encode("utf-8") # username=all_data["user"]["screen_name"] # tweet=" ".join(re.findall("[a-zA-Z]+", tweet)) tweet = pre_process(tweet) print("TEXT:::", tweet) X_newinput = lm.load_X_newinput(tweet) # Y_pred='0' Y_pred = str(model.predict_classes(X_newinput)[0][0]) # Y_pred2=str(model2.predict_classes(X_newinput)[0][0]) # print(X_newinput) print("ANSWER IS: ", Y_pred) # self.listWidget.addItem(tweet) # myQCustomQWidget = QCustomQWidget() # myQCustomQWidget.setIcon('earthquake.png') # self.delegate = ItemDelegate() username = all_data["user"]["screen_name"] self.win_draw.pushButtonPlot.setText(username) # self.win_draw.matplotlibWidget.axis.clear() # self.win_draw.matplotlibWidget.axis.plot(random.sample(range(0, 20),10)) # xx=[[0, 0 ,0,0 ,0 ,0 ,0 ,0 ,0]] # xx = np.array(xx) # modelRNN=lr.load_modelRNN() # print ("RNN_LSTM ANSWER", lr.RNN_predict(modelRNN,xx)[0,0]) # self.win_draw.matplotlibWidget.canvas.draw() # self.win_draw.on_pushButtonPlot_clicked(5,10) if Y_pred == '0': # and 1488499200000 < long(timestamp_ms) < 1488758400000: # self.listWidget.addItem(created_at + ": "+tweet) self.listWidget.scrollToBottom() self.win_draw.Accumulate(timestamp_ms) self.addnew_Item_listWidget(created_at + ": " + tweet) # self.win_draw.RealtimePlot() else: # 1488499200000 < long(timestamp_ms) < 1488758400000: # self.listWidget_2.addItem(created_at + ": "+tweet) self.listWidget_2.scrollToBottom() # self.win_draw.Accumulate(timestamp_ms) self.addnew_Item_listWidget_2(created_at + ": " + tweet) # self.win_draw.RealtimePlot() # plt.axis([ 0, 70, -20,20]) # plt.xlabel('Time') # plt.ylabel('Sentiment') # plt.plot([t],20,'go',[t] ,30,'ro') # plt.show() line = f.readline() f.close() print ("# Tweets Imported:", tweet_count)