def on_data(self, data):
#global model
try:
with open(self.outfile, 'a') as f:
#if 'korea' in data:
f.write(data)
#print(data)
all_data=json.loads(data)
tweet=all_data["text"].encode("utf-8")
#username=all_data["user"]["screen_name"]
tweet=" ".join(re.findall("[a-zA-Z]+", tweet))
print("TEXT:::", tweet)
#model=lm.load_model()
#X_newinput=[[1, 40445, 55824, 52346, 67196, 72368, 76113, 49609, 45222, 77017, 18685, 43795, 44566, 20744, 18336, 71466, 16945, 42630, 23024, 56334, 58251, 53247, 67560]]
#X_newinput = sequence.pad_sequences(X_newinput, 100)
X_newinput = lm.load_X_newinput(tweet)
print(X_newinput)
print("ANSWER IS: ", model.predict_classes(X_newinput))
# t0 = time.clock()
# X_newinput= aidr2.load_and_numberize_data2(input_tweet,path=data_dir, nb_words=max_features, init_type=init_type, embfile=emb_file, validate_train_merge=0, #map_labels_to_five_class=0)
# print ("Thoi gian map to vector ", time.clock() - t0)
# print ( X_newinput)
# X_newinput = sequence.pad_sequences(X_newinput, maxlen)#Quan them vao--------------------
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
time.sleep(5)
return True
def on_data(self, data):
#global model
try:
with open("streamming_quan.json", 'a') as f:
f.write(data)
#print(data)
all_data = json.loads(data)
tweet = all_data["text"].encode("utf-8")
#username=all_data["user"]["screen_name"]
tweet = " ".join(re.findall("[a-zA-Z]+", tweet))
print("TEXT:::", tweet)
X_newinput = lm.load_X_newinput(tweet)
Y_pred = str(model.predict_classes(X_newinput)[0][0])
print(X_newinput)
print("ANSWER IS: ", Y_pred)
#self.listWidget.addItem(tweet)
if Y_pred == '0':
self.listWidget.addItem(tweet)
else:
self.listWidget_2.addItem(tweet)
return tweet #True
except BaseException as e:
print("Error on_data: %s" % str(e))
time.sleep(5)
return True
def on_data(self, data):
global keywords, limits, lang
try:
with open("streamming_quan.json", 'a') as f:
if 'created_at' in data and '"lang":"en"' in data and (
check_limit(data.lower(), limits)): # ('' in data.lower() or '' in data.lower()) :
t = int(calctime(initime))
# f.write(data)
# print(data)
send_tweets_to_spark(data, conn)
all_data = json.loads(data)
tweet1 = all_data["text"].encode("utf-8")
created_at = all_data['created_at'].encode("utf-8")
timestamp_ms = all_data['timestamp_ms'].encode("utf-8")
# username=all_data["user"]["screen_name"]
# tweet=" ".join(re.findall("[a-zA-Z]+", tweet))
tweet = pre_process(tweet1)
print("TEXT:::", tweet)
X_newinput = lm.load_X_newinput(tweet)
# Y_pred='0'
Y_pred = str(model.predict(X_newinput)[0][0])
# Y_pred2=str(model2.predict_classes(X_newinput)[0][0])
# print(X_newinput)
print("ANSWER IS: ", Y_pred)
# self.listWidget.addItem(tweet)
# myQCustomQWidget = QCustomQWidget()
# myQCustomQWidget.setIcon('earthquake.png')
# self.delegate = ItemDelegate()
username = all_data["user"]["screen_name"]
self.win_draw.pushButtonPlot.setText(username)
# self.win_draw.matplotlibWidget.axis.clear()
# self.win_draw.matplotlibWidget.axis.plot(random.sample(range(0, 20),10))
# x=[[3, 3 ,3,3 ,3 ,3 ,3 ,3 ,3]]
# x = np.array(x)
# modelRNN=lr.load_modelRNN()
# print ("ANSERRRRRRRRR", lr.RNN_predict(modelRNN,x)[0,0])
# self.win_draw.matplotlibWidget.canvas.draw()
# self.win_draw.on_pushButtonPlot_clicked(5,10)
if Y_pred == '0':
# self.listWidget.addItem(created_at + ": "+tweet)
self.win_draw.Accumulate(timestamp_ms)
self.addnew_Item_listWidget(created_at + ": " + tweet)
self.listWidget.scrollToBottom()
# self.win_draw.RealtimePlot()
else:
# self.listWidget_2.addItem(created_at + ": "+tweet)
self.addnew_Item_listWidget_2(created_at + ": " + tweet)
self.listWidget_2.scrollToBottom()
# self.win_draw.Accumulate(timestamp_ms)
# self.win_draw.RealtimePlot()
# plt.axis([ 0, 70, -20,20])
# plt.xlabel('Time')
# plt.ylabel('Sentiment')
# plt.plot([t],20,'go',[t] ,30,'ro')
# plt.show()
return tweet # True
except BaseException as e:
print("Error on_data: %s" % str(e))
time.sleep(5)
return True
def run(self):
global keywords, limits, lang,conn
# print(check_limit("i love uou", limits))
tweet_count = 0
file_in = 'stream_earthquake8_ngay21thag9.json'
# keyword_list=['quanap5']
print("RUN DEMO:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::RUN DEMO")
f = open(file_in, 'r')
line = f.readline()
while line and self.win_draw.check_stopDemo == 0:
if 'created_at' in line and '"lang":"en"' in line and (
check_limit(line.lower(), limits)): # ('korea' in line.lower() or 'kor' in line.lower()) :
# t=int(calctime(initime))
# f.write(data)
# print(data)
tweet_count += 1
send_tweets_to_spark(line, conn)
print (conn)
print ("AlREADY read: ", tweet_count)
all_data = json.loads(line)
tweet = all_data["text"].encode("utf-8")
created_at = all_data['created_at'].encode("utf-8")
timestamp_ms = all_data['timestamp_ms'].encode("utf-8")
# username=all_data["user"]["screen_name"]
# tweet=" ".join(re.findall("[a-zA-Z]+", tweet))
tweet = pre_process(tweet)
print("TEXT:::", tweet)
X_newinput = lm.load_X_newinput(tweet)
# Y_pred='0'
Y_pred = str(model.predict_classes(X_newinput)[0][0])
# Y_pred2=str(model2.predict_classes(X_newinput)[0][0])
# print(X_newinput)
print("ANSWER IS: ", Y_pred)
# self.listWidget.addItem(tweet)
# myQCustomQWidget = QCustomQWidget()
# myQCustomQWidget.setIcon('earthquake.png')
# self.delegate = ItemDelegate()
username = all_data["user"]["screen_name"]
self.win_draw.pushButtonPlot.setText(username)
# self.win_draw.matplotlibWidget.axis.clear()
# self.win_draw.matplotlibWidget.axis.plot(random.sample(range(0, 20),10))
# xx=[[0, 0 ,0,0 ,0 ,0 ,0 ,0 ,0]]
# xx = np.array(xx)
# modelRNN=lr.load_modelRNN()
# print ("RNN_LSTM ANSWER", lr.RNN_predict(modelRNN,xx)[0,0])
# self.win_draw.matplotlibWidget.canvas.draw()
# self.win_draw.on_pushButtonPlot_clicked(5,10)
if Y_pred == '0': # and 1488499200000 < long(timestamp_ms) < 1488758400000:
# self.listWidget.addItem(created_at + ": "+tweet)
self.listWidget.scrollToBottom()
self.win_draw.Accumulate(timestamp_ms)
self.addnew_Item_listWidget(created_at + ": " + tweet)
# self.win_draw.RealtimePlot()
else: # 1488499200000 < long(timestamp_ms) < 1488758400000:
# self.listWidget_2.addItem(created_at + ": "+tweet)
self.listWidget_2.scrollToBottom()
# self.win_draw.Accumulate(timestamp_ms)
self.addnew_Item_listWidget_2(created_at + ": " + tweet)
# self.win_draw.RealtimePlot()
# plt.axis([ 0, 70, -20,20])
# plt.xlabel('Time')
# plt.ylabel('Sentiment')
# plt.plot([t],20,'go',[t] ,30,'ro')
# plt.show()
line = f.readline()
f.close()
print ("# Tweets Imported:", tweet_count)