class Tagger():
def __init__(self, data_path, train_dir):
self.twit = Twit()
print("Model upload....")
self.model = model(self.twit.vocab_size)
self.sess = tf.Session()
print('checkpoint upload...')
ckpt = tf.train.get_checkpoint_state(train_dir)
print('model restore,,,')
self.model.saver.restore(self.sess, ckpt.model_checkpoint_path)
print('initialize end!')
def run(self):
sys.stdout.write('> ')
sys.stdout.flush()
line = sys.stdin.readline()
while line:
tags = self.recommend({'text': line.strip()})
for i in tags:
print('#' + i, end=' ')
print('\n')
sys.stdout.write('\n> ')
sys.stdout.flush()
line = sys.stdin.readline()
def recommend(self, post):
#언젠가 유저가 그림도 줄 수 있으니, string이 아닌 dict으로 처리
print("유저 POST : {}".format(post['text']))
msg = post['text'].replace('#', '').lower()
tok = nt.pos_tag(nt.word_tokenize(msg))
tok = [i[0] for i in tok]
enc_input = self.twit.tokens_to_id(tok)
dec_input = []
curr_seq = 0
for i in range(1): #FLAGS.max_decode_len):
top_k, outputs = self._decode(enc_input, dec_input)
candis = []
top_k_indices = top_k.indices[0][0]
final_recommend = []
for candi in top_k_indices:
if candi not in [
self.twit.EOS_KEY, self.twit.PAD_KEY, self.twit.UNK_KEY
]:
candis.append(self.twit.voca_list[candi])
final_recommend.append(candi)
final_recommend = final_recommend[:FLAGS.map_k]
reply = self.twit.decode([final_recommend], True)
return reply[0]
def _decode(self, enc_input, dec_input):
if type(dec_input) == np.ndarray:
dec_input = dec_input.tolist()
input_len = int(math.ceil((len(enc_input) + 1) * 1.5))
enc_input, dec_input, _ = self.twit.transform(enc_input, dec_input,
input_len)
return self.model.predict(self.sess, [enc_input], [dec_input])
def __init__(self, data_path, train_dir):
self.twit = Twit()
print("Model upload....")
self.model = model(self.twit.vocab_size)
self.sess = tf.Session()
print('checkpoint upload...')
ckpt = tf.train.get_checkpoint_state(train_dir)
print('model restore,,,')
self.model.saver.restore(self.sess, ckpt.model_checkpoint_path)
print('initialize end!')
def get_elsetwits(self, usrhandle):
connection = dbapi2.connect(current_app.config['dsn'])
cursor = connection.cursor()
cursor.execute("""SELECT ID FROM USERS WHERE USERNAME=%s""", (usrhandle,))
userid=cursor.fetchone()
cursor.execute("""SELECT tweets.title,
tweets.context,
tweets.tweetid,
users.username,
tweets.numberoflikes,
tweets.numberofrts,
tweets.isrt,
userprofile.username AS rtowner
FROM tweets
RIGHT JOIN users ON users.id = tweets.userid
RIGHT JOIN userprofile ON userprofile.id = tweets.rtownerid
WHERE tweets.userid = %s AND tweets.isrt = %s
UNION
SELECT tweets.title,
tweets.context,
tweets.tweetid,
users.username,
tweets.numberoflikes,
tweets.numberofrts,
tweets.isrt,
userprofile.username AS rtowner
FROM tweets
RIGHT JOIN users ON users.id = tweets.userid
RIGHT JOIN userprofile ON userprofile.id = tweets.rtownerid
WHERE tweets.rtownerid = %s AND tweets.isrt = %s
ORDER BY TWEETID DESC""", (userid, 0, userid, 1))
twit = [(Twit(title, context, twitid, userhandle, numberoflikes, numberofrts, isrt, rtowner))
for title, context, twitid, userhandle, numberoflikes, numberofrts, isrt, rtowner in cursor]
return twit
def __init__(self):
self.cfg = Config()
self.t = Terminal()
self.tw = Twit(self.cfg.key('consumer_key'),
self.cfg.key('consumer_secret'),
self.cfg.key('access_token'),
self.cfg.key('access_token_secret'))
self.name = self.cfg.get('name')
self.background_char = self.cfg.get('background_char')
self.color_a = self.cfg.get('color_a')
self.color_b = self.cfg.get('color_b')
self.last_message = self.cfg.msg('footer').format(
a=self.t.color(self.color_a),
b=self.t.color(self.color_b),
quit=self.t.underline('quit'))
def main(_):
print('시작~')
twit = Twit()
if FLAGS.train:
print('model train start')
train(twit, batch_size=FLAGS.batch_size, epoch=FLAGS.epoch)
elif FLAGS.test:
print('model test start')
test(twit, batch_size=FLAGS.batch_size)
def NB():
#clf = GaussianNB()
clf = MultinomialNB()
twit = Twit()
X,y,X_test,y_test = [],[],[],[]
for t in twit.twits:
for i,h in enumerate(t['tag_vec']):
if h in [twit.PAD_KEY,twit.UNK_KEY]: continue
X.append(t['vec'])
y.append([h])
for t in twit.test:
X_test.append(t['vec'])
y_test.append([])
for i,h in enumerate(t['tag_vec']):
if h in [twit.PAD_KEY,twit.UNK_KEY]: continue
y_test[-1].append(h)
print("CHECK COPLETE")
X = np.array(X)
y = np.array(y)
clf.fit(X,y)
res = clf.predict(np.array(X_test))
prob = clf.predict_log_proba(np.array(X_test))
samples = random.sample(range(len(X_test)),10)
for i in samples:
t=' '.join(twit.decode([X_test[i]])[0]) #raw text
idx = t.index('_PAD_')
t=t[:idx] #text padding 제거
top_n = prob[i].argsort()[-len(y_test[i]):][::-1] #정답과 같은 개수의 추천 받는다
print('원문 : ',t)
print('예측 : ',twit.voca_list[res[i]])
print('예측 후보 : ',[twit.voca_list[w] for w in top_n])
print('정답 : ',[twit.voca_list[j] for j in y_test[i]])
print()
def getLikedTweets(username):
connection=dbapi2.connect(current_app.config['dsn'])
cursor=connection.cursor()
cursor.execute("""SELECT ID FROM USERS WHERE USERNAME=%s""",(username,))
temp=cursor.fetchone()
userid=temp[0]
cursor.execute("""SELECT T.TWEETID,T.USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOwnerID,TWEETOWNERNAME,RTOWNERNAME,LIKETIME
FROM LIKES JOIN (SELECT TWEETID,USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOwnerID,U1.USERNAME AS TWEETOWNERNAME,
U2.USERNAME AS RTOWNERNAME FROM TWEETS JOIN USERS AS U1 ON U1.ID=TWEETS.USERID JOIN USERS AS U2 ON U2.ID=TWEETS.RTOWNERID)
AS T ON LIKES.TWEETID=T.TWEETID WHERE LIKES.USERID=%s ORDER BY LIKETIME DESC""",(userid,))
likedTweets = [(Twit(title, context, twitid, userhandle, numberoflikes, numberofrts, isrt, rtowner))
for twitid,userid,context,title, numberoflikes,numberofrts,isrt, rtownerid,userhandle,rtowner,liketime in cursor]
return likedTweets
def get_twit(self, twitid):
connection = dbapi2.connect(current_app.config['dsn'])
cursor = connection.cursor()
cursor.execute("""SELECT tweets.title,
tweets.context,
tweets.tweetid,
users.username,
tweets.numberoflikes,
tweets.numberofrts,
tweets.isrt,
userprofile.username AS rtowner
FROM tweets
RIGHT JOIN users ON users.id = tweets.userid
RIGHT JOIN userprofile ON userprofile.id = tweets.rtownerid
WHERE tweets.tweetid = %s""", [twitid],)
title, context, twitid, userhandle, numberoflikes, numberofrts, isrt, rtowner = cursor.fetchone()
twits=Twit(title, context, twitid, userhandle, numberoflikes, numberofrts, isrt, rtowner)
return twits
def getTweets(self):
connection = dbapi2.connect(current_app.config['dsn'])
cursor = connection.cursor()
cursor.execute(
"""SELECT LISTID FROM LISTS WHERE NAME=%s AND CreatorID=%s""",
(self.name, self.creatorid))
temp = cursor.fetchone()
listid = temp[0]
cursor.close()
cursor = connection.cursor()
cursor.execute(
"""SELECT TWEETID,R.USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOWNERID,
TWEETOWNERNAME,RTOWNERNAME FROM (SELECT USERNAME,USERID FROM USERS JOIN LISTMEMBERS ON USERS.ID=LISTMEMBERS.USERID
WHERE ((USERTYPE='Insider' OR USERTYPE='Owner') AND LISTID=%s )) AS R JOIN
(SELECT TWEETID,USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOwnerID,U1.USERNAME AS TWEETOWNERNAME,
U2.USERNAME AS RTOWNERNAME FROM TWEETS
JOIN USERS AS U1 ON U1.ID=TWEETS.USERID JOIN USERS AS U2 ON U2.ID=TWEETS.RTOWNERID ) AS Y
ON R.USERID=Y.USERID
UNION
SELECT TWEETID,E.USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOWNERID,
TWEETOWNERNAME,RTOWNERNAME
FROM
(SELECT USERNAME,USERID FROM USERS JOIN LISTMEMBERS ON USERS.ID=LISTMEMBERS.USERID
WHERE ((USERTYPE='Insider' OR USERTYPE='Owner') AND LISTID=%s )) AS E
JOIN
(SELECT TWEETID,USERID,CONTEXT,TITLE,NUMBEROFLIKES,NUMBEROFRTS,isRT,RTOwnerID,U3.USERNAME AS TWEETOWNERNAME,U4.USERNAME AS RTOWNERNAME FROM TWEETS
JOIN USERS AS U3 ON U3.ID=TWEETS.USERID JOIN USERS AS U4 ON U4.ID=TWEETS.RTOWNERID ) AS Z
ON E.USERID=Z.RTOwnerID
ORDER BY TWEETID DESC""", (listid, listid))
twit = [(Twit(title, context, twitid, userhandle, numberoflikes,
numberofrts, isrt, rtowner))
for twitid, userid, context, title, numberoflikes, numberofrts,
isrt, rtownerid, userhandle, rtowner in cursor]
cursor.close()
connection.close()
return twit
class App:
def __init__(self):
self.cfg = Config()
self.t = Terminal()
self.tw = Twit(self.cfg.key('consumer_key'),
self.cfg.key('consumer_secret'),
self.cfg.key('access_token'),
self.cfg.key('access_token_secret'))
self.name = self.cfg.get('name')
self.background_char = self.cfg.get('background_char')
self.color_a = self.cfg.get('color_a')
self.color_b = self.cfg.get('color_b')
self.last_message = self.cfg.msg('footer').format(
a=self.t.color(self.color_a),
b=self.t.color(self.color_b),
quit=self.t.underline('quit'))
def colorize(self, key, message=False):
return self.cfg.msg(key).format(a=self.t.color(self.color_a),
b=self.t.color(self.color_b),
message=message)
def decode(self, message):
msg = message.strip().split()
cmd_color = ['color', 'c']
cmd_twit = ['twit', 't']
cmd_help = ['help', 'h']
cmd_name = ['name']
# COLOR
if msg[0] in cmd_color and len(msg) == 3:
if msg[1].isdigit() and msg[2].isdigit():
self.color_a = int(msg[1])
self.color_b = int(msg[2])
return self.colorize('done')
return self.colorize('error')
# TWIT
elif msg[0] in cmd_twit and len(msg) > 1:
ret = self.tw.twit(message.replace(msg[0], '').strip())
if ret:
return self.colorize('twit', ret)
else:
return self.colorize('error')
# NAME
elif msg[0] in cmd_name and len(msg) > 1:
self.name = ' {name} '.format(
name=message.replace(msg[0], '').strip())
return self.colorize('done')
# DEFAULT
else:
return self.colorize('error')
def decorate_background(self):
with self.t.location(0, 0):
for row in range(self.t.height):
for col in range(self.t.width):
print(self.t.blue_on_bright_blue(self.background_char),
end='')
def terminal(self):
os.system('clear')
self.decorate_background()
with self.t.location(4, 2):
print(
self.t.on_bright_blue(
self.cfg.msg('title').format(a=self.t.color(self.color_a),
b=self.t.color(self.color_b),
title=self.t.underline(
self.name),
ver=self.cfg.get('version'))))
if self.last_message:
with self.t.location(4, self.t.height - 2):
print(self.t.on_bright_blue(self.last_message))
with self.t.location(4, self.t.height - 4):
message = input(self.colorize('prompt'))
if message == 'quit':
return False
else:
self.last_message = self.decode(message)
return True