def checkForum(self, url):
pubg_news = feedparser.parse(url)
# check 5 last news
n = 5
for x in range(0, n):
link = pubg_news['entries'][x]['link']
id = pubg_news['entries'][x]['id']
title = pubg_news['entries'][x]['title']
post_date = pubg_news['entries'][x]['published']
image = "https://content.invisioncic.com/r273030/monthly_2017_07/PUBG_Logo_color_RGB5.png.05a3748e744a03bcc597cc88afef9533.png"
with Database() as dbconn:
c = dbconn.cursor()
c.execute('SELECT id FROM news WHERE id = ?', (id, ))
data = c.fetchone()
if (data is None):
if self.checkDate(post_date) is True:
e = Embed()
entry = e.make_embed(link, title, image)
self.postWebhook(entry)
c.execute('INSERT INTO news VALUES (?, ?)',
(id, datetime.now()))
dbconn.commit()
def prepare():
config = ConfigRNN.instance()
embed = Embed()
embedding_model = embed.get_embedding_model()
if config.EMBED_METHOD == "DEFAULT":
model = ReviewParser()
else:
model = ReviewParser(
pretrained=torch.from_numpy(embedding_model.wv.vectors).float())
evaluator = RNNEvaluator(model)
return evaluator, embed
async def userinfo(self, ctx):
person = ctx.message.mentions[0].id
person = await self.bot.get_user_info(person)
name = person.name
discriminator = person.discriminator
avatar = person.avatar_url
id = person.id
embed = Embed(title="%s#%s" % (name, discriminator),
description="id:%s" % (id),
color=0xE0FFFF)
embed.set_thumbnail(url=avatar)
await self.bot.send_message(ctx.message.channel, embed=embed)
def __init__(self, train_dir="./data/train", classifier_dir="./model"):
self.emb = Embed()
self.train_dir = train_dir
self.classifier_dir = classifier_dir
self.extensions = ['.png', '.jpg']
self.rate = 0.0001
self.epochs = 200000
self.strip = 50
def inference(in_dir, model_dir):
emb = Embed()
LABELS = load_label_info("./data/train/train_label.txt")
w, b = load_coefs(model_dir)
cnts = [0, 0, 0, 0, 0]
# scan image in input
sys.stdout.write(" Scan folder for classification...\n")
files = [
f for f in os.listdir(in_dir)
if os.path.isfile(os.path.join(in_dir, f))
]
sys.stdout.write(" total files {}\n".format(len(files)))
for f in files:
fn, ext = os.path.splitext(f)
if ext.lower() == '.png':
# Convert image file to jpg
im = cv2.imread(os.path.join(in_dir, f))
new_fn = os.path.join(in_dir, fn + '.jpg')
cv2.imwrite(new_fn, im)
os.remove(os.path.join(in_dir, f))
elif ext.lower() == '.jpg':
new_fn = os.path.join(in_dir, f)
else:
continue
# Extract the feature vector per each image
feature = emb.get_feature_from_image(new_fn)
feature = feature.tolist()
label_idx = classify(feature, w, b)
# move the labeled image to folder named with its label(string)
cnts[label_idx] += 1
label = LABELS[label_idx]
label_dir = os.path.join(in_dir, label)
if not os.path.exists(label_dir):
os.makedirs(label_dir)
out_fn = os.path.join(label_dir, f)
if os.path.exists(out_fn):
os.remove(new_fn)
else:
os.rename(new_fn, out_fn)
sys.stdout.write(" label({}) : file{}\n".format(label, new_fn))
return cnts
async def post(self, ctx, select=0):
if select == 0:
conn = psycopg2.connect(DATABASE_URL)
cur = conn.cursor()
cur.execute('select * from post')
rows = cur.fetchall()
for row in rows:
num = row[0]
author = row[1]
head = row[2]
await self.bot.send_message(
ctx.message.channel,
"%s. %s - by %s\n" % (num, head, author))
conn.close()
await self.bot.send_message(
ctx.message.channel,
_("%s글 (번호)를 입력하시면 글을 보실수 있어요!") % prefix.get())
await self.bot.send_message(
ctx.message.channel,
_("글을 쓰시려면 %s써줘 (번호)와 제목을 한 줄에 쓰세요!") % prefix.get())
else:
conn = psycopg2.connect(DATABASE_URL)
cur = conn.cursor()
cur.execute('select * from post where num = {0}'.format(select))
row = cur.fetchone()
num = row[0]
author = row[1]
head = row[2]
body = row[3]
conn.close()
embed = Embed(title="%s. %s" % (num, head),
description="\nby %s\n%s" % (author, body),
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
async def telnet(self, ctx, host: str, port: int):
telnet = telnetlib.Telnet(host, port)
await self.bot.send_message(ctx.message.channel,
_('%s:%s에 연결 중...') % (host, port))
while True:
body = await self.bot.wait_for_message(timeout=60.0,
author=ctx.message.author)
if body is None:
await self.bot.send_message(ctx.message.channel,
_('응답이 없어서 종료되었습니다.'))
break
else:
if body.content.startswith(_('종료')):
await self.bot.send_message(ctx.message.channel,
_('종료되었습니다.'))
break
else:
try:
line = body.content + "\n"
telnet.write(line.encode('ascii'))
await asyncio.sleep(1)
response = telnet.read_very_eager()
embed = Embed(title="%s:%s" % (host, port),
description="%s" %
(response.decode('ascii')),
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel,
embed=embed)
except:
await self.bot.send_message(ctx.message.channel,
_('오류가 발생했습니다!'))
return
async def osu(self, ctx, *user):
username = ''
i = 0
for word in user:
i += 1
if word == user[-1] and len(user) == i:
username += str(word)
else:
username += str(word) + ' '
api = OsuApi(osuapikey, connector=ReqConnector())
results = api.get_user(username)
userid = results[0].user_id
thumbnail = "https://s.ppy.sh/a/" + str(userid)
embed = Embed(title="%s" % (username),
description="id:%s" % (userid),
color=0xE0FFFF)
embed.set_thumbnail(url=thumbnail)
await self.bot.send_message(ctx.message.channel, embed=embed)
await self.bot.send_message(ctx.message.channel,
"https://osu.ppy.sh/users/%d" % userid)
def main():
config = ConfigRNN.instance()
embed = Embed()
# TODO(kyungsoo): Make this working.
embedding_model = embed.get_embedding_model()
if embedding_model == "DEFAULT":
model = RNN()
else:
vectors = embedding_model.wv.vectors
# Add padding for masking.
vectors = np.append(np.array([100 * [0]]), vectors, axis=0)
model = RNN(torch.from_numpy(vectors).float())
optimizer = torch.optim.SGD(model.parameters(),
lr=config.LEARNING_RATE,
weight_decay=config.WEIGHT_DECAY)
trainer = RNNEvaluator(model, optimizer)
# TODO(kyungsoo): Make this working.
review_vector = embed.review2vec(sys.argv[0])
print(trainer.evaluate(review_vector=review_vector))
async def module(self, ctx):
i = 0
body = ""
for plugin in pluginlist.get()["default"]:
i += 1
body += plugin + " "
if i % 5 == 0:
body += '\n'
embed = Embed(title=_("사용중인 모듈 리스트"), description=body, color=0x00FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
i = 0
body = ""
for plugin in pluginlist.get()["non-default"]:
i += 1
body += plugin + " "
if i % 5 == 0:
body += '\n'
embed = Embed(title=_("사용 가능한 모듈 리스트"),
description=body,
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
await self.bot.say("sorry! module setting for each channel is not yet."
)
async def exec(self, ctx, *, command):
if ctx.message.author.id == owner:
try:
print(command)
result = subprocess.check_output(command, shell=True)
print(result)
embed = Embed(title=_("명령어 실행: %s") % (command),
description="%s" % (result.decode('ascii')),
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
except Exception as e:
print(e)
await self.bot.send_message(ctx.message.channel,
_('오류가 발생했습니다!'))
else:
await self.bot.send_message(ctx.message.channel,
_('권한이 없습니다!\n개발자만 접근 가능합니다.'))
async def log(self, message, channel, key):
listener = message.author
try:
body = await self.bot.wait_for_message(channel=channel)
if body is None:
await self.bot.send_message(listener, _('응답이 없어서 종료되었습니다.'))
pass
else:
if str(body.content) == str(key) and body.author.name != self.bot.user.name:
await self.bot.send_message(listener, _('비밀번호 입력으로 인해 종료되었습니다.'))
elif body.server == None:
pass
else:
await bot_log(_('\n서버:%s\n채널:%s\n작성자:%s\n%s\n') % (body.server, body.channel, body.author, body.content))
embed = Embed(title=_("log"), description=_('\n서버:%s\n\n채널:%s\n\n작성자:%s\n\n%s') % (
body.server, body.channel, body.author, body.content), color=0xE0FFFF)
await self.bot.send_message(message.author, embed=embed)
await self.log(message, channel, key)
except Exception as e:
print(e)
async def alohsaydCore(self, ctx, num: int):
conn = psycopg2.connect(DATABASE_URL, sslmode='require')
with conn:
try:
cur = conn.cursor()
cur.execute("select * from alohsayd where num={0}".format(num))
rows = cur.fetchall()
row = rows[0]
print(row)
num = row[0]
head = row[1]
body = row[2]
head = head.replace("\n", "")
embed = Embed(title="%s" % head,
description="\n%s" % (body),
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
except:
await self.bot.send_message(ctx.message.channel,
_("파일을 찾지 못했어요!"))
pass
async def write(self, ctx, *, head=None):
conn = psycopg2.connect(DATABASE_URL)
with conn:
try:
cur = conn.cursor()
num = cur.lastlowid
num += 1
except:
num = 1
pass
if not head:
await self.bot.send_message(ctx.message.channel, _("제목이 없습니다."))
else:
await self.bot.send_message(ctx.message.channel, _("내용을 말해주세요!"))
body = await self.bot.wait_for_message(timeout=600,
author=ctx.message.author)
body = body.content
author = ctx.message.author.name
embed = Embed(title="%s" % head,
description="\nby %s\n%s" % (author, body),
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
await self.postinsert("post", num, author, head, body)
async def whoami(self, ctx):
embed = Embed(title=_("당신은 혹시..."),
description=_("\n%s\n\n이신가요?!?!") % ctx.message.author,
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
async def naesungwiki(self, ctx):
embed = Embed(title=_("내성위키"),
description="https://naesung.wiki",
color=0xE0FFFF)
await self.bot.send_message(ctx.message.channel, embed=embed)
async def info(self, ctx):
ownername = await self.bot.get_user_info(owner)
ownername = ownername.name
modstring = ""
for a_mod in mod:
modname = await self.bot.get_user_info(a_mod)
modname = modname.name
modstring += modname + " "
if not modstring:
modstring = "None"
users = 0
for s in self.bot.servers:
users += len(s.members)
now = datetime.now().date()
dday = date(2018, 8, 6)
result = now - dday
embed = Embed(title=_("%s 정보") % self.bot.user.name,
description=instructions.get(),
color=0x1ef7fa)
embed.set_thumbnail(url=self.bot.user.avatar_url)
embed.add_field(name=_("봇 운영자"), value=ownername, inline=True)
embed.add_field(name=_("봇 부운영자"), value=modstring, inline=True)
embed.add_field(name=_("서버 수"),
value=len(self.bot.servers),
inline=True)
embed.add_field(name=_("사용자 수"), value=users, inline=True)
embed.add_field(name=_("접두사"), value=prefix.get(), inline=True)
embed.add_field(name=_("DPSBot의 나이"),
value=_("최초 공개로부터 %s일 지났습니다.") % str(result.days),
inline=True)
embed.set_footer(text=_("Powered by Team ttakkku"))
await self.bot.send_message(ctx.message.channel, embed=embed)
async def info(self, ctx):
ownername = await self.bot.get_user_info(owner)
ownername = ownername.name
modstring = ""
for a_mod in mod:
modname = await self.bot.get_user_info(a_mod)
modname = modname.name
modstring += modname + " "
if not modstring:
modstring = "None"
users = 0
for s in self.bot.servers:
users += len(s.members)
now = datetime.now().date()
dday = date(2018, 8, 6)
result = now - dday
embed = Embed(title=_("%s 정보") % self.bot.user.name,
description=instructions.get(),
color=0x1ef7fa)
embed.set_thumbnail(url=self.bot.user.avatar_url)
embed.add_field(name=_("봇 운영자"), value=ownername, inline=True)
embed.add_field(name=_("봇 부운영자"), value=modstring, inline=True)
embed.add_field(name=_("서버 수"),
value=len(self.bot.servers),
inline=True)
embed.add_field(name=_("사용자 수"), value=users, inline=True)
embed.add_field(name=_("접두사"), value=prefix.get(), inline=True)
embed.add_field(name="Bot Official site",
value="https://dps0340.xyz",
inline=True)
embed.add_field(name="Bot Official git",
value="https://github.com/dpsbot-project/DPSBot",
inline=True)
embed.add_field(name="Bot Official server",
value="https://discordapp.com/invite/b9aEEaE",
inline=True)
embed.add_field(
name="invite this bot",
value=
"https://discordapp.com/oauth2/authorize?client_id=588671532944064524&permissions=8&scope=bot",
inline=True)
embed.add_field(name=_("DPSBot의 나이"),
value=_("최초 공개로부터 %s일 지났습니다.") % str(result.days),
inline=True)
embed.set_footer(text=_("DPSbot X Naesungbot"))
await self.bot.send_message(ctx.message.channel, embed=embed)
import pandas as pd
import embed
from embed import Embed
if __name__ == "__main__":
data = pd.read_csv('datasets/processed_data.csv')
emb = Embed(data)
emb.train()
print("Trained")
#print(emb.sentences)
#emb.get_embeddings().shape
print(emb.search('Alex', k=10))
print("Found!")
#return embed.get_embeddings()
class Train:
def __init__(self, train_dir="./data/train", classifier_dir="./model"):
self.emb = Embed()
self.train_dir = train_dir
self.classifier_dir = classifier_dir
self.extensions = ['.png', '.jpg']
self.rate = 0.0001
self.epochs = 200000
self.strip = 50
def train_data(self):
if not os.path.isdir(self.train_dir):
sys.stderr.write("Not exist folder for training data\n")
sys.exit(1)
sub_dirs = []
childs = os.listdir(self.train_dir)
for child in childs:
child_path = os.path.join(self.train_dir, child)
if os.path.isdir(child_path):
sub_dirs.append(child)
sub_dirs.sort()
labels = sub_dirs
tails = []
for i in range(len(sub_dirs)):
line = np.zeros((len(sub_dirs)), dtype=np.uint8)
line[i] = 1
tails.append(line.tolist())
"""
tails = [[1., 0., 0., 0., 0.],
[0., 1., 0., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 1., 0.],
[0., 0., 0., 0., 1.]]
"""
sys.stdout.write("\n Scanning folder: {}\n".format(self.train_dir))
features = []
for sub_dir_name in sub_dirs:
sub_dir_path = os.path.join(self.train_dir, sub_dir_name)
count = 0
for fn in os.listdir(sub_dir_path):
path = os.path.join(sub_dir_path, fn)
if not os.path.isfile(path) or os.path.splitext(path)[1] not in self.extensions:
continue
try:
# Extract the feature vector per each image
feature = self.emb.get_feature_from_image(path)
sys.stdout.write("\r" + path)
sys.stdout.flush()
except Exception as e:
print(e)
continue
line = feature.tolist()
line.extend(tails[sub_dirs.index(sub_dir_name)])
features.append(line)
count += 1
# if count > 10: # for only testing
# break
sys.stdout.write("\nLabel: {}, Counts: {}\n".format(sub_dir_name, count))
# write the train_data.csv file on the same location
train_data_path = os.path.join(self.train_dir, "train_data.csv")
if sys.version_info[0] == 2: # py 2x
with open(train_data_path, 'wb') as fp: # for python 2x
wr = csv.writer(fp, delimiter=',')
wr.writerows(features)
elif sys.version_info[0] == 3: # py 3x
with open(train_data_path, 'w', newline='') as fp: # for python 3x
wr = csv.writer(fp, delimiter=',')
wr.writerows(features)
# write the train_label.txt on the same location
train_label_path = os.path.join(self.train_dir, "train_label.txt")
with open(train_label_path, 'w') as fp:
for label in labels:
fp.write(label + "\n")
sys.stdout.write("Create the train_data.csv successfully!\n")
return train_data_path
def load_label_info(self):
train_label_path = os.path.join(self.train_dir, "train_label.txt")
labels = []
with open(train_label_path, 'r') as fp:
for line in fp:
line.replace('\n', '')
labels.append(line)
return labels
def train(self, bRestore):
in_dir = self.train_dir
out_dir = self.classifier_dir
learning_rate = self.rate
epochs = self.epochs
strip = self.strip
train_data_path = os.path.join(in_dir, 'train_data.csv')
ckpt_path = os.path.join(out_dir, 'model_bin.ckpt')
labels = self.load_label_info()
x_data = []
y_data = []
""" Loading training data from csv files """
print('[Step 1] Loading training data ...')
# for python 2x
with open(train_data_path) as fp:
csv_reader = csv.reader(fp, delimiter=',')
for row in csv_reader:
x_data.append([float(row[i]) for i in range(0, len(row) - len(labels))])
y_data.append([float(row[i]) for i in range(len(row) - len(labels), len(row))])
print("total features :" + str(len(x_data)))
print("length of feature :" + str(len(x_data[0])))
print("length of label :" + str(len(y_data[0])))
""" Placeholder """
print('[Step 2] Placeholder')
x = tf.placeholder('float', [None, 2048]) # len(feature) = 2048
y = tf.placeholder('float', [None, len(labels)]) # len(Directions) = 7 : classes
W1 = tf.get_variable('W1', shape=[2048, len(labels)], initializer=xaver_init(2048, len(labels)))
b1 = tf.Variable(tf.zeros([len(labels)]))
activation = tf.add(tf.matmul(x, W1), b1)
t1 = tf.nn.softmax(activation)
""" Minimize error using cross entropy """
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=activation, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Gradient Descent
""" Initializing the variables """
print('[Step 3] Initializing the variables.')
if sys.version_info[0] == 3:
init = tf.initialize_all_variables() # python 3x
elif sys.version_info[0] == 2:
init = tf.global_variables_initializer() # python 2x
sess = tf.Session()
sess.run(init)
saver = tf.train.Saver()
sys.stdout.write("bRestore:{}".format(bRestore))
if bRestore:
print('Loading the last learning Session.')
saver.restore(sess, ckpt_path)
""" Training cycle """
print('[Step 4] Training...')
for step in range(epochs):
sess.run(optimizer, feed_dict={x: x_data, y: y_data})
if step % strip == 0:
ret = sess.run(t1, feed_dict={x: x_data})
acc1 = acc(sess.run(tf.arg_max(ret, 1)), sess.run(tf.arg_max(y_data, 1))) * 100
sys.stdout.write("\t{} {} {}\n".format(step, sess.run(cost, feed_dict={x: x_data, y: y_data}), acc1))
saver.save(sess, ckpt_path)
print('Optimization Finished!')
def save_coefs(self):
out_dir = self.classifier_dir
w_coef_path = os.path.join(out_dir, 'w.csv')
b_coef_path = os.path.join(out_dir, 'b.csv')
ckpt_path = os.path.join(out_dir, 'model_bin.ckpt')
labels = self.load_label_info()
""" Placeholder """
sys.stdout.write('Placeholder\n')
W1 = tf.get_variable('W1', shape=[2048, len(labels)], initializer=xaver_init(2048, len(labels)))
b1 = tf.Variable(tf.zeros([len(labels)]))
""" Initializing the variables """
sys.stdout.write('Initializing the variables.\n')
if sys.version_info[0] == 3:
init = tf.global_variables_initializer() # python 3x
elif sys.version_info[0] == 2:
init = tf.initialize_all_variables() # python 2x
sess = tf.Session()
sess.run(init)
saver = tf.train.Saver()
sys.stdout.write('Loading the last learning Session.\n')
saver.restore(sess, ckpt_path)
ret_W = sess.run(W1)
ret_b = sess.run(b1)
sys.stdout.write('Saving the coefs W, b\n')
sys.stdout.write("{}, {}\n".format(len(ret_W), len(ret_W[0])))
sys.stdout.write("{}\n".format(len(ret_b)))
if sys.version_info[0] == 2:
with open(w_coef_path, 'wb') as fp: # for python 2x
wr = csv.writer(fp, delimiter=',')
wr.writerows(ret_W)
with open(b_coef_path, 'wb') as fp: # for python 2x
wr = csv.writer(fp, delimiter=',')
wr.writerow(ret_b)
if sys.version_info[0] == 3:
with open(w_coef_path, 'w', newline='') as fp: # for python 3x
wr = csv.writer(fp, delimiter=',')
wr.writerows(ret_W)
with open(b_coef_path, 'w', newline='') as fp: # for python 3x
wr = csv.writer(fp, delimiter=',')
wr.writerow(ret_b)
sys.stdout.write('Saving coeficient files Finished!\n')