def checkForum(self, url): pubg_news = feedparser.parse(url) # check 5 last news n = 5 for x in range(0, n): link = pubg_news['entries'][x]['link'] id = pubg_news['entries'][x]['id'] title = pubg_news['entries'][x]['title'] post_date = pubg_news['entries'][x]['published'] image = "https://content.invisioncic.com/r273030/monthly_2017_07/PUBG_Logo_color_RGB5.png.05a3748e744a03bcc597cc88afef9533.png" with Database() as dbconn: c = dbconn.cursor() c.execute('SELECT id FROM news WHERE id = ?', (id, )) data = c.fetchone() if (data is None): if self.checkDate(post_date) is True: e = Embed() entry = e.make_embed(link, title, image) self.postWebhook(entry) c.execute('INSERT INTO news VALUES (?, ?)', (id, datetime.now())) dbconn.commit()
def prepare(): config = ConfigRNN.instance() embed = Embed() embedding_model = embed.get_embedding_model() if config.EMBED_METHOD == "DEFAULT": model = ReviewParser() else: model = ReviewParser( pretrained=torch.from_numpy(embedding_model.wv.vectors).float()) evaluator = RNNEvaluator(model) return evaluator, embed
async def userinfo(self, ctx): person = ctx.message.mentions[0].id person = await self.bot.get_user_info(person) name = person.name discriminator = person.discriminator avatar = person.avatar_url id = person.id embed = Embed(title="%s#%s" % (name, discriminator), description="id:%s" % (id), color=0xE0FFFF) embed.set_thumbnail(url=avatar) await self.bot.send_message(ctx.message.channel, embed=embed)
def __init__(self, train_dir="./data/train", classifier_dir="./model"): self.emb = Embed() self.train_dir = train_dir self.classifier_dir = classifier_dir self.extensions = ['.png', '.jpg'] self.rate = 0.0001 self.epochs = 200000 self.strip = 50
def inference(in_dir, model_dir): emb = Embed() LABELS = load_label_info("./data/train/train_label.txt") w, b = load_coefs(model_dir) cnts = [0, 0, 0, 0, 0] # scan image in input sys.stdout.write(" Scan folder for classification...\n") files = [ f for f in os.listdir(in_dir) if os.path.isfile(os.path.join(in_dir, f)) ] sys.stdout.write(" total files {}\n".format(len(files))) for f in files: fn, ext = os.path.splitext(f) if ext.lower() == '.png': # Convert image file to jpg im = cv2.imread(os.path.join(in_dir, f)) new_fn = os.path.join(in_dir, fn + '.jpg') cv2.imwrite(new_fn, im) os.remove(os.path.join(in_dir, f)) elif ext.lower() == '.jpg': new_fn = os.path.join(in_dir, f) else: continue # Extract the feature vector per each image feature = emb.get_feature_from_image(new_fn) feature = feature.tolist() label_idx = classify(feature, w, b) # move the labeled image to folder named with its label(string) cnts[label_idx] += 1 label = LABELS[label_idx] label_dir = os.path.join(in_dir, label) if not os.path.exists(label_dir): os.makedirs(label_dir) out_fn = os.path.join(label_dir, f) if os.path.exists(out_fn): os.remove(new_fn) else: os.rename(new_fn, out_fn) sys.stdout.write(" label({}) : file{}\n".format(label, new_fn)) return cnts
async def post(self, ctx, select=0): if select == 0: conn = psycopg2.connect(DATABASE_URL) cur = conn.cursor() cur.execute('select * from post') rows = cur.fetchall() for row in rows: num = row[0] author = row[1] head = row[2] await self.bot.send_message( ctx.message.channel, "%s. %s - by %s\n" % (num, head, author)) conn.close() await self.bot.send_message( ctx.message.channel, _("%s글 (번호)를 입력하시면 글을 보실수 있어요!") % prefix.get()) await self.bot.send_message( ctx.message.channel, _("글을 쓰시려면 %s써줘 (번호)와 제목을 한 줄에 쓰세요!") % prefix.get()) else: conn = psycopg2.connect(DATABASE_URL) cur = conn.cursor() cur.execute('select * from post where num = {0}'.format(select)) row = cur.fetchone() num = row[0] author = row[1] head = row[2] body = row[3] conn.close() embed = Embed(title="%s. %s" % (num, head), description="\nby %s\n%s" % (author, body), color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed)
async def telnet(self, ctx, host: str, port: int): telnet = telnetlib.Telnet(host, port) await self.bot.send_message(ctx.message.channel, _('%s:%s에 연결 중...') % (host, port)) while True: body = await self.bot.wait_for_message(timeout=60.0, author=ctx.message.author) if body is None: await self.bot.send_message(ctx.message.channel, _('응답이 없어서 종료되었습니다.')) break else: if body.content.startswith(_('종료')): await self.bot.send_message(ctx.message.channel, _('종료되었습니다.')) break else: try: line = body.content + "\n" telnet.write(line.encode('ascii')) await asyncio.sleep(1) response = telnet.read_very_eager() embed = Embed(title="%s:%s" % (host, port), description="%s" % (response.decode('ascii')), color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) except: await self.bot.send_message(ctx.message.channel, _('오류가 발생했습니다!')) return
async def osu(self, ctx, *user): username = '' i = 0 for word in user: i += 1 if word == user[-1] and len(user) == i: username += str(word) else: username += str(word) + ' ' api = OsuApi(osuapikey, connector=ReqConnector()) results = api.get_user(username) userid = results[0].user_id thumbnail = "https://s.ppy.sh/a/" + str(userid) embed = Embed(title="%s" % (username), description="id:%s" % (userid), color=0xE0FFFF) embed.set_thumbnail(url=thumbnail) await self.bot.send_message(ctx.message.channel, embed=embed) await self.bot.send_message(ctx.message.channel, "https://osu.ppy.sh/users/%d" % userid)
def main(): config = ConfigRNN.instance() embed = Embed() # TODO(kyungsoo): Make this working. embedding_model = embed.get_embedding_model() if embedding_model == "DEFAULT": model = RNN() else: vectors = embedding_model.wv.vectors # Add padding for masking. vectors = np.append(np.array([100 * [0]]), vectors, axis=0) model = RNN(torch.from_numpy(vectors).float()) optimizer = torch.optim.SGD(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) trainer = RNNEvaluator(model, optimizer) # TODO(kyungsoo): Make this working. review_vector = embed.review2vec(sys.argv[0]) print(trainer.evaluate(review_vector=review_vector))
async def module(self, ctx): i = 0 body = "" for plugin in pluginlist.get()["default"]: i += 1 body += plugin + " " if i % 5 == 0: body += '\n' embed = Embed(title=_("사용중인 모듈 리스트"), description=body, color=0x00FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) i = 0 body = "" for plugin in pluginlist.get()["non-default"]: i += 1 body += plugin + " " if i % 5 == 0: body += '\n' embed = Embed(title=_("사용 가능한 모듈 리스트"), description=body, color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) await self.bot.say("sorry! module setting for each channel is not yet." )
async def exec(self, ctx, *, command): if ctx.message.author.id == owner: try: print(command) result = subprocess.check_output(command, shell=True) print(result) embed = Embed(title=_("명령어 실행: %s") % (command), description="%s" % (result.decode('ascii')), color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) except Exception as e: print(e) await self.bot.send_message(ctx.message.channel, _('오류가 발생했습니다!')) else: await self.bot.send_message(ctx.message.channel, _('권한이 없습니다!\n개발자만 접근 가능합니다.'))
async def log(self, message, channel, key): listener = message.author try: body = await self.bot.wait_for_message(channel=channel) if body is None: await self.bot.send_message(listener, _('응답이 없어서 종료되었습니다.')) pass else: if str(body.content) == str(key) and body.author.name != self.bot.user.name: await self.bot.send_message(listener, _('비밀번호 입력으로 인해 종료되었습니다.')) elif body.server == None: pass else: await bot_log(_('\n서버:%s\n채널:%s\n작성자:%s\n%s\n') % (body.server, body.channel, body.author, body.content)) embed = Embed(title=_("log"), description=_('\n서버:%s\n\n채널:%s\n\n작성자:%s\n\n%s') % ( body.server, body.channel, body.author, body.content), color=0xE0FFFF) await self.bot.send_message(message.author, embed=embed) await self.log(message, channel, key) except Exception as e: print(e)
async def alohsaydCore(self, ctx, num: int): conn = psycopg2.connect(DATABASE_URL, sslmode='require') with conn: try: cur = conn.cursor() cur.execute("select * from alohsayd where num={0}".format(num)) rows = cur.fetchall() row = rows[0] print(row) num = row[0] head = row[1] body = row[2] head = head.replace("\n", "") embed = Embed(title="%s" % head, description="\n%s" % (body), color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) except: await self.bot.send_message(ctx.message.channel, _("파일을 찾지 못했어요!")) pass
async def write(self, ctx, *, head=None): conn = psycopg2.connect(DATABASE_URL) with conn: try: cur = conn.cursor() num = cur.lastlowid num += 1 except: num = 1 pass if not head: await self.bot.send_message(ctx.message.channel, _("제목이 없습니다.")) else: await self.bot.send_message(ctx.message.channel, _("내용을 말해주세요!")) body = await self.bot.wait_for_message(timeout=600, author=ctx.message.author) body = body.content author = ctx.message.author.name embed = Embed(title="%s" % head, description="\nby %s\n%s" % (author, body), color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed) await self.postinsert("post", num, author, head, body)
async def whoami(self, ctx): embed = Embed(title=_("당신은 혹시..."), description=_("\n%s\n\n이신가요?!?!") % ctx.message.author, color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed)
async def naesungwiki(self, ctx): embed = Embed(title=_("내성위키"), description="https://naesung.wiki", color=0xE0FFFF) await self.bot.send_message(ctx.message.channel, embed=embed)
async def info(self, ctx): ownername = await self.bot.get_user_info(owner) ownername = ownername.name modstring = "" for a_mod in mod: modname = await self.bot.get_user_info(a_mod) modname = modname.name modstring += modname + " " if not modstring: modstring = "None" users = 0 for s in self.bot.servers: users += len(s.members) now = datetime.now().date() dday = date(2018, 8, 6) result = now - dday embed = Embed(title=_("%s 정보") % self.bot.user.name, description=instructions.get(), color=0x1ef7fa) embed.set_thumbnail(url=self.bot.user.avatar_url) embed.add_field(name=_("봇 운영자"), value=ownername, inline=True) embed.add_field(name=_("봇 부운영자"), value=modstring, inline=True) embed.add_field(name=_("서버 수"), value=len(self.bot.servers), inline=True) embed.add_field(name=_("사용자 수"), value=users, inline=True) embed.add_field(name=_("접두사"), value=prefix.get(), inline=True) embed.add_field(name=_("DPSBot의 나이"), value=_("최초 공개로부터 %s일 지났습니다.") % str(result.days), inline=True) embed.set_footer(text=_("Powered by Team ttakkku")) await self.bot.send_message(ctx.message.channel, embed=embed)
async def info(self, ctx): ownername = await self.bot.get_user_info(owner) ownername = ownername.name modstring = "" for a_mod in mod: modname = await self.bot.get_user_info(a_mod) modname = modname.name modstring += modname + " " if not modstring: modstring = "None" users = 0 for s in self.bot.servers: users += len(s.members) now = datetime.now().date() dday = date(2018, 8, 6) result = now - dday embed = Embed(title=_("%s 정보") % self.bot.user.name, description=instructions.get(), color=0x1ef7fa) embed.set_thumbnail(url=self.bot.user.avatar_url) embed.add_field(name=_("봇 운영자"), value=ownername, inline=True) embed.add_field(name=_("봇 부운영자"), value=modstring, inline=True) embed.add_field(name=_("서버 수"), value=len(self.bot.servers), inline=True) embed.add_field(name=_("사용자 수"), value=users, inline=True) embed.add_field(name=_("접두사"), value=prefix.get(), inline=True) embed.add_field(name="Bot Official site", value="https://dps0340.xyz", inline=True) embed.add_field(name="Bot Official git", value="https://github.com/dpsbot-project/DPSBot", inline=True) embed.add_field(name="Bot Official server", value="https://discordapp.com/invite/b9aEEaE", inline=True) embed.add_field( name="invite this bot", value= "https://discordapp.com/oauth2/authorize?client_id=588671532944064524&permissions=8&scope=bot", inline=True) embed.add_field(name=_("DPSBot의 나이"), value=_("최초 공개로부터 %s일 지났습니다.") % str(result.days), inline=True) embed.set_footer(text=_("DPSbot X Naesungbot")) await self.bot.send_message(ctx.message.channel, embed=embed)
import pandas as pd import embed from embed import Embed if __name__ == "__main__": data = pd.read_csv('datasets/processed_data.csv') emb = Embed(data) emb.train() print("Trained") #print(emb.sentences) #emb.get_embeddings().shape print(emb.search('Alex', k=10)) print("Found!") #return embed.get_embeddings()
class Train: def __init__(self, train_dir="./data/train", classifier_dir="./model"): self.emb = Embed() self.train_dir = train_dir self.classifier_dir = classifier_dir self.extensions = ['.png', '.jpg'] self.rate = 0.0001 self.epochs = 200000 self.strip = 50 def train_data(self): if not os.path.isdir(self.train_dir): sys.stderr.write("Not exist folder for training data\n") sys.exit(1) sub_dirs = [] childs = os.listdir(self.train_dir) for child in childs: child_path = os.path.join(self.train_dir, child) if os.path.isdir(child_path): sub_dirs.append(child) sub_dirs.sort() labels = sub_dirs tails = [] for i in range(len(sub_dirs)): line = np.zeros((len(sub_dirs)), dtype=np.uint8) line[i] = 1 tails.append(line.tolist()) """ tails = [[1., 0., 0., 0., 0.], [0., 1., 0., 0., 0.], [0., 0., 1., 0., 0.], [0., 0., 0., 1., 0.], [0., 0., 0., 0., 1.]] """ sys.stdout.write("\n Scanning folder: {}\n".format(self.train_dir)) features = [] for sub_dir_name in sub_dirs: sub_dir_path = os.path.join(self.train_dir, sub_dir_name) count = 0 for fn in os.listdir(sub_dir_path): path = os.path.join(sub_dir_path, fn) if not os.path.isfile(path) or os.path.splitext(path)[1] not in self.extensions: continue try: # Extract the feature vector per each image feature = self.emb.get_feature_from_image(path) sys.stdout.write("\r" + path) sys.stdout.flush() except Exception as e: print(e) continue line = feature.tolist() line.extend(tails[sub_dirs.index(sub_dir_name)]) features.append(line) count += 1 # if count > 10: # for only testing # break sys.stdout.write("\nLabel: {}, Counts: {}\n".format(sub_dir_name, count)) # write the train_data.csv file on the same location train_data_path = os.path.join(self.train_dir, "train_data.csv") if sys.version_info[0] == 2: # py 2x with open(train_data_path, 'wb') as fp: # for python 2x wr = csv.writer(fp, delimiter=',') wr.writerows(features) elif sys.version_info[0] == 3: # py 3x with open(train_data_path, 'w', newline='') as fp: # for python 3x wr = csv.writer(fp, delimiter=',') wr.writerows(features) # write the train_label.txt on the same location train_label_path = os.path.join(self.train_dir, "train_label.txt") with open(train_label_path, 'w') as fp: for label in labels: fp.write(label + "\n") sys.stdout.write("Create the train_data.csv successfully!\n") return train_data_path def load_label_info(self): train_label_path = os.path.join(self.train_dir, "train_label.txt") labels = [] with open(train_label_path, 'r') as fp: for line in fp: line.replace('\n', '') labels.append(line) return labels def train(self, bRestore): in_dir = self.train_dir out_dir = self.classifier_dir learning_rate = self.rate epochs = self.epochs strip = self.strip train_data_path = os.path.join(in_dir, 'train_data.csv') ckpt_path = os.path.join(out_dir, 'model_bin.ckpt') labels = self.load_label_info() x_data = [] y_data = [] """ Loading training data from csv files """ print('[Step 1] Loading training data ...') # for python 2x with open(train_data_path) as fp: csv_reader = csv.reader(fp, delimiter=',') for row in csv_reader: x_data.append([float(row[i]) for i in range(0, len(row) - len(labels))]) y_data.append([float(row[i]) for i in range(len(row) - len(labels), len(row))]) print("total features :" + str(len(x_data))) print("length of feature :" + str(len(x_data[0]))) print("length of label :" + str(len(y_data[0]))) """ Placeholder """ print('[Step 2] Placeholder') x = tf.placeholder('float', [None, 2048]) # len(feature) = 2048 y = tf.placeholder('float', [None, len(labels)]) # len(Directions) = 7 : classes W1 = tf.get_variable('W1', shape=[2048, len(labels)], initializer=xaver_init(2048, len(labels))) b1 = tf.Variable(tf.zeros([len(labels)])) activation = tf.add(tf.matmul(x, W1), b1) t1 = tf.nn.softmax(activation) """ Minimize error using cross entropy """ cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=activation, labels=y)) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Gradient Descent """ Initializing the variables """ print('[Step 3] Initializing the variables.') if sys.version_info[0] == 3: init = tf.initialize_all_variables() # python 3x elif sys.version_info[0] == 2: init = tf.global_variables_initializer() # python 2x sess = tf.Session() sess.run(init) saver = tf.train.Saver() sys.stdout.write("bRestore:{}".format(bRestore)) if bRestore: print('Loading the last learning Session.') saver.restore(sess, ckpt_path) """ Training cycle """ print('[Step 4] Training...') for step in range(epochs): sess.run(optimizer, feed_dict={x: x_data, y: y_data}) if step % strip == 0: ret = sess.run(t1, feed_dict={x: x_data}) acc1 = acc(sess.run(tf.arg_max(ret, 1)), sess.run(tf.arg_max(y_data, 1))) * 100 sys.stdout.write("\t{} {} {}\n".format(step, sess.run(cost, feed_dict={x: x_data, y: y_data}), acc1)) saver.save(sess, ckpt_path) print('Optimization Finished!') def save_coefs(self): out_dir = self.classifier_dir w_coef_path = os.path.join(out_dir, 'w.csv') b_coef_path = os.path.join(out_dir, 'b.csv') ckpt_path = os.path.join(out_dir, 'model_bin.ckpt') labels = self.load_label_info() """ Placeholder """ sys.stdout.write('Placeholder\n') W1 = tf.get_variable('W1', shape=[2048, len(labels)], initializer=xaver_init(2048, len(labels))) b1 = tf.Variable(tf.zeros([len(labels)])) """ Initializing the variables """ sys.stdout.write('Initializing the variables.\n') if sys.version_info[0] == 3: init = tf.global_variables_initializer() # python 3x elif sys.version_info[0] == 2: init = tf.initialize_all_variables() # python 2x sess = tf.Session() sess.run(init) saver = tf.train.Saver() sys.stdout.write('Loading the last learning Session.\n') saver.restore(sess, ckpt_path) ret_W = sess.run(W1) ret_b = sess.run(b1) sys.stdout.write('Saving the coefs W, b\n') sys.stdout.write("{}, {}\n".format(len(ret_W), len(ret_W[0]))) sys.stdout.write("{}\n".format(len(ret_b))) if sys.version_info[0] == 2: with open(w_coef_path, 'wb') as fp: # for python 2x wr = csv.writer(fp, delimiter=',') wr.writerows(ret_W) with open(b_coef_path, 'wb') as fp: # for python 2x wr = csv.writer(fp, delimiter=',') wr.writerow(ret_b) if sys.version_info[0] == 3: with open(w_coef_path, 'w', newline='') as fp: # for python 3x wr = csv.writer(fp, delimiter=',') wr.writerows(ret_W) with open(b_coef_path, 'w', newline='') as fp: # for python 3x wr = csv.writer(fp, delimiter=',') wr.writerow(ret_b) sys.stdout.write('Saving coeficient files Finished!\n')