def handle_command(command, channel, model, embeddings, metadata):
if command.startswith("###welcome"):
response = command[3:]
else:
sentence = command.strip()
if "who are you" in sentence.lower():
response = "read this :nerd_face: http://arxiv.org/abs/1506.05869"
else:
response = utils.get_translation(sess, model, sentence, embeddings,
_buckets, metadata)
slack_client.api_call("chat.postMessage",
channel=channel,
text=response,
as_user=True)
async def translation(self, ctx: Context, *args):
language_config = get_language_config_by_id(ctx.guild.id)
if args:
if args[0] not in get_possible_translations(ctx.guild.id):
await ctx.send(embed=Embed(
description=language_config.not_supported_translation,
color=0xff0000))
return
translation = config.translation
translation[str(ctx.guild.id)] = args[0]
config.save("translation", translation)
await ctx.send(embed=Embed(
description=language_config.f_translation_successfully_changed(
get_translation(ctx.guild.id)),
color=0x34bdeb))
return
embed = Embed(title=language_config.f_translation_settings_embed_title(
ctx.guild.name),
description=language_config.
f_translation_settings_embed_description(
get_translation(ctx.guild.id), await
get_prefix(ctx.message)),
color=0x34bdeb)
embed.add_field(name=language_config.supported_translations,
value="\n".join([
f"`{i}`"
for i in get_possible_translations(ctx.guild.id)
]))
await ctx.send(embed=embed)
async def on_message(self, message: Message):
content = message.content
queries = get_bible_queries(content)
for query in queries:
translation = get_language_config_by_id(
message.guild.id)._translations
book = query.split(" ")[0]
try:
book_name = translation[book]
translation = get_language_config_by_id(message.guild.id)
try:
request = BibleRequest(book_name,
query.split(" ")[1],
get_translation(message.guild.id))
except JSONDecodeError:
await message.channel.send(embed=Embed(
description=translation.quote_not_available,
color=0xff0000))
continue
embed = Embed(title=request.book_name + " " +
str(request.chapter))
embed.set_footer(text="Bible quoted using getbible.net",
icon_url=self.bot.user.avatar_url)
embed.timestamp = datetime.utcnow()
for verse in request.verses.keys():
embed.add_field(name=str(verse),
value=request.verses[verse],
inline=False)
await message.channel.send(embed=embed)
except KeyError:
translation = get_language_config_by_id(message.guild.id)
await message.channel.send(embed=Embed(
description=translation.book_abbr_not_available + "\n" +
translation.book_abbr_info + "\n[" +
translation.abbreviation_url + "](" +
translation.abbreviation_url + ")",
color=0xff0000))
def run_sample_decode(infer_model, infer_sess, hparams, src_data, tgt_data):
sample_id = random.randint(0, len(src_data) - 1)
with infer_model.graph.as_default():
infer_model.model.reinitializer_or_load_model(infer_sess,
hparams.model_dir)
iterator_feed_dict = {
infer_model.src_placeholder: [src_data[sample_id]],
infer_model.batch_size_placeholder: 1
}
infer_sess.run(infer_model.iterator.initializer,
feed_dict=iterator_feed_dict)
nmt_outputs = infer_model.model.decode(infer_sess)
if hparams.beam_width > 0:
nmt_outputs = nmt_outputs[0]
translation = utils.get_translation(nmt_outputs, 0, EOS)
print('src: %s' % src_data[sample_id])
print('ref: %s' % tgt_data[sample_id])
print('nmt: ' + translation)
def translate(hparams, string):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
infer_model = create_nmt_model.create_infer_model(hparams)
infer_sess = tf.Session(graph=infer_model.graph, config=config)
with infer_model.graph.as_default():
infer_model.model.reinitializer_or_load_model(infer_sess,
hparams.model_dir)
iterator_feed_dict = {
infer_model.src_placeholder: [string],
infer_model.batch_size_placeholder: 1
}
infer_sess.run(infer_model.iterator.initializer,
feed_dict=iterator_feed_dict)
nmt_outputs = infer_model.model.decode(infer_sess)
if hparams.beam_width > 0:
nmt_outputs = nmt_outputs[0]
translation = utils.get_translation(nmt_outputs, 0, hparams.eos)
return translation
async def settings(self, ctx: Context):
"""
Manage the Settings of BibleBot
"""
if ctx.invoked_subcommand is None:
message = await ctx.send(embed=Embed(
description="Gathering Information ..."))
settings = get_language_config_by_id(ctx.guild.id)
embed = Embed(
title=settings.f_general_settings_embed_title(ctx.guild.name),
description=settings.f_general_settings_embed_description(
await get_prefix(ctx.message)),
color=0x34bdeb)
embed.add_field(
name=settings.general_settings_embed_field_prefix_title,
value=settings.
f_general_settings_embed_field_prefix_description(
await get_prefix(ctx.message), await
get_prefix(ctx.message)))
embed.add_field(
name=settings.general_settings_embed_field_language_title,
value=settings.
f_general_settings_embed_field_language_description(
get_language(ctx.guild.id), await get_prefix(ctx.message)))
embed.add_field(
name=settings.general_settings_embed_field_translation_title,
value=settings.
f_general_settings_embed_field_translation_description(
get_translation(ctx.guild.id), await
get_prefix(ctx.message)))
await message.edit(embed=embed)
__version__ = '1.0'
__license__ = "GNU AGPLv3 or Proprietary (see LICENSE.txt)"
__version_info__ = (1, 0)
__author__ = 'Dan McDougall <*****@*****.**>'
# Python stdlib
import os, sys, logging, re, time
from datetime import datetime, timedelta
from functools import partial
# Our stuff
from gateone import BaseHandler
from utils import get_translation, mkdir_p, shell_command, which, json_encode
from utils import noop
_ = get_translation()
# Tornado stuff
import tornado.web
import tornado.ioloop
# Globals
OPENSSH_VERSION = None
DROPBEAR_VERSION = None
OPEN_SUBCHANNELS = {}
SUBCHANNEL_TIMEOUT = timedelta(
minutes=5) # How long to wait before auto-closing
READY_STRING = "GATEONE_SSH_EXEC_CMD_CHANNEL_READY"
READY_MATCH = re.compile("^%s$" % READY_STRING, re.MULTILINE)
OUTPUT_MATCH = re.compile("^{rs}.+^{rs}$".format(rs=READY_STRING),
re.MULTILINE | re.DOTALL)
before I attempt to get a custom service principal working with other
applications. This is because SSH uses the HOST/ prinicipal which is often
taken care of automatically via most Kerberos management tools (including AD).
If you can get SSO working with SSH you can get SSO working with anything else.
Class Docstrings
================
"""
# Standard library modules
import logging, base64
# Import our own stuff
from utils import get_translation
# Enable localization support
_ = get_translation()
# 3rd party modules
import tornado.httpserver
import tornado.ioloop
import tornado.web
import kerberos
# NOTE: For some reason if I set this as just an 'object' it doesn't work.
class KerberosAuthMixin(tornado.web.RequestHandler):
"""
Authenticates users via Kerberos-based Single Sign-On. Requires that you
define 'sso_realm' and 'sso_service' in your Tornado Application settings.
For example::
settings = dict(
def main():
start = time.time()
parser = args.parse_args()
# run some checks on arguments
check_args(parser)
# format logging
log_name = os.path.join(
parser.run_log,
'{}_run_log_{}.log'.format(parser.experiment,
dt.now().strftime("%Y%m%d_%H%M")))
log.basicConfig(filename=log_name,
format='%(asctime)s | %(name)s -- %(message)s',
level=log.INFO)
os.chmod(log_name, parser.access_mode)
# set devise to CPU if available
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
log.info("Starting experiment {} VN -> EN NMT on {}.".format(
parser.experiment, device))
# set seed for replication
random.seed(parser.seed)
np.random.seed(parser.seed)
torch.manual_seed(parser.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(parser.seed)
log.info("For reproducibility, the seed is set to {}.".format(parser.seed))
# set file paths
source_name = parser.source_name
target_name = parser.target_name
# get saved models dir
base_saved_models_dir = parser.save_dir
saved_models_dir = os.path.join(base_saved_models_dir,
source_name + '2' + target_name)
plots_dir = parser.plots_dir
log.info("We will save the models in this directory: {}".format(
saved_models_dir))
log.info("We will save the plots in this directory: {}".format(plots_dir))
# get data dir
main_data_path = parser.data_dir
path_to_train_data = {
'source': main_data_path + 'train.' + source_name,
'target': main_data_path + 'train.' + target_name
}
path_to_dev_data = {
'source': main_data_path + 'dev.' + source_name,
'target': main_data_path + 'dev.' + target_name
}
# get language objects
saved_language_model_dir = os.path.join(saved_models_dir, 'lang_obj')
# get dictionary of datasets
dataset_dict = {
'train':
nmt_dataset.LanguagePair(source_name=source_name,
target_name=target_name,
filepath=path_to_train_data,
lang_obj_path=saved_language_model_dir,
minimum_count=1),
'dev':
nmt_dataset.LanguagePair(source_name=source_name,
target_name=target_name,
filepath=path_to_dev_data,
lang_obj_path=saved_language_model_dir,
minimum_count=1)
}
# get max sentence length by 99% percentile
MAX_LEN = int(dataset_dict['train'].main_df['source_len'].quantile(0.9999))
log.info("MAX_LEN (99th Percentile) = {}".format(MAX_LEN))
batchSize = parser.batch_size
log.info("Batch size = {}.".format(batchSize))
dataloader_dict = {
'train':
DataLoader(dataset_dict['train'],
batch_size=batchSize,
collate_fn=partial(nmt_dataset.vocab_collate_func,
MAX_LEN=MAX_LEN),
shuffle=True,
num_workers=0),
'dev':
DataLoader(dataset_dict['dev'],
batch_size=batchSize,
collate_fn=partial(nmt_dataset.vocab_collate_func,
MAX_LEN=MAX_LEN),
shuffle=True,
num_workers=0)
}
# Configuration
source_lang_obj = dataset_dict['train'].source_lang_obj
target_lang_obj = dataset_dict['train'].target_lang_obj
source_vocab = dataset_dict['train'].source_lang_obj.n_words
target_vocab = dataset_dict['train'].target_lang_obj.n_words
hidden_size = parser.hidden_size
rnn_layers = parser.rnn_layers
lr = parser.learning_rate
longest_label = parser.longest_label
gradient_clip = parser.gradient_clip
num_epochs = parser.epochs
log.info(
"The source vocab ({}) has {} words and target vocab ({}) has {} words"
.format(source_name, source_vocab, target_name, target_vocab))
# encoder model
encoder_rnn = nnet_models_new.EncoderRNN(input_size=source_vocab,
hidden_size=hidden_size,
numlayers=rnn_layers)
# decoder model
decoder_rnn = nnet_models_new.DecoderRNN(output_size=target_vocab,
hidden_size=hidden_size,
numlayers=rnn_layers)
# seq2seq model
nmt_rnn = nnet_models_new.seq2seq(
encoder_rnn,
decoder_rnn,
lr=lr,
hiddensize=hidden_size,
numlayers=hidden_size,
target_lang=dataset_dict['train'].target_lang_obj,
longest_label=longest_label,
clip=gradient_clip,
device=device)
log.info(
"Seq2Seq Model with the following parameters: batch_size = {}, learning_rate = {}, hidden_size = {}, rnn_layers = {}, lr = {}, longest_label = {}, gradient_clip = {}, num_epochs = {}, source_name = {}, target_name = {}"
.format(batchSize, lr, hidden_size, rnn_layers, lr, longest_label,
gradient_clip, num_epochs, source_name, target_name))
# do we want to train again?
train_again = False
saved_file_name = 'no_attn_bs{}_lr{}_hs_{}_rnnlayer{}'.format(
batchSize, lr, hidden_size, rnn_layers)
# check if there is a saved model and if we want to train again
if os.path.exists(utils.get_full_filepath(saved_models_dir,
'rnn')) and (not train_again):
log.info("Retrieving saved model from {}".format(
utils.get_full_filepath(saved_models_dir, 'rnn')))
nmt_rnn = torch.load(utils.get_full_filepath(saved_models_dir, 'rnn'),
map_location=global_variables.device)
# train model again
else:
log.info("Check if this path exists: {}".format(
utils.get_full_filepath(saved_models_dir, saved_file_name)))
log.info("It does not exist! Starting to train...")
utils.train_model(dataloader_dict,
nmt_rnn,
num_epochs=num_epochs,
saved_model_path=saved_models_dir,
enc_type=saved_file_name)
log.info("Total time is: {} min : {} s".format((time.time() - start) // 60,
(time.time() - start) % 60))
log.info("We will save the models in this directory: {}".format(
saved_models_dir))
# generate translations
use_cuda = True
utils.get_translation(nmt_rnn, 'I love to watch science movies on Mondays',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(nmt_rnn,
'I want to be the best friend that I can be',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(nmt_rnn, 'I love you', source_lang_obj, use_cuda,
source_name, target_name)
utils.get_translation(
nmt_rnn,
'I love football, I like to watch it with my friends. It is always a great time.',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(
nmt_rnn,
'I do not know what I would do without pizza, it is very tasty to eat. If I could have any food in the world it would probably be pizza.',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(
nmt_rnn,
'Trump is the worst president in all of history. He can be a real racist and say very nasty things to people of color.',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(nmt_rnn, 'Thank you very much.', source_lang_obj,
use_cuda, source_name, target_name)
utils.get_translation(nmt_rnn, 'Think about your own choices.',
source_lang_obj, use_cuda, source_name, target_name)
utils.get_translation(
nmt_rnn,
'I recently did a survey with over 2,000 Americans , and the average number of choices that the typical American reports making is about 70 in a typical day .',
source_lang_obj, use_cuda, source_name, target_name)
# export plot
log.info("Exported Binned Bleu Score Plot to {}!".format(plots_dir))
_, _, fig = utils.get_binned_bl_score(nmt_rnn,
dataset_dict['dev'],
plots_dir,
batchSize=batchSize)
from utils import get_translation, Generator, sp
import sentencepiece as spm
data_token_path = "./en_hi_t2t_v3/token_data/"
model_checkpoint = "./en_hi_t2t_v3/checkpoints/checkpoint9.pt"
sp_path = "./sentencepiece.bpe.model"
srclang = "en" ### hi for hi2en
gen = Generator(data_token_path, model_checkpoint)
print("Model Checkpoint Load Complete")
sp.load(sp_path)
print("Tokenizer Load Complete\n")
while True:
text = input("English Text : ")
outtext = get_translation(gen, sp, text, srclang)
print("Hindi Text : %s\n" % outtext)
from tornado.options import define, options
# You need 'options' to get define()'d values
# Globals
SESSIONS = {} # This will get replaced with gateone.py's SESSIONS dict
# NOTE: The overwriting of SESSIONS happens inside of gateone.py as part of
# the application initialization process.
APPLICATION_PATH = os.path.split(__file__)[0] # Path to our application
web_handlers = [] # Populated at the bottom of this file
example_log = go_logger("gateone.example") # Our app's logger
# NOTE: You can pass additional metadata to logs which will be JSON-encoded
# when your messages are logged. Examples of how to do this are further along
# in this file...
# Localization support
_ = get_translation() # You don't *have* to do this but it is a good idea
# This is how you add command-line options to Gate One:
define(
"example_option", # NOTE: underscores are the preferred word separator here
default=True,
help=_("Doesn't do anything (just an example from the example app)."))
# You could then reference this option like so:
# print(options.example_option)
ALLOW = True # Used by polcy_example() below
def policy_test_example(cls, policy):
"""
An example policy-checking function. It will return ``True`` if conditions
def getTranslation(self, text):
return get_translation(self.gen, self.tokObj.sp, text, "hi")
from tornado.options import define, options
# You need 'options' to get define()'d values
# Globals
SESSIONS = {} # This will get replaced with gateone.py's SESSIONS dict
# NOTE: The overwriting of SESSIONS happens inside of gateone.py as part of
# the application initialization process.
APPLICATION_PATH = os.path.split(__file__)[0] # Path to our application
web_handlers = [] # Populated at the bottom of this file
example_log = go_logger("gateone.example") # Our app's logger
# NOTE: You can pass additional metadata to logs which will be JSON-encoded
# when your messages are logged. Examples of how to do this are further along
# in this file...
# Localization support
_ = get_translation() # You don't *have* to do this but it is a good idea
# This is how you add command-line options to Gate One:
define(
"example_option", # NOTE: underscores are the preferred word separator here
default=True,
help=_("Doesn't do anything (just an example from the example app).")
)
# You could then reference this option like so:
# print(options.example_option)
ALLOW = True # Used by polcy_example() below
def policy_test_example(cls, policy):
"""
An example policy-checking function. It will return ``True`` if conditions
are met. ``False`` if not.