async def on_ready(self):
"""pull autoresponses from the db, then set activity"""
await self.initialize_user_commands()
logger.info('Logged on as {0}!'.format(self.user))
await self.change_presence(activity=discord.Activity(
name=f"the tragedy of darth plagueis the wise {self.shard_id}", type=2))
await self.manager_client.guild_update(self.shard_id, self.guilds_as_dicts)
def train_entry():
logger.info("Building model...")
bert_config = BertConfig.from_json_file(config.bert_config)
lr = config.learning_rate
# num_train_steps = int(train_dataset.features_size/ config.batch_size * config.epochs)
# warm_up = int(num_train_steps * config.warmup_proportion)
model = get_model(config.model_package, config.model_name,
config.model_class_name).to(device)
ema = EMA(config.ema_decay)
params = get_model_trainabel_param(ema, model)
optimizer = get_optimizer(lr, params=params)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, config.T_MAX,
config.min_lr)
epochs = config.epochs
best_f1 = best_em = patience = 0
start_index = 0 if not config.is_continue else config.continue_checkpoint
for epoch in range(config.start_epoch, epochs):
logger.info(f"Epoch: {epoch}")
train(model, optimizer, scheduler, ema, bert_config, start_index,
get_steps("train", config.mode), epoch)
if config.is_test_with_test_dev_dataset:
valid(model, bert_config, epoch)
metrics = test(model, bert_config, epoch)
async def on_guild_join(self, guild):
logger.info(f" -- JOINED NEW GUILD: {guild.name} -- ")
try:
await self.manager_client.guild_update(self.guilds_as_message)
except Exception:
logger.info(
f"Shard {self.shard_id} failed to send manager its guild list")
def netbios_scan(ipaddress_list, timeout=0.5):
TYPE_UNKNOWN = 0x01
TYPE_WORKSTATION = 0x00
TYPE_CLIENT = 0x03
TYPE_SERVER = 0x20
TYPE_DOMAIN_MASTER = 0x1B
TYPE_DOMAIN_CONTROLLER = 0x1C
TYPE_MASTER_BROWSER = 0x1D
TYPE_BROWSER = 0x1E
TYPE_NETDDE = 0x1F
TYPE_STATUS = 0x21
format_str = "{:<16}{:<20}{:<20}{:<20}{}".format("IPAddress", "DOMAIN",
"NetbiosName", "MAC",
"Nets")
logger.info(format_str)
for ipaddress in ipaddress_list:
n = NetBIOS()
result = {
"ip": ipaddress,
"domain": None,
"name": None,
"nets": [],
"mac": None
}
try:
entries = n.getnodestatus('*', ipaddress, timeout=timeout)
result['mac'] = n.getmacaddress()
except Exception as E:
continue
for entrie in entries:
if entrie["TYPE"] == TYPE_SERVER:
result["name"] = entrie["NAME"].strip().decode('latin-1')
elif entrie["TYPE"] == TYPE_WORKSTATION:
result["domain"] = entrie["NAME"].strip().decode('latin-1')
elif entrie["TYPE"] == TYPE_CLIENT:
result["TYPE_CLIENT"] = entrie["NAME"].strip().decode(
'latin-1')
elif entrie["TYPE"] == TYPE_DOMAIN_MASTER:
result["TYPE_DOMAIN_MASTER"] = entrie["NAME"].strip().decode(
'latin-1')
elif entrie["TYPE"] == TYPE_DOMAIN_CONTROLLER:
result["TYPE_DOMAIN_CONTROLLER"] = entrie["NAME"].strip(
).decode('latin-1')
elif entrie["TYPE"] == TYPE_MASTER_BROWSER:
result["TYPE_MASTER_BROWSER"] = entrie["NAME"].strip().decode(
'latin-1')
elif entrie["TYPE"] == TYPE_STATUS:
result["TYPE_STATUS"] = entrie["NAME"].strip().decode(
'latin-1')
try:
if result.get("name") is not None:
resp = n.name_query_request(result.get("name"), ipaddress)
result["nets"].extend(resp.entries)
except Exception as e:
pass
format_str = "{:<16}{:<20}{:<20}{:<20}{}".format(
ipaddress, result.get("domain"), result.get("name"),
result.get("mac"), result.get("nets"))
logger.warning(format_str)
def __init__(self, total_shards):
logger.info(f"Number of shards: {total_shards}")
self.hoarfrost_gen = HoarFrostGenerator()
self.total_shards = total_shards
self.registered = [False for _ in range(total_shards)]
self.last_checkin = {}
self.store = {}
def classify_data():
from lib.models import QANet
logger.info("Building model...")
train_dataset = get_dataset("train")
model = get_model().to(device)
classify(model, train_dataset)
async def emojis(self, ctx):
"""
List currently cached emojis.
Enclose the name (case sensitive) of cached emoji in `:`s to auto-load it into a message
"""
settings = self.bot.settings[ctx.guild]
if not settings.manage_emojis:
message = f"The emoji manager is disabled, you can enable it in `{settings.command_prefix}settings`"
await ctx.send(message)
else:
# message = '```\n • ' + '\n • '.join(self.managers[ctx.guild.id].list_unloaded()) + '```\n'
logger.debug("generating list image")
file = self.managers[ctx.guild.id].list_unloaded()
if file is not None:
logger.debug("file is good")
message = "Enclose the name (case sensitive) of cached emoji in `:`s to auto-load it into a message"
# msg = await ctx.send(message, file=discord.File(file, "cool.png"))
try:
data = await self.bot.manager_client.publish_file(
iter([message_type.File(file=file.getvalue())]))
except Exception:
logger.info(
f"Shard {self.bot.shard_id} failed to upload emoji")
await ctx.send("Failed to generate cached emoji preview")
return
em = discord.Embed(title="Cached Emojis",
description=ctx.guild.name)
# em.set_image(url=msg.attachments[0].url)
em.set_image(url=data.url)
em.color = 0x7b8fb7
em.set_footer(text=message)
await ctx.send(embed=em)
else:
await ctx.send("No cached emoji")
def docker_build(**kwargs):
for line in docker_client_api.build(**kwargs):
line = json.loads(line.decode('utf-8'))
if 'stream' in line:
line = line['stream'].rstrip('\n')
if line:
logger.info(line)
def __init__(self, **kwargs):
self.session = get_session()
self.asyncpg_wrapper = AsyncConnWrapper()
self.deletable_messages = []
self.hoarfrost_gen = HoarFrostGenerator()
logger.debug("registering with manager...")
manager_client = grpc_client.get_blocking_client('manager:50051')
while True:
try:
shard_info = manager_client.register(message.RegisterRequest())
break
except Exception:
logger.info("Trying to get shard id from manager")
time.sleep(3)
self.shard_id = shard_info.shard_id
shard_dict = {
'shard_id': shard_info.shard_id,
'shard_count': shard_info.shard_count
}
logger.info(f"Got shard_id {self.shard_id}")
kwargs.update(shard_dict)
super().__init__(**kwargs)
async def on_member_join(self, member):
logger.info("%s joined guild: %s" % (member.name, member.guild.name))
settings = self.bot.settings[member.guild]
try:
default_role = discord.utils.get(member.guild.roles, id=settings.default_role_id)
await member.add_roles(default_role)
except Exception:
logger.exception("could not add %s to %s" % (member.display_name, 'default role'))
def register(self, request, context):
"""Returns the next shard id that needs to be filled as well as the total shards"""
if all(self.registered):
raise Exception("Shard trying to register even though we're full")
i = next(i for i in range(self.total_shards) if not self.registered[i])
logger.info(
f"Shard requested id, assigning {i + 1}/{self.total_shards}...")
self.registered[i] = True
return message.ShardInfo(shard_id=i, shard_count=self.total_shards)
async def register(self):
"""Returns the next shard id that needs to be filled as well as the total shards"""
if all(self.registered):
raise Exception("Shard trying to register even though we're full")
i = next(i for i in range(self.total_shards) if not self.registered[i])
logger.info(f'Shard requested id, assigning {i + 1}/{self.total_shards}...')
self.store[i] = {'shard_id': i}
self.registered[i] = True
return {'shard_id': i, 'shard_count': self.total_shards}
async def heartbeat(self):
await self.wait_until_ready()
while not self.is_closed():
await asyncio.sleep(0.5)
try:
await self.manager_client.checkin(
message.ShardID(shard_id=self.shard_id))
except Exception:
logger.info(
f"Shard {self.shard_id} failed to checkin with manager")
def __init__(self, total_shards):
"""
Instantiates a new manager server that handles some
number of shards.
"""
logger.info(f"Number of shards: {total_shards}")
self.hoarfrost_gen = HoarFrostGenerator()
self.total_shards = total_shards
self.registered = [False for _ in range(total_shards)]
self.last_checkin = dict()
self.store = dict()
async def on_message(self, msg):
"""Execute commands, then trigger autoresponses"""
logger.info(
'Message from {0.author} in {0.guild.name}: {0.content}'.format(
msg))
if msg.author == self.user:
return
# check for real commands
await self.process_commands(msg)
async def publish_file(self, location: str = 'assets', name: str = '', filetype: str = 'png', data: str = ''):
assert data != ''
if name == '':
name = str(self.hoarfrost_gen.generate())
directory = f'/var/www/{location}'
if not os.path.exists(directory):
os.makedirs(directory)
with open(f'{directory}/{name}.{filetype}', 'wb') as f:
logger.info(f'writing {directory}/{name}.{filetype}')
f.write(base64.b64decode(data))
return {'url': f'https://cdn.{domain_name}/{location}/{name}.{filetype}'}
def test_model(bert_config, losses, model, valid_result, epoch, label="valid"):
logger.info("start_step test or valid:")
tokenizer, dev_dataloader = load_data(bert_config, config.dev_dir1)
softmax = torch.nn.Softmax(dim=-1)
log_sofmax = torch.nn.LogSoftmax(dim=-1)
exact_match_total = 0
f1_total = 0
exact_match = 0
f1 = 0
with torch.no_grad():
for step, batch in enumerate(dev_dataloader):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, start_positions, end_positions = batch
start_embeddings, end_embeddings = model(input_ids, input_mask,
segment_ids)
# start_embeddings = mask(start_embeddings, input_span_mask, -1)
# end_embeddings = mask(end_embeddings, input_span_mask, -1)
loss1 = F.nll_loss(log_sofmax(start_embeddings),
start_positions,
reduction='mean')
loss2 = F.nll_loss(log_sofmax(end_embeddings),
end_positions,
reduction='mean')
loss = (loss1 + loss2) / 2
logger.info(f"Origin Loss: {loss}, step: {step}")
losses.append(loss.item())
pre_start, pre_end, probabilities = find_max_proper_batch(
softmax(start_embeddings), softmax(end_embeddings))
cur_res = convert_pre_res(input_ids, pre_start, pre_end,
start_positions, end_positions,
probabilities, tokenizer)
record_info(valid_result=cur_res,
epoch=epoch,
is_continue=True,
r_type=label)
visual_data(model,
loss,
loss,
None,
epoch,
step,
exact_match_total,
f1_total,
exact_match,
f1,
label=label)
loss = np.mean(losses)
exact_match_total, f1_total, exact_match, f1 = evaluate_valid_result(
valid_result, 0, 0, len(dev_dataloader.dataset))
# metrics["loss"] = loss
return loss, exact_match, f1
async def on_ready(self):
"""pull autoresponses from the db, then set activity"""
await self.asyncpg_wrapper.connect()
logger.info('Logged on as {0}!'.format(self.user))
await self.change_presence(activity=discord.Activity(
name=f"the tragedy of darth plagueis the wise {self.shard_id}",
type=2))
try:
await self.manager_client.guild_update(self.guilds_as_message)
except Exception:
logger.info(
f"Shard {self.shard_id} failed to send manager its guild list")
async def on_connect(self, sid: str, environ: dict):
logger.debug(f"{environ['REMOTE_ADDR']} has connected with sid: {sid}")
request = environ['aiohttp.request']
try:
jwt = JWT(token=request.cookies['token'])
except (InvalidTokenError, KeyError):
logger.info(
"No valid token found, logging into unprivileged gateway...")
else:
logger.info("Found valid token, logging into elevated gateway...")
sio.enter_room(sid, f"{sid}_auth")
async with sio.session(sid) as session:
session['jwt'] = jwt
async def list_guilds(self):
"""Update the manager with the guilds that we know about"""
await self.wait_until_ready()
while not self.is_closed():
logger.info("Current guilds:")
for guild in self.guilds:
if guild.me.display_name == 'archit.us':
try:
await guild.me.edit(nick='architus')
except discord.Forbidden:
logger.warning(f"couldn't change nickname in {guild.name}")
logger.info("{} - {} ({})".format(guild.name, guild.id, guild.member_count))
await asyncio.sleep(600)
def netbios_interface(ipaddress_list, timeout, pool):
format_str = "{:<16}{:<20}{:<20}{:<20}{}".format("IPAddress", "DOMAIN",
"NetbiosName", "MAC",
"Nets")
logger.info(format_str)
tasks = []
for ipaddress in ipaddress_list:
task = pool.spawn(netbios_scan, ipaddress, timeout * 2)
tasks.append(task)
if len(tasks) >= 300:
joinall(tasks)
tasks = []
joinall(tasks)
async def on_message(self, msg):
"""Execute commands, then trigger autoresponses"""
logger.info('Message from {0.author} in {0.guild.name}: {0.content}'.format(msg))
if msg.author == self.user:
return
# check for real commands
await self.process_commands(msg)
# check for user commands
for command in self.user_commands[msg.guild.id]:
if (command.triggered(msg.content)):
await command.execute(msg)
break
async def guild_count(self):
try:
resp = await self.bot.manager_client.guild_count(
message.GuildCountRequest())
return {
'guild_count': resp.guild_count,
'user_count': resp.user_count
}, sc.OK_200
except Exception:
logger.info(
f"Shard {self.bot.shard_id} failed to get guild count from manager"
)
return {
'guild_count': -1,
'user_count': -1
}, sc.INTERNAL_SERVER_ERROR_500
async def url(self):
if self.str_url == "":
with BytesIO() as buf:
self.im.save(buf, format="PNG")
binary = buf.getvalue()
try:
data = await self.bot.manager_client.publish_file(
iter([
message.File(
location="emojis",
name=f"{self.id}",
file=binary)]))
except Exception:
logger.info(f"Shard {self.bot.shard_id} failed to upload emoji")
return None
self.str_url = data.url
return self.str_url
async def on_request_elevation(self, sid: str, nonce: int):
logger.debug(f"{sid} requesting elevation...")
try:
jwt = JWT(token=auth_nonces[nonce])
del auth_nonces[nonce]
except (InvalidTokenError, KeyError):
logger.info(
f"{sid} requested room elevation but didn't provide a valid jwt"
)
await sio.emit('elevation_return',
{'message': "Missing or invalid jwt"},
room=sid)
else:
logger.debug(f"valid nonce provided, granting access...")
sio.enter_room(sid, f"{sid}_auth")
await sio.emit('elevation_return', {'message': "success"},
room=sid)
async with sio.session(sid) as session:
session['token'] = jwt
def __init__(self, **kwargs):
self.user_commands = {}
self.session = get_session()
self.deletable_messages = []
self.hoarfrost_gen = HoarFrostGenerator()
manager_client = blocking_rpc_client.shardRPC()
# wait for manager to come up; this is scuffed
import time
time.sleep(2)
logger.debug("asking for shard id")
shard_info, sc = manager_client.register(routing_key='manager_rpc')
self.shard_id = shard_info['shard_id']
logger.info(f"Got shard_id {self.shard_id}")
kwargs.update(shard_info)
super().__init__(**kwargs)
def valid(model, bert_config, epoch=0):
model.eval()
valid_result = []
losses = []
logger.info("start valid:")
loss, extract_match, f1 = test_model(bert_config,
losses,
model,
valid_result,
epoch,
label="valid")
record_info(
loss,
f1=f1,
em=extract_match,
# valid_result=valid_result,
epoch=epoch,
r_type="valid")
logger.info("VALID loss {:8f} F1 {:8f} EM {:8f}\n".format(
loss, f1, extract_match))
def publish_file(self, request_iterator, context):
"""Missing associated documentation comment in .proto file"""
first = next(request_iterator)
filetype = "png" if first.filetype == "" else first.filetype
name = first.name
if name == "":
name = str(self.hoarfrost_gen.generate())
location = first.location
if location == "":
location = "assets"
directory = f"/var/www/{location}"
if not os.path.exists(directory):
os.makedirs(directory)
with open(f"{directory}/{name}.{filetype}", "wb") as f:
logger.info(f"Writing {directory}/{name}.{filetype}")
f.write(first.file)
for datum in request_iterator:
f.write(datum.file)
return message.Url(
url=f"https://cdn.{domain_name}/{location}/{name}.{filetype}")
async def tag_autbot_guilds(self, guild_list, user_id: int):
try:
all_guilds = [
guild for guild in await self.bot.manager_client.all_guilds(
message.AllGuildsRequest())
]
except Exception:
logger.info(
f"Shard {self.bot.shard_id} failed to get guild list from manager"
)
return {'guilds': []}, sc.INTERNAL_SERVER_ERROR_500
for guild_dict in guild_list:
for guild in all_guilds:
if guild.id == int(guild_dict['id']):
guild_dict['has_architus'] = True
guild_dict['architus_admin'] = user_id in guild.admin_ids
break
else:
guild_dict.update({
'has_architus': False,
'architus_admin': False
})
return {'guilds': guild_list}, sc.OK_200
async def starboard_post(self, message, guild):
"""Post a message in the starboard channel"""
starboard_ch = discord.utils.get(guild.text_channels, name='starboard')
if message.id in self.starboarded_messages or not starboard_ch or message.author == self.bot.user:
return
logger.info("Starboarding message: " + message.content)
self.starboarded_messages.append(message.id)
# TODO save author's image so this doesn't break when the change it
em = discord.Embed(title=timezone_aware_format(message.created_at),
description=message.content,
colour=0x42f468)
img = await message.author.avatar_url.read()
data = await self.bot.manager_client.publish_file(
message_type.File(location='avatars',
name=str(message.author.id),
file=img))
em.set_author(name=message.author.display_name, icon_url=data.url)
if message.embeds:
em.set_image(url=message.embeds[0].url)
elif message.attachments:
em.set_image(url=message.attachments[0].url)
await starboard_ch.send(embed=em)
