async def on_ready(self): """pull autoresponses from the db, then set activity""" await self.initialize_user_commands() logger.info('Logged on as {0}!'.format(self.user)) await self.change_presence(activity=discord.Activity( name=f"the tragedy of darth plagueis the wise {self.shard_id}", type=2)) await self.manager_client.guild_update(self.shard_id, self.guilds_as_dicts)
def train_entry(): logger.info("Building model...") bert_config = BertConfig.from_json_file(config.bert_config) lr = config.learning_rate # num_train_steps = int(train_dataset.features_size/ config.batch_size * config.epochs) # warm_up = int(num_train_steps * config.warmup_proportion) model = get_model(config.model_package, config.model_name, config.model_class_name).to(device) ema = EMA(config.ema_decay) params = get_model_trainabel_param(ema, model) optimizer = get_optimizer(lr, params=params) scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, config.T_MAX, config.min_lr) epochs = config.epochs best_f1 = best_em = patience = 0 start_index = 0 if not config.is_continue else config.continue_checkpoint for epoch in range(config.start_epoch, epochs): logger.info(f"Epoch: {epoch}") train(model, optimizer, scheduler, ema, bert_config, start_index, get_steps("train", config.mode), epoch) if config.is_test_with_test_dev_dataset: valid(model, bert_config, epoch) metrics = test(model, bert_config, epoch)
async def on_guild_join(self, guild): logger.info(f" -- JOINED NEW GUILD: {guild.name} -- ") try: await self.manager_client.guild_update(self.guilds_as_message) except Exception: logger.info( f"Shard {self.shard_id} failed to send manager its guild list")
def netbios_scan(ipaddress_list, timeout=0.5): TYPE_UNKNOWN = 0x01 TYPE_WORKSTATION = 0x00 TYPE_CLIENT = 0x03 TYPE_SERVER = 0x20 TYPE_DOMAIN_MASTER = 0x1B TYPE_DOMAIN_CONTROLLER = 0x1C TYPE_MASTER_BROWSER = 0x1D TYPE_BROWSER = 0x1E TYPE_NETDDE = 0x1F TYPE_STATUS = 0x21 format_str = "{:<16}{:<20}{:<20}{:<20}{}".format("IPAddress", "DOMAIN", "NetbiosName", "MAC", "Nets") logger.info(format_str) for ipaddress in ipaddress_list: n = NetBIOS() result = { "ip": ipaddress, "domain": None, "name": None, "nets": [], "mac": None } try: entries = n.getnodestatus('*', ipaddress, timeout=timeout) result['mac'] = n.getmacaddress() except Exception as E: continue for entrie in entries: if entrie["TYPE"] == TYPE_SERVER: result["name"] = entrie["NAME"].strip().decode('latin-1') elif entrie["TYPE"] == TYPE_WORKSTATION: result["domain"] = entrie["NAME"].strip().decode('latin-1') elif entrie["TYPE"] == TYPE_CLIENT: result["TYPE_CLIENT"] = entrie["NAME"].strip().decode( 'latin-1') elif entrie["TYPE"] == TYPE_DOMAIN_MASTER: result["TYPE_DOMAIN_MASTER"] = entrie["NAME"].strip().decode( 'latin-1') elif entrie["TYPE"] == TYPE_DOMAIN_CONTROLLER: result["TYPE_DOMAIN_CONTROLLER"] = entrie["NAME"].strip( ).decode('latin-1') elif entrie["TYPE"] == TYPE_MASTER_BROWSER: result["TYPE_MASTER_BROWSER"] = entrie["NAME"].strip().decode( 'latin-1') elif entrie["TYPE"] == TYPE_STATUS: result["TYPE_STATUS"] = entrie["NAME"].strip().decode( 'latin-1') try: if result.get("name") is not None: resp = n.name_query_request(result.get("name"), ipaddress) result["nets"].extend(resp.entries) except Exception as e: pass format_str = "{:<16}{:<20}{:<20}{:<20}{}".format( ipaddress, result.get("domain"), result.get("name"), result.get("mac"), result.get("nets")) logger.warning(format_str)
def __init__(self, total_shards): logger.info(f"Number of shards: {total_shards}") self.hoarfrost_gen = HoarFrostGenerator() self.total_shards = total_shards self.registered = [False for _ in range(total_shards)] self.last_checkin = {} self.store = {}
def classify_data(): from lib.models import QANet logger.info("Building model...") train_dataset = get_dataset("train") model = get_model().to(device) classify(model, train_dataset)
async def emojis(self, ctx): """ List currently cached emojis. Enclose the name (case sensitive) of cached emoji in `:`s to auto-load it into a message """ settings = self.bot.settings[ctx.guild] if not settings.manage_emojis: message = f"The emoji manager is disabled, you can enable it in `{settings.command_prefix}settings`" await ctx.send(message) else: # message = '```\n • ' + '\n • '.join(self.managers[ctx.guild.id].list_unloaded()) + '```\n' logger.debug("generating list image") file = self.managers[ctx.guild.id].list_unloaded() if file is not None: logger.debug("file is good") message = "Enclose the name (case sensitive) of cached emoji in `:`s to auto-load it into a message" # msg = await ctx.send(message, file=discord.File(file, "cool.png")) try: data = await self.bot.manager_client.publish_file( iter([message_type.File(file=file.getvalue())])) except Exception: logger.info( f"Shard {self.bot.shard_id} failed to upload emoji") await ctx.send("Failed to generate cached emoji preview") return em = discord.Embed(title="Cached Emojis", description=ctx.guild.name) # em.set_image(url=msg.attachments[0].url) em.set_image(url=data.url) em.color = 0x7b8fb7 em.set_footer(text=message) await ctx.send(embed=em) else: await ctx.send("No cached emoji")
def docker_build(**kwargs): for line in docker_client_api.build(**kwargs): line = json.loads(line.decode('utf-8')) if 'stream' in line: line = line['stream'].rstrip('\n') if line: logger.info(line)
def __init__(self, **kwargs): self.session = get_session() self.asyncpg_wrapper = AsyncConnWrapper() self.deletable_messages = [] self.hoarfrost_gen = HoarFrostGenerator() logger.debug("registering with manager...") manager_client = grpc_client.get_blocking_client('manager:50051') while True: try: shard_info = manager_client.register(message.RegisterRequest()) break except Exception: logger.info("Trying to get shard id from manager") time.sleep(3) self.shard_id = shard_info.shard_id shard_dict = { 'shard_id': shard_info.shard_id, 'shard_count': shard_info.shard_count } logger.info(f"Got shard_id {self.shard_id}") kwargs.update(shard_dict) super().__init__(**kwargs)
async def on_member_join(self, member): logger.info("%s joined guild: %s" % (member.name, member.guild.name)) settings = self.bot.settings[member.guild] try: default_role = discord.utils.get(member.guild.roles, id=settings.default_role_id) await member.add_roles(default_role) except Exception: logger.exception("could not add %s to %s" % (member.display_name, 'default role'))
def register(self, request, context): """Returns the next shard id that needs to be filled as well as the total shards""" if all(self.registered): raise Exception("Shard trying to register even though we're full") i = next(i for i in range(self.total_shards) if not self.registered[i]) logger.info( f"Shard requested id, assigning {i + 1}/{self.total_shards}...") self.registered[i] = True return message.ShardInfo(shard_id=i, shard_count=self.total_shards)
async def register(self): """Returns the next shard id that needs to be filled as well as the total shards""" if all(self.registered): raise Exception("Shard trying to register even though we're full") i = next(i for i in range(self.total_shards) if not self.registered[i]) logger.info(f'Shard requested id, assigning {i + 1}/{self.total_shards}...') self.store[i] = {'shard_id': i} self.registered[i] = True return {'shard_id': i, 'shard_count': self.total_shards}
async def heartbeat(self): await self.wait_until_ready() while not self.is_closed(): await asyncio.sleep(0.5) try: await self.manager_client.checkin( message.ShardID(shard_id=self.shard_id)) except Exception: logger.info( f"Shard {self.shard_id} failed to checkin with manager")
def __init__(self, total_shards): """ Instantiates a new manager server that handles some number of shards. """ logger.info(f"Number of shards: {total_shards}") self.hoarfrost_gen = HoarFrostGenerator() self.total_shards = total_shards self.registered = [False for _ in range(total_shards)] self.last_checkin = dict() self.store = dict()
async def on_message(self, msg): """Execute commands, then trigger autoresponses""" logger.info( 'Message from {0.author} in {0.guild.name}: {0.content}'.format( msg)) if msg.author == self.user: return # check for real commands await self.process_commands(msg)
async def publish_file(self, location: str = 'assets', name: str = '', filetype: str = 'png', data: str = ''): assert data != '' if name == '': name = str(self.hoarfrost_gen.generate()) directory = f'/var/www/{location}' if not os.path.exists(directory): os.makedirs(directory) with open(f'{directory}/{name}.{filetype}', 'wb') as f: logger.info(f'writing {directory}/{name}.{filetype}') f.write(base64.b64decode(data)) return {'url': f'https://cdn.{domain_name}/{location}/{name}.{filetype}'}
def test_model(bert_config, losses, model, valid_result, epoch, label="valid"): logger.info("start_step test or valid:") tokenizer, dev_dataloader = load_data(bert_config, config.dev_dir1) softmax = torch.nn.Softmax(dim=-1) log_sofmax = torch.nn.LogSoftmax(dim=-1) exact_match_total = 0 f1_total = 0 exact_match = 0 f1 = 0 with torch.no_grad(): for step, batch in enumerate(dev_dataloader): batch = tuple(t.to(device) for t in batch) input_ids, input_mask, segment_ids, start_positions, end_positions = batch start_embeddings, end_embeddings = model(input_ids, input_mask, segment_ids) # start_embeddings = mask(start_embeddings, input_span_mask, -1) # end_embeddings = mask(end_embeddings, input_span_mask, -1) loss1 = F.nll_loss(log_sofmax(start_embeddings), start_positions, reduction='mean') loss2 = F.nll_loss(log_sofmax(end_embeddings), end_positions, reduction='mean') loss = (loss1 + loss2) / 2 logger.info(f"Origin Loss: {loss}, step: {step}") losses.append(loss.item()) pre_start, pre_end, probabilities = find_max_proper_batch( softmax(start_embeddings), softmax(end_embeddings)) cur_res = convert_pre_res(input_ids, pre_start, pre_end, start_positions, end_positions, probabilities, tokenizer) record_info(valid_result=cur_res, epoch=epoch, is_continue=True, r_type=label) visual_data(model, loss, loss, None, epoch, step, exact_match_total, f1_total, exact_match, f1, label=label) loss = np.mean(losses) exact_match_total, f1_total, exact_match, f1 = evaluate_valid_result( valid_result, 0, 0, len(dev_dataloader.dataset)) # metrics["loss"] = loss return loss, exact_match, f1
async def on_ready(self): """pull autoresponses from the db, then set activity""" await self.asyncpg_wrapper.connect() logger.info('Logged on as {0}!'.format(self.user)) await self.change_presence(activity=discord.Activity( name=f"the tragedy of darth plagueis the wise {self.shard_id}", type=2)) try: await self.manager_client.guild_update(self.guilds_as_message) except Exception: logger.info( f"Shard {self.shard_id} failed to send manager its guild list")
async def on_connect(self, sid: str, environ: dict): logger.debug(f"{environ['REMOTE_ADDR']} has connected with sid: {sid}") request = environ['aiohttp.request'] try: jwt = JWT(token=request.cookies['token']) except (InvalidTokenError, KeyError): logger.info( "No valid token found, logging into unprivileged gateway...") else: logger.info("Found valid token, logging into elevated gateway...") sio.enter_room(sid, f"{sid}_auth") async with sio.session(sid) as session: session['jwt'] = jwt
async def list_guilds(self): """Update the manager with the guilds that we know about""" await self.wait_until_ready() while not self.is_closed(): logger.info("Current guilds:") for guild in self.guilds: if guild.me.display_name == 'archit.us': try: await guild.me.edit(nick='architus') except discord.Forbidden: logger.warning(f"couldn't change nickname in {guild.name}") logger.info("{} - {} ({})".format(guild.name, guild.id, guild.member_count)) await asyncio.sleep(600)
def netbios_interface(ipaddress_list, timeout, pool): format_str = "{:<16}{:<20}{:<20}{:<20}{}".format("IPAddress", "DOMAIN", "NetbiosName", "MAC", "Nets") logger.info(format_str) tasks = [] for ipaddress in ipaddress_list: task = pool.spawn(netbios_scan, ipaddress, timeout * 2) tasks.append(task) if len(tasks) >= 300: joinall(tasks) tasks = [] joinall(tasks)
async def on_message(self, msg): """Execute commands, then trigger autoresponses""" logger.info('Message from {0.author} in {0.guild.name}: {0.content}'.format(msg)) if msg.author == self.user: return # check for real commands await self.process_commands(msg) # check for user commands for command in self.user_commands[msg.guild.id]: if (command.triggered(msg.content)): await command.execute(msg) break
async def guild_count(self): try: resp = await self.bot.manager_client.guild_count( message.GuildCountRequest()) return { 'guild_count': resp.guild_count, 'user_count': resp.user_count }, sc.OK_200 except Exception: logger.info( f"Shard {self.bot.shard_id} failed to get guild count from manager" ) return { 'guild_count': -1, 'user_count': -1 }, sc.INTERNAL_SERVER_ERROR_500
async def url(self): if self.str_url == "": with BytesIO() as buf: self.im.save(buf, format="PNG") binary = buf.getvalue() try: data = await self.bot.manager_client.publish_file( iter([ message.File( location="emojis", name=f"{self.id}", file=binary)])) except Exception: logger.info(f"Shard {self.bot.shard_id} failed to upload emoji") return None self.str_url = data.url return self.str_url
async def on_request_elevation(self, sid: str, nonce: int): logger.debug(f"{sid} requesting elevation...") try: jwt = JWT(token=auth_nonces[nonce]) del auth_nonces[nonce] except (InvalidTokenError, KeyError): logger.info( f"{sid} requested room elevation but didn't provide a valid jwt" ) await sio.emit('elevation_return', {'message': "Missing or invalid jwt"}, room=sid) else: logger.debug(f"valid nonce provided, granting access...") sio.enter_room(sid, f"{sid}_auth") await sio.emit('elevation_return', {'message': "success"}, room=sid) async with sio.session(sid) as session: session['token'] = jwt
def __init__(self, **kwargs): self.user_commands = {} self.session = get_session() self.deletable_messages = [] self.hoarfrost_gen = HoarFrostGenerator() manager_client = blocking_rpc_client.shardRPC() # wait for manager to come up; this is scuffed import time time.sleep(2) logger.debug("asking for shard id") shard_info, sc = manager_client.register(routing_key='manager_rpc') self.shard_id = shard_info['shard_id'] logger.info(f"Got shard_id {self.shard_id}") kwargs.update(shard_info) super().__init__(**kwargs)
def valid(model, bert_config, epoch=0): model.eval() valid_result = [] losses = [] logger.info("start valid:") loss, extract_match, f1 = test_model(bert_config, losses, model, valid_result, epoch, label="valid") record_info( loss, f1=f1, em=extract_match, # valid_result=valid_result, epoch=epoch, r_type="valid") logger.info("VALID loss {:8f} F1 {:8f} EM {:8f}\n".format( loss, f1, extract_match))
def publish_file(self, request_iterator, context): """Missing associated documentation comment in .proto file""" first = next(request_iterator) filetype = "png" if first.filetype == "" else first.filetype name = first.name if name == "": name = str(self.hoarfrost_gen.generate()) location = first.location if location == "": location = "assets" directory = f"/var/www/{location}" if not os.path.exists(directory): os.makedirs(directory) with open(f"{directory}/{name}.{filetype}", "wb") as f: logger.info(f"Writing {directory}/{name}.{filetype}") f.write(first.file) for datum in request_iterator: f.write(datum.file) return message.Url( url=f"https://cdn.{domain_name}/{location}/{name}.{filetype}")
async def tag_autbot_guilds(self, guild_list, user_id: int): try: all_guilds = [ guild for guild in await self.bot.manager_client.all_guilds( message.AllGuildsRequest()) ] except Exception: logger.info( f"Shard {self.bot.shard_id} failed to get guild list from manager" ) return {'guilds': []}, sc.INTERNAL_SERVER_ERROR_500 for guild_dict in guild_list: for guild in all_guilds: if guild.id == int(guild_dict['id']): guild_dict['has_architus'] = True guild_dict['architus_admin'] = user_id in guild.admin_ids break else: guild_dict.update({ 'has_architus': False, 'architus_admin': False }) return {'guilds': guild_list}, sc.OK_200
async def starboard_post(self, message, guild): """Post a message in the starboard channel""" starboard_ch = discord.utils.get(guild.text_channels, name='starboard') if message.id in self.starboarded_messages or not starboard_ch or message.author == self.bot.user: return logger.info("Starboarding message: " + message.content) self.starboarded_messages.append(message.id) # TODO save author's image so this doesn't break when the change it em = discord.Embed(title=timezone_aware_format(message.created_at), description=message.content, colour=0x42f468) img = await message.author.avatar_url.read() data = await self.bot.manager_client.publish_file( message_type.File(location='avatars', name=str(message.author.id), file=img)) em.set_author(name=message.author.display_name, icon_url=data.url) if message.embeds: em.set_image(url=message.embeds[0].url) elif message.attachments: em.set_image(url=message.attachments[0].url) await starboard_ch.send(embed=em)