async def user_prefix(self, ctx, prefix=None): """Sets your custom user prefix! Your custom prefix will persist across servers and will be permanent through any crashes/downtime. This does not override the global prefix (a!) or any server prefixes. Instead, this is just a prefix you can use in addition to the global/server prefix. Setting the prefix to "none" (or any capitalization thereof) instead removes the prefix. If run without a prefix, it instead sends your current custom prefix.""" if prefix is None: if ctx.message.author.id in self.bot.bot_prefixes["user"].keys(): await ctx.send("**{}** is your custom prefix.".format( self.bot.bot_prefixes["user"][ctx.message.author.id])) else: await ctx.send("You don't have a custom prefix... yet!") else: if prefix.lower() == "none": self.bot.bot_prefixes["user"].pop(ctx.message.author.id) else: self.bot.bot_prefixes["user"][ctx.message.author.id] = prefix dump_yaml(self.bot.bot_prefixes, "prefixes.yml") shutil.copy2( "prefixes.yml", "drive/My Drive/bot_files/among_us_bot/prefixes.yml", ) await ctx.send("Done! Your custom prefix {}.".format( f"is now {prefix}" if prefix.lower() != "none" else "has been unset"))
async def delete(self, ctx): "Deletes automatically set up categories, channels, etc." if ctx.guild.id in self.game_setup: deleting_msg = await ctx.send( "Deleting auto-set-up categories, channels, and roles...") game_setup = self.game_setup[ctx.guild.id] del self.game_setup[ctx.guild.id] for game_map in game_setup.values(): if not isinstance(game_map, int): for id in game_map["channels"].values(): if ctx.guild.get_channel(id) is not None: if ctx.guild.get_channel(id).category is not None: await ctx.guild.get_channel( id).category.delete( reason="Among Us Bot auto-deletion") await ctx.guild.get_channel(id).delete( reason="Among Us Bot auto-deletion") in_game_role = ctx.guild.get_role(int(game_setup["in_game_role"])) if in_game_role is not None: await in_game_role.delete(reason="Among Us Bot auto-deletion") dead_role = ctx.guild.get_role(int(game_setup["dead_role"])) if dead_role is not None: await dead_role.delete(reason="Among Us Bot auto-deletion") dump_yaml(self.game_setup, "game-setup.yml") await deleting_msg.edit(content="Done!") await asyncio.sleep(10) await deleting_msg.delete()
def train(self): """Perform training.""" ## archive code and configs if self.archive_code: utils.archive_code(os.path.join(self.checkpoint_dir, 'archived_code.zip'), filetypes=['.py', '.yml']) utils.dump_yaml(os.path.join(self.checkpoint_dir, 'configs.yml'), self.cfgs) ## initialize start_epoch = 0 self.metrics_trace.reset() self.model.to_device(self.device) self.model.init_optimizers() ## resume from checkpoint if self.resume: start_epoch = self.load_checkpoint(optim=True) ## initialize tensorboardX logger if self.use_logger: from tensorboardX import SummaryWriter self.logger = SummaryWriter( os.path.join(self.checkpoint_dir, 'logs', datetime.now().strftime("%Y%m%d-%H%M%S"))) ## cache one batch for visualization self.viz_input = self.val_loader.__iter__().__next__() ## run epochs print( f"{self.model.model_name}: optimizing to {self.num_epochs} epochs") for epoch in range(start_epoch, self.num_epochs): self.train_loader, self.val_loader, self.test_loader = self.get_data_loader_func( self.cfgs) self.train_iter_per_epoch = len(self.train_loader) self.current_epoch = epoch metrics = self.run_epoch(self.train_loader, epoch) self.metrics_trace.append("train", metrics) with torch.no_grad(): metrics = self.run_epoch(self.val_loader, epoch, is_validation=True) self.metrics_trace.append("val", metrics) if (epoch + 1) % self.save_checkpoint_freq == 0: self.save_checkpoint(epoch + 1, optim=True) self.metrics_trace.plot( pdf_path=os.path.join(self.checkpoint_dir, 'metrics.pdf')) self.metrics_trace.save( os.path.join(self.checkpoint_dir, 'metrics.json')) print(f"Training completed after {epoch+1} epochs.")
def _guild_prefix(self, _id, prefix): if prefix.lower() == "none": self.bot.bot_prefixes["guild"].pop(_id) else: self.bot.bot_prefixes["guild"][_id] = prefix dump_yaml(self.bot.bot_prefixes, "prefixes.yml") shutil.copy2("prefixes.yml", "drive/My Drive/bot_files/among_us_bot/prefixes.yml")
async def setup(self, ctx): 'Automatically set up the game, or edit setup options manually. Only if you have the "manage channels" and "manage roles" permissions, though!' if ctx.invoked_subcommand is None: setup_msg = await ctx.send( "Setting up game categories, channels, and roles...") category_overwrites = discord.PermissionOverwrite( **{k: v for k, v in iter(discord.Permissions.none())}) in_game_role = await ctx.guild.create_role( name="In-Game", reason="Among Us Bot setup") dead_role = await ctx.guild.create_role( name="Dead", reason="Among Us Bot setup") game_setup = { "in_game_role": in_game_role.id, "dead_role": dead_role.id, } for map_name, game_map in self.maps.items(): category = await ctx.guild.create_category( map_name, overwrites={ role: category_overwrites for role in ctx.guild.roles }, reason="Among Us Bot setup", ) meeting_chan = await category.create_text_channel( "meeting-chat", reason="Among Us Bot setup") room_chans = {"Meeting": meeting_chan.id} for room in game_map.keys(): room_chan = await category.create_text_channel( "-".join(room.lower().split(" ")), reason="Among Us Bot setup", ) room_chans[room] = room_chan.id game_setup[map_name] = { "category": category.id, "channels": room_chans, } self.game_setup[ctx.guild.id] = game_setup dump_yaml(self.game_setup, "game-setup.yml") await setup_msg.edit(content="Done setting up!") await asyncio.sleep(10) await setup_msg.delete()
def create_on_demand_configuration(self, group): nodegroup = deepcopy(self.nodegroupDefaults) recursive_dict_copy(group['nodegroupOverrides'], nodegroup) nodegroup['availabilityZones'] = group['availabilityZones'] if group['instances']: if len(group['instances']) > 1: raise Exception(""" Cannot create an on-demand nodegroup with more than one instance type!\nGroup causing error:\n """ + dump_yaml(group)) else: nodegroup['instanceType'] = group['instances'][0] else: raise Exception("No instances in group!\nGroup causing error:\n" + dump_yaml(group)) return nodegroup
def dump_eksctl_config(self): if self.eksctl_config: return dump_yaml(self.eksctl_config) else: print("eksctl configuration not set!", file=sys.stderr) print("Trying now...", file=sys.stderr) self.create_eksctl_config() return self.dump_eksctl_config()
def _dump_configuration(self): """ Writes the active configuration to the configuration file. """ config = { 'physical_ports': { 'incoming': self._incoming_port, 'outgoing': self._outgoing_port, }, 'mode': self._mode.name, 'flow_active_time_secs': self._flow_active_time_secs, 'time_to_keep_stats_secs': self._time_to_keep_stats_secs, 'flow_starts_retrieval_interval_secs': self._flow_starts_retrieval_interval_secs, 'firewall_dpid': self._firewall_dpid, 'blacklist_rules': [rule.as_dict() for rule in self._blacklist_rules], 'whitelist_rules': [rule.as_dict() for rule in self._whitelist_rules], } utils.dump_yaml(self.CONFIG_FILE_PATH, config)
def _create_out_dir(self): mkdirne(self.args.save_dir) timestamp = datetime.now().strftime('%y%m%d_%H%M') tag = self.args.tag commit = subprocess.run( ['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE).stdout.decode('utf-8')[:6] if commit == '': commit = 'nohead' out_dir = os.path.join( self.args.save_dir, '_'.join( (timestamp, commit, self.args.model, tag))) mkdirne(out_dir) cfg_file = os.path.join(out_dir, 'config.yml') dump_yaml(self.args, cfg_file) return out_dir
def _dump_configuration(self): """ Writes the active configuration to the configuration file. """ config = { 'physical_ports': { 'incoming': self._incoming_port, 'outgoing': self._outgoing_port, }, 'mode': self._mode.name, 'flow_active_time_secs': self._flow_active_time_secs, 'time_to_keep_stats_secs': self._time_to_keep_stats_secs, 'flow_starts_retrieval_interval_secs': self._flow_starts_retrieval_interval_secs, 'firewall_dpid': self._firewall_dpid, 'blacklist_rules': [rule.as_dict() for rule in self._blacklist_rules], 'whitelist_rules': [rule.as_dict() for rule in self._whitelist_rules], } utils.dump_yaml(self.CONFIG_FILE_PATH, config)
def prepare_model(args, vocabs, resume_from=None): if resume_from is None: resume_from = dict() model_path = args.model_path if resume_from.get("model_args") is not None: temp_path = tempfile.mkstemp()[1] utils.dump_yaml(resume_from["model_args"], temp_path) torchmodels.register_packages(models) mdl_cls = torchmodels.create_model_cls(models.jlu, model_path) mdl = mdl_cls( hidden_dim=args.hidden_dim, word_dim=args.word_dim, num_words=len(vocabs[0]), num_slots=len(vocabs[1]), num_intents=len(vocabs[2]) ) mdl.reset_parameters() if resume_from.get("model") is not None: mdl.load_state_dict(resume_from["model"]) else: embeds.load_embeddings(args, vocabs[0], mdl.embeddings()) return mdl
def _add_instance(server): """ Given a server dict from the layout, merge configuration from the YAML into the defaults provided in defaults.py. Returns a dict that looks like {label: data} """ try: label = server['label'] except KeyError: raise ConfigurationError('Server without a label encountered! ' 'Data was:\n' '{}'.format(utils.dump_yaml(server))) # Apply extra config from yaml file extra_info = server.get('instance_info', {}) filtered_info = _filter_info(extra_info) instance_data = utils.update_recursive(DEFAULT_INSTANCE, filtered_info) log.debug(utils.dump_json(instance_data)) # Logic for special cases az = extra_info.get('availability_zone', 'us-east-1a') instance_data['subnet_id'] = \ '${{terraform_remote_state.subnets.output.{}}}'.format(az) instance_data['tags'].update({'label': label}) instance_data['tags'].update({'id': label + '_${var.run_id}'}) ebs_data = _get_ebs_block_devices(extra_info) log.debug(utils.dump_json(ebs_data)) instance_data.update(ebs_data) provisioners = _get_provisioners(extra_info, label) log.debug(utils.dump_json(provisioners)) instance_data.update(provisioners) security_groups = _get_instance_security_groups(extra_info) log.debug(utils.dump_json(security_groups)) instance_data.update(security_groups) return {label: instance_data}
def _add_instance(server): """ Given a server dict from the layout, merge configuration from the YAML into the defaults provided in defaults.py. Returns a dict that looks like {label: data} """ try: label = server['label'] except KeyError: raise ConfigurationError('Server without a label encountered! ' 'Data was:\n' '{}'.format(utils.dump_yaml(server))) # Apply extra config from yaml file extra_info = server.get('instance_info', {}) filtered_info = _filter_info(extra_info) instance_data = utils.update_recursive(DEFAULT_INSTANCE, filtered_info) log.debug(utils.dump_json(instance_data)) # Logic for special cases az = extra_info.get('availability_zone', 'us-east-1a') instance_data['subnet_id'] = \ '${{terraform_remote_state.subnets.output.{}}}'.format(az) instance_data['tags'].update({'label': label}) instance_data['tags'].update({'id': label + '_${var.run_id}'}) ebs_data = _get_ebs_block_devices(extra_info) log.debug(utils.dump_json(ebs_data)) instance_data.update(ebs_data) provisioners = _get_provisioners(extra_info, label) log.debug(utils.dump_json(provisioners)) instance_data.update(provisioners) security_groups = _get_instance_security_groups(extra_info) log.debug(utils.dump_json(security_groups)) instance_data.update(security_groups) return {label: instance_data}
def view_task(task_id, fmt, dir, ext): task_id = int(task_id) data = w.get_task(id=task_id)[1] uuid = data['uuid'] with open( '.'.join([os.path.join(dir, uuid), ext])) as f: notation = f.read() if fmt == 'note': logger.info('returning note text for {0}'.format(task_id)) print( notation ) else: logger.info('returning full task information for {0} in {1} format.'.format(task_id, fmt)) if fmt == 'yaml': data['notation'] = '\n' + notation output = dump_yaml(data) elif fmt == 'json': data['notation'] = notation output = json.dumps(data, indent=3) print( output )
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. """ import discord, os, asyncio from discord.ext import commands from utils import load_yaml, dump_yaml from checks import is_bot_owner # Code to run at startup if not os.path.exists("prefixes.yml"): dump_yaml({"global": "a!", "guild": {}, "user": {}}, "prefixes.yml") # Load prefixes bot_prefixes = load_yaml("prefixes.yml") print(bot_prefixes) config = load_yaml("config.yml") # Gets possible prefixes to use async def prefix_callable(bot, message): guild = message.guild prefix_results = [] if guild: if str(guild.id) in bot_prefixes["guild"].keys(): prefix_results.append(bot_prefixes["guild"][str(guild.id)])
f'{name}_loss{valid_loss:.5f}_epoch{str(epoch).zfill(3)}.pth' ) torch.save(trainer.weights, path) scheduler.step() # type: ignore logger.info(f'EPOCH: [{epoch}/{cfg.num_epochs}]') logger.info( f'TRAIN LOSS: {train_loss:.8f}, VALID LOSS: {valid_loss:.8f}' ) logger.info( f'TRAIN mIoU: {train_iou:.8f}, VALID mIoU: {valid_iou:.8f}' ) cfg_path: str = os.path.join(save_dir, 'config.yml') utils.dump_yaml(cfg_path, cfg.todict()) # Plot metrics plt.plot(metrics['train_loss'], label='train') plt.plot(metrics['valid_loss'], label='valid') plt.title('Loss curve') plt.legend() plt.savefig(os.path.join(save_dir, 'loss.png')) plt.clf() plt.plot(metrics['train_iou'], label='train') plt.plot(metrics['valid_iou'], label='valid') plt.title('mIoU curve') plt.legend() plt.savefig(os.path.join(save_dir, 'mIoU.png'))
def create_spot_configuration(self, group): nodegroup = deepcopy(self.nodegroupDefaults) recursive_dict_copy(group['nodegroupOverrides'], nodegroup) nodegroup['availabilityZones'] = group['availabilityZones'] if group['instances']: if 'instancesDistribution' not in nodegroup.keys(): nodegroup['instancesDistribution'] = {} instances_distribution = nodegroup['instancesDistribution'] if 'onDemandBaseCapacity' not in instances_distribution.keys(): instances_distribution['onDemandBaseCapacity'] = 0 if 'onDemandPercentageAboveBaseCapacity' not in instances_distribution.keys( ): instances_distribution[ 'onDemandPercentageAboveBaseCapacity'] = 0 instances_distribution['instanceTypes'] = group['instances'] # find maximum price among the Spot prices of all instances in this group max_prices = [] for instance in group['instances']: max_prices.append(self.region_information[instance] ['spot_pricing']['maxPrice']) max_price = max(max_prices) # set maximum price and over pay by a bit instances_distribution['maxPrice'] = max_price * ( 1. + self.config['overPayBy'] / 100) # hack to get around spot instances with only 1 instance in its group # include the most expensive instance in its family and keep the max price # the same so that the expensive one is never scheduled if len(group['instances']) == 1: family_prices = [] for instance, instance_information in self.region_information.items( ): instance_family = instance.split(".")[0] group_family = group['instances'][0].split(".")[0] same_family = instance_family == group_family if same_family: family_prices.append( (instance, self.region_information[instance] ['spot_pricing']['maxPrice'])) most_expensive_in_family = sorted(family_prices, key=lambda x: x[1], reverse=True)[0] most_expensive_in_family_instance_name = most_expensive_in_family[ 0] instances_distribution['instanceTypes'].append( most_expensive_in_family_instance_name) # hack to avoid nodegroup with two of the most expensive instances # ...just don't allow that as a spot configuration if group['instances'][ 0] == most_expensive_in_family_instance_name: print( f"WARNING: instance {group['instances'][0]} is the most expensive in it's family and cannot be in a spot group by itself!", file=sys.stderr) return None else: raise Exception("No instances in group!\nGroup causing error:\n" + dump_yaml(group)) return nodegroup
train_losses[k].append(v) train_losses['loss_sum'].append(sum([v for v in train_loss.values()])) for k, v in valid_loss.items(): valid_losses[k].append(v) valid_losses['loss_sum'].append(sum([v for v in valid_loss.values()])) logger.info(f'EPOCH: [{epoch + 1}/{cfg.num_epochs}]') logger.info('TRAIN_LOSS:') logger.info(train_loss) logger.info('VALID_LOSS') logger.info(valid_loss) weights_path: str = os.path.join(save_dir, 'weights', 'weights.pth') torch.save(model.state_dict(), weights_path) utils.dump_yaml(os.path.join(save_dir, 'config.yml'), cfg.todict()) # Plot metrics keys: Tuple[str, ...] = ('loss_classifier', 'loss_box_reg', 'loss_mask', 'loss_objectness', 'loss_rpn_box_reg', 'loss_sum') plt.tight_layout() plt.figure(figsize=(12, 8)) for idx, key in enumerate(keys): plt.subplot(2, 3, idx + 1) plt.plot(train_losses[key], label='train') plt.plot(valid_losses[key], label='valid') plt.title(key) plt.legend() plt.savefig(os.path.join(save_dir, 'fig.png')) plt.clf()
def render_task_list(query, dir, ext, fmt): if fmt in ['note', 'yaml']: print(dump_yaml(list_tasks(query, dir, ext))) elif fmt == 'json': for doc in list_tasks(query, dir, ext): print(json.dumps(doc, indent=2))
try: exp_name = utils.make_experiment_name(args.debug) result_dir = utils.RESULTS_BASE_DIR / exp_name os.mkdir(result_dir) logger = mylogger.get_mylogger(filename=result_dir / 'log') sandesh.send(f'start: {exp_name}') logger.debug(f'created: {result_dir}') logger.debug('loading data ...') train_feat_path = utils.FEATURE_DIR / 'baseline_features.pkl' X = utils.load_pickle(train_feat_path) print(X.columns) features_list = utils.load_yaml(args.feature) utils.dump_yaml(features_list, result_dir / 'features_list.yml') all_features = features_list['features']['original'] + \ features_list['features']['generated'] categorical_feat = features_list['categorical_features'] logger.debug(all_features) logger.debug(f'features num: {len(all_features)}') utils.dump_yaml(features_list, result_dir / 'features_list.yml') # X_test = X_test[all_features] # sandesh.send(args.config) config = utils.load_yaml(args.config) logger.debug(config) utils.dump_yaml(config, result_dir / 'model_config.yml') model_params = config['model_params']
def dump_defaults(defaults): dump_yaml(DEFAULTS_PATH, defaults)
def check_perms(ctx, command_name=None, cog_name=None, suppress=False): ret = False # inits GLOBALS = utils.load_yaml(utils.GLOBAL_PERMS_FILE) cmd = ctx.command.name if command_name is None else command_name if cog_name is None: cog = ctx.cog.qualified_name if ctx.cog is not None else "none" else: cog = cog_name # check global admin if _is_global_admin(ctx, GLOBALS): ret |= True # if dm, check dm perms in global_perms file, else guild perms file if ctx.guild is None: ret |= _check(cmd=cmd, cog=cog, perm_dict=GLOBALS['dm'], flags=GLOBALS['flags'], ctx=ctx, is_dm=True) utils.dump_yaml(GLOBALS, utils.GLOBAL_PERMS_FILE) else: # check guild owner if ctx.author.id == ctx.guild.owner.id: ret |= True # check guild admin member = ctx.guild.get_member(ctx.author.id) perms = member.permissions_in(ctx.channel) if perms.administrator: ret |= True # load guild perms perms_file = f"{utils.PERMS_DIR}{str(ctx.guild.id)}.yaml" if os.path.exists(perms_file): perms_dict = utils.load_yaml(perms_file) else: perms_dict = GLOBALS['default_perms'] utils.dump_yaml(perms_dict, perms_file) # check guild perms if not suppress and not ret: ret |= _check(cmd=cmd, cog=cog, perm_dict=perms_dict, flags=perms_dict['flags'], ctx=ctx, is_dm=False) else: try: ret |= _check(cmd=cmd, cog=cog, perm_dict=perms_dict, flags=perms_dict['flags'], ctx=ctx, is_dm=False) except PermissionError: ret |= False utils.dump_yaml(perms_dict, perms_file) return ret