def main() -> None: # Read args parser = argparse.ArgumentParser(prog=config.PACKAGE_NAME, description="ETag investigation") parser.add_argument( "--config-path", required=True, help="Configuration file path, e.g. /some/dir/config.yaml") instance_config_path = Path(parser.parse_args().config_path) # Read user config log.debug("Reading instance configuration file %s", instance_config_path) instance_config = YAML().load(instance_config_path) instance_config = json.loads(json.dumps( instance_config)) # Recursively use a dict as the data structure. # Log user config logged_instance_config = instance_config.copy() del logged_instance_config["feeds"] log.info( "Read user configuration file %s having excerpted configuration %s for %s channels %s with %s feeds.", instance_config_path, logged_instance_config, len(instance_config["feeds"]), list(instance_config["feeds"]), len([ feed for channel in instance_config["feeds"].values() for feed in channel ]), ) config.INSTANCE = instance_config investigate_etags()
def main() -> None: # Read args parser = argparse.ArgumentParser(prog=config.PACKAGE_NAME, description="IRC RSS feed posting bot") parser.add_argument( '--config-path', required=True, help='Configuration file path, e.g. /some/dir/config.yaml') instance_config_path = Path(parser.parse_args().config_path) # Read user config log.debug('Reading instance configuration file %s', instance_config_path) instance_config = YAML().load(instance_config_path) instance_config = json.loads(json.dumps( instance_config)) # Recursively use a dict as the data structure. # Log user config logged_instance_config = instance_config.copy() del logged_instance_config['feeds'] log.info( 'Read user configuration file %s having excerpted configuration %s for %s channels %s with %s feeds.', instance_config_path, logged_instance_config, len(instance_config['feeds']), list(instance_config['feeds']), len([ feed for channel in instance_config['feeds'].values() for feed in channel ])) for channel, channel_config in instance_config['feeds'].items(): for feed, feed_config in channel_config.items(): log.info('%s has feed %s having config: %s', channel, feed, feed_config) # Set alerts channel if 'alerts_channel' not in instance_config: instance_config[ 'alerts_channel'] = config.ALERTS_CHANNEL_FORMAT_DEFAULT instance_config['alerts_channel'] = instance_config[ 'alerts_channel'].format(nick=instance_config['nick']) if instance_config['alerts_channel'] not in instance_config['feeds']: instance_config['feeds'][instance_config['alerts_channel']] = {} # Process user config instance_config['dir'] = instance_config_path.parent instance_config['nick:casefold'] = instance_config['nick'].casefold() instance_config['channels:casefold'] = [ channel.casefold() for channel in instance_config['feeds'] ] instance_config['repeated_urls'] = { url for url, count in collections.Counter( feed_cfg['url'] for channel_cfg in instance_config['feeds'].values() for feed_cfg in channel_cfg.values()).items() if count > 1 } config.INSTANCE = instance_config # Start bot Bot()
def load_config() -> None: """Read and load the instance configuration.""" # Read args parser = argparse.ArgumentParser(prog=config.PACKAGE_NAME, description="IRC URL title posting bot") parser.add_argument( "--config-path", required=True, help="Configuration file path, e.g. /some/dir/config.yaml") instance_config_path = Path(parser.parse_args().config_path) # Read user config log.debug("Reading instance configuration file %s", instance_config_path) instance_config = YAML().load(instance_config_path) instance_config = json.loads(json.dumps( instance_config)) # Recursively use a dict as the data structure. # Log user config logged_instance_config = instance_config.copy() if "sites" in logged_instance_config: del logged_instance_config["sites"] log.info( "Read user configuration file %s having excerpted configuration: %s", instance_config_path, logged_instance_config) for site, site_config in instance_config.get("sites", {}).items(): log.info("User configuration for site %s is: %s", site, site_config) # Set alerts channel if "alerts_channel" not in instance_config: instance_config[ "alerts_channel"] = config.ALERTS_CHANNEL_FORMAT_DEFAULT instance_config["alerts_channel"] = instance_config[ "alerts_channel"].format(nick=instance_config["nick"]) if instance_config["alerts_channel"] not in instance_config["channels"]: instance_config["channels"].append(instance_config["alerts_channel"]) # Process user config instance_config["nick:casefold"] = instance_config["nick"].casefold() instance_config["channels:casefold"] = [ channel.casefold() for channel in instance_config["channels"] ] instance_config["ignores:casefold"] = [ ignore.casefold() for ignore in instance_config.get("ignores", []) ] # Process blacklist blacklists = instance_config["blacklist"] = instance_config.get( "blacklist", {}) blacklists["title"] = set(blacklists.get("title", set())) blacklists["title"] = {entry.casefold() for entry in blacklists["title"]} blacklists["url"] = set(blacklists.get("url", set())) blacklists["url"] = {entry.casefold() for entry in blacklists["url"]} config.INSTANCE = instance_config
def load_config(self): """ Load configuration from path specified by the framework. Configuration file is in YAML format. """ config_path = efb_utils.get_config_path(self.channel_id) if not config_path.exists(): raise FileNotFoundError( self._("Config File does not exist. ({path})").format( path=config_path)) with config_path.open() as f: data = YAML().load(f) # Verify configuration if not isinstance(data.get('token', None), str): raise ValueError(self._('Telegram bot token must be a string')) if isinstance(data.get('admins', None), int): data['admins'] = [data['admins']] if isinstance(data.get('admins', None), str) and data['admins'].isdigit(): data['admins'] = [int(data['admins'])] if not isinstance(data.get('admins', None), list) or not data['admins']: raise ValueError( self. _("Admins' user IDs must be a list of one number or more." )) for i in range(len(data['admins'])): if isinstance(data['admins'][i], str) and data['admins'][i].isdigit(): data['admins'][i] = int(data['admins'][i]) if not isinstance(data['admins'][i], int): raise ValueError( self. _('Admin ID is expected to be an int, but {data} is found.' ).format(data=data['admins'][i])) self.config = data.copy()
def load_instance_config(log_details: bool = True) -> None: # pylint: disable=too-many-locals """Read and load the instance configuration.""" # Read args parser = argparse.ArgumentParser(prog=config.PACKAGE_NAME, description="IRC RSS feed posting bot") parser.add_argument( "--config-path", required=True, help="Configuration file path, e.g. /some/dir/config.yaml") instance_config_path = Path(parser.parse_args().config_path) # Read instance config log.debug("Reading instance configuration file %s", instance_config_path) instance_config = YAML().load(instance_config_path) instance_config = json.loads(json.dumps( instance_config)) # Recursively use a dict as the data structure. log.info("Read user configuration file %s", instance_config_path) if "taxonomies" in instance_config: del instance_config["taxonomies"] if instance_config.get("tracemalloc"): TraceMalloc().start() if not instance_config["feeds"]: instance_config["feeds"] = {} url_counter = collections.Counter( feed_url for channel_cfg in instance_config["feeds"].values() for feed_cfg in channel_cfg.values() for feed_url in ensure_list(feed_cfg["url"])) if log_details: # Log instance config logged_instance_config = instance_config.copy() del logged_instance_config["feeds"] log.info( "The excerpted configuration for %s channels with %s feeds having %s unique URLs is:\n%s", len(instance_config["feeds"]), len([ feed for channel in instance_config["feeds"].values() for feed in channel ]), len(url_counter), logged_instance_config, ) # Log channel config for channel, channel_config in instance_config["feeds"].items(): feed_names = sorted(channel_config) log.info("%s has %s feeds: %s", channel, len(feed_names), ", ".join(feed_names)) for feed, feed_config in channel_config.items(): log.debug("%s has feed %s having config: %s", channel, feed, feed_config) # Log unused channel colors unclear_colors = {"white", "black", "grey", "silver"} clear_colors = config.IRC_COLORS - unclear_colors for channel, channel_config in instance_config["feeds"].items(): if not (used_colors := { fg_color for feed_config in channel_config.values() if (fg_color := feed_config.get("style", {}).get( "name", {}).get("fg")) is not None }): log.info("%s has no foreground colors in use.", channel) continue if not (unused_colors := clear_colors - used_colors): # pylint: disable=superfluous-parens log.info("%s has all foreground colors in use.", channel) continue log.info("%s has %s unused foreground colors: %s", channel, len(unused_colors), ", ".join(sorted(unused_colors)))