def extract_config(section: SectionProxy, config_filename: str) -> Dict[str, str]: AlertRouter.validate_config_fields_existence(section, config_filename) if "twilio" in config_filename: return { 'id': section.get('id'), 'parent_ids': [x for x in section.get('parent_ids').split(",") if x.strip()], 'info': False, 'warning': False, 'error': False, 'critical': True, } return { 'id': section.get('id'), 'parent_ids': [x for x in section.get('parent_ids').split(",") if x.strip()], 'info': section.getboolean('info'), 'warning': section.getboolean('warning'), 'error': section.getboolean('error'), 'critical': section.getboolean('critical'), }
def from_config_parser(cls, name: str, config_section: configparser.SectionProxy): """Parse a configuration from a configparser section""" # Default use_auth to True use_auth = DEFAULT_USE_AUTH try: use_auth = config_section.getboolean(CONFIG_USE_AUTH) except ValueError: pass # Get all secrets secrets = {} for config_key, config_val in config_section.items(): if config_key.startswith(CREDENTIALS_SECRET_PREFIX): # Of form "secret_GRAPHNAME" - parse GRAPHNAME graph_name = config_key.replace(CREDENTIALS_SECRET_PREFIX, "", 1) secrets[graph_name] = config_val return cls( name=name, server=config_section.get(CONFIG_SERVER_KEY, ""), client_version=config_section.get(CONFIG_CLIENT_VERSION_KEY, ""), restpp_port=config_section.get(CONFIG_RESTPP_PORT, DEFAULT_RESTPP_PORT).__str__(), gs_port=config_section.get(CONFIG_GS_PORT, DEFAULT_GS_PORT).__str__(), use_auth=use_auth, username=config_section.get(CREDENTIALS_USERNAME_KEY, ""), password=config_section.get(CREDENTIALS_PASSWORD_KEY, ""), secrets=secrets)
def __init__(self, aws_config: SectionProxy, dwh_config: SectionProxy): KEY = aws_config.get('KEY') SECRET = aws_config.get('SECRET') REGION = dwh_config.get('DWH_REGION') self.__iam = boto3.client( 'iam', region_name=REGION, aws_access_key_id=KEY, aws_secret_access_key=SECRET ) self.__redshift = boto3.client( 'redshift', region_name=REGION, aws_access_key_id=KEY, aws_secret_access_key=SECRET ) self.__ec2 = boto3.resource( 'ec2', region_name=REGION, aws_access_key_id=KEY, aws_secret_access_key=SECRET ) self.dwh_config = dwh_config self.__redshift_cluster_creation_timeout = 600 self.__redshift_cluster_creation_waiting_period = 60 self.__redshift_cluster_failed_statuses = ['failed', 'unavailable', 'deleting', 'hardware-failure'] self.__redshift_cluster_acceptable_statuses = ['available']
def build_scheduler(cfg_section: SectionProxy, optimizer): name = cfg_section['Name'] args = [optimizer] + loads(cfg_section['Args']) if cfg_section.get('Args') is not None else [] kwargs = loads(cfg_section['Kwargs']) if cfg_section.get('Kwargs') is not None else {} optim_constr = getattr(lr_scheduler, name) return optim_constr(*args, **kwargs)
def __init__(self, config: SectionProxy, method: str, query_string: bytes, response_start: Callable[..., None], response_body: Callable[..., None], response_done: Callable[..., None], error_log: Callable[[str], int] = sys.stderr.write) -> None: self.config = config # type: SectionProxy self.charset_bytes = self.config['charset'].encode('ascii') self.method = method # Request method to the UX; bytes self.response_start = response_start self.response_body = response_body self._response_done = response_done self.error_log = error_log # function to log errors to self.test_uri = None # type: str self.req_hdrs = None # type: StrHeaderListType self.format = None # type: str self.test_id = None # type: str self.check_name = None # type: str self.descend = None # type: bool self.save = None # type: bool self.timeout = None # type: Any self.referer_spam_domains = [] # type: List[str] if config.get("limit_origin_tests", ""): if self._origin_period == None: self._origin_period = config.getfloat("limit_origin_period", fallback=1) * 3600 thor.schedule(self._origin_period, self.ratelimit_cleanup) if config.get("referer_spam_domains", ""): self.referer_spam_domains = [i.strip() for i in \ config["referer_spam_domains"].split()] self.run(query_string)
def build_optimizer(cfg_section: SectionProxy, net_params): name = cfg_section['Name'] args = [net_params] + loads(cfg_section['Args']) if cfg_section.get('Args') is not None else [] kwargs = loads(cfg_section['Kwargs']) if cfg_section.get('Kwargs') is not None else {} # dynamically get optimizer constructor optim_constr = getattr(optim, name) return optim_constr(*args, **kwargs)
def __init__(self, config: SectionProxy, experiment): super().__init__( filename=config.get("name"), directory=config.get("path"), activation_level=config.getint("level"), experiment=experiment, name=config.get("name"), encrypt=config.getboolean("encrypt", fallback=False), )
def create( cls, name: str, config: configparser.SectionProxy, ) -> "LogindSessionsIdle": types = config.get("types", fallback="tty,x11,wayland").split(",") types = [t.strip() for t in types] states = config.get("states", fallback="active,online").split(",") states = [t.strip() for t in states] return cls(name, types, states)
def translate_bootstrap_servers( section: configparser.SectionProxy) -> List[str]: server_tpl = "{}" if section.get("bootstrap_domain"): server_tpl += "." + section["bootstrap_domain"] if section.get("bootstrap_port"): server_tpl += ":" + section["bootstrap_port"] return [ server_tpl.format(server.strip()) for server in section["bootstrap_hosts"].split(",") ]
def read_authorization_settings( self, authorization_section: configparser.SectionProxy): """ read settings from given sections, if no value found, no value would change :param authorization_section: the authorization in config :return: None """ self.domain = authorization_section.get('domain', self.domain) self.auth_token = authorization_section.get('auth_token', self.auth_token) self.upload_repo_id = authorization_section.get( 'upload_repo_id', self.upload_repo_id)
def _init_client(self, config: SectionProxy): ca_file = config.get("ca_file_path") if config.getboolean( "use_ssl") else None client = pymongo.MongoClient( host=config.get("host"), port=config.getint("port"), username=config.get("user"), password=config.get("password"), tls=config.getboolean("use_ssl"), tlsCAFile=ca_file, authSource=config.get("auth_source"), ) return client
def create(cls, name: str, config: configparser.SectionProxy) -> "Users": with warnings.catch_warnings(): warnings.simplefilter("ignore", FutureWarning) try: user_regex = re.compile(config.get("name", fallback=r".*")) terminal_regex = re.compile( config.get("terminal", fallback=r".*")) host_regex = re.compile(config.get("host", fallback=r".*")) return cls(name, user_regex, terminal_regex, host_regex) except re.error as error: raise ConfigurationError( "Regular expression is invalid: {}".format(error), ) from error
def __createInstanceFromSettings( base_class: Type[T], section: configparser.SectionProxy) -> Optional[T]: instance = Storage.__getInstance(base_class, section.get("type", fallback="UNSET")) if instance is None: return None if hasattr(instance, "name"): instance.name = section.get("name", fallback=instance.name) for setting in instance: if setting.type.key in section: setting.value = section[setting.type.key] return instance
def build_feature_extractor(cfg_section: SectionProxy): name = cfg_section.get('Type').lower() if name == "dlib": return DlibFeatureExtractor(cfg_section.get('ShapePredictor'), cfg_section.get('Extractor')) elif name == "senet": return Senet50FeatureExtractor(cfg_section.get("Detections"), cfg_section.get('Snapshot')) elif name == "vgg": return VggFeatureExtractor(cfg_section.getint("MaxpoolIdx"), cfg_section.getint("NoActivation")) else: raise Exception("Wrong feature extractor type.")
def __get_option_with_default_value(section: configparser.SectionProxy, option, default_value, data_type: str = ""): if section is not None and option in section: try: if data_type == "bool": temp = section.getboolean(option) elif data_type == "int": temp = section.getint(option) elif data_type == "float": temp = section.getfloat(option) else: temp = section.get(option) if option is None: log.warning( f"Setting '{option}' is present but has no value. Please refer to 'config.default' for an example config." ) return default_value else: return temp except ValueError: return default_value log.error( f"Setting '{option}' is missing from config. Please refer to 'config.default' for an example config." ) return default_value
def standalone_main(config: SectionProxy) -> None: """Run REDbot as a standalone Web server.""" # load static files static_types = { b'.js': b'text/javascript', b'.css': b'text/css', b'.png': b'image/png', } static_files = {} for root, dirs, files in os.walk(config['static_dir']): for name in files: try: path = os.path.join(root, name) uri = os.path.relpath(path, config['static_dir']) static_files[b"/static/%s" % uri.encode('utf-8')] = open(path, 'rb').read() except IOError: sys.stderr.write("* Problem loading %s\n" % path) def red_handler(x: EventEmitter) -> None: @thor.events.on(x) def request_start(method: bytes, uri: bytes, req_hdrs: RawHeaderListType) -> None: p_uri = urlsplit(uri) if p_uri.path in static_files: headers = [] file_ext = os.path.splitext(p_uri.path)[1].lower() content_encoding = static_types.get(file_ext, b'application/octet-stream') headers.append((b'Content-Encoding', content_encoding)) headers.append((b'Cache-Control', b'max-age=300')) x.response_start(b"200", b"OK", headers) x.response_body(static_files[p_uri.path]) x.response_done([]) elif p_uri.path == b"/": try: RedWebUi(config, method.decode(config['charset']), p_uri.query, x.response_start, x.response_body, x.response_done) except Exception: sys.stderr.write(""" *** FATAL ERROR REDbot has encountered a fatal error which it really, really can't recover from in standalone server mode. Details follow. """) import traceback traceback.print_exc() thor.stop() sys.exit(1) else: x.response_start(b"404", b"Not Found", []) x.response_done([]) server = thor.http.HttpServer(config.get('host', ''), int(config['port'])) server.on('exchange', red_handler) try: thor.run() except KeyboardInterrupt: sys.stderr.write("Stopping...\n") thor.stop()
def get_password(section: configparser.SectionProxy) -> str: password = section.get("password", None) password_cmd = section.get("password_command", None) if password and password_cmd: raise ConfigError( "password or password_command are mutually exclusive") elif password_cmd: process = subprocess.run(password_cmd, capture_output=True, shell=True) if process.returncode != 0: raise ConfigError( f"Password command '{password_cmd}' exited with {process.returncode}" ) password = process.stdout.decode("utf-8").rstrip() elif not password: raise ConfigError("No imap password or password_command set") return password
def _parse_option( self, option_key: str, option_value: str, template: LinterOptions, section: SectionProxy, ) -> Any: template_key = option_key.replace('-', '_') option_type: Any = option_types.get(template_key) processed_value: Any if not option_type: dv = getattr(template, template_key, None) if dv is None: if self.report_unrecognized_options: print( f'Unrecognized option: {option_key} = {option_value}', file=sys.stderr, ) return None option_type = type(dv) try: if option_type is bool: processed_value = section.getboolean(template_key) elif callable(option_type): processed_value = option_type(section.get(option_key)) else: print( f'Cannot determine what type {option_key} should have', file=sys.stderr, ) return None except (ValueError, argparse.ArgumentTypeError) as error: print(f'{option_key}: {error}', file=sys.stderr) return None return processed_value
def __init__(self, config: SectionProxy, method: str, query_string: bytes, response_start: Callable[..., None], response_body: Callable[..., None], response_done: Callable[..., None], error_log: Callable[[str], int] = sys.stderr.write) -> None: self.config = config # type: SectionProxy self.charset_bytes = self.config['charset'].encode('ascii') self.method = method # Request method to the UX; bytes self.response_start = response_start self.response_body = response_body self._response_done = response_done self.error_log = error_log # function to log errors to self.test_uri = None # type: str self.req_hdrs = None # type: StrHeaderListType self.format = None # type: str self.test_id = None # type: str self.check_name = None # type: str self.descend = None # type: bool self.save = None # type: bool self.timeout = None # type: Any self.referer_spam_domains = [] # type: List[str] if config.get("referer_spam_domains", ""): self.referer_spam_domains = [i.strip() for i in \ config["referer_spam_domains"].split()] self.run(query_string)
def read_general_settings(self, general_section: configparser.SectionProxy): """ read settings from given sections, if no value found, no value would change better be called after the reset so the variables won't be None :param general_section: the general section in config :return: None """ self.type = general_section.get('type', self.type) self.size = general_section.getint('size', self.size) self.timeout = general_section.getint('timeout', self.timeout) self.paste_format = general_section.get('paste_format', self.paste_format).replace( '<&br />', '\n') self.log_save_path = general_section.get('log_path', self.log_save_path)
def _load_config(config_section: configparser.SectionProxy): supplied_keys = set(config_section.keys()) missing_keys = _REQUIRED_KEYS - supplied_keys if missing_keys: raise _errors.StreamConfigMissingKeysError(missing_keys) unexpected_keys = supplied_keys - (_REQUIRED_KEYS | _OPTIONAL_KEYS) if unexpected_keys: raise _errors.StreamConfigUnexpectedKeysError(unexpected_keys) # Make sure to_emails and to_sms_emails are valid json to_emails = config_section[_TO_EMAILS_KEY] try: json.loads(to_emails) except json.decoder.JSONDecodeError as e: raise _errors.StreamConfigToEmailsFormatError(to_emails) from e to_emails = config_section.get(_TO_SMS_EMAILS_KEY) if to_emails: try: json.loads(to_emails) except json.decoder.JSONDecodeError as e: raise _errors.StreamConfigToSmsEmailsFormatError(to_emails) from e return config_section
def from_config_section(cls, section: SectionProxy) -> Config: """Creates a credentials tuple from the respective config section. """ host = section['host'] port = section.getint('port') passwd = section.get('passwd') return cls(host, port, passwd)
def __init__( self, contest_type: str, config_section: configparser.SectionProxy ) -> None: self.name: Optional[str] = config_section.get("name", None) self.score: int = int(config_section["points"]) if contest_type == "cms": self.in_globs: List[str] = config_section["in_globs"].split()
def loadFont(self, section: SectionProxy, name: str, fontNameDef: str = "Helvetica", fontSizeDef: int = 32, isBoldDef: bool = False, isItalicDef: bool = False) -> Tuple[str, int, bool, bool]: if not isinstance(section, SectionProxy): raise TypeError("section") fontName = section.get(name + "Name", fontNameDef) fontSize = section.getint(name + "Size", fontSizeDef) isBold = section.getboolean(name + "Bold", isBoldDef) isItalic = section.getboolean(name + "Italic", isItalicDef) return (fontName, fontSize, isBold, isItalic)
def get_graph_api_object(config_section: configparser.SectionProxy, **kwargs) -> GraphAPI: _api_class = kwargs.get('api_class') or GraphAPI auth_ca_cert = config_section.get('auth_ca_cert_path', True) graph_ca_cert = config_section.get('graph_ca_cert_path', True) proxies = kwargs.get('proxies') or saq.PROXIES try: return _api_class( config_section, verify_auth=auth_ca_cert, verify_graph=graph_ca_cert, proxies=proxies, ) except Exception as e: logging.error(f"error creating Graph API object: {e.__class__} '{e}'") raise e
def define_pass_target(section: configparser.SectionProxy, request: Mapping[str, str]) -> str: """Determine the pass target by filling in potentially used variables.""" pass_target = section.get("target").replace("${host}", request["host"]) if "username" in request: pass_target = pass_target.replace("${username}", request["username"]) return pass_target
def __init__(self, config: SectionProxy, method: str, query_string: bytes, req_headers: RawHeaderListType, response_start: Callable[..., None], response_body: Callable[..., None], response_done: Callable[..., None], error_log: Callable[[str], int] = sys.stderr.write) -> None: self.config = config # type: SectionProxy self.charset_bytes = self.config['charset'].encode('ascii') self.method = method self.req_headers = req_headers self.response_start = response_start self.response_body = response_body self._response_done = response_done self.error_log = error_log # function to log errors to self.test_uri = None # type: str self.test_id = None # type: str self.robot_time = None # type: str self.robot_hmac = None # type: str self.req_hdrs = None # type: StrHeaderListType self.format = None # type: str self.check_name = None # type: str self.descend = None # type: bool self.save = None # type: bool self.save_path = None # type: str self.timeout = None # type: Any self.referer_spam_domains = [] # type: List[str] if config.get("limit_client_tests", fallback=""): if self._client_period is None: self._client_period = config.getfloat("limit_client_period", fallback=1) * 3600 thor.schedule(self._client_period, self.client_ratelimit_cleanup) if config.get("limit_origin_tests", fallback=""): if self._origin_period is None: self._origin_period = config.getfloat("limit_origin_period", fallback=1) * 3600 thor.schedule(self._origin_period, self.origin_ratelimit_cleanup) if config.get("referer_spam_domains", fallback=""): self.referer_spam_domains = [i.strip() for i in \ config["referer_spam_domains"].split()] self.run(query_string)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler: """ Create a handler of this class based on the configuration in the config section. :param section: The configuration section :param option_handler_id: Optional extra identifier :return: A handler object :rtype: OptionHandler """ # The option handler ID is our primary link prefix responsible_for_links = [] try: prefix = IPv6Network(option_handler_id) responsible_for_links.append(prefix) except ValueError: raise configparser.ParsingError("The ID of [{}] must be the primary link prefix".format(section.name)) # Add any extra prefixes additional_prefixes = section.get("additional-prefixes", "").split(" ") for additional_prefix in additional_prefixes: if not additional_prefix: continue try: prefix = IPv6Network(additional_prefix) responsible_for_links.append(prefix) except ValueError: raise configparser.ParsingError("'{}' is not a valid IPv6 prefix".format(additional_prefix)) # Get the lifetimes address_preferred_lifetime = section.getint("address-preferred-lifetime", 3600) address_valid_lifetime = section.getint("address-valid-lifetime", 7200) prefix_preferred_lifetime = section.getint("prefix-preferred-lifetime", 43200) prefix_valid_lifetime = section.getint("prefix-valid-lifetime", 86400) sqlite_filename = section.get("assignments-file") return cls( sqlite_filename, responsible_for_links, address_preferred_lifetime, address_valid_lifetime, prefix_preferred_lifetime, prefix_valid_lifetime, )
def from_config_section(cls, section: configparser.SectionProxy): domain_names = section.get('domain-names') if domain_names is None: raise configparser.NoOptionError('domain-names', section.name) domain_names = re.split('[,\t ]+', domain_names) option = cls(search_list=domain_names) option.validate() return option
def _get_config(section: configparser.SectionProxy, platform_name: str) -> PlatformConfig: paths = parse_path_list(section.get('paths', '')) chosen_emulators = tuple( f'{chosen_emulator} ({platform_name})' if chosen_emulator in {'MAME', 'Mednafen', 'VICE'} else chosen_emulator for chosen_emulator in parse_string_list(section.get('emulators', ''))) options = {} if platform_name in platforms: option_definitions = platforms[platform_name].options for k, v in option_definitions.items(): options[k] = parse_value(section, k, v.type, v.default_value) elif platform_name in manually_specified_platforms: option_definitions = manually_specified_platforms[ platform_name].options for k, v in option_definitions.items(): options[k] = parse_value(section, k, v.type, v.default_value) return PlatformConfig(platform_name, paths, chosen_emulators, options)
def __init__(self, config: SectionProxy, **kwargs): host = config.get("host") port = config.getint("port") username = config.get("user") password = config.get("password") tls = config.getboolean("use_ssl") tlsCAFile = config.get("ca_file_path") if tls else None authSource = config.get("auth_source") super().__init__( host=host, port=port, username=username, password=password, authSource=authSource, tls=tls, tlsCAFile=tlsCAFile, **kwargs, )
def __init__(self, config: SectionProxy) -> None: self.config = config self.static_files = {} # type: Dict[bytes, bytes] # Load static files self.walk_files(config['asset_dir'], b"static/") if config.get('extra_base_dir'): self.walk_files(config['extra_base_dir']) # Set up the watchdog if os.environ.get("SYSTEMD_WATCHDOG"): thor.schedule(self.watchdog_freq, self.watchdog_ping) # Set up the server server = thor.http.HttpServer(config.get('host', ''), int(config['port'])) server.on('exchange', self.red_handler) try: thor.run() except KeyboardInterrupt: sys.stderr.write("Stopping...\n") thor.stop()
def configure(self, config: configparser.SectionProxy) -> None: """See base class method.""" super().configure(config) self._regex = self._build_matcher( config.get( "regex{suffix}".format(suffix=self._option_suffix), fallback=self._regex.pattern, ))
def get_port(section: configparser.SectionProxy, default_port: int) -> int: raw_port = section.get("port", str(default_port)) try: port = int(raw_port) if port not in range(1, 65535): raise ImapError(f"Imap port is out of range (1-65535): {port}") return port except ValueError as err: raise ImapError(f"Imap port is not a valid: {port}: {err}")
def get_epd_data( config: SectionProxy) -> tuple[Optional[ctypes.CDLL], tuple[int, int]]: if (is_supported_epd(config.get('EPD_MODEL'))): if (config.getboolean('FILE_OUTPUT')): return (None, (1872, 1404)) else: return (ctypes.CDLL("lib/epd78.so"), (1872, 1404)) else: raise Exception(f'Unsupported model: {config.get("EPD_MODEL")}')
def create(cls, name: str, config: configparser.SectionProxy) -> "Mpd": try: host = config.get("host", fallback="localhost") port = config.getint("port", fallback=6600) timeout = config.getint("timeout", fallback=5) return cls(name, host, port, timeout) except ValueError as error: raise ConfigurationError( "Host port or timeout configuration wrong: {}".format( error)) from error
def factory(conf: SectionProxy, force_installation: bool): """ Parses the configuration and returns the correct Installer :param conf: the configuration of the program to install :param force_installation: if reinstalling is enabled or not :return: an Installer instance """ if "git_repo" in conf: return GitInstaller(conf, force_installation) elif "svn_repo" in conf: return SVNInstaller(conf, force_installation) elif conf.get("system_package", False): return DependenciesInstaller.factory([conf.get("name")]) elif conf.getboolean("license", False): return LicensedSourceInstaller(conf, force_installation) elif conf.get("url", None): return DownloadableSourceInstaller(conf, force_installation) elif conf.get("source", None): return SourceInstaller(conf, force_installation) else: raise RuntimeError("There was an error while configuring the Installer to use")
def allow_ssh_temporarily(_config: SectionProxy) -> None: """ Allows ssh temporarily :param _config: the config to use """ number_of_required_chains = len(_config.getlist("ports")) for i in range(1, number_of_required_chains): self.execute("-N SSH-KNOCKING-{}".format(i)) command = "-A INPUT -m state --state NEW -m tcp -p tcp" if _config.get("interface", None): command += " -i " + _config.get("interface") command += " --dport " + _config.get("ssh_port", "22") command += " -m recent --rcheck --seconds " + _config.get("timeout", 30) command += " --name SSH{}".format(len(_config.getlist("ports")) - 1) command += \ ' -m comment --comment "Allow port {} for ssh for {} if the connecting ip is in the list SSH{}"'.format( _config.get("ssh_port", "22"), _config.get("timeout", 30), len(_config.getlist("ports")) - 1 ) self.execute(command)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler: """ Create a handler of this class based on the configuration in the config section. :param section: The configuration section :param option_handler_id: Optional extra identifier :return: A handler object :rtype: OptionHandler """ address = section.get('server-address') if address is None: raise configparser.NoOptionError('server-address', section.name) address = IPv6Address(address) return cls(address)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler: """ Create a handler of this class based on the configuration in the config section. :param section: The configuration section :param option_handler_id: Optional extra identifier :return: A handler object :rtype: OptionHandler """ min_t1 = cls.str_to_time(section.get('min-t1', '0')) max_t1 = cls.str_to_time(section.get('max-t1', 'INFINITY')) factor_t1 = cls.str_to_factor(section.get('factor-t1', '0.5')) min_t2 = cls.str_to_time(section.get('min-t2', '0')) max_t2 = cls.str_to_time(section.get('max-t2', 'INFINITY')) factor_t2 = cls.str_to_factor(section.get('factor-t2', '0.8')) return cls(min_t1, max_t1, factor_t1, min_t2, max_t2, factor_t2)