def test_sx_adaptor_settings_name_not_type(caplog):
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [
{
'type': 'my-type1',
'name': 'my-fork1',
'endpoint': 'http://my-left-foot1.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
},
{
'type': 'my-type2',
'name': 'my-type1',
'endpoint': 'http://my-left-foot2.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
},
]
x = ServiceXConfigAdaptor(c)
endpoint, token = x.get_servicex_adaptor_config('my-type1')
assert endpoint == 'http://my-left-foot2.com:5000'
assert token == 'forkingshirtballs.thegoodplace.bortles'
assert len(caplog.record_tuples) == 0
async def reloadconfig_command(self, ctx):
if self._check_user_access(ctx.author, "owner"):
logger.debug("Reloading configuration...")
cc = Configuration('twitchma', __name__)
cc.set_file('./config.yaml')
self._config = cc
self._set_log_level()
def test_configured_cache_location():
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
here = Path('./servicex-dude').absolute()
c['cache_path'] = str(here)
p = get_configured_cache_path(c)
# Should default to temp directory - should work on all platforms!
assert p.exists()
assert str(p) == str(here)
def test_configured_cache_temp_location():
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['cache_path'] = '/tmp/servicex-dude'
p = get_configured_cache_path(c)
# Should default to temp directory - should work on all platforms!
assert p.exists()
assert str(p).startswith(tempfile.gettempdir())
assert 'servicex-dude' in str(p)
def run(validator: Validator, configuration: Configuration):
"""Validate config file and refresh metrics
:param validator: Validator used to verify configuration
:param configuration: Config object
"""
configuration.reload()
if not validate(validator, configuration):
logger.error('Shares field of the config file is invalid : %s',
validator.errors)
return
expose_metrics(configuration)
def test_sx_adaptor_settings_backend_name_requested_with_unlabeled_type(
caplog):
'Request None for a backend name'
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'endpoint': 'http://my-left-foot.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
}]
x = ServiceXConfigAdaptor(c)
with pytest.raises(ServiceXException) as e:
_ = x.get_servicex_adaptor_config('xaod')
assert 'Unable to find' in str(e)
def test_sx_adaptor_settings_name_worng(caplog):
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'type': 'my-type',
'name': 'my-fork',
'endpoint': 'http://my-left-foot.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
}]
x = ServiceXConfigAdaptor(c)
with pytest.raises(ServiceXException) as e:
x.get_servicex_adaptor_config('my-type')
assert 'Unable to find type my-type' in str(e)
def test_cache_expansion_username():
'''On windows this will expand one way, and on linux another. So need to be a little careful here!
'''
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['cache_path'] = '/tmp/servicex_${UserName}'
# Get the right answer, depending on the definition of USER
u_name = os.environ['USER'] if 'USER' in os.environ else os.environ[
'UserName']
path_name = f'servicex_{u_name}'
p = get_configured_cache_path(c)
# Should default to temp directory - should work on all platforms!
assert p.name == path_name
def test_sx_adaptor_settings_backend_name_unlabeled_type():
'Request None for a backend name'
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'type': 'xaod',
'endpoint': 'http://my-left-foot.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
}, {
'endpoint': 'http://my-left-foot.com:5001',
'token': 'forkingshirtballs.thegoodplace.bortles1'
}]
x = ServiceXConfigAdaptor(c)
endpoint, token = x.get_servicex_adaptor_config()
assert endpoint == 'http://my-left-foot.com:5001'
assert token == 'forkingshirtballs.thegoodplace.bortles1'
def test_sx_adaptor_settings_backend_name_requested_with_unlabeled_type(
caplog):
'Request None for a backend name'
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'endpoint': 'http://my-left-foot.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
}]
x = ServiceXConfigAdaptor(c)
endpoint, token = x.get_servicex_adaptor_config('xaod')
assert endpoint == 'http://my-left-foot.com:5000'
assert token == 'forkingshirtballs.thegoodplace.bortles'
assert caplog.record_tuples[0][2] == "No 'xaod' backend type found, " \
"using http://my-left-foot.com:5000 - please add to " \
"the configuration file (e.g. servicex.yaml)"
def test_sx_adaptor_settings_no_backend_name_requested_or_listed(caplog):
'Request None for a backend name'
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'endpoint': 'http://my-left-foot.com:5000',
'token': 'forkingshirtballs.thegoodplace.bortles'
}]
x = ServiceXConfigAdaptor(c)
endpoint, token = x.get_servicex_adaptor_config()
assert endpoint == 'http://my-left-foot.com:5000'
assert token == 'forkingshirtballs.thegoodplace.bortles'
assert caplog.record_tuples[0][2] == "No backend type requested, " \
"using http://my-left-foot.com:5000 - please be " \
"explicit " \
"in the ServiceXDataset constructor"
def app_init(app):
CORS(app)
config = Configuration(config_root, __name__)
app.config['ldap_servers'] = config['ldap_servers'].get(dict)
keys = ['ldap_mocked', 'swagger']
for key in keys:
app.logger.info(f'{key} = {config[key].get(dict)}')
app.config[key] = config[key].get(dict)
app_register_blueprints(app)
def test_sx_adaptor_settings_env():
from confuse import Configuration
c = Configuration('bogus', 'bogus')
c.clear()
c['api_endpoints'] = [{
'type': '${SXTYPE}',
'endpoint': '${ENDPOINT}:5000',
'token': '${SXTOKEN}',
}]
from os import environ
environ['ENDPOINT'] = 'http://tachi.com'
environ['SXTYPE'] = 'mcrn'
environ['SXTOKEN'] = 'protomolecule'
x = ServiceXConfigAdaptor(c)
endpoint, token = x.get_servicex_adaptor_config('mcrn')
assert endpoint == 'http://tachi.com:5000'
assert token == 'protomolecule'
async def test_configuration():
c = Configuration("distribd", __name__)
c["webhooks"].set([{
"url": "http://foo.bar.svc:1234",
"matcher": "foo:.*"
}])
wh = WebhookManager(c)
assert wh.webhooks[0]["url"] == "http://foo.bar.svc:1234"
assert wh.webhooks[0]["matcher"].search("bar:badger") is None
assert wh.webhooks[0]["matcher"].search("foo:badger") is not None
def test_exchange_receive_ancestor():
c = Configuration("distribd", __name__)
c["node"]["identifier"].set("node1")
c["raft"]["address"].set("127.0.0.1")
c["raft"]["port"].set(8080)
c["registry"]["default"]["address"].set("127.0.0.1")
c["registry"]["default"]["port"].set(9080)
s = Seeder(c, 1, None)
s.process_state_change({
"raft": {
"address": "127.0.0.1",
"port": 8080
},
"registry.default": {
"address": "127.0.0.1",
"port": 9080
},
})
result = s.exchange_gossip({
"node1": {
"raft": {
"address": "127.0.0.1",
"port": 8081
},
"registry": {
"address": "127.0.0.1",
"port": 9081
},
"generation": 25,
}
})
assert result == {
"node1": {
"raft": {
"address": "127.0.0.1",
"port": 8080
},
"registry": {
"address": "127.0.0.1",
"port": 9080
},
"generation": 1,
}
}
def load_config() -> Configuration:
template = {
"bot_token": String(),
"users_group_id": String(),
"database": {
"host": String(default="localhost"),
"port": Integer(default=5432),
"name": String(default="postgres"),
"username": String(default="postgres"),
"password": String(default=""),
"timeout": Number(default=60.0),
"wait": Number(default=30.0),
"pool": {
"minsize": Integer(default=2),
"maxsize": Integer(default=10),
"recycle": Integer(default=-1)
}
},
"pki": {
"ca": Filename(default="certs/ca.crt"),
"cert": Filename(default="certs/root.crt"),
"pkey": Filename(default="certs/root.key"),
"passphrase": String(default=None),
"tls_auth": Filename(default="certs/ta.key")
},
"server": {
"host": String(default="127.0.0.1"),
"port": Integer(default=1443)
},
"default": {
"max_devices": Integer(default=6)
}
}
config = Configuration("VpnBot", __name__)
config.set(load_secrets())
config.set_args(parse_args(), dots=True)
log.info("Configuration loaded")
return config.get(template)
def parse_args(config: confuse.Configuration) -> confuse.Configuration:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-r",
"--repository",
default=config["repository"].get(),
help="Git repository URL",
)
parser.add_argument("-b",
"--branch",
default=config["branch"].get("str"),
help="Git branch")
parser.add_argument(
"-p",
"--path",
default=config["path"].get(),
help="Path inside the git repository",
)
default_replace = config["replace_existing"].get(bool)
parser.add_argument(
"-f",
"--replace-existing",
default=default_replace,
action="store_true",
help="Replace the existing file?",
)
parser.add_argument(
"-c",
"--configuration-files",
nargs="+",
help="Configuration files to fetch",
default=config["configuration_files"].get(list),
)
update_gitignore = config["update_gitignore"].get(bool)
parser.add_argument(
"-u",
"--update-gitignore",
default=update_gitignore,
action="store_true",
help="Add configuration file to the .gitignore.",
)
default_insecure = config["insecure"].get(bool)
parser.add_argument(
"-k",
"--insecure",
default=default_insecure,
action="store_true",
help="Accept self signed certificate?",
)
parser.add_argument("--no-replace-existing",
dest="replace_existing",
action="store_false")
default_verbose = config["verbose"].get(bool)
parser.add_argument(
"-v",
"--verbose",
default=default_verbose,
action="store_true",
help="Display additional information?",
)
parser.add_argument("--no-verbose", dest="verbose", action="store_false")
args = parser.parse_args()
config.set_args(args)
return config
"""
configuration.reload()
if not validate(validator, configuration):
logger.error('Shares field of the config file is invalid : %s',
validator.errors)
return
expose_metrics(configuration)
if __name__ == "__main__":
logger.info('SuiviBourse is running !')
# Load config
config = Configuration('SuiviBourse', __name__)
# Load schema file
with open(Path(__file__).parent / "schema.yaml", encoding='UTF-8') as f:
dataSchema = yaml.safe_load(f)
shares_validator = Validator(dataSchema)
try:
# Start up the server to expose the metrics.
prometheus_client.start_http_server(
int(os.getenv('SB_METRICS_PORT', default='8081')))
# Schedule run the job on startup.
run(shares_validator, config)
# Start scheduler
scheduler = BlockingScheduler()
scheduler.add_job(run,
def __init__(self, from_config=None):
if from_config:
self._config = LocalConfig(from_config.settings)
else:
self._config = Configuration("PRESC", read=False)
self.reset_defaults()
class PrescConfig:
"""
Wrapper around a confuse Configuration object.
This is used for managing config options in PRESC, including the global config.
Attributes
----------
from_config : PrescConfig
A PrescConfig instance to override. If None, the config is initialized to the default settings.
"""
def __init__(self, from_config=None):
if from_config:
self._config = LocalConfig(from_config.settings)
else:
self._config = Configuration("PRESC", read=False)
self.reset_defaults()
def reset_defaults(self):
"""Reset all options to their defaults."""
self._config.clear()
self.update_from_file(DEFAULT_CONFIG_PATH)
def update_from_file(self, file_path):
"""Override current settings with those in the given YAML file."""
self._config.set_file(str(file_path))
def set(self, settings):
"""Update one or more config options.
These should be specified in a dict, either mirroring the nested
structure of the configuration file, or as flat key-value pairs using
dots to indicate nested namespaces.
Examples
--------
``config.set({"report": {"title": "My Report", "author": "Me"}})``
``config.set({"report.title": "My Report", "report.author": "Me"})``
"""
if not isinstance(settings, dict):
raise PrescError("Config settings must be specified in a dict")
self._config.set_args(settings, dots=True)
@property
def settings(self):
"""Access the underlying confuse object."""
return self._config
def dump(self):
"""Dump the current config in YAML format."""
return self._config.dump()
# Make option access work on the PrescConfig:
def __getitem__(self, key):
return self._config.__getitem__(key)
def get(self, template=None):
# If template is None, defer to the underlying default arg.
template_arg = {}
if template:
template_arg["template"] = template
return self._config.get(**template_arg)
def flatten(self):
return self._config.flatten()
def __init__(self, appname, modname=None, loader=yaml_util.Loader):
Configuration.__init__(self, appname, modname, loader)
async def lightbot_command(self, ctx):
logger.debug(f"command: '{ctx.message.clean_content}'")
args = ctx.message.clean_content[len(ctx.command.name):].lstrip(' ').split(' ')
for arg in args:
logger.debug(f"checking keyword: {arg}")
index = [
i for i, item in enumerate(self._config['commands']['keyword_mapping']['mappings']) if item['word'].get() == arg
]
if len(index) == 0:
logger.debug(f" - Could not locate keyword '{arg}' in mapping")
continue
elif len(index) > 1:
logger.error(
f" - You have defined more than one mapping that uses the same keyword '{arg}'. Using first found."
)
mapping = self._config['commands']['keyword_mapping']['mappings'][index[0]].get()
if self._check_user_access(ctx.message.author, mapping['access']) and \
self._check_rate_limit(ctx, mapping['timeout'], mapping['word']):
await self._telnet_send(mapping['command'], ctx)
logger.debug("######################################################")
if __name__ == "__main__":
c = Configuration('twitchma', __name__)
c.set_file('./config.yaml')
TwitchMa(c).run()
def to_dict(config: confuse.Configuration) -> dict:
"""Convert confuse Configuration object to dict."""
yaml_config = config.dump(redact=True)
dct = yaml.safe_load(StringIO(yaml_config))
return dct
def __init__(
self,
key,
config: confuse.Configuration,
log_root: Log = None,
**kwargs,
):
"""Set base variables.
Mainly the descriptions
"""
# Sets logging
super().__init__(log_root=log_root)
log = self.log
log.debug(f"{__name__=}")
self.key: str = key
self.config_cf: confuse.Configuration = config
# Override the YAML with a dictionary
# https://confuse.readthedocs.io/en/latest/
# https://www.pythoncentral.io/how-to-check-if-a-list-tuple-or-dictionary-is-empty-in-python/
if kwargs:
for k, v in kwargs.items():
if "index" in k:
k = k.split("_")[0]
ind = [
i for i in config["Model"][self.key]["index"].get()
if f"({k})" in i
][0]
args = {"Dimension": {ind: v}}
else:
args = {"Model": {self.key: {k: v}}}
config.set_args(args, dots=True)
self.data_cf: confuse.Configuration = config["Model"][key]
self.dimension_cf: confuse.Configuration = config["Dimension"]
self.index_cf: confuse.Configuration = self.data_cf["index"]
self._array: np.ndArray = None
self._df: pd.DataFrame = None
self._narrow: pd.DataFrame = None
# if there is no value it is a calculate amount and fill with NaNs
try:
self._array = np.array(self.data_cf["array"].get())
# user create exceptions must include the import module name
# https://www.geeksforgeeks.org/user-defined-exceptions-python-examples/
except confuse.NotFoundError:
log.debug(f"set null array based on {self.index_cf.get()=}")
shape = [
len(self.dimension_cf[x].get()) for x in self.index_cf.get()
]
log.debug(f"of {shape=}")
self._array = np.empty(
[len(self.dimension_cf[x].get()) for x in self.index_cf.get()])
log.debug(f"{self._array=}")
self.set_df()
def test_configured_cache_default():
from confuse import Configuration
c = Configuration('servicex', 'servicex')
assert c['cache_path'].exists()
def update_config(config: confuse.Configuration):
"""Update the config file."""
open(config.user_config_path(), 'w').write(config.dump())
from confuse import Configuration
config = Configuration("HeimsnetBot", __name__)
config.set_file("config.yaml")
def get_config(args: typing.Optional[Namespace] = None,
modname: str = PROGRAM_NAME) -> AttrDict:
"""Get the config dict, layering env and args over defaults."""
config = Configuration(PROGRAM_NAME, modname=modname, read=False)
try:
config.read()
except Exception as exc:
LOG.warning(exc)
if args and args.comicbox and args.comicbox.config:
config.set_file(args.comicbox.config)
config.set_env()
if args:
config.set_args(args)
ad = config.get(TEMPLATE)
if not isinstance(ad, AttrDict):
raise ValueError()
if ad.comicbox.paths:
ad.comicbox.paths = sorted(set(ad.comicbox.paths))
return ad.comicbox
def app_init(app):
CORS(app)
config = Configuration(config_root, __name__)
app.config['ldap_service_url'] = config['ldap_service_url'].get(str)
app.config['swagger'] = config['swagger'].get(dict)
app_register_blueprints(app)
def write_to_file(config: confuse.Configuration) -> None:
"""Write updated config to file."""
config_filename = os.path.join(config.config_dir(),
confuse.CONFIG_FILENAME)
with open(config_filename, "w", encoding="UTF-8") as f:
f.write(config.dump())
def update_config(config: confuse.Configuration):
open(config.user_config_path(), 'w').write(config.dump())