def test_from_environment_dict_required(monkeypatch): """Raise an error where we require the environment to provide a value.""" with pytest.raises(OSError): EXPECTED_CONFIG = {'HOME': None, 'LANGUAGE': 'en_US'} monkeypatch.delenv("HOME", raising=False) monkeypatch.setenv("LANGUAGE", "ja_JP") from_environment(EXPECTED_CONFIG)
def main() -> None: """Configure a monitoring component from the environment and set it running.""" config = from_environment(EXPECTED_CONFIG) # configure structured logging for the application structured_formatter = StructuredFormatter(component_type='Monitoring', ndjson=True) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setFormatter(structured_formatter) root_logger = logging.getLogger(None) root_logger.setLevel(logging.NOTSET) root_logger.addHandler(stream_handler) logger = logging.getLogger("lta.monitoring") monitors = [] loop = asyncio.get_event_loop() for name in MONITOR_NAMES: if check_bool(cast(str, config['ENABLE_' + name])): logger.info(f"Setting up monitor {name}") kwargs = { n.split('_', 1)[-1].lower(): config[n] for n in config if n.startswith(name) } kwargs.update({ 'lta_rest_url': config['LTA_REST_URL'], 'lta_rest_token': config['LTA_REST_TOKEN'], }) m = MONITOR_NAMES[name](**kwargs) # type: ignore[arg-type] monitors.append(m) loop.create_task(m.run()) logger.info("Starting asyncio loop") loop.run_forever()
def test_from_environment_list(monkeypatch): """Return a dictionary with a list of environment variables.""" monkeypatch.setenv("HOME", "/home/tux") monkeypatch.setenv("LANGUAGE", "ja_JP") obj = from_environment(["HOME", "LANGUAGE"]) assert len(obj.keys()) == 2 assert obj["HOME"] == "/home/tux" assert obj["LANGUAGE"] == "ja_JP"
def test_from_environment_dict(monkeypatch): """Return a dictionary where we override one default but leave the other.""" EXPECTED_CONFIG = {'HOME': '/home/tux', 'LANGUAGE': 'en_US'} monkeypatch.delenv("HOME", raising=False) monkeypatch.setenv("LANGUAGE", "ja_JP") obj = from_environment(EXPECTED_CONFIG) assert len(obj.keys()) == 2 assert obj["HOME"] == "/home/tux" assert obj["LANGUAGE"] == "ja_JP"
def __init__(self, duration: Optional[int] = None): """Create a SiteGlobusProxy object.""" # load what we can from the environment self.cfg = from_environment(PROXY_CONFIG) # remove anything optional that wasn't specified cfg_keys = list(self.cfg.keys()) for key in cfg_keys: if self.cfg[key] == EMPTY_STRING_SENTINEL_VALUE: del self.cfg[key] # ensure duration is converted to an integer value if "GLOBUS_PROXY_DURATION" in self.cfg: self.cfg["GLOBUS_PROXY_DURATION"] = int( self.cfg["GLOBUS_PROXY_DURATION"]) # ensure we have at least an empty string for passphrase if "GLOBUS_PROXY_PASSPHRASE" not in self.cfg: self.cfg["GLOBUS_PROXY_PASSPHRASE"] = "" # override the duration if specified during construction if duration: self.cfg['GLOBUS_PROXY_DURATION'] = duration
def runner() -> None: """Configure a Deleter component from the environment and set it running.""" # obtain our configuration from the environment config = from_environment(EXPECTED_CONFIG) # configure structured logging for the application structured_formatter = StructuredFormatter( component_type='Deleter', component_name=config["COMPONENT_NAME"], # type: ignore[arg-type] ndjson=True) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setFormatter(structured_formatter) root_logger = logging.getLogger(None) root_logger.setLevel(logging.NOTSET) root_logger.addHandler(stream_handler) logger = logging.getLogger("lta.deleter") # create our Deleter service deleter = Deleter(config, logger) # type: ignore[arg-type] # let's get to work deleter.logger.info("Adding tasks to asyncio loop") loop = asyncio.get_event_loop() loop.create_task(status_loop(deleter)) loop.create_task(work_loop(deleter))
def test_from_environment_none(): """Fail with a TypeError if we don't ask for any environment variables.""" with pytest.raises(TypeError): from_environment(None)
def test_from_environment_key(monkeypatch): """Return a dictionary with a single environment variable.""" monkeypatch.setenv("LANGUAGE", "ja_JP") obj = from_environment("LANGUAGE") assert len(obj.keys()) == 1 assert obj["LANGUAGE"] == "ja_JP"
def test_from_environment_empty(): """Return an empty dictionary if we ask for no environment variables.""" obj = from_environment([]) assert len(obj.keys()) == 0
def test_from_environment_missing_list(monkeypatch): """Fail with an OSError if we ask for an environment variable that does not exist on a list that we provide.""" with pytest.raises(OSError): monkeypatch.delenv("PAN_GALACTIC_GARGLE_BLASTER", raising=False) from_environment(["PAN_GALACTIC_GARGLE_BLASTER"])
def test_from_environment_scalar_list(): """Fail with a TypeError if we don't ask for any environment variables.""" with pytest.raises(TypeError): from_environment([10, 20, 30, 40, 50])
import logging from typing import cast try: from typing import TypedDict except ImportError: from typing_extensions import TypedDict from wipac_dev_tools import from_environment from wipac_dev_tools.enviro_tools import KeySpec class _TypedConfig(TypedDict): OTEL_EXPORTER_OTLP_ENDPOINT: str WIPACTEL_EXPORT_STDOUT: bool WIPACTEL_LOGGING_LEVEL: str WIPACTEL_SERVICE_NAME_PREFIX: str defaults: _TypedConfig = { "OTEL_EXPORTER_OTLP_ENDPOINT": "", "WIPACTEL_EXPORT_STDOUT": False, "WIPACTEL_LOGGING_LEVEL": "WARNING", "WIPACTEL_SERVICE_NAME_PREFIX": "", } CONFIG = cast(_TypedConfig, from_environment(cast(KeySpec, defaults))) LOGGER = logging.getLogger("wipac-telemetry") LOGGER.setLevel(CONFIG["WIPACTEL_LOGGING_LEVEL"])
def start(debug: bool = False) -> RestServer: """Start a LTA DB service.""" config = from_environment(EXPECTED_CONFIG) # logger = logging.getLogger('lta.rest') for name in config: if name not in LOGGING_DENY_LIST: logging.info(f"{name} = {config[name]}") else: logging.info(f"{name} = REDACTED") for name in ["OTEL_EXPORTER_OTLP_ENDPOINT", "WIPACTEL_EXPORT_STDOUT"]: if name in os.environ: logging.info(f"{name} = {os.environ[name]}") else: logging.info(f"{name} = NOT SPECIFIED") args = RestHandlerSetup({ # type: ignore 'auth': { 'secret': config['LTA_AUTH_SECRET'], 'issuer': config['LTA_AUTH_ISSUER'], 'algorithm': config['LTA_AUTH_ALGORITHM'], }, 'debug': debug }) args['check_claims'] = CheckClaims(int(config['LTA_MAX_CLAIM_AGE_HOURS'])) # configure access to MongoDB as a backing store mongo_user = quote_plus(cast(str, config["LTA_MONGODB_AUTH_USER"])) mongo_pass = quote_plus(cast(str, config["LTA_MONGODB_AUTH_PASS"])) mongo_host = config["LTA_MONGODB_HOST"] mongo_port = int(config["LTA_MONGODB_PORT"]) mongo_db = cast(str, config["LTA_MONGODB_DATABASE_NAME"]) lta_mongodb_url = f"mongodb://{mongo_host}:{mongo_port}/{mongo_db}" if mongo_user and mongo_pass: lta_mongodb_url = f"mongodb://{mongo_user}:{mongo_pass}@{mongo_host}:{mongo_port}/{mongo_db}" ensure_mongo_indexes(lta_mongodb_url, mongo_db) motor_client = MotorClient(lta_mongodb_url) args['db'] = motor_client[mongo_db] # See: https://github.com/WIPACrepo/rest-tools/issues/2 max_body_size = int(config["LTA_MAX_BODY_SIZE"]) server = RestServer(debug=debug, max_body_size=max_body_size) # type: ignore[no-untyped-call] server.add_route(r'/', MainHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles', BundlesHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles/actions/bulk_create', BundlesActionsBulkCreateHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles/actions/bulk_delete', BundlesActionsBulkDeleteHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles/actions/bulk_update', BundlesActionsBulkUpdateHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles/actions/pop', BundlesActionsPopHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Bundles/(?P<bundle_id>\w+)', BundlesSingleHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Metadata', MetadataHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Metadata/actions/bulk_create', MetadataActionsBulkCreateHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Metadata/actions/bulk_delete', MetadataActionsBulkDeleteHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/Metadata/(?P<metadata_id>\w+)', MetadataSingleHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/TransferRequests', TransferRequestsHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/TransferRequests/(?P<request_id>\w+)', TransferRequestSingleHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/TransferRequests/actions/pop', TransferRequestActionsPopHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/status', StatusHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/status/nersc', StatusNerscHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/status/(?P<component>\w+)', StatusComponentHandler, args) # type: ignore[no-untyped-call] server.add_route(r'/status/(?P<component>\w+)/count', StatusComponentCountHandler, args) # type: ignore[no-untyped-call] server.startup(address=config['LTA_REST_HOST'], port=int(config['LTA_REST_PORT'])) # type: ignore[no-untyped-call] return server