def test_from_environment_dict_required(monkeypatch):
"""Raise an error where we require the environment to provide a value."""
with pytest.raises(OSError):
EXPECTED_CONFIG = {'HOME': None, 'LANGUAGE': 'en_US'}
monkeypatch.delenv("HOME", raising=False)
monkeypatch.setenv("LANGUAGE", "ja_JP")
from_environment(EXPECTED_CONFIG)
def main() -> None:
"""Configure a monitoring component from the environment and set it running."""
config = from_environment(EXPECTED_CONFIG)
# configure structured logging for the application
structured_formatter = StructuredFormatter(component_type='Monitoring',
ndjson=True)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setFormatter(structured_formatter)
root_logger = logging.getLogger(None)
root_logger.setLevel(logging.NOTSET)
root_logger.addHandler(stream_handler)
logger = logging.getLogger("lta.monitoring")
monitors = []
loop = asyncio.get_event_loop()
for name in MONITOR_NAMES:
if check_bool(cast(str, config['ENABLE_' + name])):
logger.info(f"Setting up monitor {name}")
kwargs = {
n.split('_', 1)[-1].lower(): config[n]
for n in config if n.startswith(name)
}
kwargs.update({
'lta_rest_url': config['LTA_REST_URL'],
'lta_rest_token': config['LTA_REST_TOKEN'],
})
m = MONITOR_NAMES[name](**kwargs) # type: ignore[arg-type]
monitors.append(m)
loop.create_task(m.run())
logger.info("Starting asyncio loop")
loop.run_forever()
def test_from_environment_list(monkeypatch):
"""Return a dictionary with a list of environment variables."""
monkeypatch.setenv("HOME", "/home/tux")
monkeypatch.setenv("LANGUAGE", "ja_JP")
obj = from_environment(["HOME", "LANGUAGE"])
assert len(obj.keys()) == 2
assert obj["HOME"] == "/home/tux"
assert obj["LANGUAGE"] == "ja_JP"
def test_from_environment_dict(monkeypatch):
"""Return a dictionary where we override one default but leave the other."""
EXPECTED_CONFIG = {'HOME': '/home/tux', 'LANGUAGE': 'en_US'}
monkeypatch.delenv("HOME", raising=False)
monkeypatch.setenv("LANGUAGE", "ja_JP")
obj = from_environment(EXPECTED_CONFIG)
assert len(obj.keys()) == 2
assert obj["HOME"] == "/home/tux"
assert obj["LANGUAGE"] == "ja_JP"
def __init__(self, duration: Optional[int] = None):
"""Create a SiteGlobusProxy object."""
# load what we can from the environment
self.cfg = from_environment(PROXY_CONFIG)
# remove anything optional that wasn't specified
cfg_keys = list(self.cfg.keys())
for key in cfg_keys:
if self.cfg[key] == EMPTY_STRING_SENTINEL_VALUE:
del self.cfg[key]
# ensure duration is converted to an integer value
if "GLOBUS_PROXY_DURATION" in self.cfg:
self.cfg["GLOBUS_PROXY_DURATION"] = int(
self.cfg["GLOBUS_PROXY_DURATION"])
# ensure we have at least an empty string for passphrase
if "GLOBUS_PROXY_PASSPHRASE" not in self.cfg:
self.cfg["GLOBUS_PROXY_PASSPHRASE"] = ""
# override the duration if specified during construction
if duration:
self.cfg['GLOBUS_PROXY_DURATION'] = duration
def runner() -> None:
"""Configure a Deleter component from the environment and set it running."""
# obtain our configuration from the environment
config = from_environment(EXPECTED_CONFIG)
# configure structured logging for the application
structured_formatter = StructuredFormatter(
component_type='Deleter',
component_name=config["COMPONENT_NAME"], # type: ignore[arg-type]
ndjson=True)
stream_handler = logging.StreamHandler(sys.stdout)
stream_handler.setFormatter(structured_formatter)
root_logger = logging.getLogger(None)
root_logger.setLevel(logging.NOTSET)
root_logger.addHandler(stream_handler)
logger = logging.getLogger("lta.deleter")
# create our Deleter service
deleter = Deleter(config, logger) # type: ignore[arg-type]
# let's get to work
deleter.logger.info("Adding tasks to asyncio loop")
loop = asyncio.get_event_loop()
loop.create_task(status_loop(deleter))
loop.create_task(work_loop(deleter))
def test_from_environment_none():
"""Fail with a TypeError if we don't ask for any environment variables."""
with pytest.raises(TypeError):
from_environment(None)
def test_from_environment_key(monkeypatch):
"""Return a dictionary with a single environment variable."""
monkeypatch.setenv("LANGUAGE", "ja_JP")
obj = from_environment("LANGUAGE")
assert len(obj.keys()) == 1
assert obj["LANGUAGE"] == "ja_JP"
def test_from_environment_empty():
"""Return an empty dictionary if we ask for no environment variables."""
obj = from_environment([])
assert len(obj.keys()) == 0
def test_from_environment_missing_list(monkeypatch):
"""Fail with an OSError if we ask for an environment variable that does not exist on a list that we provide."""
with pytest.raises(OSError):
monkeypatch.delenv("PAN_GALACTIC_GARGLE_BLASTER", raising=False)
from_environment(["PAN_GALACTIC_GARGLE_BLASTER"])
def test_from_environment_scalar_list():
"""Fail with a TypeError if we don't ask for any environment variables."""
with pytest.raises(TypeError):
from_environment([10, 20, 30, 40, 50])
import logging
from typing import cast
try:
from typing import TypedDict
except ImportError:
from typing_extensions import TypedDict
from wipac_dev_tools import from_environment
from wipac_dev_tools.enviro_tools import KeySpec
class _TypedConfig(TypedDict):
OTEL_EXPORTER_OTLP_ENDPOINT: str
WIPACTEL_EXPORT_STDOUT: bool
WIPACTEL_LOGGING_LEVEL: str
WIPACTEL_SERVICE_NAME_PREFIX: str
defaults: _TypedConfig = {
"OTEL_EXPORTER_OTLP_ENDPOINT": "",
"WIPACTEL_EXPORT_STDOUT": False,
"WIPACTEL_LOGGING_LEVEL": "WARNING",
"WIPACTEL_SERVICE_NAME_PREFIX": "",
}
CONFIG = cast(_TypedConfig, from_environment(cast(KeySpec, defaults)))
LOGGER = logging.getLogger("wipac-telemetry")
LOGGER.setLevel(CONFIG["WIPACTEL_LOGGING_LEVEL"])
def start(debug: bool = False) -> RestServer:
"""Start a LTA DB service."""
config = from_environment(EXPECTED_CONFIG)
# logger = logging.getLogger('lta.rest')
for name in config:
if name not in LOGGING_DENY_LIST:
logging.info(f"{name} = {config[name]}")
else:
logging.info(f"{name} = REDACTED")
for name in ["OTEL_EXPORTER_OTLP_ENDPOINT", "WIPACTEL_EXPORT_STDOUT"]:
if name in os.environ:
logging.info(f"{name} = {os.environ[name]}")
else:
logging.info(f"{name} = NOT SPECIFIED")
args = RestHandlerSetup({ # type: ignore
'auth': {
'secret': config['LTA_AUTH_SECRET'],
'issuer': config['LTA_AUTH_ISSUER'],
'algorithm': config['LTA_AUTH_ALGORITHM'],
},
'debug': debug
})
args['check_claims'] = CheckClaims(int(config['LTA_MAX_CLAIM_AGE_HOURS']))
# configure access to MongoDB as a backing store
mongo_user = quote_plus(cast(str, config["LTA_MONGODB_AUTH_USER"]))
mongo_pass = quote_plus(cast(str, config["LTA_MONGODB_AUTH_PASS"]))
mongo_host = config["LTA_MONGODB_HOST"]
mongo_port = int(config["LTA_MONGODB_PORT"])
mongo_db = cast(str, config["LTA_MONGODB_DATABASE_NAME"])
lta_mongodb_url = f"mongodb://{mongo_host}:{mongo_port}/{mongo_db}"
if mongo_user and mongo_pass:
lta_mongodb_url = f"mongodb://{mongo_user}:{mongo_pass}@{mongo_host}:{mongo_port}/{mongo_db}"
ensure_mongo_indexes(lta_mongodb_url, mongo_db)
motor_client = MotorClient(lta_mongodb_url)
args['db'] = motor_client[mongo_db]
# See: https://github.com/WIPACrepo/rest-tools/issues/2
max_body_size = int(config["LTA_MAX_BODY_SIZE"])
server = RestServer(debug=debug, max_body_size=max_body_size) # type: ignore[no-untyped-call]
server.add_route(r'/', MainHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles', BundlesHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles/actions/bulk_create', BundlesActionsBulkCreateHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles/actions/bulk_delete', BundlesActionsBulkDeleteHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles/actions/bulk_update', BundlesActionsBulkUpdateHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles/actions/pop', BundlesActionsPopHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Bundles/(?P<bundle_id>\w+)', BundlesSingleHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Metadata', MetadataHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Metadata/actions/bulk_create', MetadataActionsBulkCreateHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Metadata/actions/bulk_delete', MetadataActionsBulkDeleteHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/Metadata/(?P<metadata_id>\w+)', MetadataSingleHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/TransferRequests', TransferRequestsHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/TransferRequests/(?P<request_id>\w+)', TransferRequestSingleHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/TransferRequests/actions/pop', TransferRequestActionsPopHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/status', StatusHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/status/nersc', StatusNerscHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/status/(?P<component>\w+)', StatusComponentHandler, args) # type: ignore[no-untyped-call]
server.add_route(r'/status/(?P<component>\w+)/count', StatusComponentCountHandler, args) # type: ignore[no-untyped-call]
server.startup(address=config['LTA_REST_HOST'],
port=int(config['LTA_REST_PORT'])) # type: ignore[no-untyped-call]
return server
