def validate(self) -> None:
"""Extended validation.
- Only 'text/plain' MIME type is alowed for `input_pipe_format`.
- Only 'charset' and 'errors' MIME parameters are alowed for `input_pipe_format`.
- Only `LOG_FORMAT` MIME type is allowed for `output_pipe_format`.
Raises:
Error: When any check fails.
"""
super().validate()
# Input Pipe format
fmt = self.input_pipe_format
if fmt.value.mime_type != MIME_TYPE_TEXT:
raise Error(
f"MIME type '{fmt.value.mime_type}' is not a valid input format"
)
# MIME parameters
for param in fmt.value.params.keys():
if param not in ('charset', 'errors'):
raise Error(f"Unknown MIME parameter '{param}' in pipe format")
# Output Pipe format
fmt = self.output_pipe_format
if fmt.value.mime_type != MIME_TYPE_PROTO:
raise Error(
f"MIME type '{fmt.value.mime_type}' is not a valid output format"
)
if fmt.value.params['type'] != LOG_PROTO:
raise Error(
f"Unsupported protobuf type '{fmt.value.params['type']}'")
def validate(self) -> None:
"""Extended validation.
- whether all required options have value other than None
- that 'input_pipe_format' MIME type is 'text/plain'
- that 'output_pipe_format' MIME type is 'application/x.fb.proto'
- that exactly one from 'func' or 'template' options have a value
- that 'func' option value could be compiled
Raises:
Error: When any check fails.
"""
super().validate()
#
if self.output_pipe_format.value.mime_type != MIME_TYPE_TEXT:
raise Error(f"Only '{MIME_TYPE_TEXT}' output format allowed.")
if self.input_pipe_format.value is None:
if self.input_pipe_mode.value is SocketMode.CONNECT:
raise Error(
"Input pipe: `format` is required for CONNECT mode.")
else:
if self.input_pipe_format.value.mime_type != MIME_TYPE_PROTO:
raise Error(f"Only '{MIME_TYPE_PROTO}' input format allowed.")
if 'type' not in self.input_pipe_format.value.params:
raise Error(
f"Missing required 'type' MIME parameter in input format.")
if not is_msg_registered(proto_class := self.input_pipe_format.
value.params.get('type')):
raise Error(f"Unknown protobuf message type '{proto_class}'")
def open(self, channel: Channel, address: ZMQAddress, agent: AgentDescriptor,
peer_uid: UUID) -> None:
"""Open connection to Firebird Butler service.
Arguments:
channel: Channel used for communication with service.
address: Service endpoint address.
agent: Client agent identification.
peer_uid: Client peer ID.
"""
assert isinstance(channel.protocol, FBSPClient)
self.channel = channel
self.session = channel.connect(address)
self.protocol = channel.protocol
self.protocol.send_hello(channel, self.session, agent,
PeerDescriptor(peer_uid, os.getpid(), platform.node()),
self.token.next())
msg = self.channel.receive(self.timeout)
if isinstance(msg, ErrorMessage):
raise self.protocol.exception_for(msg)
elif msg is TIMEOUT:
raise TimeoutError()
elif msg is INVALID:
raise Error("Invalid response from service")
elif not isinstance(msg, WelcomeMessage):
raise Error(f"Unexpected {msg.msg_type.name} message from service")
def handle_accept_client(self, session: FBDPSession) -> None:
"""Event handler executed when client connects to the data pipe via OPEN message.
Arguments:
session: Session associated with client.
The session attributes `data_pipe`, `pipe_socket`, `data_format` and `params`
contain information sent by client, and the event handler validates the request.
If request should be rejected, it raises the `StopError` exception with `code`
attribute containing the `ErrorCode` to be returned in CLOSE message.
Important:
Base implementation validates pipe identification and pipe socket, and converts
data format from string to MIME (in session).
The descendant class that overrides this method must call super() as first
action.
"""
if session.pipe != self.pipe:
raise Error(f"Unknown data pipe '{session.data_pipe}'",
code=ErrorCode.PIPE_ENDPOINT_UNAVAILABLE)
# We're CONSUMER server, so clients can only attach to our INPUT
elif session.socket is not PipeSocket.INPUT:
raise Error(f"'{session.socket}' socket not available",
code=ErrorCode.PIPE_ENDPOINT_UNAVAILABLE)
# We work with MIME formats, so we'll convert the format specification to MIME
session.data_format = MIME(session.data_format)
def validate(self) -> None:
"""Extended validation.
- Exactly one filter definition must be provided.
- Only 'text/plain' MIME types are alowed for input and output `pipe_format`.
- Only 'charset' and 'errors' MIME parameters are alowed for input and output
`pipe_format`.
- Regex is valid.
"""
super().validate()
# Pipe format
for fmt in (self.input_pipe_format, self.output_pipe_format):
if fmt.value.mime_type != MIME_TYPE_TEXT:
raise Error(
f"MIME type '{fmt.value.mime_type}' is not a valid input format"
)
# MIME parameters
for param in fmt.value.params.keys():
if param not in ('charset', 'errors'):
raise Error(
f"Unknown MIME parameter '{param}' in pipe format")
# Filters
defined = 0
for opt in (self.regex, self.expr, self.func):
if opt.value is not None:
defined += 1
if defined != 1:
raise Error(
"Configuration must contain exactly one filter definition.")
# regex
if self.regex.value is not None:
re.compile(self.regex.value)
def validate(self) -> None:
"""Extended validation.
- `pipe_format` is required for CONNECT `pipe_mode`.
"""
super().validate()
if self.input_pipe_mode.value is SocketMode.CONNECT and self.input_pipe_format.value is None:
raise Error("'input_pipe_format' required for CONNECT pipe mode.")
if self.output_pipe_mode.value is SocketMode.CONNECT and self.output_pipe_format.value is None:
raise Error("'output_pipe_format' required for CONNECT pipe mode.")
def validate(self) -> None:
"""Extended validation.
- Only 'text/plain' MIME type is alowed for `pipe_format`.
- Only 'charset' and 'errors' MIME parameters are alowed for `pipe_format`.
"""
super().validate()
# Pipe format
if self.pipe_format.value.mime_type != MIME_TYPE_TEXT:
raise Error("Only 'text/plain' pipe format supported")
for param in self.pipe_format.value.params.keys():
if param not in ('charset', 'errors'):
raise Error(
f"Unsupported MIME parameter '{param}' in pipe format")
def validate(self) -> None:
"""Extended validation.
- Only FileOpenMode.WRITE is allowed for stdout and stderr.
- FileOpenMode.READ is not supported.
"""
super().validate()
if (self.filename.value.lower() in ['stdout', 'stderr']
and self.file_mode.value != FileOpenMode.WRITE):
raise Error("STD[OUT|ERR] support only WRITE open mode")
if self.file_mode.value not in (FileOpenMode.APPEND,
FileOpenMode.CREATE,
FileOpenMode.RENAME,
FileOpenMode.WRITE):
raise Error(
f"File open mode '{self.file_mode.value.name}' not supported")
def validate(self) -> None:
"""Extended validation."""
super().validate()
if (self.filename.value.lower() in ['stdin', 'stdout', 'stderr']
and self.file_mode.value
not in [FileOpenMode.WRITE, FileOpenMode.READ]):
raise Error("STD[IN|OUT|ERR] support only READ and WRITE modes")
def configure(self, *, section: str = SECTION_BUNDLE) -> None:
"""
Arguments:
section: Configuration section with bundle definition.
"""
svc_cfg: ServiceExecConfig = ServiceExecConfig(section)
bundle_cfg: ServiceBundleConfig = ServiceBundleConfig(section)
bundle_cfg.load_config(self.config)
bundle_cfg.validate()
# Assign Peer IDs to service sections (instances)
peer_uids = {
section.name: uuid.uuid1()
for section in bundle_cfg.agents.value
}
self.config[SECTION_PEER_UID].update(
(k, v.hex) for k, v in peer_uids.items())
#
#
for svc_cfg in bundle_cfg.agents.value:
svc_cfg.validate()
if svc_cfg.agent.value in self.available_services:
controller = ThreadController(
self.available_services[svc_cfg.agent.value],
name=svc_cfg.name,
peer_uid=peer_uids[svc_cfg.name],
manager=self.mngr)
self.services.append(controller)
else:
self.services.clear()
raise Error(f"Unknonw agent in section '{svc_cfg.name}'")
def aquire_resources(self) -> None:
"""Aquire resources required by component (open files, connect to other services etc.).
Must raise an exception when resource aquisition fails.
"""
get_logger(self).info("Aquiring resources...")
if self._fail_on is FailOn.RESOURCE_AQUISITION:
raise Error("Service configured to fail")
def receive(self) -> FBSPMessage:
"""Receive one message from service.
Raises:
TimeoutError: When timeout expires.
Error: When `Channel.receive()` returns INVALID sentinel, or service closes
connection with CLOSE message.
"""
msg = self.channel.receive(self.timeout)
if isinstance(msg, ErrorMessage):
raise self.protocol.exception_for(msg)
elif msg is TIMEOUT:
raise TimeoutError()
elif msg is INVALID:
raise Error("Invalid response from service")
elif msg.msg_type is MsgType.CLOSE:
raise Error("Connection closed by service")
return msg
def validate(self) -> None:
"""Extended validation.
- `block_size` is positive or -1.
"""
super().validate()
# Blocik size
if (self.block_size.value < -1) or (self.block_size.value == 0):
raise Error("'block_size' must be positive or -1")
def get_service_desciptors(uid: str = None) -> List[ServiceDescriptor]:
"""Returns list of service descriptors for registered services.
"""
result = []
for e in chain.from_iterable([i(uid) for i in _iterators]):
try:
result.append(e.load())
except Exception as exc:
raise Error(f"Descriptor loading failed: {str(e)}") from exc
return result
def start_activities(self) -> None:
"""Start normal component activities.
Must raise an exception when start fails.
"""
get_logger(self).info("Starting activities...")
if self._fail_on is FailOn.ACTIVITY_START:
raise Error("Service configured to fail")
if self._schedule:
for delay in self._schedule:
self.schedule(partial(self.action, delay), delay)
def initialize(self, config: DummyConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
get_logger(self).info("Initialization...")
self._fail_on: FailOn = config.fail_on.value
get_logger(self).info("{fail_on=}", fail_on=self._fail_on)
self._schedule: List[int] = config.schedule.value
get_logger(self).info("{schedule=}", schedule=self._schedule)
if self._fail_on is FailOn.INIT:
raise Error("Service configured to fail")
def terminate(self) -> None:
"""Terminate the service.
Terminate should be called ONLY when call to stop() (with sensible timeout) fails.
Does nothing when service is not running.
Raises:
Error: When service termination fails.
"""
if self.is_running():
tid = ctypes.c_long(self.runtime.ident)
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
tid, ctypes.py_object(SystemExit))
if res == 0:
raise Error(
"Service termination failed due to invalid thread ID.")
if res != 1:
# if it returns a number greater than one, you're in trouble,
# and you should call it again with exc=NULL to revert the effect
ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
raise Error(
"Service termination failed due to PyThreadState_SetAsyncExc failure"
)
def validate(self) -> None:
"""Extended validation.
- that 'input_format' MIME type is 'application/x.fb.proto'
- that 'output_format' MIME type is 'application/x.fb.proto;type=saturnin.protobuf.common.GenDataRecord'
- that 'aggregate' values have format '<aggregate_func>:<field_spec>', and
<aggregate_func> is from supported functions
"""
super().validate()
#
if self.input_pipe_format.value.mime_type != MIME_TYPE_PROTO:
raise Error(f"Only '{MIME_TYPE_PROTO}' input format allowed.")
if self.output_pipe_format.value != AGGREGATE_FORMAT:
raise Error(f"Only '{AGGREGATE_FORMAT}' output format allowed.")
#
for spec in self.aggregate.value:
l = spec.split(':')
if len(l) != 2:
raise Error("The 'aggregate' values must have '<aggregate_func>:<field_spec>' format")
func_name = l[0].lower()
if ' as ' in func_name:
func_name, _ = func_name.split(' as ')
if func_name not in AGGREGATE_FUNCTIONS:
raise Error(f"Unknown aggregate function '{func_name}'")
def run(self, args: Namespace) -> None:
"""Command execution.
Arguments:
args: Collected argument values.
"""
paths: List[str] = []
if self.pth.is_file():
paths.extend(Path(x) for x in self.pth.read_text().splitlines())
if args.add is not None:
path: Path = Path(args.add).absolute()
if path in paths:
raise Error(f"Path '{path}' already linked to site-packages")
self.out(f"Linking '{path}' to site-packages ... ")
paths.append(path)
self.pth.write_text(''.join(f'{str(x)}\n' for x in paths))
self.out("OK\n")
elif args.remove is not None:
path: Path = Path(args.remove).absolute()
if path not in paths:
raise Error(f"Path '{path}' is not linked to site-packages")
self.out(f"Unlinking '{path}' from site-packages ... ")
paths.remove(path)
self.pth.write_text(''.join(f'{str(x)}\n' for x in paths))
self.out("OK\n")
# Remove registration of services from removed directory
write_svc = False
while (i := find(directory=str(path))) >= 0:
self.out(f"Removing registration of {path} ... ")
site.svc_registry.pop(i)
write_svc = True
self.out('OK\n')
if write_svc:
self.out("Saving service registry ... ")
site.save_svc_registry()
self.out("OK\n")
def validate(self) -> None:
"""Extended validation.
- Only 'text/plain' MIME type is alowed for `file_format`.
- Only 'text/plain' and 'application/x.fb.proto' MIME types are alowed for `pipe_format`.
- Only 'charset' and 'errors' MIME parameters are alowed for `file_format` and
`pipe_format` specifications of type 'plain/text'.
- Only FileOpenMode.WRITE is allowed for stdout and stderr.
- FileOpenMode.READ is not supported.
"""
super().validate()
if (self.filename.value.lower() in ['stdout', 'stderr']
and self.file_mode.value != FileOpenMode.WRITE):
raise Error("STD[OUT|ERR] support only WRITE open mode")
if self.file_mode.value not in (FileOpenMode.APPEND,
FileOpenMode.CREATE,
FileOpenMode.RENAME,
FileOpenMode.WRITE):
raise Error(
f"File open mode '{self.file_mode.value.name}' not supported")
# File format
if self.file_format.value.mime_type != MIME_TYPE_TEXT:
raise Error(f"Only '{MIME_TYPE_TEXT}' file format supported")
for param in self.file_format.value.params.keys():
if param not in ('charset', 'errors'):
raise Error(f"Unknown MIME parameter '{param}' in file format")
# Pipe format
if self.pipe_format.value.mime_type not in SUPPORTED_MIME:
raise Error(
f"MIME type '{self.pipe_format.value.mime_type}' is not a valid input format"
)
if self.pipe_format.value.mime_type == MIME_TYPE_TEXT:
for param in self.pipe_format.value.params.keys():
if param not in ('charset', 'errors'):
raise Error(
f"Unknown MIME parameter '{param}' in pipe format")
elif self.pipe_format.value.mime_type == MIME_TYPE_PROTO:
for param in self.pipe_format.value.params.keys():
if param != 'type':
raise Error(
f"Unknown MIME parameter '{param}' in pipe format")
proto_class = self.pipe_format.value.params.get('type')
if not is_msg_registered(proto_class):
raise Error(f"Unknown protobuf message type '{proto_class}'")
class ProtoPrinterConfig(DataFilterConfig):
"""Data printer microservice configuration.
"""
def __init__(self, name: str):
super().__init__(name)
#
self.output_pipe_format.default = MIME('text/plain;charset=utf-8')
self.output_pipe_format.set_value(MIME('text/plain;charset=utf-8'))
#
self.template: StrOption = \
StrOption('template', "Text formatting template")
self.func: PyCallableOption = \
PyCallableOption('func',
"Function that returns text representation of data",
'def f(data: Any, utils: TransformationUtilities) -> str:\n ...\n')
def validate(self) -> None:
"""Extended validation.
- whether all required options have value other than None
- that 'input_pipe_format' MIME type is 'text/plain'
- that 'output_pipe_format' MIME type is 'application/x.fb.proto'
- that exactly one from 'func' or 'template' options have a value
- that 'func' option value could be compiled
Raises:
Error: When any check fails.
"""
super().validate()
#
if self.output_pipe_format.value.mime_type != MIME_TYPE_TEXT:
raise Error(f"Only '{MIME_TYPE_TEXT}' output format allowed.")
if self.input_pipe_format.value is None:
if self.input_pipe_mode.value is SocketMode.CONNECT:
raise Error(
"Input pipe: `format` is required for CONNECT mode.")
else:
if self.input_pipe_format.value.mime_type != MIME_TYPE_PROTO:
raise Error(f"Only '{MIME_TYPE_PROTO}' input format allowed.")
if 'type' not in self.input_pipe_format.value.params:
raise Error(
f"Missing required 'type' MIME parameter in input format.")
if not is_msg_registered(proto_class := self.input_pipe_format.
value.params.get('type')):
raise Error(f"Unknown protobuf message type '{proto_class}'")
def register_command(self, cmd: Command) -> None:
"""Registers command.
Arguments:
cmd: Command to be registered.
Raises:
Error: If command is already registered.
"""
if cmd.name in self.commands:
raise Error(f"Command {cmd.name} already registered")
self.commands[cmd.name] = cmd
cmd_parser = self.subparsers.add_parser(cmd.name,
description=cmd.description,
help=cmd.description)
cmd.set_arguments(self, cmd_parser)
cmd.out = self.out
cmd_parser.set_defaults(runner=cmd.run)
cmd_parser.set_defaults(err_handler=cmd.on_error)
def validate(self) -> None:
"""Extended validation.
- Input/output format is required and must be MIME_TYPE_PROTO
- Input and output MIME 'type' params are present and the same
- At least one from '*_func' / '*_expr' options have a value
- Only one from include / exclude methods is defined
- '*_func' option value could be compiled
"""
super().validate()
#
for fmt in [self.output_pipe_format, self.input_pipe_format]:
if fmt.value is not None:
if fmt.value.mime_type != MIME_TYPE_PROTO:
raise Error(f"Only '{MIME_TYPE_PROTO}' format allowed for '{fmt.name}' option.")
if not fmt.value.params.get('type'):
raise Error(f"The 'type' parameter not found in '{fmt.name}' option.")
#
if self.input_pipe_format.value is not None \
and self.output_pipe_format.value is not None \
and self.output_pipe_format.value.params.get('type') != self.input_pipe_format.value.params.get('type'):
raise Error(f"The 'type' parameter value must be the same for both MIME format options.")
#
defined = 0
for opt in [self.include_func, self.exclude_func, self.include_expr, self.exclude_expr]:
if opt.value is not None:
defined += 1
if defined == 0:
raise Error("At least one filter specification option must have a value")
#
for func, expr in [(self.include_func, self.include_expr),
(self.exclude_func, self.exclude_expr)]:
if expr.value and func.value:
raise Error(f"Options '{expr.name}' and '{func.name}' are mutually exclusive")
#
for func in [self.include_func, self.exclude_func]:
try:
func.value
except Exception as exc:
raise Error(f"Invalid code definition in '{func.name}' option") from exc
def release_resources(self) -> None:
"""Release resources aquired by component (close files, disconnect from other services etc.)
"""
get_logger(self).info("Releasing resources...")
if self._fail_on is FailOn.RESOURCE_RELEASE:
raise Error("Service configured to fail")
def stop_activities(self) -> None:
"""Stop component activities.
"""
get_logger(self).info("Stopping activities...")
if self._fail_on is FailOn.ACTIVITY_STOP:
raise Error("Service configured to fail")
if self.input_pipe_format.value.mime_type != MIME_TYPE_PROTO:
raise Error(f"Only '{MIME_TYPE_PROTO}' input format allowed.")
if 'type' not in self.input_pipe_format.value.params:
raise Error(
f"Missing required 'type' MIME parameter in input format.")
if not is_msg_registered(proto_class := self.input_pipe_format.
value.params.get('type')):
raise Error(f"Unknown protobuf message type '{proto_class}'")
#
defined = 0
for opt in (self.template, self.func):
if opt.value is not None:
defined += 1
if defined != 1:
raise Error(
"Configuration must contain either 'template' or 'func' option"
)
#
try:
self.func.value
except Exception as exc:
raise Error("Invalid code definition in 'func' option") from exc
# Service description
SERVICE_AGENT: AgentDescriptor = \
AgentDescriptor(uid=SERVICE_UID,
name="saturnin.proto.printer",
version=SERVICE_VERSION,
vendor_uid=VENDOR_UID,
def install_package(self, args: List, zfile: zipfile.ZipFile,
path: zipfile.Path):
work_dir = site.scheme.tmp / 'import'
if work_dir.exists():
self.remove_dir(work_dir)
work_dir.mkdir()
#
root_files = [PKG_TOML]
# pyproject.toml
pkg_file: zipfile.Path = path / PKG_TOML
if not pkg_file.exists():
raise Error(f"Invalid package: File {_file} not found")
toml_data = pkg_file.read_text()
pkg_toml: PyProjectTOML = PyProjectTOML(toml.loads(toml_data))
self.out(f"Found: {pkg_toml.original_name}-{pkg_toml.version}\n")
#
add_component = False
cmp: Dict = self.find_component(uid=pkg_toml.uid)
if cmp is None:
cmp = {}
top_level = self.get_new_toplevel()
cmp[CMP_UID] = pkg_toml.uid
cmp[CMP_NAME] = pkg_toml.original_name
cmp[CMP_PACKAGE] = pkg_toml.name
cmp[CMP_VERSION] = pkg_toml.version
cmp[CMP_DESCRIPTION] = pkg_toml.description
cmp[CMP_DESCRIPTOR] = f'{top_level}.{pkg_toml.descriptor}'
cmp[CMP_TOPLEVEL] = top_level
add_component = True
else:
top_level = cmp[CMP_TOPLEVEL]
vn = version.parse(pkg_toml.version)
vo = version.parse(cmp[CMP_VERSION])
if vn < vo:
self.out(
f"Newer version {cmp[CMP_VERSION]} already installed\n")
return
elif vn == vo:
self.out(f"Version {cmp[CMP_VERSION]} already installed\n")
return
cmp[CMP_NAME] = pkg_toml.original_name
cmp[CMP_PACKAGE] = pkg_toml.name
cmp[CMP_VERSION] = pkg_toml.version
cmp[CMP_DESCRIPTION] = pkg_toml.description
cmp[CMP_DESCRIPTOR] = f'{top_level}.{pkg_toml.descriptor}'
# write toml
target: Path = work_dir / PKG_TOML
target.write_text(toml_data)
# setup.cfg
cfg: ConfigParser = ConfigParser(interpolation=ExtendedInterpolation())
pkg_file = path / SETUP_CFG
if pkg_file.exists():
root_files.append(SETUP_CFG)
cfg.read_string(pkg_file.read_text())
pkg_toml.make_setup_cfg(cfg)
if not cfg.has_section(SEC_OPTIONS):
cfg.add_section(SEC_OPTIONS)
cfg[SEC_OPTIONS]['packages'] = top_level
if not cfg.has_section(SEC_ENTRYPOINTS):
cfg.add_section(SEC_ENTRYPOINTS)
if pkg_toml.component_type == 'service':
cfg[SEC_ENTRYPOINTS][
'saturnin.service'] = f'\n{cmp[CMP_UID]} = {cmp[CMP_DESCRIPTOR]}'
elif pkg_toml.component_type == 'application':
cfg[SEC_ENTRYPOINTS][
'saturnin.application'] = f'\n{cmp[CMP_UID]} = {cmp[CMP_DESCRIPTOR]}'
target = work_dir / SETUP_CFG
with target.open('w') as f:
cfg.write(f)
# README
if pkg_toml.readme is not None:
readme_file = None
if isinstance(pkg_toml.readme, str):
readme_file = pkg_toml.readme
elif 'file' in pkg_toml.readme:
readme_file = pkg_toml.readme['file']
if readme_file is not None:
root_files.append(readme_file)
target = work_dir / readme_file
pkg_file = path / readme_file
target.write_bytes(pkg_file.read_bytes())
# LICENSE
if pkg_toml.license is not None:
if (license_file := pkg_toml.license.get('file')) is not None:
root_files.append(license_file)
target = work_dir / license_file
pkg_file = path / license_file
target.write_bytes(pkg_file.read_bytes())
