def test_logging_of_os_errors(monkeypatch, tmpdir, caplog):
"""Filesystem watcher can fail due to limits, and it should be logged."""
def raiser(self):
raise OSError('inotify watch limit reached')
monkeypatch.setattr(
Observer,
name='start',
value=raiser,
)
dir_watcher = DirectoryWatcher(
directory=tmpdir,
on_modified=lambda x: x,
)
caplog.clear()
dir_watcher.start()
assert 'inotify watch limit reached' in caplog.record_tuples[0][2]
dir_watcher.stop()
class ModuleManager:
"""
A manager for operating on a set of modules.
:param config: Global configuration options.
:param modules: Dictionary containing globally defined modules.
:param context: Global context.
:param directory: Directory containing global configuration.
:param dry_run: If file system actions should be printed and skipped.
"""
def __init__(
self,
config: AstralityYAMLConfigDict = {},
modules: Dict[str, ModuleConfigDict] = {},
context: Context = Context(),
directory: Path = Path(__file__).parent / 'tests' / 'test_config',
dry_run: bool = False,
) -> None:
"""Initialize a ModuleManager object from `astrality.yml` dict."""
self.config_directory = directory
self.application_config = config
self.application_context = context
self.dry_run = dry_run
self.startup_done = False
self.last_module_events: Dict[str, str] = {}
# Get module configurations which are externally defined
self.global_modules_config = GlobalModulesConfig(
config=config.get('modules', {}),
config_directory=self.config_directory,
)
self.reprocess_modified_files = \
self.global_modules_config.reprocess_modified_files
self.modules: Dict[str, Module] = {}
# Insert externally managed modules
for external_module_source \
in self.global_modules_config.external_module_sources:
# Insert context defined in external configuration
module_context = external_module_source.context(
context=self.application_context,
)
self.application_context.reverse_update(module_context)
module_configs = external_module_source.modules(
context=self.application_context,
)
module_directory = external_module_source.directory
for module_name, module_config in module_configs.items():
if module_name \
not in self.global_modules_config.enabled_modules:
continue
if not Module.valid_module(
name=module_name,
config=module_config,
requires_timeout=self.global_modules_config.
requires_timeout, # noqa
requires_working_directory=module_directory,
):
continue
module = Module(
name=module_name,
module_config=module_config,
module_directory=module_directory,
replacer=self.interpolate_string,
context_store=self.application_context,
global_modules_config=self.global_modules_config,
dry_run=dry_run,
)
self.modules[module.name] = module
# Insert modules defined in `astrality.yml`
for module_name, module_config in modules.items():
# Check if this module should be included
if module_name not in self.global_modules_config.enabled_modules:
continue
if not Module.valid_module(
name=module_name,
config=module_config,
requires_timeout=self.global_modules_config.requires_timeout,
requires_working_directory=self.config_directory,
):
continue
module = Module(
name=module_name,
module_config=module_config,
module_directory=self.config_directory,
replacer=self.interpolate_string,
context_store=self.application_context,
global_modules_config=self.global_modules_config,
dry_run=dry_run,
)
self.modules[module.name] = module
# Remove modules which depends on other missing modules
Requirement.pop_missing_module_dependencies(self.modules)
# Initialize the config directory watcher, but don't start it yet
self.directory_watcher = DirectoryWatcher(
directory=self.config_directory,
on_modified=self.file_system_modified,
)
logger.info('Enabled modules: ' + ', '.join(self.modules.keys()))
def module_events(self) -> Dict[str, str]:
"""Return dict containing the event of all modules."""
module_events = {}
for module_name, module in self.modules.items():
module_events[module_name] = module.event_listener.event()
return module_events
def finish_tasks(self) -> None:
"""
Finish all due tasks defined by the managed modules.
The order of finishing tasks is as follows:
1) Import any relevant context sections.
2) Compile all templates with the new section.
3) Run startup commands, if it is not already done.
4) Run on_event commands, if it is not already done for this
module events combination.
"""
if not self.startup_done:
# Save the last event configuration, such that on_event
# is only run when the event *changes*
self.last_module_events = self.module_events()
# Perform setup actions not yet executed
self.setup()
# Perform all startup actions
self.startup()
elif self.last_module_events != self.module_events():
# One or more module events have changed, execute the event blocks
# of these modules.
for module_name, event in self.module_events().items():
if not self.last_module_events[module_name] == event:
logger.info(
f'[module/{module_name}] New event "{event}". '
'Executing actions.',
)
self.execute(
action='all',
block='on_event',
module=self.modules[module_name],
)
self.last_module_events[module_name] = event
def has_unfinished_tasks(self) -> bool:
"""Return True if there are any module tasks due."""
if not self.startup_done:
return True
else:
return self.last_module_events != self.module_events()
def time_until_next_event(self) -> timedelta:
"""Time left until first event change of any of the modules managed."""
try:
return min(
module.event_listener.time_until_next_event()
for module
in self.modules.values()
)
except ValueError:
return timedelta.max
def execute(
self,
action: str,
block: str,
module: Optional[Module] = None,
) -> None:
"""
Execute action(s) specified in managed modules.
The module actions are executed according to their specified priority.
First import context, then symlink, and so on...
:param action: Action to be perfomed. If given 'all', then all actions
will be performed.
:param block: Action block to be executed, for example 'on_exit'.
:module: Specific module to be executed. If not provided, then all
managed modules will be executed.
"""
assert block in ('on_setup', 'on_startup', 'on_event', 'on_exit')
modules: Iterable[Module]
if isinstance(module, Module):
modules = (module, )
else:
modules = self.modules.values()
if action == 'all':
all_actions = filter(
lambda x: x != 'trigger',
ActionBlock.action_types.keys(),
)
else:
all_actions = (action,) # type: ignore
for specific_action in all_actions:
for module in modules:
module.execute(
action=specific_action,
block=block,
dry_run=self.dry_run,
)
def setup(self) -> None:
"""
Run setup actions specified by the managed modules, not yet executed.
"""
self.execute(action='all', block='on_setup')
def startup(self):
"""
Run all startup actions specified by the managed modules.
Also starts the directory watcher in $ASTRALITY_CONFIG_HOME.
"""
assert not self.startup_done
self.execute(action='all', block='on_startup')
self.directory_watcher.start()
self.startup_done = True
def exit(self):
"""
Run all exit tasks specified by the managed modules.
Also close all temporary file handlers created by the modules.
"""
self.execute(action='all', block='on_exit')
# Stop watching config directory for file changes
self.directory_watcher.stop()
def on_modified(self, modified: Path) -> bool:
"""
Perform actions when a watched file is modified.
:return: Returns True if on_modified block was triggered.
"""
assert modified.is_absolute()
triggered = False
for module in self.modules.values():
if modified not in module.action_blocks['on_modified']:
continue
triggered = True
logger.info(
f'[module/{module.name}] on_modified:{modified} triggered.',
)
module.execute(
action='all',
block='on_modified',
path=modified,
dry_run=self.dry_run,
)
return triggered
def file_system_modified(self, modified: Path) -> None:
"""
Perform actions for when files within the config directory are modified.
Run any context imports, compilations, and shell commands specified
within the on_modified event block of each module.
Also, if hot_reload is True, we reinstantiate the ModuleManager object
if the application configuration has been modified.
"""
config_files = (
self.config_directory / 'astrality.yml',
self.config_directory / 'modules.yml',
self.config_directory / 'context.yml',
)
if modified in config_files:
logger.info(
f'$ASTRALITY_CONFIG_HOME/{modified.name} has been modified!',
)
self.on_application_config_modified()
return
else:
# Run any relevant on_modified blocks.
triggered = self.on_modified(modified)
if not triggered:
# Check if the modified path is a template which is supposed to
# be recompiled.
self.recompile_modified_template(modified=modified)
def on_application_config_modified(self):
"""
Reload the ModuleManager if astrality.yml has been modified.
Reloadnig the module manager only occurs if the user has configured
`hot_reload_config`.
"""
if not self.application_config.get(
'astrality',
{},
).get(
'hot_reload_config',
False,
):
# Hot reloading is not enabled, so we return early
logger.info('"hot_reload" disabled.')
return
# Hot reloading is enabled, get the new configuration dict
logger.info('Reloading $ASTRALITY_CONFIG_HOME...')
(
new_application_config,
new_modules,
new_context,
directory,
) = user_configuration(
config_directory=self.config_directory,
)
try:
# Reinstantiate this object
new_module_manager = ModuleManager(
config=new_application_config,
modules=new_modules,
context=new_context,
directory=directory,
)
# Run all old exit actions, since the new config is valid
self.exit()
# Swap place with the new configuration
self = new_module_manager
# Run startup commands from the new configuration
self.finish_tasks()
except Exception:
# New configuration is invalid, just keep the old one
# TODO: Test this behaviour
logger.error('New configuration detected, but it is invalid!')
pass
def recompile_modified_template(self, modified: Path):
"""
Recompile any modified template if configured.
This requires setting the global setting:
reprocess_modified_files: true
"""
if not self.reprocess_modified_files:
return
# Run any compile action a new if that compile action uses the modifed
# path as a template.
for module in self.modules.values():
for action_block in module.all_action_blocks():
for compile_action in action_block._compile_actions:
if modified in compile_action:
compile_action.execute(dry_run=self.dry_run)
for stow_action in action_block._stow_actions:
if modified in stow_action:
stow_action.execute(dry_run=self.dry_run)
# TODO: Test this branch
for copy_action in action_block._copy_actions:
if modified in copy_action:
copy_action.execute(dry_run=self.dry_run)
def interpolate_string(self, string: str) -> str:
"""
Process configuration string before using it.
This function is passed as a reference to all modules, making them
perform the replacement instead. For now, the ModuleManager does not
change the string at all, but we will leave it here in case we would
want to interpolate strings from the ModuleManager level instead of
the Module level.
:param string: String to be processed.
:return: Processed string.
"""
return string
@property
def keep_running(self) -> bool:
"""Return True if ModuleManager needs to keep running."""
if self.reprocess_modified_files:
return True
if any(module.keep_running for module in self.modules.values()):
return True
current_process = psutil.Process()
children = current_process.children(recursive=False)
return bool(children)
def __len__(self) -> int:
"""Return the number of managed modules."""
return len(self.modules)
def __del__(self) -> None:
"""Close filesystem watcher if enabled."""
if hasattr(self, 'directory_watcher'):
self.directory_watcher.stop()
class ModuleManager:
"""A manager for operating on a set of modules."""
def __init__(self, config: ApplicationConfig) -> None:
"""Initialize a ModuleManager object from `astrality.yml` dict."""
self.config_directory = Path(config['_runtime']['config_directory'])
self.temp_directory = Path(config['_runtime']['temp_directory'])
self.application_config = config
self.application_context: Dict[str, Resolver] = {}
self.startup_done = False
self.last_module_events: Dict[str, str] = {}
# Get module configurations which are externally defined
self.global_modules_config = GlobalModulesConfig( # type: ignore
config=config.get('config/modules', {}),
config_directory=self.config_directory,
)
self.recompile_modified_templates = \
self.global_modules_config.recompile_modified_templates
self.modules: Dict[str, Module] = {}
# Application context is used in compiling external config sources
application_context = context(config)
# Insert externally managed modules
for external_module_source \
in self.global_modules_config.external_module_sources:
module_directory = external_module_source.directory
module_configs = external_module_source.config(
context=application_context, )
# Insert context defined in external configuration
self.application_context.update(context(module_configs))
for section, options in module_configs.items():
module_config = {section: options}
if not Module.valid_class_section(
section=module_config,
requires_timeout=self.global_modules_config.
requires_timeout, # noqa
requires_working_directory=module_directory,
) or section not in self.global_modules_config.enabled_modules:
continue
module = Module(
module_config=module_config,
module_directory=module_directory,
replacer=self.interpolate_string,
context_store=self.application_context,
)
self.modules[module.name] = module
# Update the context from `astrality.yml`, overwriting any defined
# contexts in external modules in the case of naming conflicts
self.application_context.update(application_context)
# Insert modules defined in `astrality.yml`
for section, options in config.items():
module_config = {section: options}
# Check if this module should be included
if not Module.valid_class_section(
section=module_config,
requires_timeout=self.global_modules_config.
requires_timeout,
requires_working_directory=self.config_directory,
) or section not in self.global_modules_config.enabled_modules:
continue
module = Module(
module_config=module_config,
module_directory=self.config_directory,
replacer=self.interpolate_string,
context_store=self.application_context,
)
self.modules[module.name] = module
# Initialize the config directory watcher, but don't start it yet
self.directory_watcher = DirectoryWatcher(
directory=self.config_directory,
on_modified=self.file_system_modified,
)
logger.info('Enabled modules: ' + ', '.join(self.modules.keys()))
def __len__(self) -> int:
"""Return the number of managed modules."""
return len(self.modules)
def module_events(self) -> Dict[str, str]:
"""Return dict containing the event of all modules."""
module_events = {}
for module_name, module in self.modules.items():
module_events[module_name] = module.event_listener.event()
return module_events
def finish_tasks(self) -> None:
"""
Finish all due tasks defined by the managed modules.
The order of finishing tasks is as follows:
1) Import any relevant context sections.
2) Compile all templates with the new section.
3) Run startup commands, if it is not already done.
4) Run on_event commands, if it is not already done for this
module events combination.
"""
if not self.startup_done:
# Save the last event configuration, such that on_event
# is only run when the event *changes*
self.last_module_events = self.module_events()
# Perform all startup actions
self.startup()
elif self.last_module_events != self.module_events():
# One or more module events have changed, execute the event blocks
# of these modules.
for module_name, event in self.module_events().items():
if not self.last_module_events[module_name] == event:
# This module has a new event
self.import_context_sections(
trigger='on_event',
module=self.modules[module_name],
)
self.compile_templates(
trigger='on_event',
module=self.modules[module_name],
)
self.run_on_event_commands(
module=self.modules[module_name], )
# Save the event
self.last_module_events[module_name] = event
def has_unfinished_tasks(self) -> bool:
"""Return True if there are any module tasks due."""
if not self.startup_done:
return True
else:
return self.last_module_events != self.module_events()
def time_until_next_event(self) -> timedelta:
"""Time left until first event change of any of the modules managed."""
return min(module.event_listener.time_until_next_event()
for module in self.modules.values())
def import_context_sections(
self,
trigger: str,
module: Optional[Module] = None,
) -> None:
"""
Import context sections defined by the managed modules.
Trigger is one of 'on_startup', 'on_event', or 'on_exit'.
This determines which event block of the module is used to get the
context import specification from.
"""
assert trigger in (
'on_startup',
'on_event',
'on_exit',
)
modules: Iterable[Module]
if isinstance(module, Module):
modules = (module, )
else:
modules = self.modules.values()
for module in modules:
module.import_context(block_name=trigger)
def compile_templates(
self,
trigger: str,
module: Optional[Module] = None,
) -> None:
"""
Compile the module templates specified by the `templates` option.
Trigger is one of 'on_startup', 'on_event', or 'on_exit'.
This determines which section of the module is used to get the compile
specification from.
"""
assert trigger in (
'on_startup',
'on_event',
'on_exit',
)
modules: Iterable[Module]
if isinstance(module, Module):
modules = (module, )
else:
modules = self.modules.values()
for module in modules:
module.compile(block_name=trigger)
def startup(self):
"""Run all startup actions specified by the managed modules."""
assert not self.startup_done
self.import_context_sections('on_startup')
self.compile_templates('on_startup')
for module in self.modules.values():
logger.info(f'[module/{module.name}] Running startup commands.')
module.run(
block_name='on_startup',
default_timeout=self.global_modules_config.run_timeout,
)
self.startup_done = True
# Start watching config directory for file changes
self.directory_watcher.start()
def run_on_event_commands(
self,
module: Module,
):
"""Run all event change commands specified by a managed module."""
logger.info(f'[module/{module.name}] Running event commands.')
module.run(
block_name='on_event',
default_timeout=self.global_modules_config.run_timeout,
)
def exit(self):
"""
Run all exit tasks specified by the managed modules.
Also close all temporary file handlers created by the modules.
"""
# First import context and compile templates
self.import_context_sections('on_exit')
self.compile_templates('on_exit')
# Then run all shell commands
for module in self.modules.values():
logger.info(f'[module/{module.name}] Running exit commands.')
module.run(
block_name='on_exit',
default_timeout=self.global_modules_config.run_timeout,
)
if hasattr(self, 'temp_files'):
for temp_file in self.temp_files:
temp_file.close()
# Prevent files from being closed again
del self.temp_files
# Stop watching config directory for file changes
self.directory_watcher.stop()
def on_modified(self, modified: Path) -> bool:
"""
Perform actions when a watched file is modified.
:return: Returns True if on_modified block was triggered.
"""
assert modified.is_absolute()
triggered = False
for module in self.modules.values():
if modified not in module.action_blocks['on_modified']:
continue
triggered = True
logger.info(
f'[module/{module.name}] on_modified:{modified} triggered.', )
# First import context sections in on_modified block
module.import_context(block_name='on_modified', path=modified)
# Now compile templates specified in on_modified block
module.compile(block_name='on_modified', path=modified)
# Lastly, run commands specified in on_modified block
logger.info(f'[module/{module.name}] Running modified commands.')
module.run(
'on_modified',
path=modified,
default_timeout=self.global_modules_config.run_timeout,
)
return triggered
def file_system_modified(self, modified: Path) -> None:
"""
Perform actions for when files within the config directory are modified.
Run any context imports, compilations, and shell commands specified
within the on_modified event block of each module.
Also, if hot_reload is True, we reinstantiate the ModuleManager object
if the application configuration has been modified.
"""
config_file = \
self.application_config['_runtime']['config_directory'] \
/ 'astrality.yml'
if modified == config_file:
self.on_application_config_modified()
return
else:
# Run any relevant on_modified blocks.
triggered = self.on_modified(modified)
if not triggered:
# Check if the modified path is a template which is supposed to
# be recompiled.
self.recompile_modified_template(modified=modified)
def on_application_config_modified(self):
"""
Reload the ModuleManager if astrality.yml has been modified.
Reloadnig the module manager only occurs if the user has configured
`hot_reload_config`.
"""
if not self.application_config['config/astrality']['hot_reload_config']:
# Hot reloading is not enabled, so we return early
return
# Hot reloading is enabled, get the new configuration dict
new_application_config = user_configuration(
config_directory=self.config_directory, )
try:
# Reinstantiate this object
new_module_manager = ModuleManager(new_application_config)
# Run all old exit actions, since the new config is valid
self.exit()
# Swap place with the new configuration
self = new_module_manager
# Run startup commands from the new configuration
self.finish_tasks()
except Exception:
# New configuration is invalid, just keep the old one
# TODO: Test this behaviour
logger.error('New configuration detected, but it is invalid!')
pass
def recompile_modified_template(self, modified: Path):
"""
Recompile any modified template if configured.
This requires setting the global setting:
recompile_modified_templates: true
"""
if not self.recompile_modified_templates:
return
# Run any compile action a new if that compile action uses the modifed
# path as a template.
for module in self.modules.values():
for action_block in module.all_action_blocks():
for compile_action in action_block._compile_actions:
if modified in compile_action:
compile_action.execute()
def interpolate_string(self, string: str) -> str:
"""
Process configuration string before using it.
This function is passed as a reference to all modules, making them
perform the replacement instead. For now, the ModuleManager does not
change the string at all, but we will leave it here in case we would
want to interpolate strings from the ModuleManager level instead of
the Module level.
:param string: String to be processed.
:return: Processed string.
"""
return string