def setUp(self): self.project = mock() self.fist_delegatee = mock() self.second_delegatee = mock() self.loader = DispatchingPluginLoader(mock, self.fist_delegatee, self.second_delegatee)
def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: self.plugin_loader = DispatchingPluginLoader( self.logger, BuiltinPluginLoader(self.logger), DownloadingPluginLoader(self.logger)) else: self.plugin_loader = plugin_loader self._plugins = [] self._pending_plugin_installs = [] self._plugins_imported = set() self._deferred_plugins = ModuleTraversalTree() self._deferred_import = False self.project = None self.project_module = None self._tools = [] python_env_registry = self._python_env_registry = PythonEnvRegistry( self) system_pyenv = PythonEnv(sys.exec_prefix, self).populate() python_env_registry["system"] = system_pyenv self._sys_path_original = list(sys.path)
def setUp(self): self.project = Mock() self.first_delegatee = Mock() self.second_delegatee = Mock() self.loader = DispatchingPluginLoader(Mock(), self.first_delegatee, self.second_delegatee)
class DispatchingPluginLoaderTest(unittest.TestCase): def setUp(self): self.project = mock() self.first_delegatee = mock() self.second_delegatee = mock() self.loader = DispatchingPluginLoader(mock, self.first_delegatee, self.second_delegatee) def test_should_raise_exception_when_all_delegatees_raise_exception(self): when(self.first_delegatee).load_plugin( self.project, "spam", None, None).thenRaise(MissingPluginException("spam")) when(self.second_delegatee).load_plugin( self.project, "spam", None, None).thenRaise(MissingPluginException("spam")) self.assertRaises(MissingPluginException, self.loader.load_plugin, self.project, "spam") verify(self.first_delegatee).load_plugin(self.project, "spam", None, None) verify(self.second_delegatee).load_plugin(self.project, "spam", None, None) def test_should_return_module_returned_by_second_loader_when_first_delegatee_raises_exception( self): result = "result" when(self.first_delegatee).load_plugin( self.project, "spam", None, None).thenRaise(MissingPluginException("spam")) when(self.second_delegatee).load_plugin(self.project, "spam", None, None).thenReturn(result) self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) verify(self.first_delegatee).load_plugin(self.project, "spam", None, None) verify(self.second_delegatee).load_plugin(self.project, "spam", None, None) def test_ensure_second_delegatee_will_not_try_when_first_delegatee_loads_plugin( self): result = "result" when(self.first_delegatee).load_plugin(self.project, "spam", None, None).thenReturn(result) self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) verify(self.first_delegatee).load_plugin(self.project, "spam", None, None) verify(self.second_delegatee, never).load_plugin(self.project, "spam", None, None)
def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: self.plugin_loader = DispatchingPluginLoader(self.logger, BuiltinPluginLoader(self.logger), DownloadingPluginLoader(self.logger)) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None
class DispatchingPluginLoaderTest(unittest.TestCase): def setUp(self): self.project = Mock() self.first_delegatee = Mock() self.second_delegatee = Mock() self.loader = DispatchingPluginLoader(Mock(), self.first_delegatee, self.second_delegatee) def test_should_raise_exception_when_all_delegatees_raise_exception(self): self.first_delegatee.load_plugin.side_effect = MissingPluginException( "spam") self.second_delegatee.load_plugin.side_effect = MissingPluginException( "spam") self.assertRaises(MissingPluginException, self.loader.load_plugin, self.project, "spam") self.first_delegatee.load_plugin.assert_called_with( self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_called_with( self.project, "spam", None, None) def test_should_return_module_returned_by_second_loader_when_first_delegatee_raises_exception( self): result = "result" self.first_delegatee.load_plugin.side_effect = MissingPluginException( "spam") self.second_delegatee.load_plugin.return_value = result self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) self.first_delegatee.load_plugin.assert_called_with( self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_called_with( self.project, "spam", None, None) def test_ensure_second_delegatee_will_not_try_when_first_delegatee_loads_plugin( self): result = "result" self.first_delegatee.load_plugin.return_value = result self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) self.first_delegatee.load_plugin.assert_called_with( self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_not_called()
def setUp(self): self.project = mock() self.fist_delegatee = mock() self.second_delegatee = mock() self.loader = DispatchingPluginLoader( mock, self.fist_delegatee, self.second_delegatee)
def setUp(self): self.project = Mock() self.first_delegatee = Mock() self.second_delegatee = Mock() self.loader = DispatchingPluginLoader( Mock(), self.first_delegatee, self.second_delegatee)
def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: builtin_plugin_loader = BuiltinPluginLoader(self.logger) installed_thirdparty_plugin_loader = ThirdPartyPluginLoader( self.logger) downloading_thirdparty_plugin_loader = DownloadingPluginLoader( self.logger) self.plugin_loader = DispatchingPluginLoader( self.logger, builtin_plugin_loader, installed_thirdparty_plugin_loader, downloading_thirdparty_plugin_loader) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None
class DispatchingPluginLoaderTest (unittest.TestCase): def setUp(self): self.project = mock() self.fist_delegatee = mock() self.second_delegatee = mock() self.loader = DispatchingPluginLoader( mock, self.fist_delegatee, self.second_delegatee) def test_should_raise_exception_when_all_delgatees_raise_exception(self): when(self.fist_delegatee).load_plugin( self.project, "spam").thenRaise(MissingPluginException("spam")) when(self.second_delegatee).load_plugin( self.project, "spam").thenRaise(MissingPluginException("spam")) self.assertRaises( MissingPluginException, self.loader.load_plugin, self.project, "spam") verify(self.fist_delegatee).load_plugin(self.project, "spam") verify(self.second_delegatee).load_plugin(self.project, "spam") def test_should_return_module_returned_by_second_loader_when_first_delgatee_raises_exception(self): result = "result" when(self.fist_delegatee).load_plugin( self.project, "spam").thenRaise(MissingPluginException("spam")) when(self.second_delegatee).load_plugin( self.project, "spam").thenReturn(result) self.assertEquals( result, self.loader.load_plugin(self.project, "spam")) verify(self.fist_delegatee).load_plugin(self.project, "spam") verify(self.second_delegatee).load_plugin(self.project, "spam") def test_ensure_second_delegatee_is_not_trie_when_first_delegatee_loads_plugin(self): result = "result" when(self.fist_delegatee).load_plugin( self.project, "spam").thenReturn(result) self.assertEquals( result, self.loader.load_plugin(self.project, "spam")) verify(self.fist_delegatee).load_plugin(self.project, "spam") verify(self.second_delegatee, never).load_plugin(self.project, "spam")
def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: builtin_plugin_loader = BuiltinPluginLoader(self.logger) thirdparty_plugin_loader = ThirdPartyPluginLoader(self.logger) self.plugin_loader = DispatchingPluginLoader( self.logger, builtin_plugin_loader, thirdparty_plugin_loader) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None
class DispatchingPluginLoaderTest(unittest.TestCase): def setUp(self): self.project = Mock() self.first_delegatee = Mock() self.second_delegatee = Mock() self.loader = DispatchingPluginLoader( Mock(), self.first_delegatee, self.second_delegatee) def test_should_raise_exception_when_all_delegatees_raise_exception(self): self.first_delegatee.load_plugin.side_effect = MissingPluginException("spam") self.second_delegatee.load_plugin.side_effect = MissingPluginException("spam") self.assertRaises( MissingPluginException, self.loader.load_plugin, self.project, "spam") self.first_delegatee.load_plugin.assert_called_with(self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_called_with(self.project, "spam", None, None) def test_should_return_module_returned_by_second_loader_when_first_delegatee_raises_exception(self): result = "result" self.first_delegatee.load_plugin.side_effect = MissingPluginException("spam") self.second_delegatee.load_plugin.return_value = result self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) self.first_delegatee.load_plugin.assert_called_with(self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_called_with(self.project, "spam", None, None) def test_ensure_second_delegatee_will_not_try_when_first_delegatee_loads_plugin(self): result = "result" self.first_delegatee.load_plugin.return_value = result self.assertEquals(result, self.loader.load_plugin(self.project, "spam")) self.first_delegatee.load_plugin.assert_called_with(self.project, "spam", None, None) self.second_delegatee.load_plugin.assert_not_called()
class Reactor(object): _current_instance = None @staticmethod def current_instance(): return Reactor._current_instance def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: builtin_plugin_loader = BuiltinPluginLoader(self.logger) installed_thirdparty_plugin_loader = ThirdPartyPluginLoader( self.logger) downloading_thirdparty_plugin_loader = DownloadingPluginLoader( self.logger) self.plugin_loader = DispatchingPluginLoader( self.logger, builtin_plugin_loader, installed_thirdparty_plugin_loader, downloading_thirdparty_plugin_loader) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None def require_plugin(self, plugin, version=None): if plugin not in self._plugins: try: self._plugins.append(plugin) self.import_plugin(plugin, version) except: # NOQA self._plugins.remove(plugin) raise def get_plugins(self): return self._plugins def get_tasks(self): return self.execution_manager.tasks def validate_project(self): validation_messages = self.project.validate() if len(validation_messages) > 0: raise ProjectValidationFailedException(validation_messages) def prepare_build(self, property_overrides=None, project_directory=".", project_descriptor="build.py", exclude_optional_tasks=None, exclude_tasks=None, exclude_all_optional=False): if not property_overrides: property_overrides = {} Reactor._current_instance = self project_directory, project_descriptor = self.verify_project_directory( project_directory, project_descriptor) self.logger.debug("Loading project module from %s", project_descriptor) self.project = Project(basedir=project_directory) self.project_module = self.load_project_module(project_descriptor) self.apply_project_attributes() self.override_properties(property_overrides) self.logger.debug("Have loaded plugins %s", ", ".join(self._plugins)) self.collect_tasks_and_actions_and_initializers(self.project_module) self.execution_manager.resolve_dependencies(exclude_optional_tasks, exclude_tasks, exclude_all_optional) def build(self, tasks=None, environments=None): if not tasks: tasks = [] else: tasks = as_list(tasks) if not environments: environments = [] execution_plan = self.create_execution_plan(tasks, environments) self.build_execution_plan(tasks, execution_plan) def create_execution_plan(self, tasks, environments): Reactor._current_instance = self if environments: self.logger.info("Activated environments: %s", ", ".join(environments)) self.execution_manager.execute_initializers(environments, logger=self.logger, project=self.project) self.log_project_properties() self.validate_project() if not len(tasks): if self.project.default_task: tasks += as_list(self.project.default_task) else: raise PyBuilderException("No default task given.") return self.execution_manager.build_execution_plan(tasks) def build_execution_plan(self, tasks, execution_plan): self.logger.debug("Execution plan is %s", ", ".join([task.name for task in execution_plan])) self.logger.info("Building %s version %s%s", self.project.name, self.project.version, get_dist_version_string(self.project)) self.logger.info("Executing build in %s", self.project.basedir) if len(tasks) == 1: self.logger.info("Going to execute task %s", tasks[0]) else: list_of_tasks = ", ".join(tasks) self.logger.info("Going to execute tasks: %s", list_of_tasks) task_execution_summaries = self.execution_manager.execute_execution_plan( execution_plan, logger=self.logger, project=self.project, reactor=self) return BuildSummary(self.project, task_execution_summaries) def execute_task(self, task_name): execution_plan = self.execution_manager.build_execution_plan(task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def execute_task_shortest_plan(self, task_name): execution_plan = self.execution_manager.build_shortest_execution_plan( task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def override_properties(self, property_overrides): for property_override in property_overrides: self.project.set_property(property_override, property_overrides[property_override]) def log_project_properties(self): formatted = "" for key in sorted(self.project.properties): formatted += "\n%40s : %s" % (key, self.project.get_property(key)) self.logger.debug("Project properties: %s", formatted) def import_plugin(self, plugin, version=None): self.logger.debug("Loading plugin '%s'%s", plugin, " version %s" % version if version else "") plugin_module = self.plugin_loader.load_plugin(self.project, plugin, version) self.collect_tasks_and_actions_and_initializers(plugin_module) def collect_tasks_and_actions_and_initializers(self, project_module): injected_task_dependencies = dict() def normalize_candidate_name(candidate): if hasattr(candidate, NAME_ATTRIBUTE): return getattr(candidate, NAME_ATTRIBUTE) elif hasattr(candidate, "__name__"): return candidate.__name__ def add_task_dependency(names, depends_on, optional): for name in as_list(names): if not isinstance(name, basestring): name = normalize_candidate_name(name) if name not in injected_task_dependencies: injected_task_dependencies[name] = list() injected_task_dependencies[name].append( TaskDependency(depends_on, optional)) for name in dir(project_module): candidate = getattr(project_module, name) if hasattr(candidate, TASK_ATTRIBUTE) and getattr( candidate, TASK_ATTRIBUTE): dependents = getattr( candidate, DEPENDENTS_ATTRIBUTE) if hasattr( candidate, DEPENDENTS_ATTRIBUTE) else None if dependents: dependents = list(as_list(dependents)) for d in dependents: if isinstance(d, optional): d = d() add_task_dependency(d, candidate, True) else: add_task_dependency(d, candidate, False) for name in dir(project_module): candidate = getattr(project_module, name) name = normalize_candidate_name(candidate) description = getattr(candidate, DESCRIPTION_ATTRIBUTE) if hasattr( candidate, DESCRIPTION_ATTRIBUTE) else "" if hasattr(candidate, TASK_ATTRIBUTE) and getattr( candidate, TASK_ATTRIBUTE): dependencies = getattr( candidate, DEPENDS_ATTRIBUTE) if hasattr( candidate, DEPENDS_ATTRIBUTE) else None task_dependencies = list() if dependencies: dependencies = list(as_list(dependencies)) for d in dependencies: if isinstance(d, optional): d = as_list(d()) task_dependencies.extend( [TaskDependency(item, True) for item in d]) else: task_dependencies.append(TaskDependency(d)) # Add injected if name in injected_task_dependencies: task_dependencies.extend(injected_task_dependencies[name]) self.logger.debug("Found task '%s' with dependencies %s", name, task_dependencies) self.execution_manager.register_task( Task(name, candidate, task_dependencies, description)) elif hasattr(candidate, ACTION_ATTRIBUTE) and getattr( candidate, ACTION_ATTRIBUTE): before = getattr(candidate, BEFORE_ATTRIBUTE) if hasattr( candidate, BEFORE_ATTRIBUTE) else None after = getattr(candidate, AFTER_ATTRIBUTE) if hasattr( candidate, AFTER_ATTRIBUTE) else None only_once = False if hasattr(candidate, ONLY_ONCE_ATTRIBUTE): only_once = getattr(candidate, ONLY_ONCE_ATTRIBUTE) teardown = False if hasattr(candidate, TEARDOWN_ATTRIBUTE): teardown = getattr(candidate, TEARDOWN_ATTRIBUTE) self.logger.debug("Found action %s", name) self.execution_manager.register_action( Action(name, candidate, before, after, description, only_once, teardown)) elif hasattr(candidate, INITIALIZER_ATTRIBUTE) and getattr( candidate, INITIALIZER_ATTRIBUTE): environments = [] if hasattr(candidate, ENVIRONMENTS_ATTRIBUTE): environments = getattr(candidate, ENVIRONMENTS_ATTRIBUTE) self.execution_manager.register_initializer( Initializer(name, candidate, environments, description)) def apply_project_attributes(self): self.propagate_property("name") self.propagate_property("version") self.propagate_property("default_task") self.propagate_property("summary") self.propagate_property("home_page") self.propagate_property("description") self.propagate_property("authors") self.propagate_property("license") self.propagate_property("url") def propagate_property(self, property): if hasattr(self.project_module, property): value = getattr(self.project_module, property) setattr(self.project, property, value) @staticmethod def load_project_module(project_descriptor): try: return imp.load_source("build", project_descriptor) except ImportError as e: raise PyBuilderException( "Error importing project descriptor %s: %s" % (project_descriptor, e)) @staticmethod def verify_project_directory(project_directory, project_descriptor): project_directory = os.path.abspath(project_directory) if not os.path.exists(project_directory): raise PyBuilderException("Project directory does not exist: %s", project_directory) if not os.path.isdir(project_directory): raise PyBuilderException( "Project directory is not a directory: %s", project_directory) project_descriptor_full_path = os.path.join(project_directory, project_descriptor) if not os.path.exists(project_descriptor_full_path): raise PyBuilderException( "Project directory does not contain descriptor file: %s", project_descriptor_full_path) if not os.path.isfile(project_descriptor_full_path): raise PyBuilderException("Project descriptor is not a file: %s", project_descriptor_full_path) return project_directory, project_descriptor_full_path
class Reactor(object): _current_instance = None @staticmethod def current_instance(): return Reactor._current_instance @staticmethod def _set_current_instance(reactor): Reactor._current_instance = reactor def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: self.plugin_loader = DispatchingPluginLoader(self.logger, BuiltinPluginLoader(self.logger), DownloadingPluginLoader(self.logger)) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None def require_plugin(self, plugin, version=None, plugin_module_name=None): if plugin not in self._plugins: try: self._plugins.append(plugin) self.import_plugin(plugin, version, plugin_module_name) except: # NOQA self._plugins.remove(plugin) raise def get_plugins(self): return self._plugins def get_tasks(self): return self.execution_manager.tasks def validate_project(self): validation_messages = self.project.validate() if len(validation_messages) > 0: raise ProjectValidationFailedException(validation_messages) def prepare_build(self, property_overrides=None, project_directory=".", project_descriptor="build.py", exclude_optional_tasks=None, exclude_tasks=None, exclude_all_optional=False): if not property_overrides: property_overrides = {} Reactor._set_current_instance(self) project_directory, project_descriptor = self.verify_project_directory( project_directory, project_descriptor) self.logger.debug("Loading project module from %s", project_descriptor) self.project = Project(basedir=project_directory) self.project_module = self.load_project_module(project_descriptor) self.apply_project_attributes() self.override_properties(property_overrides) self.logger.debug("Have loaded plugins %s", ", ".join(self._plugins)) self.collect_tasks_and_actions_and_initializers(self.project_module) self.execution_manager.resolve_dependencies(exclude_optional_tasks, exclude_tasks, exclude_all_optional) def build(self, tasks=None, environments=None): if not tasks: tasks = [] else: tasks = as_list(tasks) if not environments: environments = [] execution_plan = self.create_execution_plan(tasks, environments) return self.build_execution_plan(tasks, execution_plan) def create_execution_plan(self, tasks, environments): Reactor._set_current_instance(self) if environments: self.logger.info( "Activated environments: %s", ", ".join(environments)) self.execution_manager.execute_initializers( environments, logger=self.logger, project=self.project) self.log_project_properties() self.validate_project() if not len(tasks): if self.project.default_task: tasks += as_list(self.project.default_task) else: raise PyBuilderException("No default task given.") return self.execution_manager.build_execution_plan(tasks) def build_execution_plan(self, tasks, execution_plan): self.logger.debug("Execution plan is %s", ", ".join( [task.name for task in execution_plan])) self.logger.info( "Building %s version %s%s", self.project.name, self.project.version, get_dist_version_string(self.project)) self.logger.info("Executing build in %s", self.project.basedir) if len(tasks) == 1: self.logger.info("Going to execute task %s", tasks[0]) else: list_of_tasks = ", ".join(tasks) self.logger.info("Going to execute tasks: %s", list_of_tasks) task_execution_summaries = self.execution_manager.execute_execution_plan( execution_plan, logger=self.logger, project=self.project, reactor=self) return BuildSummary(self.project, task_execution_summaries) def execute_task(self, task_name): execution_plan = self.execution_manager.build_execution_plan(task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def execute_task_shortest_plan(self, task_name): execution_plan = self.execution_manager.build_shortest_execution_plan(task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def override_properties(self, property_overrides): for property_override in property_overrides: self.project.set_property( property_override, property_overrides[property_override]) def log_project_properties(self): formatted = "" for key in sorted(self.project.properties): formatted += "\n%40s : %s" % (key, self.project.get_property(key)) self.logger.debug("Project properties: %s", formatted) def import_plugin(self, plugin, version=None, plugin_module_name=None): self.logger.debug("Loading plugin '%s'%s", plugin, " version %s" % version if version else "") plugin_module = self.plugin_loader.load_plugin(self.project, plugin, version, plugin_module_name) self.collect_tasks_and_actions_and_initializers(plugin_module) def collect_tasks_and_actions_and_initializers(self, project_module): injected_task_dependencies = {} def normalize_candidate_name(candidate): return getattr(candidate, NAME_ATTRIBUTE, candidate.__name__ if hasattr(candidate, "__name__") else None) def add_task_dependency(names, depends_on, optional): for name in as_list(names): if not isinstance(name, basestring): name = normalize_candidate_name(name) if name not in injected_task_dependencies: injected_task_dependencies[name] = list() injected_task_dependencies[name].append(TaskDependency(depends_on, optional)) for name in dir(project_module): candidate = getattr(project_module, name) name = normalize_candidate_name(candidate) if getattr(candidate, TASK_ATTRIBUTE, None): dependents = getattr(candidate, DEPENDENTS_ATTRIBUTE, None) if dependents: dependents = list(as_list(dependents)) for d in dependents: if isinstance(d, optional): d = d() add_task_dependency(d, name, True) else: add_task_dependency(d, name, False) for name in dir(project_module): candidate = getattr(project_module, name) name = normalize_candidate_name(candidate) description = getattr(candidate, DESCRIPTION_ATTRIBUTE, "") if getattr(candidate, TASK_ATTRIBUTE, None): dependencies = getattr(candidate, DEPENDS_ATTRIBUTE, None) task_dependencies = list() if dependencies: dependencies = list(as_list(dependencies)) for d in dependencies: if isinstance(d, optional): d = as_list(d()) task_dependencies.extend([TaskDependency(item, True) for item in d]) else: task_dependencies.append(TaskDependency(d)) # Add injected if name in injected_task_dependencies: task_dependencies.extend(injected_task_dependencies[name]) del injected_task_dependencies[name] self.logger.debug("Found task '%s' with dependencies %s", name, task_dependencies) self.execution_manager.register_task( Task(name, candidate, task_dependencies, description)) elif getattr(candidate, ACTION_ATTRIBUTE, None): before = getattr(candidate, BEFORE_ATTRIBUTE, None) after = getattr(candidate, AFTER_ATTRIBUTE, None) only_once = getattr(candidate, ONLY_ONCE_ATTRIBUTE, False) teardown = getattr(candidate, TEARDOWN_ATTRIBUTE, False) self.logger.debug("Found action %s", name) self.execution_manager.register_action( Action(name, candidate, before, after, description, only_once, teardown)) elif getattr(candidate, INITIALIZER_ATTRIBUTE, None): environments = getattr(candidate, ENVIRONMENTS_ATTRIBUTE, []) self.execution_manager.register_initializer( Initializer(name, candidate, environments, description)) self.execution_manager.register_late_task_dependencies(injected_task_dependencies) def apply_project_attributes(self): self.propagate_property("name") self.propagate_property("version") self.propagate_property("default_task") self.propagate_property("summary") self.propagate_property("home_page") self.propagate_property("description") self.propagate_property("authors") self.propagate_property("license") self.propagate_property("url") def propagate_property(self, property): if hasattr(self.project_module, property): value = getattr(self.project_module, property) setattr(self.project, property, value) @staticmethod def load_project_module(project_descriptor): try: return imp.load_source("build", project_descriptor) except ImportError as e: raise PyBuilderException( "Error importing project descriptor %s: %s" % (project_descriptor, e)) @staticmethod def verify_project_directory(project_directory, project_descriptor): project_directory = os.path.abspath(project_directory) if not os.path.exists(project_directory): raise PyBuilderException( "Project directory does not exist: %s", project_directory) if not os.path.isdir(project_directory): raise PyBuilderException( "Project directory is not a directory: %s", project_directory) project_descriptor_full_path = os.path.join( project_directory, project_descriptor) if not os.path.exists(project_descriptor_full_path): raise PyBuilderException( "Project directory does not contain descriptor file: %s", project_descriptor_full_path) if not os.path.isfile(project_descriptor_full_path): raise PyBuilderException( "Project descriptor is not a file: %s", project_descriptor_full_path) return project_directory, project_descriptor_full_path
class Reactor: _current_instance = None @staticmethod def current_instance(): return Reactor._current_instance @staticmethod def _set_current_instance(reactor): Reactor._current_instance = reactor def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: self.plugin_loader = DispatchingPluginLoader( self.logger, BuiltinPluginLoader(self.logger), DownloadingPluginLoader(self.logger)) else: self.plugin_loader = plugin_loader self._plugins = [] self._pending_plugin_installs = [] self._plugins_imported = set() self._deferred_plugins = ModuleTraversalTree() self._deferred_import = False self.project = None self.project_module = None self._tools = [] python_env_registry = self._python_env_registry = PythonEnvRegistry( self) system_pyenv = PythonEnv(sys.exec_prefix, self).populate() python_env_registry["system"] = system_pyenv self._sys_path_original = list(sys.path) def require_plugin(self, plugin, version=None, plugin_module_name=None): if plugin not in self._plugins: self._plugins.append(plugin) plugin_def = PluginDef(plugin, version, plugin_module_name) self._deferred_plugins.add_plugin(plugin_def) self._pending_plugin_installs.append(plugin_def) def get_plugins(self): return self._plugins def get_tasks(self): return self.execution_manager.tasks def validate_project(self): validation_messages = self.project.validate() if len(validation_messages) > 0: raise ProjectValidationFailedException(validation_messages) def prepare_build(self, property_overrides=None, project_directory=".", project_descriptor="build.py", exclude_optional_tasks=None, exclude_tasks=None, exclude_all_optional=False, reset_plugins=False, offline=False): if not property_overrides: property_overrides = {} Reactor._set_current_instance(self) project_directory, project_descriptor = self.verify_project_directory( project_directory, project_descriptor) self.logger.debug("Loading project module from %s", project_descriptor) self.project = Project(basedir=project_directory, offline=offline) self._setup_plugin_directory(reset_plugins) self._setup_deferred_plugin_import() # This is really a way to make sure we can install `billiard` as a dependency # before any of the plugins actually initialize if PY2 and not IS_WIN: self.require_plugin( "pypi:billiard", "~=3.6.2", plugin_module_name="pybuilder.plugins.billiard_plugin") self.project_module = self.load_project_module(project_descriptor) self._load_deferred_plugins() self._collect_project_annotations() self.apply_project_attributes() self.override_properties(property_overrides) self.logger.debug("Have loaded plugins %s", ", ".join(self._plugins)) self.collect_project_annotations(self.project_module) self.execution_manager.resolve_dependencies(exclude_optional_tasks, exclude_tasks, exclude_all_optional) self._remove_deferred_plugin_import() def build(self, tasks=None, environments=None): if not tasks: tasks = [] else: tasks = as_list(tasks) if not environments: environments = [] execution_plan = self.create_execution_plan(tasks, environments) execution_summary = self.build_execution_plan(tasks, execution_plan) self.execution_manager.execute_finalizers(environments, logger=self.logger, project=self.project, reactor=self) return execution_summary def create_execution_plan(self, tasks, environments): Reactor._set_current_instance(self) if environments: self.logger.info("Activated environments: %s", ", ".join(environments)) self.execution_manager.execute_initializers(environments, logger=self.logger, project=self.project, reactor=self) self.log_project_properties() self.validate_project() tasks = self._prepare_tasks(tasks) return self.execution_manager.build_execution_plan(tasks) def build_execution_plan(self, tasks, execution_plan): self.logger.debug("Execution plan is %s", ", ".join([task.name for task in execution_plan])) self.logger.info("Building %s version %s%s", self.project.name, self.project.version, get_dist_version_string(self.project)) self.logger.info("Executing build in %s", self.project.basedir) if len(tasks) == 1: self.logger.info("Going to execute task %s", tasks[0]) else: list_of_tasks = ", ".join(tasks) self.logger.info("Going to execute tasks: %s", list_of_tasks) task_execution_summaries = self.execution_manager.execute_execution_plan( execution_plan, logger=self.logger, project=self.project, reactor=self) return BuildSummary(self.project, task_execution_summaries) def execute_task(self, task_name): execution_plan = self.execution_manager.build_execution_plan(task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def execute_task_shortest_plan(self, task_name): execution_plan = self.execution_manager.build_shortest_execution_plan( task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def override_properties(self, property_overrides): for property_override in property_overrides: self.project.set_property(property_override, property_overrides[property_override]) def log_project_properties(self): formatted = "" for key in sorted(self.project.properties): formatted += "\n%40s : %s" % (key, self.project.get_property(key)) self.logger.debug("Project properties: %s", formatted) def import_plugin(self, plugin_def): if self._pending_plugin_installs: self.plugin_loader.install_plugin(self, self._pending_plugin_installs) del self._pending_plugin_installs[:] if plugin_def not in self._plugins_imported: self.logger.debug( "Loading plugin '%s'%s", plugin_def.name, " version %s" % plugin_def.version if plugin_def.version else "") plugin_module = self.plugin_loader.load_plugin( self.project, plugin_def) self._plugins_imported.add(plugin_def) self._deferred_plugins.set_module(plugin_module) def collect_project_annotations(self, project_module): injected_task_dependencies = {} def add_task_dependency(names, depends_on, optional): for name in as_list(names): if not isinstance(name, basestring): name = self.normalize_candidate_name(name) if name not in injected_task_dependencies: injected_task_dependencies[name] = list() injected_task_dependencies[name].append( TaskDependency(depends_on, optional)) for name in dir(project_module): candidate = getattr(project_module, name) name = self.normalize_candidate_name(candidate) if getattr(candidate, TASK_ATTRIBUTE, None): dependents = getattr(candidate, DEPENDENTS_ATTRIBUTE, None) if dependents: dependents = list(as_list(dependents)) for d in dependents: if isinstance(d, optional): d = d() add_task_dependency(d, name, True) else: add_task_dependency(d, name, False) for name in dir(project_module): candidate = getattr(project_module, name) name = self.normalize_candidate_name(candidate) description = getattr(candidate, DESCRIPTION_ATTRIBUTE, "") if getattr(candidate, TASK_ATTRIBUTE, None): dependencies = getattr(candidate, DEPENDS_ATTRIBUTE, None) task_dependencies = list() if dependencies: dependencies = list(as_list(dependencies)) for d in dependencies: if isinstance(d, optional): d = as_list(d()) task_dependencies.extend( [TaskDependency(item, True) for item in d]) else: task_dependencies.append(TaskDependency(d)) # Add injected if name in injected_task_dependencies: task_dependencies.extend(injected_task_dependencies[name]) del injected_task_dependencies[name] self.logger.debug("Found task '%s' with dependencies %s", name, task_dependencies) self.execution_manager.register_task( Task(name, candidate, task_dependencies, description)) elif getattr(candidate, ACTION_ATTRIBUTE, None): before = getattr(candidate, BEFORE_ATTRIBUTE, None) after = getattr(candidate, AFTER_ATTRIBUTE, None) only_once = getattr(candidate, ONLY_ONCE_ATTRIBUTE, False) teardown = getattr(candidate, TEARDOWN_ATTRIBUTE, False) self.logger.debug("Found action %s", name) self.execution_manager.register_action( Action(name, candidate, before, after, description, only_once, teardown)) elif getattr(candidate, INITIALIZER_ATTRIBUTE, None): environments = getattr(candidate, ENVIRONMENTS_ATTRIBUTE, []) self.execution_manager.register_initializer( Initializer(name, candidate, environments, description)) elif getattr(candidate, FINALIZER_ATTRIBUTE, None): environments = getattr(candidate, ENVIRONMENTS_ATTRIBUTE, []) self.execution_manager.register_finalizer( Finalizer(name, candidate, environments, description)) self.execution_manager.register_late_task_dependencies( injected_task_dependencies) def apply_project_attributes(self): self.propagate_property("name") self.propagate_property("version") self.propagate_property("default_task") self.propagate_property("summary") self.propagate_property("description") self.propagate_property("author") self.propagate_property("authors") self.propagate_property("maintainer") self.propagate_property("maintainers") self.propagate_property("license") self.propagate_property("url") self.propagate_property("urls") self.propagate_property("explicit_namespaces") self.propagate_property("requires_python") self.propagate_property("obsoletes") def propagate_property(self, property): if hasattr(self.project_module, property): value = getattr(self.project_module, property) setattr(self.project, property, value) def _prepare_tasks(self, tasks): if not len(tasks): if self.project.default_task: tasks += as_list(self.project.default_task) else: raise PyBuilderException("No default task given.") else: new_tasks = [ task for task in tasks if task[0] not in ("+", "^") or task in ("+", "^") ] append_tasks = [ task[1:] for task in tasks if task[0] == "+" and task != "+" ] remove_tasks = [ task[1:] for task in tasks if task[0] == "^" and task != "^" ] if len(new_tasks): del tasks[:] tasks.extend(new_tasks) tasks.extend(append_tasks) for task in remove_tasks: try: tasks.remove(task) except ValueError: pass else: del tasks[:] if self.project.default_task: tasks += as_list(self.project.default_task) tasks += append_tasks for task in remove_tasks: try: tasks.remove(task) except ValueError: pass return tasks @staticmethod def normalize_candidate_name(candidate): return getattr( candidate, NAME_ATTRIBUTE, candidate.__name__ if hasattr(candidate, "__name__") else None) @staticmethod def load_project_module(project_descriptor): try: return imp.load_source("build", project_descriptor) except ImportError as e: raise PyBuilderException( "Error importing project descriptor %s: %s" % (project_descriptor, e)) @staticmethod def verify_project_directory(project_directory, project_descriptor): project_directory = np(project_directory) if not os.path.exists(project_directory): raise PyBuilderException("Project directory does not exist: %s", project_directory) if not os.path.isdir(project_directory): raise PyBuilderException( "Project directory is not a directory: %s", project_directory) project_descriptor_full_path = jp(project_directory, project_descriptor) if not os.path.exists(project_descriptor_full_path): raise PyBuilderException( "Project directory does not contain descriptor file: %s", project_descriptor_full_path) if not os.path.isfile(project_descriptor_full_path): raise PyBuilderException("Project descriptor is not a file: %s", project_descriptor_full_path) return project_directory, project_descriptor_full_path def add_tool(self, tool): self._tools.append(tool) def remove_tool(self, tool): self._tools.remove(tool) @property def tools(self): return self._tools @property def python_env_registry(self): return self._python_env_registry @property def pybuilder_venv(self): return self._python_env_registry["pybuilder"] def _setup_plugin_directory(self, reset_plugins): per = self.python_env_registry system_env = per["system"] plugin_dir = self._plugin_dir = np( jp(self.project.basedir, ".pybuilder", "plugins", system_env.versioned_dir_name)) self.logger.debug("Setting up plugins VEnv at '%s'%s", plugin_dir, " (resetting)" if reset_plugins else "") plugin_env = per["pybuilder"] = PythonEnv( plugin_dir, self).create_venv(with_pip=True, symlinks=system_env.venv_symlinks, upgrade=True, clear=(reset_plugins or system_env.is_pypy), offline=self.project.offline) prepend_env_to_path(plugin_env, sys.path) patch_mp_pyb_env(plugin_env) def _setup_deferred_plugin_import(self): self._old_import = __import__ try: __builtins__["__import__"] = self.__import_with_plugins except TypeError: # PyPy setattr(__builtins__, "__import__", self.__import_with_plugins) self.logger.debug("Patched __import__ system to defer plugin loading") def _remove_deferred_plugin_import(self): try: __builtins__["__import__"] = self._old_import except TypeError: # PyPy setattr(__builtins__, "__import__", self._old_import) def __import_with_plugins(self, *args, **kwargs): try: return self._old_import(*args, **kwargs) except ImportError: if self._load_deferred_plugins(): return self._old_import(*args, **kwargs) else: raise def _load_deferred_plugins(self): if not self._deferred_import: self._deferred_import = True try: while True: mods = self._deferred_plugins.get_mods() for deferred_plugin in self._deferred_plugins.traverse(): self.import_plugin(deferred_plugin[0]) new_mods = self._deferred_plugins.get_mods() if mods == new_mods: break return True finally: self._deferred_import = False def _collect_project_annotations(self): for deferred_plugin in self._deferred_plugins.traverse(): self.collect_project_annotations(deferred_plugin[1])
class Reactor(object): _current_instance = None @staticmethod def current_instance(): return Reactor._current_instance def __init__(self, logger, execution_manager, plugin_loader=None): self.logger = logger self.execution_manager = execution_manager if not plugin_loader: builtin_plugin_loader = BuiltinPluginLoader(self.logger) thirdparty_plugin_loader = ThirdPartyPluginLoader(self.logger) self.plugin_loader = DispatchingPluginLoader( self.logger, builtin_plugin_loader, thirdparty_plugin_loader) else: self.plugin_loader = plugin_loader self._plugins = [] self.project = None def require_plugin(self, plugin): if plugin not in self._plugins: try: self._plugins.append(plugin) self.import_plugin(plugin) except: self._plugins.remove(plugin) raise def get_plugins(self): return self._plugins def get_tasks(self): return self.execution_manager.tasks def validate_project(self): validation_messages = self.project.validate() if len(validation_messages) > 0: raise ProjectValidationFailedException(validation_messages) def prepare_build(self, property_overrides={}, project_directory=".", project_descriptor="build.py"): Reactor._current_instance = self project_directory, project_descriptor = self.verify_project_directory( project_directory, project_descriptor) self.logger.debug("Loading project module from %s", project_descriptor) self.project = Project(basedir=project_directory) self.project_module = self.load_project_module(project_descriptor) self.apply_project_attributes() self.override_properties(property_overrides) self.logger.debug("Have loaded plugins %s", ", ".join(self._plugins)) self.collect_tasks_and_actions_and_initializers(self.project_module) self.execution_manager.resolve_dependencies() def build(self, tasks=[], environments=[]): Reactor._current_instance = self if environments: self.logger.info( "Activated environments: %s", ", ".join(environments)) self.execution_manager.execute_initializers( environments, logger=self.logger, project=self.project) self.log_project_properties() self.validate_project() tasks = as_list(tasks) if not len(tasks): if self.project.default_task: tasks += as_list(self.project.default_task) else: raise PyBuilderException("No default task given.") execution_plan = self.execution_manager.build_execution_plan(tasks) self.logger.debug("Execution plan is %s", ", ".join( [task.name for task in execution_plan])) self.logger.info( "Building %s version %s", self.project.name, self.project.version) self.logger.info("Executing build in %s", self.project.basedir) if len(tasks) == 1: self.logger.info("Going to execute task %s", tasks[0]) else: list_of_tasks = ", ".join(tasks) self.logger.info("Going to execute tasks: %s", list_of_tasks) task_execution_summaries = self.execution_manager.execute_execution_plan( execution_plan, logger=self.logger, project=self.project, reactor=self) return BuildSummary(self.project, task_execution_summaries) def execute_task(self, task_name): execution_plan = self.execution_manager.build_execution_plan(task_name) self.execution_manager.execute_execution_plan(execution_plan, logger=self.logger, project=self.project, reactor=self) def override_properties(self, property_overrides): for property_override in property_overrides: self.project.set_property( property_override, property_overrides[property_override]) def log_project_properties(self): formatted = "" for key in sorted(self.project.properties): formatted += "\n%40s : %s" % (key, self.project.get_property(key)) self.logger.debug("Project properties: %s", formatted) def import_plugin(self, plugin): self.logger.debug("Loading plugin '%s'", plugin) plugin_module = self.plugin_loader.load_plugin(self.project, plugin) self.collect_tasks_and_actions_and_initializers(plugin_module) def collect_tasks_and_actions_and_initializers(self, project_module): for name in dir(project_module): candidate = getattr(project_module, name) if hasattr(candidate, NAME_ATTRIBUTE): name = getattr(candidate, NAME_ATTRIBUTE) elif hasattr(candidate, "__name__"): name = candidate.__name__ description = getattr(candidate, DESCRIPTION_ATTRIBUTE) if hasattr( candidate, DESCRIPTION_ATTRIBUTE) else "" if hasattr(candidate, TASK_ATTRIBUTE) and getattr(candidate, TASK_ATTRIBUTE): dependencies = getattr(candidate, DEPENDS_ATTRIBUTE) if hasattr( candidate, DEPENDS_ATTRIBUTE) else None self.logger.debug("Found task %s", name) self.execution_manager.register_task( Task(name, candidate, dependencies, description)) elif hasattr(candidate, ACTION_ATTRIBUTE) and getattr(candidate, ACTION_ATTRIBUTE): before = getattr(candidate, BEFORE_ATTRIBUTE) if hasattr( candidate, BEFORE_ATTRIBUTE) else None after = getattr(candidate, AFTER_ATTRIBUTE) if hasattr( candidate, AFTER_ATTRIBUTE) else None only_once = False if hasattr(candidate, ONLY_ONCE_ATTRIBUTE): only_once = getattr(candidate, ONLY_ONCE_ATTRIBUTE) self.logger.debug("Found action %s", name) self.execution_manager.register_action( Action(name, candidate, before, after, description, only_once)) elif hasattr(candidate, INITIALIZER_ATTRIBUTE) and getattr(candidate, INITIALIZER_ATTRIBUTE): environments = [] if hasattr(candidate, ENVIRONMENTS_ATTRIBUTE): environments = getattr(candidate, ENVIRONMENTS_ATTRIBUTE) self.execution_manager.register_initializer( Initializer(name, candidate, environments, description)) def apply_project_attributes(self): self.propagate_property("name") self.propagate_property("version") self.propagate_property("default_task") self.propagate_property("summary") self.propagate_property("home_page") self.propagate_property("description") self.propagate_property("authors") self.propagate_property("license") self.propagate_property("url") def propagate_property(self, property): if hasattr(self.project_module, property): value = getattr(self.project_module, property) setattr(self.project, property, value) def load_project_module(self, project_descriptor): try: return imp.load_source("build", project_descriptor) except ImportError as e: raise PyBuilderException( "Error importing project descriptor %s: %s" % (project_descriptor, e)) def verify_project_directory(self, project_directory, project_descriptor): project_directory = os.path.abspath(project_directory) if not os.path.exists(project_directory): raise PyBuilderException( "Project directory does not exist: %s", project_directory) if not os.path.isdir(project_directory): raise PyBuilderException( "Project directory is not a directory: %s", project_directory) project_descriptor_full_path = os.path.join( project_directory, project_descriptor) if not os.path.exists(project_descriptor_full_path): raise PyBuilderException( "Project directory does not contain descriptor file: %s", project_descriptor_full_path) if not os.path.isfile(project_descriptor_full_path): raise PyBuilderException( "Project descriptor is not a file: %s", project_descriptor_full_path) return project_directory, project_descriptor_full_path