def test_get_names_of_hot_installed_package(self): pip_args = ["install", TEST_ENTRY_POINT_PATH] run_pip(pip_args) reload_site_packages() for syspath in sys.path: pkg_resources.working_set.add_entry(syspath) try: self.manager = EntryPointManager('entry.point.test') names = self.manager.get_plugin_names() self.assertIn("entry_point_1", names) self.assertIn("entry_point_2", names) except Exception, e: raise e
def __init__(self, job_uuid, name, config, api_key, entry_point_group_name="hoplite.jobs", port=5000): """ @param job_uuid unique identifier for this job @param name the name of the job, corresponds to the plugin name @param config dictionary object containing configuration for the specific job """ self.port = port self.uuid = job_uuid self.name = name self.config = config self._api_key = api_key self._status = {} self._process = None self._started = False self._killed = False self._pipe_to_self = None self._pipe_to_process = None # TODO: We need this workaround because in tests I create jobs that # don't have a corresponding loaded entry point # At some point the tests should be refactored to use jobs that exist # and we can get rid of this code module = EntryPointManager().get_plugin_module_by_name(name) logger_name = module.__name__ if module is not None else name self._logger = server_logging.get_job_logger(logger_name, uuid=self.uuid) self._entry_point_group_name = entry_point_group_name
def create_app(group_name='hoplite.jobs'): app = Flask(__name__) app.register_blueprint(site_bp, url_prefix='/') app.register_blueprint(jobs_bp, url_prefix='/jobs') app.register_blueprint(job_plugins_bp, url_prefix='/job_plugins') hoplite.api.helpers.manager = JobManager(EntryPointManager(group_name)) return app
class TestEntryPointManager(unittest2.TestCase): @classmethod def setUpClass(cls): # Install hoplite so that its entry points are added pip_args = ["install", os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')] run_pip(pip_args) reload_site_packages() @classmethod def tearDownClass(cls): pip_args = ["uninstall", "hoplite", "-y"] run_pip(pip_args) def setUp(self): self.manager = EntryPointManager() def test_get_names_with_default_group_name(self): names = self.manager.get_plugin_names() self.assertIn('hoplite.plugins.download_folder_from_ftp_job', names) self.assertIn('hoplite.plugins.download_network_folder_job', names) self.assertIn('hoplite.plugins.upload_to_network_folder_job', names) def test_get_names_of_hot_installed_package(self): pip_args = ["install", TEST_ENTRY_POINT_PATH] run_pip(pip_args) reload_site_packages() for syspath in sys.path: pkg_resources.working_set.add_entry(syspath) try: self.manager = EntryPointManager('entry.point.test') names = self.manager.get_plugin_names() self.assertIn("entry_point_1", names) self.assertIn("entry_point_2", names) except Exception, e: raise e finally:
def job_wrapper(pipe_to_parent, entry_point_name, config, status_updater, entry_point_group_name='hoplite.jobs', uuid=''): """ A picklable function that is used to start the job. It loads the specified module and calls run on it with the correct parameters. In the event of an error occurring in the job, the error is bubbled up to the highest parent of the job. This is done by encapsulating each exception into a JobFailedError, which is raised to cause the bubbling action. Since Exceptions aren't picklable, information from previous exceptions is put into a dictionary. This exception bubbling is useful for situations in which jobs are used to call other jobs. The stack trace for each "level" is saved, and the entire list of jobs with their respective traces can be displayed at the top level (where the JobFailedError is handled). """ module = EntryPointManager( entry_point_group_name).get_plugin_module_by_name(entry_point_name) logger = server_logging.get_job_logger(module.__name__, uuid) try: module.run(config, status_updater) except JobFailedError as e: logger.error( "A job raised an exception and it was not caught." " Address: {0} UUID: {1}".format(e.addr, e.uuid)) logger.error( "Exception Traceback: {0}".format( traceback.format_tb(e.traceback_object))) _, _, tb = sys.exc_info() traceback_object = tb exception_dictionary = { "address": e.addr, "uuid": e.uuid, "traceback": e.traceback_object, "previous_exception": e.previous_exception } pass_to_parent = { "traceback": traceback_object, "previous_exception": exception_dictionary } pipe_to_parent.send(pass_to_parent) except Exception as e: except_type, except_class, tb = sys.exc_info() traceback_object = tb type_string = str(except_type) try: pickled_exception = pickle.dumps(e) except pickle.PicklingError: pickled_exception = None exception_dictionary = { "type": type_string, "message": e.message, "exception_object": pickled_exception } pass_to_parent = { "traceback": pickle.dumps(traceback_object), "previous_exception": exception_dictionary } logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} " "except class:{3} traceback:{4}".format( uuid, entry_point_name, except_type, except_class, traceback.format_tb(tb))) pipe_to_parent.send(pass_to_parent) logger.debug("Finished running UUID:{0}".format(uuid))
def job_wrapper(pipe_to_parent, entry_point_name, config, status_updater, entry_point_group_name='hoplite.jobs', uuid=''): """ A picklable function that is used to start the job. It loads the specified module and calls run on it with the correct parameters. In the event of an error occurring in the job, the error is bubbled up to the highest parent of the job. This is done by encapsulating each exception into a JobFailedError, which is raised to cause the bubbling action. Since Exceptions aren't picklable, information from previous exceptions is put into a dictionary. This exception bubbling is useful for situations in which jobs are used to call other jobs. The stack trace for each "level" is saved, and the entire list of jobs with their respective traces can be displayed at the top level (where the JobFailedError is handled). """ module = EntryPointManager( entry_point_group_name).get_plugin_module_by_name(entry_point_name) logger = server_logging.get_job_logger(module.__name__, uuid) try: module.run(config, status_updater) except JobFailedError as e: logger.error("A job raised an exception and it was not caught." " Address: {0} UUID: {1}".format(e.addr, e.uuid)) logger.error("Exception Traceback: {0}".format( traceback.format_tb(e.traceback_object))) _, _, tb = sys.exc_info() traceback_object = tb exception_dictionary = { "address": e.addr, "uuid": e.uuid, "traceback": e.traceback_object, "previous_exception": e.previous_exception } pass_to_parent = { "traceback": traceback_object, "previous_exception": exception_dictionary } pipe_to_parent.send(pass_to_parent) except Exception as e: except_type, except_class, tb = sys.exc_info() traceback_object = tb type_string = str(except_type) try: pickled_exception = pickle.dumps(e) except pickle.PicklingError: pickled_exception = None exception_dictionary = { "type": type_string, "message": e.message, "exception_object": pickled_exception } pass_to_parent = { "traceback": pickle.dumps(traceback_object), "previous_exception": exception_dictionary } logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} " "except class:{3} traceback:{4}".format( uuid, entry_point_name, except_type, except_class, traceback.format_tb(tb))) pipe_to_parent.send(pass_to_parent) logger.debug("Finished running UUID:{0}".format(uuid))
def setUp(self): super(TestJobManager, self).setUp() self.manager = JobManager(EntryPointManager('hoplite.test_jobs'))
def setUp(self): self.manager = EntryPointManager()