Пример #1
0
 def test_get_names_of_hot_installed_package(self):
     pip_args = ["install", TEST_ENTRY_POINT_PATH]
     run_pip(pip_args)
     reload_site_packages()
     for syspath in sys.path:
         pkg_resources.working_set.add_entry(syspath)
     try:
         self.manager = EntryPointManager('entry.point.test')
         names = self.manager.get_plugin_names()
         self.assertIn("entry_point_1", names)
         self.assertIn("entry_point_2", names)
     except Exception, e:
         raise e
Пример #2
0
 def __init__(self,
              job_uuid,
              name,
              config,
              api_key,
              entry_point_group_name="hoplite.jobs",
              port=5000):
     """
     @param job_uuid unique identifier for this job
     @param name the name of the job, corresponds to the plugin name
     @param config dictionary object containing configuration for the
         specific job
     """
     self.port = port
     self.uuid = job_uuid
     self.name = name
     self.config = config
     self._api_key = api_key
     self._status = {}
     self._process = None
     self._started = False
     self._killed = False
     self._pipe_to_self = None
     self._pipe_to_process = None
     # TODO: We need this workaround because in tests I create jobs that
     # don't have a corresponding loaded entry point
     # At some point the tests should be refactored to use jobs that exist
     # and we can get rid of this code
     module = EntryPointManager().get_plugin_module_by_name(name)
     logger_name = module.__name__ if module is not None else name
     self._logger = server_logging.get_job_logger(logger_name,
                                                  uuid=self.uuid)
     self._entry_point_group_name = entry_point_group_name
Пример #3
0
def create_app(group_name='hoplite.jobs'):
    app = Flask(__name__)
    app.register_blueprint(site_bp, url_prefix='/')
    app.register_blueprint(jobs_bp, url_prefix='/jobs')
    app.register_blueprint(job_plugins_bp, url_prefix='/job_plugins')
    hoplite.api.helpers.manager = JobManager(EntryPointManager(group_name))
    return app
Пример #4
0
class TestEntryPointManager(unittest2.TestCase):
    @classmethod
    def setUpClass(cls):
        # Install hoplite so that its entry points are added
        pip_args = ["install", os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')]
        run_pip(pip_args)
        reload_site_packages()

    @classmethod
    def tearDownClass(cls):
        pip_args = ["uninstall", "hoplite", "-y"]
        run_pip(pip_args)

    def setUp(self):
        self.manager = EntryPointManager()

    def test_get_names_with_default_group_name(self):
        names = self.manager.get_plugin_names()
        self.assertIn('hoplite.plugins.download_folder_from_ftp_job', names)
        self.assertIn('hoplite.plugins.download_network_folder_job', names)
        self.assertIn('hoplite.plugins.upload_to_network_folder_job', names)

    def test_get_names_of_hot_installed_package(self):
        pip_args = ["install", TEST_ENTRY_POINT_PATH]
        run_pip(pip_args)
        reload_site_packages()
        for syspath in sys.path:
            pkg_resources.working_set.add_entry(syspath)
        try:
            self.manager = EntryPointManager('entry.point.test')
            names = self.manager.get_plugin_names()
            self.assertIn("entry_point_1", names)
            self.assertIn("entry_point_2", names)
        except Exception, e:
            raise e
        finally:
Пример #5
0
def job_wrapper(pipe_to_parent, entry_point_name, config, status_updater,
                entry_point_group_name='hoplite.jobs', uuid=''):
    """
    A picklable function that is used to start the job. It loads the specified
    module and calls run on it with the correct parameters.

    In the event of an error occurring in the job, the error is bubbled up to
    the highest parent of the job. This is done by encapsulating each exception
    into a JobFailedError, which is raised to cause the bubbling action. Since
    Exceptions aren't picklable, information from previous exceptions is put
    into a dictionary.

    This exception bubbling is useful for situations in which jobs are used to
    call other jobs. The stack trace for each "level" is saved, and the entire
    list of jobs with their respective traces can be displayed at the top level
    (where the JobFailedError is handled).
    """
    module = EntryPointManager(
        entry_point_group_name).get_plugin_module_by_name(entry_point_name)
    logger = server_logging.get_job_logger(module.__name__, uuid)
    try:
        module.run(config, status_updater)
    except JobFailedError as e:
        logger.error(
            "A job raised an exception and it was not caught."
            " Address: {0} UUID: {1}".format(e.addr, e.uuid))
        logger.error(
            "Exception Traceback: {0}".format(
                traceback.format_tb(e.traceback_object)))
        _, _, tb = sys.exc_info()
        traceback_object = tb
        exception_dictionary = {
            "address": e.addr,
            "uuid": e.uuid,
            "traceback": e.traceback_object,
            "previous_exception": e.previous_exception
        }
        pass_to_parent = {
            "traceback": traceback_object,
            "previous_exception": exception_dictionary
        }
        pipe_to_parent.send(pass_to_parent)
    except Exception as e:
        except_type, except_class, tb = sys.exc_info()
        traceback_object = tb
        type_string = str(except_type)
        try:
            pickled_exception = pickle.dumps(e)
        except pickle.PicklingError:
            pickled_exception = None

        exception_dictionary = {
            "type": type_string,
            "message": e.message,
            "exception_object": pickled_exception
        }
        pass_to_parent = {
            "traceback": pickle.dumps(traceback_object),
            "previous_exception": exception_dictionary
        }
        logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} "
                     "except class:{3} traceback:{4}".format(
                        uuid, entry_point_name,
                        except_type, except_class, traceback.format_tb(tb)))
        pipe_to_parent.send(pass_to_parent)
    logger.debug("Finished running UUID:{0}".format(uuid))
Пример #6
0
def job_wrapper(pipe_to_parent,
                entry_point_name,
                config,
                status_updater,
                entry_point_group_name='hoplite.jobs',
                uuid=''):
    """
    A picklable function that is used to start the job. It loads the specified
    module and calls run on it with the correct parameters.

    In the event of an error occurring in the job, the error is bubbled up to
    the highest parent of the job. This is done by encapsulating each exception
    into a JobFailedError, which is raised to cause the bubbling action. Since
    Exceptions aren't picklable, information from previous exceptions is put
    into a dictionary.

    This exception bubbling is useful for situations in which jobs are used to
    call other jobs. The stack trace for each "level" is saved, and the entire
    list of jobs with their respective traces can be displayed at the top level
    (where the JobFailedError is handled).
    """
    module = EntryPointManager(
        entry_point_group_name).get_plugin_module_by_name(entry_point_name)
    logger = server_logging.get_job_logger(module.__name__, uuid)
    try:
        module.run(config, status_updater)
    except JobFailedError as e:
        logger.error("A job raised an exception and it was not caught."
                     " Address: {0} UUID: {1}".format(e.addr, e.uuid))
        logger.error("Exception Traceback: {0}".format(
            traceback.format_tb(e.traceback_object)))
        _, _, tb = sys.exc_info()
        traceback_object = tb
        exception_dictionary = {
            "address": e.addr,
            "uuid": e.uuid,
            "traceback": e.traceback_object,
            "previous_exception": e.previous_exception
        }
        pass_to_parent = {
            "traceback": traceback_object,
            "previous_exception": exception_dictionary
        }
        pipe_to_parent.send(pass_to_parent)
    except Exception as e:
        except_type, except_class, tb = sys.exc_info()
        traceback_object = tb
        type_string = str(except_type)
        try:
            pickled_exception = pickle.dumps(e)
        except pickle.PicklingError:
            pickled_exception = None

        exception_dictionary = {
            "type": type_string,
            "message": e.message,
            "exception_object": pickled_exception
        }
        pass_to_parent = {
            "traceback": pickle.dumps(traceback_object),
            "previous_exception": exception_dictionary
        }
        logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} "
                     "except class:{3} traceback:{4}".format(
                         uuid, entry_point_name, except_type, except_class,
                         traceback.format_tb(tb)))
        pipe_to_parent.send(pass_to_parent)
    logger.debug("Finished running UUID:{0}".format(uuid))
Пример #7
0
 def setUp(self):
     super(TestJobManager, self).setUp()
     self.manager = JobManager(EntryPointManager('hoplite.test_jobs'))
Пример #8
0
 def setUp(self):
     self.manager = EntryPointManager()