def test_get_names_of_hot_installed_package(self):
pip_args = ["install", TEST_ENTRY_POINT_PATH]
run_pip(pip_args)
reload_site_packages()
for syspath in sys.path:
pkg_resources.working_set.add_entry(syspath)
try:
self.manager = EntryPointManager('entry.point.test')
names = self.manager.get_plugin_names()
self.assertIn("entry_point_1", names)
self.assertIn("entry_point_2", names)
except Exception, e:
raise e
def __init__(self,
job_uuid,
name,
config,
api_key,
entry_point_group_name="hoplite.jobs",
port=5000):
"""
@param job_uuid unique identifier for this job
@param name the name of the job, corresponds to the plugin name
@param config dictionary object containing configuration for the
specific job
"""
self.port = port
self.uuid = job_uuid
self.name = name
self.config = config
self._api_key = api_key
self._status = {}
self._process = None
self._started = False
self._killed = False
self._pipe_to_self = None
self._pipe_to_process = None
# TODO: We need this workaround because in tests I create jobs that
# don't have a corresponding loaded entry point
# At some point the tests should be refactored to use jobs that exist
# and we can get rid of this code
module = EntryPointManager().get_plugin_module_by_name(name)
logger_name = module.__name__ if module is not None else name
self._logger = server_logging.get_job_logger(logger_name,
uuid=self.uuid)
self._entry_point_group_name = entry_point_group_name
def create_app(group_name='hoplite.jobs'):
app = Flask(__name__)
app.register_blueprint(site_bp, url_prefix='/')
app.register_blueprint(jobs_bp, url_prefix='/jobs')
app.register_blueprint(job_plugins_bp, url_prefix='/job_plugins')
hoplite.api.helpers.manager = JobManager(EntryPointManager(group_name))
return app
class TestEntryPointManager(unittest2.TestCase):
@classmethod
def setUpClass(cls):
# Install hoplite so that its entry points are added
pip_args = ["install", os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')]
run_pip(pip_args)
reload_site_packages()
@classmethod
def tearDownClass(cls):
pip_args = ["uninstall", "hoplite", "-y"]
run_pip(pip_args)
def setUp(self):
self.manager = EntryPointManager()
def test_get_names_with_default_group_name(self):
names = self.manager.get_plugin_names()
self.assertIn('hoplite.plugins.download_folder_from_ftp_job', names)
self.assertIn('hoplite.plugins.download_network_folder_job', names)
self.assertIn('hoplite.plugins.upload_to_network_folder_job', names)
def test_get_names_of_hot_installed_package(self):
pip_args = ["install", TEST_ENTRY_POINT_PATH]
run_pip(pip_args)
reload_site_packages()
for syspath in sys.path:
pkg_resources.working_set.add_entry(syspath)
try:
self.manager = EntryPointManager('entry.point.test')
names = self.manager.get_plugin_names()
self.assertIn("entry_point_1", names)
self.assertIn("entry_point_2", names)
except Exception, e:
raise e
finally:
def job_wrapper(pipe_to_parent, entry_point_name, config, status_updater,
entry_point_group_name='hoplite.jobs', uuid=''):
"""
A picklable function that is used to start the job. It loads the specified
module and calls run on it with the correct parameters.
In the event of an error occurring in the job, the error is bubbled up to
the highest parent of the job. This is done by encapsulating each exception
into a JobFailedError, which is raised to cause the bubbling action. Since
Exceptions aren't picklable, information from previous exceptions is put
into a dictionary.
This exception bubbling is useful for situations in which jobs are used to
call other jobs. The stack trace for each "level" is saved, and the entire
list of jobs with their respective traces can be displayed at the top level
(where the JobFailedError is handled).
"""
module = EntryPointManager(
entry_point_group_name).get_plugin_module_by_name(entry_point_name)
logger = server_logging.get_job_logger(module.__name__, uuid)
try:
module.run(config, status_updater)
except JobFailedError as e:
logger.error(
"A job raised an exception and it was not caught."
" Address: {0} UUID: {1}".format(e.addr, e.uuid))
logger.error(
"Exception Traceback: {0}".format(
traceback.format_tb(e.traceback_object)))
_, _, tb = sys.exc_info()
traceback_object = tb
exception_dictionary = {
"address": e.addr,
"uuid": e.uuid,
"traceback": e.traceback_object,
"previous_exception": e.previous_exception
}
pass_to_parent = {
"traceback": traceback_object,
"previous_exception": exception_dictionary
}
pipe_to_parent.send(pass_to_parent)
except Exception as e:
except_type, except_class, tb = sys.exc_info()
traceback_object = tb
type_string = str(except_type)
try:
pickled_exception = pickle.dumps(e)
except pickle.PicklingError:
pickled_exception = None
exception_dictionary = {
"type": type_string,
"message": e.message,
"exception_object": pickled_exception
}
pass_to_parent = {
"traceback": pickle.dumps(traceback_object),
"previous_exception": exception_dictionary
}
logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} "
"except class:{3} traceback:{4}".format(
uuid, entry_point_name,
except_type, except_class, traceback.format_tb(tb)))
pipe_to_parent.send(pass_to_parent)
logger.debug("Finished running UUID:{0}".format(uuid))
def job_wrapper(pipe_to_parent,
entry_point_name,
config,
status_updater,
entry_point_group_name='hoplite.jobs',
uuid=''):
"""
A picklable function that is used to start the job. It loads the specified
module and calls run on it with the correct parameters.
In the event of an error occurring in the job, the error is bubbled up to
the highest parent of the job. This is done by encapsulating each exception
into a JobFailedError, which is raised to cause the bubbling action. Since
Exceptions aren't picklable, information from previous exceptions is put
into a dictionary.
This exception bubbling is useful for situations in which jobs are used to
call other jobs. The stack trace for each "level" is saved, and the entire
list of jobs with their respective traces can be displayed at the top level
(where the JobFailedError is handled).
"""
module = EntryPointManager(
entry_point_group_name).get_plugin_module_by_name(entry_point_name)
logger = server_logging.get_job_logger(module.__name__, uuid)
try:
module.run(config, status_updater)
except JobFailedError as e:
logger.error("A job raised an exception and it was not caught."
" Address: {0} UUID: {1}".format(e.addr, e.uuid))
logger.error("Exception Traceback: {0}".format(
traceback.format_tb(e.traceback_object)))
_, _, tb = sys.exc_info()
traceback_object = tb
exception_dictionary = {
"address": e.addr,
"uuid": e.uuid,
"traceback": e.traceback_object,
"previous_exception": e.previous_exception
}
pass_to_parent = {
"traceback": traceback_object,
"previous_exception": exception_dictionary
}
pipe_to_parent.send(pass_to_parent)
except Exception as e:
except_type, except_class, tb = sys.exc_info()
traceback_object = tb
type_string = str(except_type)
try:
pickled_exception = pickle.dumps(e)
except pickle.PicklingError:
pickled_exception = None
exception_dictionary = {
"type": type_string,
"message": e.message,
"exception_object": pickled_exception
}
pass_to_parent = {
"traceback": pickle.dumps(traceback_object),
"previous_exception": exception_dictionary
}
logger.error("Job UUID:{0} Type:{1} Finished with except type:{2} "
"except class:{3} traceback:{4}".format(
uuid, entry_point_name, except_type, except_class,
traceback.format_tb(tb)))
pipe_to_parent.send(pass_to_parent)
logger.debug("Finished running UUID:{0}".format(uuid))
def setUp(self):
super(TestJobManager, self).setUp()
self.manager = JobManager(EntryPointManager('hoplite.test_jobs'))
def setUp(self):
self.manager = EntryPointManager()