def test(self):
asynch_dispatcher = AsyncDispatcher()
asynch_result = [None]
test_exception_message = "testing exception handling"
wait_for_exception = Event()
def test_work():
print "sending " + " hello"
return "hello"
def handler(param):
print "got asynch result: " + param
asynch_result[0] = param
def raise_exception():
raise Exception(test_exception_message)
def handle_exception(ex):
asynch_result[0] = ex
wait_for_exception.set()
task = asynch_dispatcher.run_as_asynch(test_work, handler)
self.assertEqual(task.get_result(), "hello")
self.assertEqual(asynch_result[0], "hello")
task = asynch_dispatcher.run_as_asynch(raise_exception, on_error=handle_exception)
try:
task.get_result()
self.fail("should not reach this line")
except Exception as ex:
# check that the test exception has been thrown
self.assertTrue(test_exception_message in ex.message)
# wait for exception handler to fire
wait_for_exception.wait(timeout=5)
self.assertIsInstance(asynch_result[0], Exception)
self.assertEquals(asynch_result[0].message, test_exception_message)
def __init__(self, environment, config):
self._environment = environment
self._environment.update(
{'rest_api_req_timeout': REST_API_REQ_TIMEOUT})
self._config = config
self._application_registrar = application_registrar.HbaseApplicationRegistrar(
environment['hbase_thrift_server'])
self._application_summary_registrar = application_summary_registrar.HBaseAppplicationSummary(
environment['hbase_thrift_server'])
self._yarn_connection = YarnConnection(self._environment)
self._summary_aggregator = ComponentSummaryAggregator()
self._component_creators = {}
self.dispatcher = AsyncDispatcher(num_threads=4)
def __init__(self, repository, package_registrar, application_registrar,
environment, config):
self._repository = repository
self._package_registrar = package_registrar
self._application_registrar = application_registrar
self._environment = environment
self._config = config
self._application_creator = application_creator.ApplicationCreator(
config, environment, environment['namespace'])
self._package_parser = PackageParser()
self._package_progress = {}
self._lock = threading.RLock()
# load number of threads from config file:
number_of_threads = self._config["deployer_thread_limit"]
assert isinstance(number_of_threads, (int))
assert number_of_threads > 0
self.dispatcher = AsyncDispatcher(num_threads=number_of_threads)
self.rest_client = requests
def test(self):
asynch_dispatcher = AsyncDispatcher()
asynch_result = [None]
test_exception_message = "testing exception handling"
wait_for_exception = Event()
def test_work():
print "sending " + " hello"
return "hello"
def handler(param):
print "got asynch result: " + param
asynch_result[0] = param
def raise_exception():
raise Exception(test_exception_message)
def handle_exception(ex):
asynch_result[0] = ex
wait_for_exception.set()
task = asynch_dispatcher.run_as_asynch(test_work, handler)
self.assertEqual(task.get_result(), "hello")
self.assertEqual(asynch_result[0], "hello")
task = asynch_dispatcher.run_as_asynch(raise_exception,
on_error=handle_exception)
try:
task.get_result()
self.fail("should not reach this line")
except Exception as ex:
# check that the test exception has been thrown
self.assertTrue(test_exception_message in ex.message)
# wait for exception handler to fire
wait_for_exception.wait(timeout=5)
self.assertIsInstance(asynch_result[0], Exception)
self.assertEquals(asynch_result[0].message, test_exception_message)
def __init__(self, repository, package_registrar, application_registrar, environment, config):
self._repository = repository
self._package_registrar = package_registrar
self._application_registrar = application_registrar
self._environment = environment
self._config = config
self._application_creator = application_creator.ApplicationCreator(config, environment,
environment['namespace'])
self._package_parser = PackageParser()
self._package_progress = {}
self._lock = threading.RLock()
# load number of threads from config file:
number_of_threads = self._config["deployer_thread_limit"]
assert isinstance(number_of_threads, (int))
assert number_of_threads > 0
self.dispatcher = AsyncDispatcher(num_threads=number_of_threads)
self.rest_client = requests
class DeploymentManager(object):
def __init__(self, repository, package_registrar, application_registrar,
application_summary_registrar, environment, config):
self._repository = repository
self._package_registrar = package_registrar
self._application_registrar = application_registrar
self._environment = environment
self._config = config
self._application_creator = application_creator.ApplicationCreator(
config, environment, environment['namespace'])
self._application_summary_registrar = application_summary_registrar
self._package_parser = PackageParser()
self._package_progress = {}
self._lock = threading.RLock()
self._authorizer = authorizer_local.AuthorizerLocal()
# load number of threads from config file:
number_of_threads = self._config["deployer_thread_limit"]
assert isinstance(number_of_threads, (int))
assert number_of_threads > 0
self.dispatcher = AsyncDispatcher(num_threads=number_of_threads)
self.rest_client = requests
def _get_groups(self, user):
groups = []
if user:
try:
groups = [
g.gr_name for g in grp.getgrall() if user in g.gr_mem
]
if not pwd.getpwnam(user).pw_gid:
gid = pwd.getpwnam(user).pw_gid
groups.append(grp.getgrgid(gid).gr_name)
except:
raise Forbidden('Failed to find details for user "%s"' % user)
return groups
def _authorize(self, user_name, resource_type, resource_owner,
action_name):
qualified_action = '%s:%s' % (resource_type, action_name)
identity = {'user': user_name, 'groups': self._get_groups(user_name)}
resource = {'type': resource_type, 'owner': resource_owner}
action = {'name': qualified_action}
if not self._authorizer.authorize(identity, resource, action):
raise Forbidden('User "%s" does not have authorization for "%s"' %
(user_name, qualified_action))
def get_environment(self, user_name):
self._authorize(user_name, Resources.ENVIRONMENT, None, Actions.READ)
return self._environment
def list_packages(self, user_name):
self._authorize(user_name, Resources.PACKAGES, None, Actions.READ)
logging.info('list_deployed')
deployed = self._package_registrar.list_packages()
return deployed
def _assert_package_status(self, package, required_status):
status = self.get_package_info(package)['status']
if status != required_status:
if status == PackageDeploymentState.NOTDEPLOYED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
def list_repository(self, recency, user_name):
self._authorize(user_name, Resources.REPOSITORY, None, Actions.READ)
logging.info("list_available: %s", recency)
available = self._repository.get_package_list(user_name, recency)
return available
def _get_saved_package_data(self, package):
package_owner = None
package_exists = False
package_metadata = None
if self._package_registrar.package_exists(package):
package_metadata = self._package_registrar.get_package_metadata(
package)
logging.debug(package_metadata)
package_owner = package_metadata['metadata']['user']
package_exists = True
return package_owner, package_exists, package_metadata
def _get_package_owner(self, package):
package_owner, _, _ = self._get_saved_package_data(package)
return package_owner
def _get_application_owner(self, application):
application_owner = None
if self._application_registrar.application_has_record(application):
application_owner = self._application_registrar.get_application(
application)['overrides']['user']
return application_owner
def get_package_info(self, package, user_name=None):
package_owner, package_exists, metadata = self._get_saved_package_data(
package)
if user_name is not None:
self._authorize(user_name, Resources.PACKAGES, package_owner,
Actions.READ)
information = None
progress_state = self._get_package_progress(package)
if progress_state is not None:
properties = None
status = progress_state
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
else:
# package deploy is not in progress:
# get last package status from database
deploy_status = self._package_registrar.get_package_deploy_status(
package)
if deploy_status:
status = deploy_status["state"]
information = deploy_status["information"]
# check if package data exists in database:
if package_exists:
properties = self._package_parser.properties_from_metadata(
metadata['metadata'])
status = PackageDeploymentState.DEPLOYED
name = metadata['name']
version = metadata['version']
else:
if not deploy_status:
status = PackageDeploymentState.NOTDEPLOYED
properties = None
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
ret = {
"name": name,
"version": version,
"status": status,
"user": package_owner,
"defaults": properties,
"information": information
}
return ret
def _run_asynch_package_task(self, package_name, initial_state,
working_state, task, auth_check):
"""
Manages locks and state reporting for async background operations on packages
:param package_name: The name of the package to operate on
:param initial_state: The state to check before beginning work on the package
:param working_state: The state to set while the package operation is being carried out.
:param task: The actual work to be carried out
"""
with self._lock:
# check that package is in the right state before starting operation:
self._assert_package_status(package_name, initial_state)
auth_check()
# set the operation state before starting:
self._set_package_progress(package_name, working_state)
# this will be run in the background while taking care to release all locks and intermediate states:
def do_work_and_report_progress():
try:
# report beginning of work to external APIs:
self._state_change_event_package(package_name)
# do the actual work:
task()
finally:
# release the lock on the package:
self._clear_package_progress(package_name)
# report completion to external APIs
self._state_change_event_package(package_name)
# run everything on a background thread:
self.dispatcher.run_as_asynch(task=do_work_and_report_progress)
def deploy_package(self, package, user_name):
def auth_check():
self._authorize(user_name, Resources.PACKAGE, None, Actions.DEPLOY)
# this function will be executed in the background:
def _do_deploy():
# if this value is not changed, then it is assumed that the operation never completed
package_data_path = None
try:
package_file = package + '.tar.gz'
logging.info("deploy: %s", package)
# download package:
package_data_path = self._repository.get_package(
package_file, user_name)
# put package in database:
metadata = self._package_parser.get_package_metadata(
package_data_path)
self._application_creator.validate_package(package, metadata)
self._package_registrar.set_package(package, package_data_path,
user_name)
# set the operation status as complete
deploy_status = {
"state":
PackageDeploymentState.DEPLOYED,
"information":
"Deployed " + package + " at " + self.utc_string()
}
logging.info("deployed: %s", package)
except Exception as ex:
logging.error(str(ex))
error_message = "Error deploying " + package + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
deploy_status = {
"state": PackageDeploymentState.NOTDEPLOYED,
"information": error_message
}
raise
finally:
# report final state of operation to database:
self._package_registrar.set_package_deploy_status(
package, deploy_status)
if package_data_path is not None:
os.remove(package_data_path)
# schedule work to be done in the background:
self._run_asynch_package_task(
package_name=package,
initial_state=PackageDeploymentState.NOTDEPLOYED,
working_state=PackageDeploymentState.DEPLOYING,
task=_do_deploy,
auth_check=auth_check)
def utc_string(self):
return datetime.datetime.utcnow().isoformat()
def undeploy_package(self, package, user_name):
def auth_check():
package_owner = self._get_package_owner(package)
self._authorize(user_name, Resources.PACKAGE, package_owner,
Actions.UNDEPLOY)
# this function will be executed in the background:
def do_undeploy():
deploy_status = None
try:
logging.info("undeploy: %s", package)
self._package_registrar.delete_package(package)
logging.info("undeployed: %s", package)
except Exception as ex:
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error undeploying " + package + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
deploy_status = {
"state": PackageDeploymentState.DEPLOYED,
"information": error_message
}
raise
finally:
if deploy_status is not None:
# persist any errors in the database, but still throw them:
self._package_registrar.set_package_deploy_status(
package, deploy_status)
# schedule work to be done in the background:
self._run_asynch_package_task(
package_name=package,
initial_state=PackageDeploymentState.DEPLOYED,
working_state=PackageDeploymentState.UNDEPLOYING,
task=do_undeploy,
auth_check=auth_check)
def _set_package_progress(self, package_name, state):
"""
Marks the progress of background operations being run on the app.
:param package_name: the name of the package to be modified
:param state: the state of the background operation
"""
# currently we are using multiple threads, so this lock is added for thread saftey
with self._lock:
self._package_progress[package_name] = state
def _get_package_progress(self, package_name):
"""
:param package_name: The name of the package for which to query progress
:return: the state of the package
"""
with self._lock:
if self._is_package_in_progress(package_name):
return self._package_progress[package_name]
return None
def _is_package_in_progress(self, package_name):
"""
checks if the current package has an operation in progress
:param package_name: the name of the package to check
:return: true if the package is currently being operated on
"""
with self._lock:
return package_name in self._package_progress
def _clear_package_progress(self, package):
with self._lock:
self._package_progress.pop(package, None)
def _mark_destroying(self, package):
self._set_package_progress(package, ApplicationState.DESTROYING)
def _mark_creating(self, package):
self._set_package_progress(package, ApplicationState.CREATING)
def _mark_starting(self, package):
self._set_package_progress(package, ApplicationState.STARTING)
def _mark_stopping(self, package):
self._set_package_progress(package, ApplicationState.STOPPING)
def list_package_applications(self, package, user_name):
self._authorize(user_name, Resources.APPLICATIONS, None, Actions.READ)
logging.info('list_package_applications')
applications = self._application_registrar.list_applications_for_package(
package)
return applications
def list_applications(self, user_name):
self._authorize(user_name, Resources.APPLICATIONS, None, Actions.READ)
logging.info('list_applications')
applications = self._application_registrar.list_applications()
return applications
def _assert_application_status(self, application, required_status):
logging.debug("Checking %s is %s", application,
json.dumps(required_status))
app_info = self.get_application_info(application)
status = app_info['status']
logging.debug("Found %s is %s", application, status)
if (isinstance(required_status, list) and status not in required_status) \
or (not isinstance(required_status, list) and status != required_status):
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
logging.debug("Status for %s is OK", application)
def _assert_application_exists(self, application):
status = self.get_application_info(application)['status']
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
def start_application(self, application, user_name):
logging.info('start_application')
with self._lock:
self._assert_application_status(application,
ApplicationState.CREATED)
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION,
application_owner, Actions.START)
self._mark_starting(application)
def do_work_start():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.start_application(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.STARTED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.CREATED,
"starting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work_start)
def stop_application(self, application, user_name):
logging.info('stop_application')
with self._lock:
self._assert_application_status(application,
ApplicationState.STARTED)
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION,
application_owner, Actions.STOP)
self._mark_stopping(application)
def do_work_stop():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.stop_application(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.STARTED,
"stopping")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work_stop)
def get_application_info(self, application, user_name=None):
if user_name is not None:
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION,
application_owner, Actions.READ)
logging.info('get_application_info')
if not self._application_registrar.application_has_record(application):
record = {
'status': ApplicationState.NOTCREATED,
'information': None
}
else:
record = self._application_registrar.get_application(application)
progress_state = self._get_package_progress(application)
if progress_state is not None:
record['status'] = progress_state
return record
def get_application_detail(self, application, user_name):
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION, application_owner,
Actions.READ)
logging.info('get_application_detail')
self._assert_application_exists(application)
create_data = self._application_registrar.get_create_data(application)
record = self._application_creator.get_application_runtime_details(
application, create_data)
record['status'] = self.get_application_info(application)['status']
record['name'] = application
return record
def get_application_summary(self, application, user_name):
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION, application_owner,
Actions.READ)
logging.info('get_application_summary')
record = self._application_summary_registrar.get_summary_data(
application)
return record
# XXXX
def get_pod_logs(self, pod_name, namespace_id):
config.load_incluster_config()
logging.info('Inside pod log121 *******' + pod_name)
try:
configuration = client.Configuration()
api_client = client.ApiClient(configuration)
api_instance = client.CoreV1Api(api_client)
api_response = api_instance.read_namespaced_pod_log(
name=str(pod_name) + "-driver", namespace=namespace_id)
logging.info('Inside pod log *******' + pod_name)
logging.info(api_response)
return api_response
except ApiException as e:
print('Exception in getting status')
def get_pod_state(self, pod_name, namespace_id):
config.load_incluster_config()
logging.info('Inside pod state before try *******' + pod_name)
try:
configuration = client.Configuration()
api_client = client.ApiClient(configuration)
api_instance = client.CoreV1Api(api_client)
api_response_state = api_instance.read_namespaced_pod_status(
name=str(pod_name) + "-driver", namespace=namespace_id)
logging.info('Inside pod state *******' + pod_name)
return api_response_state.status.phase
except ApiException as e:
print('Exception in getting status')
def get_application_log(self, application, user_name):
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION, application_owner,
Actions.READ)
logging.info('get_application_log')
record = self.get_pod_logs(application, 'pnda')
return record
def get_application_state(self, application, user_name):
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION, application_owner,
Actions.READ)
logging.info('get_application_state')
record = self.get_pod_state(application, 'pnda')
return record
# XXXX
def create_application(self, package, application, overrides, user_name):
logging.info('create_application')
package_data_path = None
with self._lock:
self._assert_application_status(application,
ApplicationState.NOTCREATED)
self._assert_package_status(package,
PackageDeploymentState.DEPLOYED)
package_owner = self._get_application_owner(package)
self._authorize(user_name, Resources.PACKAGE, package_owner,
Actions.READ)
self._authorize(user_name, Resources.APPLICATION, None,
Actions.CREATE)
defaults = self.get_package_info(package)['defaults']
self._application_creator.assert_application_properties(
overrides, defaults)
package_data_path = self._package_registrar.get_package_data(
package)
self._application_registrar.create_application(
package, application, overrides, defaults)
self._mark_creating(application)
def do_work_create():
try:
self._state_change_event_application(application)
try:
package_metadata = self._package_registrar.get_package_metadata(
package)['metadata']
create_data = self._application_creator.create_application(
package_data_path, package_metadata, application,
overrides)
self._application_registrar.set_create_data(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.NOTCREATED,
"creating")
logging.error(traceback.format_exc(ex))
raise
finally:
# clear inner locks:
self._clear_package_progress(application)
self._state_change_event_application(application)
if package_data_path is not None:
os.remove(package_data_path)
self.dispatcher.run_as_asynch(task=do_work_create)
def _handle_application_error(self, application, ex, app_status,
operation):
"""
Use to handle application exceptions which should be relayed back to the user
Sets the application state to an error
:param application: The app for which to set the error
:param ex: The error
:param app_status: The status the app should be at following the error.
"""
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error %s " % operation + application + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
self._application_registrar.set_application_status(
application, app_status, error_message)
def delete_application(self, application, user_name):
logging.info('delete_application')
with self._lock:
self._assert_application_status(
application,
[ApplicationState.CREATED, ApplicationState.STARTED])
application_owner = self._get_application_owner(application)
self._authorize(user_name, Resources.APPLICATION,
application_owner, Actions.DESTROY)
self._mark_destroying(application)
def do_work_delete():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.destroy_application(
application, create_data)
self._application_registrar.delete_application(application)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.STARTED,
"deleting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work_delete)
def _state_change_event_application(self, name):
endpoint_type = "application_callback"
info = self.get_application_info(name)
self._state_change_event(name, endpoint_type, info['status'],
info['information'])
def _state_change_event_package(self, name):
endpoint_type = "package_callback"
info = self.get_package_info(name)
self._state_change_event(name, endpoint_type, info['status'],
info['information'])
def _state_change_event(self, name, endpoint_type, state, information):
callback_url = self._config[endpoint_type]
if callback_url:
logging.debug("callback: %s %s %s", endpoint_type, name, state)
callback_payload = {
"data": [{
"id": name,
"state": state,
"timestamp": milli_time()
}],
"timestamp": milli_time()
}
# add additional optional information
if information:
callback_payload["data"][0]["information"] = information
logging.debug(callback_payload)
self.rest_client.post(callback_url, json=callback_payload)
class DeploymentManager(object):
def __init__(self, repository, package_registrar, application_registrar, environment, config):
self._repository = repository
self._package_registrar = package_registrar
self._application_registrar = application_registrar
self._environment = environment
self._config = config
self._application_creator = application_creator.ApplicationCreator(config, environment,
environment['namespace'])
self._package_parser = PackageParser()
self._package_progress = {}
self._lock = threading.RLock()
# load number of threads from config file:
number_of_threads = self._config["deployer_thread_limit"]
assert isinstance(number_of_threads, (int))
assert number_of_threads > 0
self.dispatcher = AsyncDispatcher(num_threads=number_of_threads)
self.rest_client = requests
def get_environment(self):
return self._environment
def list_packages(self):
logging.info('list_deployed')
deployed = self._package_registrar.list_packages()
return deployed
def _assert_package_status(self, package, required_status):
status = self.get_package_info(package)['status']
if status != required_status:
if status == PackageDeploymentState.NOTDEPLOYED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
def list_repository(self, recency):
logging.info("list_available: %s", recency)
available = self._repository.get_package_list(recency)
return available
def get_package_info(self, package):
information = None
progress_state = self._get_package_progress(package)
if progress_state is not None:
properties = None
status = progress_state
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
else:
# package deploy is not in progress:
# get last package status from database
deploy_status = self._package_registrar.get_package_deploy_status(package)
if deploy_status:
status = deploy_status["state"]
information = deploy_status["information"]
# check if package data exists in database:
if self._package_registrar.package_exists(package):
metadata = self._package_registrar.get_package_metadata(package)
properties = self._package_parser.properties_from_metadata(metadata['metadata'])
status = PackageDeploymentState.DEPLOYED
name = metadata['name']
version = metadata['version']
else:
if not deploy_status:
status = PackageDeploymentState.NOTDEPLOYED
properties = None
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
ret = {"name": name,
"version": version,
"status": status,
"defaults": properties,
"information": information}
return ret
def _run_asynch_package_task(self, package_name, initial_state, working_state, task):
"""
Manages locks and state reporting for async background operations on packages
:param package_name: The name of the package to operate on
:param initial_state: The state to check before beginning work on the package
:param working_state: The state to set while the package operation is being carried out.
:param task: The actual work to be carried out
"""
with self._lock:
# check that package is in the right state before starting operation:
self._assert_package_status(package_name, initial_state)
# set the operation state before starting:
self._set_package_progress(package_name, working_state)
# this will be run in the background while taking care to release all locks and intermediate states:
def do_work_and_report_progress():
try:
# report beginning of work to external APIs:
self._state_change_event_package(package_name)
# do the actual work:
task()
finally:
# release the lock on the package:
self._clear_package_progress(package_name)
# report completion to external APIs
self._state_change_event_package(package_name)
# run everything on a background thread:
self.dispatcher.run_as_asynch(task=do_work_and_report_progress)
def deploy_package(self, package):
# this function will be executed in the background:
def _do_deploy():
# if this value is not changed, then it is assumed that the operation never completed
try:
package_file = package + '.tar.gz'
logging.info("deploy: %s", package)
# download package:
package_data_path = self._repository.get_package(package_file)
# put package in database:
metadata = self._package_parser.get_package_metadata(package_data_path)
self._application_creator.validate_package(package, metadata)
self._package_registrar.set_package(package, package_data_path)
# set the operation status as complete
deploy_status = {"state": PackageDeploymentState.DEPLOYED,
"information": "Deployed " + package + " at " + self.utc_string()}
logging.info("deployed: %s", package)
except Exception as ex:
logging.error(str(ex))
error_message = "Error deploying " + package + " " + str(type(ex).__name__) + ", details: " + json.dumps(str(ex))
deploy_status = {"state": PackageDeploymentState.NOTDEPLOYED, "information": error_message}
raise
finally:
# report final state of operation to database:
self._package_registrar.set_package_deploy_status(package, deploy_status)
os.remove(package_data_path)
# schedule work to be done in the background:
self._run_asynch_package_task(package_name=package,
initial_state=PackageDeploymentState.NOTDEPLOYED,
working_state=PackageDeploymentState.DEPLOYING,
task=_do_deploy)
def utc_string(self):
return datetime.datetime.utcnow().isoformat()
def undeploy_package(self, package):
# this function will be executed in the background:
def do_undeploy():
deploy_status = None
try:
logging.info("undeploy: %s", package)
self._package_registrar.delete_package(package)
logging.info("undeployed: %s", package)
except Exception as ex:
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error undeploying " + package + " " + str(type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
deploy_status = {"state": PackageDeploymentState.DEPLOYED, "information": error_message}
raise
finally:
if deploy_status is not None:
# persist any errors in the database, but still throw them:
self._package_registrar.set_package_deploy_status(package, deploy_status)
# schedule work to be done in the background:
self._run_asynch_package_task(package_name=package,
initial_state=PackageDeploymentState.DEPLOYED,
working_state=PackageDeploymentState.UNDEPLOYING,
task=do_undeploy)
def _set_package_progress(self, package_name, state):
"""
Marks the progress of background operations being run on the app.
:param package_name: the name of the package to be modified
:param state: the state of the background operation
"""
# currently we are using multiple threads, so this lock is added for thread saftey
with self._lock:
self._package_progress[package_name] = state
def _get_package_progress(self, package_name):
"""
:param package_name: The name of the package for which to query progress
:return: the state of the package
"""
with self._lock:
if self._is_package_in_progress(package_name):
return self._package_progress[package_name]
return None
def _is_package_in_progress(self, package_name):
"""
checks if the current package has an operation in progress
:param package_name: the name of the package to check
:return: true if the package is currently being operated on
"""
with self._lock:
return package_name in self._package_progress
def _clear_package_progress(self, package):
with self._lock:
self._package_progress.pop(package, None)
def _mark_destroying(self, package):
self._set_package_progress(package, ApplicationState.DESTROYING)
def _mark_creating(self, package):
self._set_package_progress(package, ApplicationState.CREATING)
def _mark_starting(self, package):
self._set_package_progress(package, ApplicationState.STARTING)
def _mark_stopping(self, package):
self._set_package_progress(package, ApplicationState.STOPPING)
def list_package_applications(self, package):
logging.info('list_package_applications')
applications = self._application_registrar.list_applications_for_package(package)
return applications
def list_applications(self):
logging.info('list_applications')
applications = self._application_registrar.list_applications()
return applications
def _assert_application_status(self, application, required_status):
app_info = self.get_application_info(application)
status = app_info['status']
if (isinstance(required_status, list) and status not in required_status) \
or (not isinstance(required_status, list) and status != required_status):
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
def _assert_application_exists(self, application):
status = self.get_application_info(application)['status']
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
def start_application(self, application):
logging.info('start_application')
with self._lock:
self._assert_application_status(application, ApplicationState.CREATED)
self._mark_starting(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(application)
self._application_creator.start_application(application, create_data)
self._application_registrar.set_application_status(application, ApplicationState.STARTED)
except Exception as ex:
self._handle_application_error(application, ex, ApplicationState.CREATED, "starting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def stop_application(self, application):
logging.info('stop_application')
with self._lock:
self._assert_application_status(application, ApplicationState.STARTED)
self._mark_stopping(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(application)
self._application_creator.stop_application(application, create_data)
self._application_registrar.set_application_status(application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex, ApplicationState.STARTED, "stopping")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def get_application_info(self, application):
logging.info('get_application_info')
if not self._application_registrar.application_has_record(application):
record = {'status': ApplicationState.NOTCREATED, 'information': None}
else:
record = self._application_registrar.get_application(application)
progress_state = self._get_package_progress(application)
if progress_state is not None:
record['status'] = progress_state
return record
def get_application_detail(self, application):
logging.info('get_application_detail')
self._assert_application_exists(application)
create_data = self._application_registrar.get_create_data(application)
record = self._application_creator.get_application_runtime_details(application, create_data)
record['status'] = self.get_application_info(application)['status']
record['name'] = application
return record
def create_application(self, package, application, overrides):
logging.info('create_application')
with self._lock:
self._assert_application_status(application, ApplicationState.NOTCREATED)
self._assert_package_status(package, PackageDeploymentState.DEPLOYED)
defaults = self.get_package_info(package)['defaults']
package_data_path = self._package_registrar.get_package_data(package)
self._application_registrar.create_application(package, application, overrides, defaults)
self._mark_creating(application)
def do_work():
try:
self._state_change_event_application(application)
try:
package_metadata = self._package_registrar.get_package_metadata(package)['metadata']
create_data = self._application_creator.create_application(
package_data_path, package_metadata, application, overrides)
self._application_registrar.set_create_data(application, create_data)
self._application_registrar.set_application_status(application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex, ApplicationState.NOTCREATED, "creating")
logging.error(traceback.format_exc(ex))
raise
finally:
# clear inner locks:
self._clear_package_progress(application)
self._state_change_event_application(application)
os.remove(package_data_path)
self.dispatcher.run_as_asynch(task=do_work)
def _handle_application_error(self, application, ex, app_status, operation):
"""
Use to handle application exceptions which should be relayed back to the user
Sets the application state to an error
:param application: The app for which to set the error
:param ex: The error
:param app_status: The status the app should be at following the error.
"""
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error %s " % operation + application + " " + str(type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
self._application_registrar.set_application_status(application, app_status, error_message)
def delete_application(self, application):
logging.info('delete_application')
with self._lock:
self._assert_application_status(application, [ApplicationState.CREATED, ApplicationState.STARTED])
self._mark_destroying(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(application)
self._application_creator.destroy_application(application, create_data)
self._application_registrar.delete_application(application)
except Exception as ex:
self._handle_application_error(application, ex, ApplicationState.STARTED, "deleting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def _state_change_event_application(self, name):
endpoint_type = "application_callback"
info = self.get_application_info(name)
self._state_change_event(name, endpoint_type, info['status'], info['information'])
def _state_change_event_package(self, name):
endpoint_type = "package_callback"
info = self.get_package_info(name)
self._state_change_event(name, endpoint_type, info['status'], info['information'])
def _state_change_event(self, name, endpoint_type, state, information):
callback_url = self._config[endpoint_type]
if callback_url:
logging.debug("callback: %s %s %s", endpoint_type, name, state)
callback_payload = {
"data": [
{
"id": name,
"state": state,
"timestamp": milli_time()
}
],
"timestamp": milli_time()
}
# add additional optional information
if information:
callback_payload["data"][0]["information"] = information
logging.debug(callback_payload)
self.rest_client.post(callback_url, json=callback_payload)
def send_accepted(self):
def finish():
self.set_status(202)
self.finish()
IOLoop.instance().add_callback(callback=finish)
def send_client_error(self, msg):
def finish():
self.set_status(400)
self.finish(msg)
IOLoop.instance().add_callback(callback=finish)
DISPATCHER = AsyncDispatcher(num_threads=10)
class SelfTestHandler(BaseHandler):
@asynchronous
def get(self):
def do_call():
self.send_result(DeployerRestClientTester().run_tests())
DISPATCHER.run_as_asynch(task=do_call, on_error=self.handle_error)
class EnvironmentHandler(BaseHandler):
@asynchronous
def get(self):
def do_call():
class DeploymentManager(object):
def __init__(self, repository, package_registrar, application_registrar,
environment, config):
self._repository = repository
self._package_registrar = package_registrar
self._application_registrar = application_registrar
self._environment = environment
self._config = config
self._application_creator = application_creator.ApplicationCreator(
config, environment, environment['namespace'])
self._package_parser = PackageParser()
self._package_progress = {}
self._lock = threading.RLock()
# load number of threads from config file:
number_of_threads = self._config["deployer_thread_limit"]
assert isinstance(number_of_threads, (int))
assert number_of_threads > 0
self.dispatcher = AsyncDispatcher(num_threads=number_of_threads)
self.rest_client = requests
def get_environment(self):
return self._environment
def list_packages(self):
logging.info('list_deployed')
deployed = self._package_registrar.list_packages()
return deployed
def _assert_package_status(self, package, required_status):
status = self.get_package_info(package)['status']
if status != required_status:
if status == PackageDeploymentState.NOTDEPLOYED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
def list_repository(self, recency):
logging.info("list_available: %s", recency)
available = self._repository.get_package_list(recency)
return available
def get_package_info(self, package):
information = None
progress_state = self._get_package_progress(package)
if progress_state is not None:
properties = None
status = progress_state
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
else:
# package deploy is not in progress:
# get last package status from database
deploy_status = self._package_registrar.get_package_deploy_status(
package)
if deploy_status:
status = deploy_status["state"]
information = deploy_status["information"]
# check if package data exists in database:
if self._package_registrar.package_exists(package):
metadata = self._package_registrar.get_package_metadata(
package)
properties = self._package_parser.properties_from_metadata(
metadata['metadata'])
status = PackageDeploymentState.DEPLOYED
name = metadata['name']
version = metadata['version']
else:
if not deploy_status:
status = PackageDeploymentState.NOTDEPLOYED
properties = None
name = package.rpartition('-')[0]
version = package.rpartition('-')[2]
ret = {
"name": name,
"version": version,
"status": status,
"defaults": properties,
"information": information
}
return ret
def _run_asynch_package_task(self, package_name, initial_state,
working_state, task):
"""
Manages locks and state reporting for async background operations on packages
:param package_name: The name of the package to operate on
:param initial_state: The state to check before beginning work on the package
:param working_state: The state to set while the package operation is being carried out.
:param task: The actual work to be carried out
"""
with self._lock:
# check that package is in the right state before starting operation:
self._assert_package_status(package_name, initial_state)
# set the operation state before starting:
self._set_package_progress(package_name, working_state)
# this will be run in the background while taking care to release all locks and intermediate states:
def do_work_and_report_progress():
try:
# report beginning of work to external APIs:
self._state_change_event_package(package_name)
# do the actual work:
task()
finally:
# release the lock on the package:
self._clear_package_progress(package_name)
# report completion to external APIs
self._state_change_event_package(package_name)
# run everything on a background thread:
self.dispatcher.run_as_asynch(task=do_work_and_report_progress)
def deploy_package(self, package):
# this function will be executed in the background:
def _do_deploy():
# if this value is not changed, then it is assumed that the operation never completed
try:
package_file = package + '.tar.gz'
logging.info("deploy: %s", package)
# download package:
package_data_path = self._repository.get_package(package_file)
# put package in database:
metadata = self._package_parser.get_package_metadata(
package_data_path)
self._application_creator.validate_package(package, metadata)
self._package_registrar.set_package(package, package_data_path)
# set the operation status as complete
deploy_status = {
"state":
PackageDeploymentState.DEPLOYED,
"information":
"Deployed " + package + " at " + self.utc_string()
}
logging.info("deployed: %s", package)
except Exception as ex:
logging.error(str(ex))
error_message = "Error deploying " + package + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
deploy_status = {
"state": PackageDeploymentState.NOTDEPLOYED,
"information": error_message
}
raise
finally:
# report final state of operation to database:
self._package_registrar.set_package_deploy_status(
package, deploy_status)
os.remove(package_data_path)
# schedule work to be done in the background:
self._run_asynch_package_task(
package_name=package,
initial_state=PackageDeploymentState.NOTDEPLOYED,
working_state=PackageDeploymentState.DEPLOYING,
task=_do_deploy)
def utc_string(self):
return datetime.datetime.utcnow().isoformat()
def undeploy_package(self, package):
# this function will be executed in the background:
def do_undeploy():
deploy_status = None
try:
logging.info("undeploy: %s", package)
self._package_registrar.delete_package(package)
logging.info("undeployed: %s", package)
except Exception as ex:
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error undeploying " + package + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
deploy_status = {
"state": PackageDeploymentState.DEPLOYED,
"information": error_message
}
raise
finally:
if deploy_status is not None:
# persist any errors in the database, but still throw them:
self._package_registrar.set_package_deploy_status(
package, deploy_status)
# schedule work to be done in the background:
self._run_asynch_package_task(
package_name=package,
initial_state=PackageDeploymentState.DEPLOYED,
working_state=PackageDeploymentState.UNDEPLOYING,
task=do_undeploy)
def _set_package_progress(self, package_name, state):
"""
Marks the progress of background operations being run on the app.
:param package_name: the name of the package to be modified
:param state: the state of the background operation
"""
# currently we are using multiple threads, so this lock is added for thread saftey
with self._lock:
self._package_progress[package_name] = state
def _get_package_progress(self, package_name):
"""
:param package_name: The name of the package for which to query progress
:return: the state of the package
"""
with self._lock:
if self._is_package_in_progress(package_name):
return self._package_progress[package_name]
return None
def _is_package_in_progress(self, package_name):
"""
checks if the current package has an operation in progress
:param package_name: the name of the package to check
:return: true if the package is currently being operated on
"""
with self._lock:
return package_name in self._package_progress
def _clear_package_progress(self, package):
with self._lock:
self._package_progress.pop(package, None)
def _mark_destroying(self, package):
self._set_package_progress(package, ApplicationState.DESTROYING)
def _mark_creating(self, package):
self._set_package_progress(package, ApplicationState.CREATING)
def _mark_starting(self, package):
self._set_package_progress(package, ApplicationState.STARTING)
def _mark_stopping(self, package):
self._set_package_progress(package, ApplicationState.STOPPING)
def list_package_applications(self, package):
logging.info('list_package_applications')
applications = self._application_registrar.list_applications_for_package(
package)
return applications
def list_applications(self):
logging.info('list_applications')
applications = self._application_registrar.list_applications()
return applications
def _assert_application_status(self, application, required_status):
app_info = self.get_application_info(application)
status = app_info['status']
if (isinstance(required_status, list) and status not in required_status) \
or (not isinstance(required_status, list) and status != required_status):
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
else:
raise ConflictingState(json.dumps({'status': status}))
def _assert_application_exists(self, application):
status = self.get_application_info(application)['status']
if status == ApplicationState.NOTCREATED:
raise NotFound(json.dumps({'status': status}))
def start_application(self, application):
logging.info('start_application')
with self._lock:
self._assert_application_status(application,
ApplicationState.CREATED)
self._mark_starting(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.start_application(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.STARTED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.CREATED,
"starting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def stop_application(self, application):
logging.info('stop_application')
with self._lock:
self._assert_application_status(application,
ApplicationState.STARTED)
self._mark_stopping(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.stop_application(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.STARTED,
"stopping")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def get_application_info(self, application):
logging.info('get_application_info')
if not self._application_registrar.application_has_record(application):
record = {
'status': ApplicationState.NOTCREATED,
'information': None
}
else:
record = self._application_registrar.get_application(application)
progress_state = self._get_package_progress(application)
if progress_state is not None:
record['status'] = progress_state
return record
def get_application_detail(self, application):
logging.info('get_application_detail')
self._assert_application_exists(application)
create_data = self._application_registrar.get_create_data(application)
record = self._application_creator.get_application_runtime_details(
application, create_data)
record['status'] = self.get_application_info(application)['status']
record['name'] = application
return record
def create_application(self, package, application, overrides):
logging.info('create_application')
with self._lock:
self._assert_application_status(application,
ApplicationState.NOTCREATED)
self._assert_package_status(package,
PackageDeploymentState.DEPLOYED)
defaults = self.get_package_info(package)['defaults']
package_data_path = self._package_registrar.get_package_data(
package)
self._application_registrar.create_application(
package, application, overrides, defaults)
self._mark_creating(application)
def do_work():
try:
self._state_change_event_application(application)
try:
package_metadata = self._package_registrar.get_package_metadata(
package)['metadata']
create_data = self._application_creator.create_application(
package_data_path, package_metadata, application,
overrides)
self._application_registrar.set_create_data(
application, create_data)
self._application_registrar.set_application_status(
application, ApplicationState.CREATED)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.NOTCREATED,
"creating")
logging.error(traceback.format_exc(ex))
raise
finally:
# clear inner locks:
self._clear_package_progress(application)
self._state_change_event_application(application)
os.remove(package_data_path)
self.dispatcher.run_as_asynch(task=do_work)
def _handle_application_error(self, application, ex, app_status,
operation):
"""
Use to handle application exceptions which should be relayed back to the user
Sets the application state to an error
:param application: The app for which to set the error
:param ex: The error
:param app_status: The status the app should be at following the error.
"""
# log error to screen:
logging.error(str(ex))
# prepare human readable message
error_message = "Error %s " % operation + application + " " + str(
type(ex).__name__) + ", details: " + json.dumps(str(ex))
# set the status:
self._application_registrar.set_application_status(
application, app_status, error_message)
def delete_application(self, application):
logging.info('delete_application')
with self._lock:
self._assert_application_status(
application,
[ApplicationState.CREATED, ApplicationState.STARTED])
self._mark_destroying(application)
def do_work():
try:
self._state_change_event_application(application)
try:
create_data = self._application_registrar.get_create_data(
application)
self._application_creator.destroy_application(
application, create_data)
self._application_registrar.delete_application(application)
except Exception as ex:
self._handle_application_error(application, ex,
ApplicationState.STARTED,
"deleting")
raise
finally:
self._clear_package_progress(application)
self._state_change_event_application(application)
self.dispatcher.run_as_asynch(task=do_work)
def _state_change_event_application(self, name):
endpoint_type = "application_callback"
info = self.get_application_info(name)
self._state_change_event(name, endpoint_type, info['status'],
info['information'])
def _state_change_event_package(self, name):
endpoint_type = "package_callback"
info = self.get_package_info(name)
self._state_change_event(name, endpoint_type, info['status'],
info['information'])
def _state_change_event(self, name, endpoint_type, state, information):
callback_url = self._config[endpoint_type]
if callback_url:
logging.debug("callback: %s %s %s", endpoint_type, name, state)
callback_payload = {
"data": [{
"id": name,
"state": state,
"timestamp": milli_time()
}],
"timestamp": milli_time()
}
# add additional optional information
if information:
callback_payload["data"][0]["information"] = information
logging.debug(callback_payload)
self.rest_client.post(callback_url, json=callback_payload)
distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Purpose: Runs blocking tornado RequestHandlers on a separate threadpool.
"""
import logging
import traceback
import tornado.ioloop
from tornado.web import asynchronous, HTTPError
from async_dispatcher import AsyncDispatcher
# the maximum number of requests to handle concurrently
MAX_CONCURRENT_REQUESTS = 20
REQUEST_HANDLER_THREAD_POOL = AsyncDispatcher(
num_threads=MAX_CONCURRENT_REQUESTS)
def asynchronize_tornado_handler(handler_class):
"""
A helper function to turn a blocking handler into an async call
:param handler_class: a tornado RequestHandler which should be made asynchronus
:return: a class which does the same work on a threadpool
"""
class AsyncTornadoHelper(handler_class):
"""
A hollow wrapper class which runs requests asynchronously on a threadpool
"""
def _do_work_and_report_error(self, work):
try:
# call the "real" method from the handler_class
class ApplicationDetailedSummary(object):
def __init__(self, environment, config):
self._environment = environment
self._environment.update(
{'rest_api_req_timeout': REST_API_REQ_TIMEOUT})
self._config = config
self._application_registrar = application_registrar.HbaseApplicationRegistrar(
environment['hbase_thrift_server'])
self._application_summary_registrar = application_summary_registrar.HBaseAppplicationSummary(
environment['hbase_thrift_server'])
self._yarn_connection = YarnConnection(self._environment)
self._summary_aggregator = ComponentSummaryAggregator()
self._component_creators = {}
self.dispatcher = AsyncDispatcher(num_threads=4)
def generate(self):
"""
Update applications detailed summary
"""
applist = self._application_registrar.list_applications()
logging.info("List of applications: %s", ', '.join(applist))
self._application_summary_registrar.sync_with_dm(applist)
apps_to_be_processed = {}
for app in applist:
apps_to_be_processed.update({app: self.generate_summary(app)})
wait_time = 0
# waiting block for all the application to get completed
while len(apps_to_be_processed) != 0:
for app_name in apps_to_be_processed.keys():
try:
apps_to_be_processed[app_name].task.get(STATUS_INTERVAL) #
del apps_to_be_processed[app_name]
except ThreadTimeoutError:
wait_time += STATUS_INTERVAL # increasing the wait time by status interval
if round(wait_time, 1) % MAX_APP_SUMMARY_TIMEOUT == 0:
# logging out list of applications whose wait time exceeds the max app summary timeout, on the interval of same max app summary timeout
# i.e. every 60 seconds as per current max app summary timeout
logging.error(
"Timeout exceeded, %s applications waiting for %d seconds",
(',').join(apps_to_be_processed.keys()),
int(wait_time))
def generate_summary(self, application):
"""
Update HBase wih recent application summary
"""
def _do_generate():
try:
create_data = self._application_registrar.get_create_data(
application)
input_data = {}
for component_name, component_data in create_data.items():
input_data[component_name] = {}
input_data[component_name][
"component_ref"] = self._load_creator(component_name)
input_data[component_name][
"component_data"] = component_data
app_data = self._summary_aggregator.get_application_summary(
application, input_data)
self._application_summary_registrar.post_to_hbase(
app_data, application)
logging.debug("Application: %s, Status: %s", application,
app_data[application]['aggregate_status'])
except Exception as ex:
logging.error(
'%s while trying to get status of application "%s"',
str(ex), application)
return self.dispatcher.run_as_asynch(task=_do_generate)
def _load_creator(self, component_type):
creator = self._component_creators.get(component_type)
if creator is None:
cls = '%s%sComponentSummary' % (component_type[0].upper(),
component_type[1:])
try:
module = import_module("plugins_summary.%s" % component_type)
self._component_creators[component_type] = getattr(module, cls)\
(self._environment, self._yarn_connection, self._application_summary_registrar)
creator = self._component_creators[component_type]
except ImportError as exception:
logging.error(
'Unable to load Creator for component type "%s" [%s]',
component_type, exception)
return creator
