class ApplicationCreator(object):
def __init__(self, config, environment, service):
self._config = config
self._environment = environment
self._service = service
self._component_creators = {}
self._name_regex = re.compile('')
self._hdfs_client = HDFS(environment['webhdfs_host'],
environment['webhdfs_port'], 'hdfs')
def assert_application_properties(self, override_properties,
default_properties):
for component_type, component_properties in default_properties.items():
creator = self._load_creator(component_type)
creator.assert_application_properties(
override_properties.get(component_type, {}),
component_properties)
def create_application(self, package_data_path, package_metadata,
application_name, property_overrides):
logging.debug("create_application: %s", application_name)
if not re.match('^[a-zA-Z0-9_-]+$', application_name):
raise FailedCreation(
'Application name %s may only contain a-z A-Z 0-9 - _' %
application_name)
user_name = property_overrides['user']
try:
pwd.getpwnam(user_name)
except KeyError:
raise FailedCreation(
'User %s does not exist. Verify that this user account exists on the machine running the deployment manager.'
% user_name)
stage_path = self._stage_package(package_data_path)
# create each class of components in the package, aggregating any
# component specific return data for destruction
create_metadata = {}
try:
for component_type, components in package_metadata[
'component_types'].items():
creator = self._load_creator(component_type)
result = creator.create_components(
stage_path, application_name, user_name, components,
property_overrides.get(component_type))
create_metadata[component_type] = result
finally:
# clean up staged package data
shutil.rmtree(stage_path)
return create_metadata
def destroy_application(self, application_name, application_create_data):
logging.debug("destroy_application: %s %s", application_name,
application_create_data)
app_hdfs_root = None
for component_type, component_create_data in application_create_data.items(
):
creator = self._load_creator(component_type)
creator.destroy_components(application_name, component_create_data)
if component_create_data and 'application_hdfs_root' in component_create_data[
0]:
app_hdfs_root = component_create_data[0][
'application_hdfs_root']
local_path = '/opt/%s/%s/' % (self._service, application_name)
if os.path.isdir(local_path):
os.rmdir(local_path)
if app_hdfs_root is not None:
self._hdfs_client.remove(app_hdfs_root, recursive=False)
def start_application(self, application_name, application_create_data):
logging.debug("start_application: %s %s", application_name,
application_create_data)
for component_type, component_create_data in application_create_data.items(
):
creator = self._load_creator(component_type)
creator.start_components(application_name, component_create_data)
def stop_application(self, application_name, application_create_data):
logging.debug("stop_application: %s %s", application_name,
application_create_data)
for component_type, component_create_data in application_create_data.items(
):
creator = self._load_creator(component_type)
creator.stop_components(application_name, component_create_data)
def validate_package(self, package_name, package_metadata):
logging.debug("validate_package: %s", json.dumps(package_metadata))
result = {}
self._validate_name(package_name, package_metadata)
for component_type, component_metadata in package_metadata[
'component_types'].items():
creator = self._load_creator(component_type)
validation_errors = creator.validate_components(component_metadata)
if validation_errors:
result[component_type] = validation_errors
if result:
raise FailedValidation(result)
def _validate_name(self, package_name, package_metadata):
parts = package_name.split('-')
if len(parts) < 2:
raise FailedValidation(
"package name must be of the form name-version e.g. name-version.1.2.3 but found %s"
% package_name)
version_parts = parts[-1].split('.')
if len(version_parts) < 3:
raise FailedValidation(
"version must be a three part major.minor.patch e.g. 1.2.3 but found %s"
% parts[-1])
if package_name != package_metadata['package_name']:
raise FailedValidation(
"package name must match name of enclosed folder but found %s and %s"
% (package_name, package_metadata['package_name']))
def get_application_runtime_details(self, application_name,
application_create_data):
logging.debug("get_application_runtime_details: %s %s",
application_name, application_create_data)
details = {}
details['yarn_applications'] = {}
for component_type, component_create_data in application_create_data.items(
):
creator = self._load_creator(component_type)
type_details = creator.get_component_runtime_details(
component_create_data)
details['yarn_applications'].update(
type_details['yarn_applications'])
return details
def _load_creator(self, component_type):
logging.debug("_load_creator %s", component_type)
creator = self._component_creators.get(component_type)
if creator is None:
module = '%s.%s' % (self._config['plugins_path'], component_type)
cls = '%s%sCreator' % (component_type[0].upper(),
component_type[1:])
try:
module = import_module("plugins.%s" % component_type)
self._component_creators[component_type] = getattr(
module, cls)(self._config, self._environment,
self._service)
creator = self._component_creators[component_type]
except ImportError as exception:
logging.error(
'Unable to load Creator for component type "%s" [%s]',
component_type, exception)
return creator
def _stage_package(self, package_data_path):
logging.debug("_stage_package")
if not os.path.isdir(self._config['stage_root']):
os.mkdir(self._config['stage_root'])
tar = tarfile.open(package_data_path)
stage_path = "%s/%s" % (self._config['stage_root'], uuid.uuid4())
tar.extractall(path=stage_path)
return stage_path
class HbasePackageRegistrar(object):
COLUMN_DEPLOY_STATUS = "cf:deploy_status"
def __init__(self, hbase_host, hdfs_host, hdfs_user, hdfs_port, package_local_dir_path):
self._hbase_host = hbase_host
self._hdfs_user = hdfs_user
self._hdfs_host = hdfs_host
self._hdfs_port = hdfs_port
self._hdfs_client = HDFS(hdfs_host, hdfs_port, hdfs_user)
self._parser = PackageParser()
self._table_name = 'platform_packages'
self._package_hdfs_dir_path = "/user/pnda/application_packages"
self._package_local_dir_path = package_local_dir_path
if self._hbase_host is not None:
connection = happybase.Connection(self._hbase_host)
try:
connection.create_table(self._table_name, {'cf': dict()})
logging.debug("packages table created")
except AlreadyExists:
logging.debug("packages table exists")
finally:
connection.close()
def set_package(self, package_name, package_data_path):
logging.debug("Storing %s", package_name)
metadata = self._parser.get_package_metadata(package_data_path)
key, data = self.generate_record(metadata)
self._write_to_hdfs(package_data_path, data['cf:package_data'])
self._write_to_db(key, data)
def set_package_deploy_status(self, package_name, deploy_status):
"""
Stores information about the progress of the deploy process of the package
:param deploy_status: the state to store
"""
logging.debug("Storing state for %s: %s", package_name, str(deploy_status))
state_as_string = json.dumps(deploy_status)
self._write_to_db(package_name, {self.COLUMN_DEPLOY_STATUS: state_as_string})
def delete_package(self, package_name):
logging.debug("Deleting %s", package_name)
package_data_hdfs_path = self._read_from_db(package_name, ['cf:package_data'])['cf:package_data']
self._hdfs_client.remove(package_data_hdfs_path)
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
table.delete(package_name)
finally:
connection.close()
def get_package_data(self, package_name):
logging.debug("Reading %s", package_name)
record = self._read_from_db(package_name, ['cf:package_data'])
if len(record) == 0:
return None
local_package_path = "%s/%s" % (self._package_local_dir_path, package_name)
self._read_from_hdfs(record['cf:package_data'], local_package_path)
return local_package_path
def get_package_metadata(self, package_name):
logging.debug("Reading %s", package_name)
package_data = self._read_from_db(
package_name, ['cf:metadata', 'cf:name', 'cf:version'])
if len(package_data) == 0:
return None
return {"metadata": json.loads(package_data["cf:metadata"]), "name": package_data[
"cf:name"], "version": package_data["cf:version"]}
def package_exists(self, package_name):
logging.debug("Checking %s", package_name)
package_data = self._read_from_db(package_name, ['cf:name'])
return len(package_data) > 0
def get_package_deploy_status(self, package_name):
"""
:param package_name: the package name to check status for
:return: The last reported progress of the deploy process for the current package
"""
logging.debug("Checking %s", package_name)
package_data = self._read_from_db(package_name, columns=[self.COLUMN_DEPLOY_STATUS])
if len(package_data) == 0:
return None
# all status is stored as json, so parse it and return it
deploy_status_as_string = package_data[self.COLUMN_DEPLOY_STATUS]
return json.loads(deploy_status_as_string)
def list_packages(self):
logging.debug("List all packages")
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
result = [key for key, _ in table.scan(columns=['cf:name'])]
finally:
connection.close()
return result
def generate_record(self, metadata):
return metadata["package_name"], {
'cf:name': '-'.join(metadata["package_name"].split("-")[:-1]),
'cf:version': metadata["package_name"].split("-")[-1],
'cf:metadata': json.dumps(metadata),
'cf:package_data': "%s/%s" % (self._package_hdfs_dir_path, metadata["package_name"])
}
def _read_from_db(self, key, columns):
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
data = table.row(key, columns=columns)
finally:
connection.close()
return data
def _read_from_hdfs(self, source_hdfs_path, dest_local_path):
self._hdfs_client.stream_file_to_disk(source_hdfs_path, dest_local_path)
def _write_to_db(self, key, data):
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
table.put(key, data)
finally:
connection.close()
def _write_to_hdfs(self, source_local_path, dest_hdfs_path):
with open(source_local_path, 'rb') as source_file:
first = True
chunk_size = 10*1024*1024
data_chunk = source_file.read(chunk_size)
while data_chunk:
if first:
self._hdfs_client.create_file(data_chunk, dest_hdfs_path)
first = False
else:
self._hdfs_client.append_file(data_chunk, dest_hdfs_path)
data_chunk = source_file.read(chunk_size)
class HbasePackageRegistrar(object):
COLUMN_DEPLOY_STATUS = "cf:deploy_status"
def __init__(self, hbase_host, hdfs_host, hdfs_user, hdfs_port,
package_local_dir_path):
self._hbase_host = hbase_host
self._hdfs_user = hdfs_user
self._hdfs_host = hdfs_host
self._hdfs_port = hdfs_port
self._hdfs_client = HDFS(hdfs_host, hdfs_port, hdfs_user)
self._parser = PackageParser()
self._table_name = 'platform_packages'
self._dm_root_dir_path = "/pnda/system/deployment-manager"
self._package_hdfs_dir_path = "%s/packages" % self._dm_root_dir_path
self._package_local_dir_path = package_local_dir_path
try:
if hdfs_host is not None:
self._hdfs_client.make_dir(self._dm_root_dir_path,
permission=755)
self._hdfs_client.make_dir(self._package_hdfs_dir_path,
permission=600)
logging.debug("packages HDFS folder created")
else:
logging.debug(
"not creating packages HDFS folder as it is not required")
except AlreadyExists:
logging.debug(
"not creating packages HDFS folder as it already exists")
if self._hbase_host is not None:
connection = happybase.Connection(self._hbase_host)
try:
connection.create_table(self._table_name, {'cf': dict()})
logging.debug("packages table created")
except AlreadyExists:
logging.debug("packages table exists")
finally:
connection.close()
def set_package(self, package_name, package_data_path, user):
logging.debug("Storing %s", package_name)
metadata = self._parser.get_package_metadata(package_data_path)
metadata['user'] = user
key, data = self.generate_record(metadata)
self._write_to_hdfs(package_data_path, data['cf:package_data'])
self._write_to_db(key, data)
def set_package_deploy_status(self, package_name, deploy_status):
"""
Stores information about the progress of the deploy process of the package
:param deploy_status: the state to store
"""
logging.debug("Storing state for %s: %s", package_name,
str(deploy_status))
state_as_string = json.dumps(deploy_status)
self._write_to_db(package_name,
{self.COLUMN_DEPLOY_STATUS: state_as_string})
def delete_package(self, package_name):
logging.debug("Deleting %s", package_name)
package_data_hdfs_path = self._read_from_db(
package_name, ['cf:package_data'])['cf:package_data']
self._hdfs_client.remove(package_data_hdfs_path)
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
table.delete(package_name)
finally:
connection.close()
def get_package_data(self, package_name):
logging.debug("Reading %s", package_name)
record = self._read_from_db(package_name, ['cf:package_data'])
if not record:
return None
local_package_path = "%s/%s" % (self._package_local_dir_path,
package_name)
self._read_from_hdfs(record['cf:package_data'], local_package_path)
return local_package_path
def get_package_metadata(self, package_name):
logging.debug("Reading %s", package_name)
data = self._read_from_db(package_name,
['cf:metadata', 'cf:name', 'cf:version'])
if not data:
return None
package_data = {
key.docode("utf-8"): value.decode("utf-8")
for key, value in data.items()
}
return {
"metadata": json.loads(package_data["cf:metadata"]),
"name": package_data["cf:name"],
"version": package_data["cf:version"]
}
def package_exists(self, package_name):
logging.debug("Checking %s", package_name)
package_data = self._read_from_db(package_name, ['cf:name'])
return len(package_data) > 0
def get_package_deploy_status(self, package_name):
"""
:param package_name: the package name to check status for
:return: The last reported progress of the deploy process for the current package
"""
logging.debug("Checking %s", package_name)
package_data = self._read_from_db(package_name,
columns=[self.COLUMN_DEPLOY_STATUS])
if not package_data:
return None
# all status is stored as json, so parse it and return it
deploy_status_as_string = package_data[self.COLUMN_DEPLOY_STATUS]
return json.loads(deploy_status_as_string)
def list_packages(self):
logging.debug("List all packages")
connection = None
try:
connection = happybase.Connection(self._hbase_host)
table = connection.table(self._table_name)
result = [key for key, _ in table.scan(columns=['cf:name'])]
except Exception as exc:
logging.debug(str(exc))
raise FailedConnection('Unable to connect to the HBase master')
finally:
if connection:
connection.close()
return result
def generate_record(self, metadata):
return metadata["package_name"], {
'cf:name':
'-'.join(metadata["package_name"].split("-")[:-1]),
'cf:version':
metadata["package_name"].split("-")[-1],
'cf:metadata':
json.dumps(metadata),
'cf:package_data':
"%s/%s" % (self._package_hdfs_dir_path, metadata["package_name"])
}
def _read_from_db(self, key, columns):
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
data = table.row(key, columns=columns)
finally:
connection.close()
return data
def _read_from_hdfs(self, source_hdfs_path, dest_local_path):
self._hdfs_client.stream_file_to_disk(source_hdfs_path,
dest_local_path)
def _write_to_db(self, key, data):
connection = happybase.Connection(self._hbase_host)
try:
table = connection.table(self._table_name)
table.put(key, data)
finally:
connection.close()
def _write_to_hdfs(self, source_local_path, dest_hdfs_path):
with open(source_local_path, 'rb') as source_file:
first = True
chunk_size = 10 * 1024 * 1024
data_chunk = source_file.read(chunk_size)
while data_chunk:
if first:
self._hdfs_client.create_file(data_chunk,
dest_hdfs_path,
permission=600)
first = False
else:
self._hdfs_client.append_file(data_chunk, dest_hdfs_path)
data_chunk = source_file.read(chunk_size)
class Creator(object):
'''
Base Functionality for Creator classes
'''
def __init__(self, config, environment, namespace):
'''
The Creator will be passed the config and environment descriptors
that are passed to the Deployment Manager when it is instantiated.
'''
self._config = config
self._environment = environment
self._namespace = namespace
self._hdfs_client = HDFS(environment['webhdfs_host'],
environment['webhdfs_port'], 'hdfs')
if 'yarn_resource_manager_host' in environment:
self._yarn_resource_manager = "%s:%s" % (
environment['yarn_resource_manager_host'],
environment['yarn_resource_manager_port'])
else:
self._yarn_resource_manager = None
if 'yarn_resource_manager_host_backup' in environment:
self._yarn_resource_manager_backup = "%s:%s" % (
environment['yarn_resource_manager_host_backup'],
environment['yarn_resource_manager_port_backup'])
else:
self._yarn_resource_manager_backup = None
def validate_component(self, components):
'''
Validates components of the package of given component type
components - a list of component maps reflecting the structure of the
components within the package and following this example.
component_detail, component_path and component_name are
guaranteed to be present, component_detail will vary according
to the contents of the package. The validation function can
do anything but it is sensible to check for presence of key
files, for example. Since the staging path is present in the
config passed to the Creator it is possible to load the actual
file data and execute more complex validation if desired.
returns - Array of messages indication what is wrong, 0 length array for no errors
'''
pass
def create_component(self, staged_component_path, application_name,
user_name, component, properties):
'''
Creates component of the package of given component type
application_name - name of the application
user_name - user to run the application as
components - a list of component maps following above example structure
properties - a full map of properties that can be used as needed
returns - create data, which can be anything but will be associated
with this application creation by the Deployment Manager. Therefore
it makes sense to include enough information to be able to
implement destroy_components.
'''
pass
def assert_application_properties(self, override_properties,
default_properties):
'''
Assert application properties before creating the application
override_properties - properties overrided by user
default_properties - default properties present in package
'''
pass
def destroy_component(self, application_name, create_data):
'''
Destroys component of the package of given component type
application_name - name of the application
create_data - as per above explanation, the data returned from the
corresponding create operation, which should be
sufficient to execute a clean destruction.
'''
pass
def start_component(self, application_name, start_data):
'''
starts component of the package of given component type
application_name - name of the application
create_data - as per above explanation, the data returned from the
corresponding create operation, which should be
sufficient to execute start command.
'''
pass
def stop_component(self, application_name, stop_data):
'''
stops component of the package of given component type
application_name - name of the application
create_data - as per above explanation, the data returned from the
corresponding create operation, which should be
sufficient to execute stop command.
'''
pass
def get_component_type(self):
'''
returns a name for the component type
'''
pass
def _instantiate_properties(self, application_name, user_name, component,
property_overrides):
logging.debug("_instantiate_properties %s %s", component,
property_overrides)
component_properties = {}
if 'properties.json' in component['component_detail']:
component_properties = component['component_detail'][
'properties.json']
props = collections.OrderedDict()
for prop in self._environment:
props['environment_' + prop] = self._environment[prop]
for prop in component_properties:
props['component_' + prop] = component_properties[prop]
for prop in property_overrides:
props['component_' + prop] = property_overrides[prop]
props['component_application'] = application_name
props['component_name'] = component['component_name']
props['component_job_name'] = '%s-%s-job' % (
props['component_application'], props['component_name'])
props[
'application_hdfs_root'] = '/pnda/system/deployment-manager/applications/%s/%s' % (
user_name, application_name)
props['component_hdfs_root'] = '%s/%s' % (
props['application_hdfs_root'], component['component_name'])
props['application_user'] = user_name
return props
def _fill_properties(self, local_file, props):
with open(local_file, "r") as myfile:
file_contents = myfile.read()
new_file_contents = string.Template(file_contents).safe_substitute(
props)
with open(local_file, "w") as myfile:
myfile.write(new_file_contents)
def _auto_fill_app_properties(self, staged_component_path, props):
app_properties_file_path = '%s/application.properties' % staged_component_path
with open(app_properties_file_path, "a") as app_properties_file:
if 'component_no_auto_props' not in props:
app_properties_file.write('\n')
for prop in props:
app_properties_file.write(
'%s=%s\n' % (prop.replace('_', '.', 1), props[prop]))
def _create_optional_descriptors(self, staged_component_path, component,
properties):
result = {}
if 'hbase.json' in component['component_detail']:
logging.debug("creating hbase.json")
self._fill_properties(
'%s/%s' % (staged_component_path, 'hbase.json'), properties)
hbase_descriptor.create(
'%s/%s' % (staged_component_path, 'hbase.json'),
self._environment)
if 'hdfs.json' in component['component_detail']:
logging.debug("creating hdfs.json")
descriptor_path = '%s/%s' % (staged_component_path, 'hdfs.json')
self._fill_properties(descriptor_path, properties)
with open(descriptor_path) as descriptor_file:
contents = descriptor_file.read()
hdfs_descriptor = json.loads(contents)
for element in hdfs_descriptor:
properties['hdfspath_' + element['name']] = element['path']
self._hdfs_client.make_dir(element['path'])
result['hdfs.json'] = hdfs_descriptor
if 'opentsdb.json' in component['component_detail']:
logging.debug("creating opentsdb.json")
self._fill_properties(
'%s/%s' % (staged_component_path, 'opentsdb.json'), properties)
opentsdb_descriptor.create(
'%s/%s' % (staged_component_path, 'opentsdb.json'),
self._environment)
return result
def _destroy_optional_descriptors(self, descriptors):
result = {}
if 'hdfs.json' in descriptors:
logging.debug("destroying hdfs.json")
hdfs_descriptor = descriptors['hdfs.json']
for element in hdfs_descriptor:
if element['delete_on_undeploy'] > 0:
dir_count = element['delete_on_undeploy']
while dir_count > 0:
dir_count = dir_count - 1
element['path'] = element['path'][:element['path'].
rfind('/')]
self._hdfs_client.remove(element['path'], recursive=True)
return result
def create_components(self, stage_path, application_name, user_name,
components, components_overrides):
results = []
for component_name, component in components.items():
staged_component_path = '%s/%s' % (stage_path,
component['component_path'])
overrides = components_overrides.get(
component_name) if components_overrides is not None else {}
overrides = {} if overrides is None else overrides
merged_props = self._instantiate_properties(
application_name, user_name, component, overrides)
descriptor_result = self._create_optional_descriptors(
staged_component_path, component, merged_props)
self._auto_fill_app_properties(staged_component_path, merged_props)
result = self.create_component(staged_component_path,
application_name, user_name,
component, merged_props)
result['component_name'] = component_name
result['application_hdfs_root'] = merged_props[
'application_hdfs_root']
result['component_job_name'] = merged_props['component_job_name']
result['descriptors'] = descriptor_result
results.append(result)
return results
def destroy_components(self, application_name, create_data):
for single_component_data in create_data:
self._destroy_optional_descriptors(
single_component_data['descriptors'])
self.destroy_component(application_name, single_component_data)
return None
def start_components(self, application_name, start_data):
for single_component_data in start_data:
self.start_component(application_name, single_component_data)
return None
def stop_components(self, application_name, stop_data):
for single_component_data in stop_data:
self.stop_component(application_name, single_component_data)
return None
def validate_components(self, components):
logging.debug("validate_components: %s", components)
result = {}
for component_name, component in components.items():
validation_errors = self.validate_component(component)
if validation_errors:
result[component_name] = validation_errors
return result
def get_component_runtime_details(self, create_data):
logging.debug("get_component_runtime_details: %s", create_data)
details = {}
yarn_applications = {}
details['yarn_applications'] = yarn_applications
all_yarn_applications = self._get_yarn_applications()
if all_yarn_applications is not None:
for single_component_data in create_data:
app_info = self._find_yarn_app_info(
all_yarn_applications,
single_component_data['component_job_name'])
if app_info is not None:
job_key = '%s-%s' % (self.get_component_type(
), single_component_data['component_name'])
yarn_applications[job_key] = {
"component": single_component_data['component_name'],
"type": self.get_component_type(),
"yarn-id": app_info['id'],
"yarn-start-time": app_info['startedTime'],
"yarn-state": app_info['state']
}
else:
logging.error(
'Failed to query application list from any resource manager')
return details
def _get_yarn_applications_from_rm(self, resource_manager):
result = None
logging.debug('Querying list of yarn applications from %s',
resource_manager)
try:
url = 'http://%s/ws/v1/cluster/apps' % resource_manager
result = requests.get(url, headers={
'Accept': 'application/json'
}).json()
except:
logging.info('Failed to query application list from %s', url)
return result
def _get_yarn_applications(self):
result = self._get_yarn_applications_from_rm(
self._yarn_resource_manager)
if result is None and self._yarn_resource_manager_backup is not None:
result = self._get_yarn_applications_from_rm(
self._yarn_resource_manager_backup)
return result
def _get_yarn_start_time(self, app_info):
try:
return int(app_info['startedTime'])
except:
return 0
def _find_yarn_app_info(self, all_yarn_applications, job_name):
result = None
if 'apps' in all_yarn_applications and 'app' in all_yarn_applications[
'apps']:
for app in all_yarn_applications['apps']['app']:
if app['name'] == job_name:
if result is None or self._get_yarn_start_time(
app) > self._get_yarn_start_time(result):
result = app
return result