def create_application(self, package_data_path, package_metadata,
application_name, property_overrides):
logging.debug("create_application: %s", application_name)
if not re.match('^[a-zA-Z0-9_-]+$', application_name):
raise FailedCreation(
'Application name %s may only contain a-z A-Z 0-9 - _' %
application_name)
stage_path = self._stage_package(package_data_path)
# create each class of components in the package, aggregating any
# component specific return data for destruction
create_metadata = {}
try:
for component_type, components in package_metadata[
'component_types'].iteritems():
creator = self._load_creator(component_type)
result = creator.create_components(
stage_path, application_name, components,
property_overrides.get(component_type))
create_metadata[component_type] = result
finally:
# clean up staged package data
shutil.rmtree(stage_path)
return create_metadata
def create_application(self, package_data_path, package_metadata,
application_name, property_overrides):
logging.debug("create_application: %s", application_name)
if not re.match('^[a-zA-Z0-9_-]+$', application_name):
raise FailedCreation(
'Application name %s may only contain a-z A-Z 0-9 - _' %
application_name)
user_name = property_overrides['user']
try:
pwd.getpwnam(user_name)
except KeyError:
raise FailedCreation(
'User %s does not exist. Verify that this user account exists on the machine running the deployment manager.'
% user_name)
stage_path = self._stage_package(package_data_path)
# create each class of components in the package, aggregating any
# component specific return data for destruction
create_metadata = {}
try:
for component_type, components in package_metadata[
'component_types'].items():
creator = self._load_creator(component_type)
result = creator.create_components(
stage_path, application_name, user_name, components,
property_overrides.get(component_type))
create_metadata[component_type] = result
finally:
#clean up staged package data
#shutil.rmtree(stage_path)
pass
return create_metadata
def _submit_oozie(self, job_properties):
logging.debug("_submit_oozie (submit only)")
result = {}
# oozie requires requests to use XML
xml_string = deployer_utils.dict_to_xml(job_properties)
oozie_url = '%s/v1/jobs' % self._environment['oozie_uri']
response = requests.post(oozie_url, data=xml_string, headers={'Content-Type': 'application/xml'})
if response.status_code >= 200 and response.status_code < 300:
result = response.json()
else:
logging.error(response.headers['oozie-error-message'])
raise FailedCreation(response.headers['oozie-error-message'])
return result
def _deploy_to_hadoop(self,
component,
properties,
staged_component_path,
remote_path,
application_user,
exclude=None):
if exclude is None:
exclude = []
exclude.extend([
'hdfs.json', 'hbase.json', 'properties.json',
'application.properties'
])
# setup queue config
try:
self._setup_queue_config(component, staged_component_path,
properties)
except Exception as ex:
logging.error(traceback.format_exc())
raise FailedCreation('Failed to set up yarn queue config: %s' %
str(ex))
# stage the component files to hdfs
self._hdfs_client.recursive_copy(staged_component_path,
remote_path,
exclude=exclude,
permission=755)
# stage the instantiated job properties back to HDFS - no functional purpose,
# just helps developers understand what has happened
effective_job_properties = deployer_utils.dict_to_props(properties)
self._hdfs_client.create_file(
effective_job_properties,
'%s/application.properties' % remote_path)
# submit to oozie
result = self._submit_oozie(properties)
self._stop_oozie(result['id'], application_user)
return result