def _validate_name(self, package_name, package_metadata):
parts = package_name.split('-')
if len(parts) < 2:
raise FailedValidation(
"package name must be of the form name-version e.g. name-version.1.2.3 but found %s"
% package_name)
version_parts = parts[-1].split('.')
if len(version_parts) < 3:
raise FailedValidation(
"version must be a three part major.minor.patch e.g. 1.2.3 but found %s"
% parts[-1])
if package_name != package_metadata['package_name']:
raise FailedValidation(
"package name must match name of enclosed folder but found %s and %s"
% (package_name, package_metadata['package_name']))
def assert_application_properties(self, override_properties,
default_properties):
# Assert application properties before creating the application
for component in default_properties:
properties = default_properties[component].copy()
properties.update(override_properties.get(component, {}))
if 'spark_version' in properties and properties[
'spark_version'] != self._config['oozie_spark_version']:
information = "Mismatch between cluster's oozie spark version (version = %s) and \
requested spark version" % self._config['oozie_spark_version']
raise FailedValidation(json.dumps({"information":
information}))
properties = None
def validate_package(self, package_name, package_metadata):
logging.debug("validate_package: %s", json.dumps(package_metadata))
result = {}
self._validate_name(package_name, package_metadata)
for component_type, component_metadata in package_metadata[
'component_types'].items():
creator = self._load_creator(component_type)
validation_errors = creator.validate_components(component_metadata)
if validation_errors:
result[component_type] = validation_errors
if result:
raise FailedValidation(result)
def get_package_metadata(self, package_data_path):
try:
logging.debug("get_package_metadata")
metadata = {}
tar = tarfile.open(package_data_path)
for name in sorted([member.name for member in tar.getmembers()]):
name_parts = name.split('/')
package_name = name_parts[0]
if len(name_parts) == 1:
# there must be exactly one package in the archive
if 'package_name' in metadata:
raise FailedValidation(
"Expected to find a single directory inside the archive, but found more than one item at the top level"
)
metadata['package_name'] = package_name
metadata['component_types'] = {}
elif len(name_parts) >= 4:
component_type = name_parts[1]
component_name = name_parts[2]
file_name = name_parts[3]
if component_type not in metadata['component_types']:
metadata['component_types'][component_type] = {}
if component_name not in metadata['component_types'][
component_type]:
metadata['component_types'][component_type][
component_name] = {
'component_name':
component_name,
'component_detail': {},
'component_path':
'%s/%s/%s' %
(package_name, component_type, component_name)
}
file_contents = {}
if file_name == 'properties.json':
file_contents = json.load(tar.extractfile(name))
metadata['component_types'][component_type][
component_name]['component_detail']['/'.join(
name_parts[3:])] = file_contents
# there must be at least one component type in the package
if 'component_types' not in metadata:
raise FailedValidation(
"Expected to find a single package directory inside the archive, but found none"
)
if len(metadata['component_types']) <= 0:
raise FailedValidation(
"Expected to find at least one component within the package directory"
)
for component_type, components in metadata[
'component_types'].items():
for component_name, component_detail in components.items():
if 'properties.json' not in component_detail[
'component_detail']:
component_detail['component_detail'][
'properties.json'] = {}
logging.debug(json.dumps(metadata))
return metadata
except FailedValidation as failure:
logging.error(traceback.format_exc())
raise failure
except Exception:
logging.error(traceback.format_exc())
raise FailedValidation(
"Unexpected error parsing the package contents")
def throwerr(_):
raise FailedValidation("Failed validation")