def _validate_name(self, package_name, package_metadata): parts = package_name.split('-') if len(parts) < 2: raise FailedValidation( "package name must be of the form name-version e.g. name-version.1.2.3 but found %s" % package_name) version_parts = parts[-1].split('.') if len(version_parts) < 3: raise FailedValidation( "version must be a three part major.minor.patch e.g. 1.2.3 but found %s" % parts[-1]) if package_name != package_metadata['package_name']: raise FailedValidation( "package name must match name of enclosed folder but found %s and %s" % (package_name, package_metadata['package_name']))
def assert_application_properties(self, override_properties, default_properties): # Assert application properties before creating the application for component in default_properties: properties = default_properties[component].copy() properties.update(override_properties.get(component, {})) if 'spark_version' in properties and properties[ 'spark_version'] != self._config['oozie_spark_version']: information = "Mismatch between cluster's oozie spark version (version = %s) and \ requested spark version" % self._config['oozie_spark_version'] raise FailedValidation(json.dumps({"information": information})) properties = None
def validate_package(self, package_name, package_metadata): logging.debug("validate_package: %s", json.dumps(package_metadata)) result = {} self._validate_name(package_name, package_metadata) for component_type, component_metadata in package_metadata[ 'component_types'].items(): creator = self._load_creator(component_type) validation_errors = creator.validate_components(component_metadata) if validation_errors: result[component_type] = validation_errors if result: raise FailedValidation(result)
def get_package_metadata(self, package_data_path): try: logging.debug("get_package_metadata") metadata = {} tar = tarfile.open(package_data_path) for name in sorted([member.name for member in tar.getmembers()]): name_parts = name.split('/') package_name = name_parts[0] if len(name_parts) == 1: # there must be exactly one package in the archive if 'package_name' in metadata: raise FailedValidation( "Expected to find a single directory inside the archive, but found more than one item at the top level" ) metadata['package_name'] = package_name metadata['component_types'] = {} elif len(name_parts) >= 4: component_type = name_parts[1] component_name = name_parts[2] file_name = name_parts[3] if component_type not in metadata['component_types']: metadata['component_types'][component_type] = {} if component_name not in metadata['component_types'][ component_type]: metadata['component_types'][component_type][ component_name] = { 'component_name': component_name, 'component_detail': {}, 'component_path': '%s/%s/%s' % (package_name, component_type, component_name) } file_contents = {} if file_name == 'properties.json': file_contents = json.load(tar.extractfile(name)) metadata['component_types'][component_type][ component_name]['component_detail']['/'.join( name_parts[3:])] = file_contents # there must be at least one component type in the package if 'component_types' not in metadata: raise FailedValidation( "Expected to find a single package directory inside the archive, but found none" ) if len(metadata['component_types']) <= 0: raise FailedValidation( "Expected to find at least one component within the package directory" ) for component_type, components in metadata[ 'component_types'].items(): for component_name, component_detail in components.items(): if 'properties.json' not in component_detail[ 'component_detail']: component_detail['component_detail'][ 'properties.json'] = {} logging.debug(json.dumps(metadata)) return metadata except FailedValidation as failure: logging.error(traceback.format_exc()) raise failure except Exception: logging.error(traceback.format_exc()) raise FailedValidation( "Unexpected error parsing the package contents")
def throwerr(_): raise FailedValidation("Failed validation")