def _raise_unsupported_bucket_operation_error(self, attr, s3_buckets_set):
"""
Raise error indicating that either
- attribute has been called with arguments which has both data
and application S3 paths in arguments
- attribute has been called with arguments that point to S3
buckets which are not managed by AWSS3CredentialsWrapper
Both cases are not supported.
:param attr: attribute/method name that has been called
:param s3_buckets_set: set of S3 buckets extracted from function arguments
"""
if s3_buckets_set.issubset(self.app_buckets + self.data_buckets):
raise M3DIllegalArgumentException(
AWSS3CredentialsWrapper.ERROR_CROSS_BUCKET_ACCESS.format(
attr=attr))
else:
# s3_buckets_set contains buckets which are not managed by AWSS3CredentialsWrapper.
managed_s3_buckets = self.app_buckets + self.data_buckets
unmanaged_s3_buckets = list(
filter(lambda bucket: bucket not in managed_s3_buckets,
s3_buckets_set))
raise M3DIllegalArgumentException(
AWSS3CredentialsWrapper.ERROR_UNMANAGED_BUCKET.format(
attr=attr, buckets=unmanaged_s3_buckets))
def validate_spark_params(spark_params):
for key in EMRSystem.SPARK_MANDATORY_KEYS:
if key not in spark_params:
raise M3DIllegalArgumentException("'{}' mandatory key of SparkParameters missing".format(key))
if not spark_params[key]:
raise M3DIllegalArgumentException("'{}' property of SparkParameters should have an assigned value".
format(key))
def validate_layers(data_layers):
if not data_layers:
raise M3DIllegalArgumentException("No data layer has been specified.")
valid_layers = DataLayers.return_all_layers()
for data_layer in data_layers:
if data_layer not in valid_layers:
raise M3DIllegalArgumentException("Not a valid data layer: {}".format(data_layer))
def validate(self, mandatory_keys):
for key in mandatory_keys:
if key not in self.parameters.keys():
raise M3DIllegalArgumentException(
"'{}' mandatory key of SparkParameters missing".format(
key))
if not self.parameters[key]:
raise M3DIllegalArgumentException(
"'{}' property of SparkParameters should have an assigned value"
.format(key))
def _validate_params(params):
if len(params.target_partitions) != len(params.regex_filename):
message = "Lengths of target_partitions and regex_filename do not match:\n{}\n{}".format(
params.target_partitions,
params.regex_filename
)
raise M3DIllegalArgumentException(message)
def validate_parameters(self):
if "source_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Source table name is missing in the acon-file")
if "target_table" not in self._parameters:
raise M3DIllegalArgumentException(
"Target table name is missing in the acon-file")
if "source_field" not in self._parameters:
raise M3DIllegalArgumentException(
"Source field name is missing in the acon-file")
if "substring_positions" not in self._parameters:
raise M3DIllegalArgumentException(
"Substring positions specification is missing in the acon-file"
)
if "select_conditions" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_conditions for unpartitioned table")
if "select_rules" in self._parameters and "target_partitions" not in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use select_rules for unpartitioned table")
if "select_conditions" in self._parameters and "select_rules" in self._parameters:
raise M3DIllegalArgumentException(
"Unable to use both select_conditions and select_rules at the same time"
)
def _validate_float(self, parameter_name, parameter_value):
"""
Validate whether the parameter was assigned a valid float value and cast it
:param parameter_name: name of the parameter
:param parameter_value: received value for the parameter
:return: casted float value
"""
float_value = float(parameter_value)
if float_value <= 0:
raise M3DIllegalArgumentException("Negative value passed for {} argument.".format(parameter_name))
return float_value
def get_layer_object(layer):
"""
Takes a string as param to return data layer object.
:param layer: string representing the name of the layer to be returned.
:return: the layer object
:exception: raises exception if the string passed does not match any layer.
"""
if layer.lower() == "landing":
return DataLayers.LANDING
elif layer.lower() == "lake":
return DataLayers.LAKE
else:
raise M3DIllegalArgumentException("Invalid layer: {}".format(layer))