def is_compound_type(type_str):
if not util.is_str(type_str):
return False
for subtype in type_str.split("|"):
if subtype not in consts.ALL_TYPES:
return False
return True
def add_tf_types(config):
if not util.is_dict(config):
return
type_fields = {}
for k, v in config.items():
if util.is_str(k) and util.is_str(v) and v in consts.COLUMN_TYPES:
type_fields[k] = v
elif util.is_dict(v):
add_tf_types(v)
elif util.is_list(v):
for sub_v in v:
add_tf_types(sub_v)
for k, v in type_fields.items():
config[k + "_tf"] = CORTEX_TYPE_TO_TF_TYPE[v]
def _expand_columns_input_dict(self, input_columns_dict):
expanded = {}
for column_name, value in input_columns_dict.items():
if util.is_str(value):
expanded[column_name] = self.column_config(value)
elif util.is_list(value):
expanded[column_name] = [self.column_config(name) for name in value]
return expanded
def _expand_feature_inputs_dict(self, input_features_dict):
expanded = {}
for feature_name, value in input_features_dict.items():
if util.is_str(value):
expanded[feature_name] = self.feature_config(value)
elif util.is_list(value):
expanded[feature_name] = [
self.feature_config(name) for name in value
]
return expanded
def create_inputs_map(values_map, input_config):
inputs = {}
for input_name, input_config_item in input_config.items():
if util.is_str(input_config_item):
inputs[input_name] = values_map[input_config_item]
elif util.is_int(input_config_item):
inputs[input_name] = values_map[input_config_item]
elif util.is_list(input_config_item):
inputs[input_name] = [values_map[f] for f in input_config_item]
else:
raise CortexException("invalid column inputs")
return inputs
def create_inputs_from_features_map(features_values_map, feature_input_config):
inputs = {}
for input_name, input_config_item in feature_input_config.items():
if util.is_str(input_config_item):
inputs[input_name] = features_values_map[input_config_item]
elif util.is_int(input_config_item):
inputs[input_name] = features_values_map[input_config_item]
elif util.is_list(input_config_item):
inputs[input_name] = [
features_values_map[f] for f in input_config_item
]
else:
raise CortexException("invalid feature inputs")
return inputs
def create_transformer_inputs_from_map(input, col_value_map):
if util.is_str(input):
if util.is_resource_ref(input):
res_name = util.get_resource_ref(input)
return col_value_map[res_name]
return input
if util.is_list(input):
replaced = []
for item in input:
replaced.append(create_transformer_inputs_from_map(item, col_value_map))
return replaced
if util.is_dict(input):
replaced = {}
for key, val in input.items():
key_replaced = create_transformer_inputs_from_map(key, col_value_map)
val_replaced = create_transformer_inputs_from_map(val, col_value_map)
replaced[key_replaced] = val_replaced
return replaced
return input
def cast_compound_type(value, type_str):
allowed_types = type_str.split("|")
if consts.VALUE_TYPE_INT in allowed_types:
if util.is_int(value):
return value
if consts.VALUE_TYPE_FLOAT in allowed_types:
if util.is_int(value):
return float(value)
if util.is_float(value):
return value
if consts.VALUE_TYPE_STRING in allowed_types:
if util.is_str(value):
return value
if consts.VALUE_TYPE_BOOL in allowed_types:
if util.is_bool(value):
return value
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(type_str), util.user_obj_str(value)
)
)
def populate_values(self, input, input_schema, preserve_column_refs):
if input is None:
if input_schema is None:
return None
if input_schema.get("_allow_null") == True:
return None
raise UserException("Null value is not allowed")
if util.is_resource_ref(input):
res_name = util.get_resource_ref(input)
if res_name in self.constants:
if self.constants[res_name].get("value") is not None:
const_val = self.constants[res_name]["value"]
elif self.constants[res_name].get("path") is not None:
const_val = self.storage.get_json_external(self.constants[res_name]["path"])
try:
return self.populate_values(const_val, input_schema, preserve_column_refs)
except CortexException as e:
e.wrap("constant " + res_name)
raise
if res_name in self.aggregates:
agg_val = self.get_obj(self.aggregates[res_name]["key"])
try:
return self.populate_values(agg_val, input_schema, preserve_column_refs)
except CortexException as e:
e.wrap("aggregate " + res_name)
raise
if res_name in self.columns:
if input_schema is not None:
col_type = self.get_inferred_column_type(res_name)
if col_type not in input_schema["_type"]:
raise UserException(
"column {}: unsupported input type (expected type {}, got type {})".format(
res_name,
util.data_type_str(input_schema["_type"]),
util.data_type_str(col_type),
)
)
if preserve_column_refs:
return input
else:
return res_name
if util.is_list(input):
elem_schema = None
if input_schema is not None:
if not util.is_list(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
elem_schema = input_schema["_type"][0]
min_count = input_schema.get("_min_count")
if min_count is not None and len(input) < min_count:
raise UserException(
"list has length {}, but the minimum allowed length is {}".format(
len(input), min_count
)
)
max_count = input_schema.get("_max_count")
if max_count is not None and len(input) > max_count:
raise UserException(
"list has length {}, but the maximum allowed length is {}".format(
len(input), max_count
)
)
casted = []
for i, elem in enumerate(input):
try:
casted.append(self.populate_values(elem, elem_schema, preserve_column_refs))
except CortexException as e:
e.wrap("index " + i)
raise
return casted
if util.is_dict(input):
if input_schema is None:
casted = {}
for key, val in input.items():
key_casted = self.populate_values(key, None, preserve_column_refs)
try:
val_casted = self.populate_values(val, None, preserve_column_refs)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key_casted] = val_casted
return casted
if not util.is_dict(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
min_count = input_schema.get("_min_count")
if min_count is not None and len(input) < min_count:
raise UserException(
"map has length {}, but the minimum allowed length is {}".format(
len(input), min_count
)
)
max_count = input_schema.get("_max_count")
if max_count is not None and len(input) > max_count:
raise UserException(
"map has length {}, but the maximum allowed length is {}".format(
len(input), max_count
)
)
is_generic_map = False
if len(input_schema["_type"]) == 1:
input_type_key = next(iter(input_schema["_type"].keys()))
if is_compound_type(input_type_key):
is_generic_map = True
generic_map_key_schema = input_schema_from_type_schema(input_type_key)
generic_map_value = input_schema["_type"][input_type_key]
if is_generic_map:
casted = {}
for key, val in input.items():
key_casted = self.populate_values(
key, generic_map_key_schema, preserve_column_refs
)
try:
val_casted = self.populate_values(
val, generic_map_value, preserve_column_refs
)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key_casted] = val_casted
return casted
# fixed map
casted = {}
for key, val_schema in input_schema["_type"].items():
if key in input:
val = input[key]
else:
if val_schema.get("_optional") is not True:
raise UserException("missing key: " + util.user_obj_str(key))
if val_schema.get("_default") is None:
continue
val = val_schema["_default"]
try:
val_casted = self.populate_values(val, val_schema, preserve_column_refs)
except CortexException as e:
e.wrap(util.user_obj_str(key))
raise
casted[key] = val_casted
return casted
if input_schema is None:
return input
if not util.is_str(input_schema["_type"]):
raise UserException(
"unsupported input type (expected type {}, got {})".format(
util.data_type_str(input_schema["_type"]), util.user_obj_str(input)
)
)
return cast_compound_type(input, input_schema["_type"])