def _run(self, cohort_config):
if "query" not in cohort_config:
raise ValueError(
dedent("""
Section: cohort_config -
key 'query' not found. You must define a cohort query."""))
query = cohort_config["query"]
if "{as_of_date}" not in query:
raise ValueError(
dedent("""
Section: cohort_config -
If 'query' is used as cohort_config,
{as_of_date} must be present"""))
if 'name' in cohort_config and not string_is_tablesafe(
cohort_config['name']):
raise ValueError(
"Section: cohort_config - "
"name should only contain letters, numbers, and underscores")
dated_query = query.replace("{as_of_date}", "2016-01-01")
logging.info("Validating cohort query")
try:
self.db_engine.execute(f"explain {dated_query}")
except Exception as e:
raise ValueError(
dedent(f"""
Section: cohort_config -
given query can not be run with a sample as_of_date .
query: "{query}"
Full error: {e}"""))
def _run(self, label_config):
logger.spam("Validating label configuration")
if not label_config:
raise ValueError(
dedent(
"""
Section: label_config -
Section not found. You must define a label config."""
)
)
if len(set(label_config.keys()).intersection({"query", "filepath"})) != 1:
raise ValueError(
dedent(
"""
Section: label_config -
keys ({label_config.keys()}) do not contain exactly one of 'filepath'
or 'query'. You must pass a filepath to a label query or include one
in the config."""
)
)
label_config = load_query_if_needed(label_config)
if "name" in label_config and not string_is_tablesafe(label_config["name"]):
raise ValueError(
"Section: label_config - "
"name should only contain lowercase letters, numbers, and underscores"
)
self._validate_query(label_config["query"])
self._validate_include_missing_labels_in_train_as(
label_config.get("include_missing_labels_in_train_as", None)
)
logger.debug("Validation of label configuration was successful")
def _validate_keys(self, aggregation_config):
logger.spam("Validating feature aggregation keys")
for key in [
"from_obj",
"intervals",
"knowledge_date_column",
"prefix",
]:
if key not in aggregation_config:
raise ValueError(
dedent(
"""
Section: feature_aggregations -
'{} required as key: aggregation config: {}""".format(
key, aggregation_config
)
)
)
if not string_is_tablesafe(aggregation_config["prefix"]):
raise ValueError(
dedent(
f"""Section: feature_aggregations -
Feature aggregation prefix should only contain
lowercase letters, numbers, and underscores.
Aggregation config: {aggregation_config}
"""
)
)
if "groups" in aggregation_config:
if aggregation_config["groups"] != [self.entity_id_column]:
raise ValueError(
dedent(
"""Specifying groupings for feature aggregation is
not supported. Features can only be grouped at the
entity_id level."""
)
)
else:
logger.warning(
dedent(
"""Specifying groupings for feature aggregation is
not supported. In the future, please exclude this key
from your feature configuration."""
)
)
logger.debug("Validation of feature aggregation keys was successful")
def _run(self, cohort_config):
logger.spam("Validating of cohort configuration")
if len(set(cohort_config.keys()).intersection({"query", "filepath"})) != 1:
raise ValueError(
dedent(
"""
Section: cohort_config -
keys ({cohort_config.keys()}) do not contain exactly one of 'filepath'
or 'query'. You must pass a filepath to a cohort query or include one
in the config."""
)
)
cohort_config = load_query_if_needed(cohort_config)
query = cohort_config["query"]
if "{as_of_date}" not in query:
raise ValueError(
dedent(
"""
Section: cohort_config -
If 'query' is used as cohort_config,
{as_of_date} must be present"""
)
)
if "name" in cohort_config and not string_is_tablesafe(cohort_config["name"]):
raise ValueError(
"Section: cohort_config - "
"name should only contain lowercase letters, numbers, and underscores"
)
dated_query = query.replace("{as_of_date}", "2016-01-01")
logger.spam("Validating cohort query via SQL EXPLAIN")
try:
self.db_engine.execute(f"explain {dated_query}")
logger.debug("Validation of cohort query was successful")
except Exception as e:
raise ValueError(
dedent(
f"""
Section: cohort_config -
given query can not be run with a sample as_of_date .
query: "{query}"
Full error: {e}"""
)
)
logger.debug("Validation of cohort configuration was successful")
def _run(self, label_config):
if not label_config:
raise ValueError(
dedent("""
Section: label_config -
Section not found. You must define a label config."""))
if "query" not in label_config:
raise ValueError(
dedent("""
Section: label_config -
key 'query' not found. You must define a label query."""))
if 'name' in label_config and not string_is_tablesafe(
label_config['name']):
raise ValueError(
"Section: label_config - "
"name should only contain letters, numbers, and underscores")
self._validate_query(label_config["query"])
self._validate_include_missing_labels_in_train_as(
label_config.get("include_missing_labels_in_train_as", None))
def _validate_keys(self, aggregation_config):
for key in [
"from_obj",
"intervals",
"groups",
"knowledge_date_column",
"prefix",
]:
if key not in aggregation_config:
raise ValueError(
dedent("""
Section: feature_aggregations -
'{} required as key: aggregation config: {}""".format(
key, aggregation_config)))
if not string_is_tablesafe(aggregation_config['prefix']):
raise ValueError(
dedent(f"""Section: feature_aggregations -
Feature aggregation prefix should only contain
lowercase letters, numbers, and underscores.
Aggregation config: {aggregation_config}
"""))
def test_string_is_not_tablesafe(s):
assert not string_is_tablesafe(s)
def _run(self, scoring_config):
logger.spam("Validating scoring configuration")
if "testing_metric_groups" not in scoring_config:
logger.warning(
"Section: scoring - No testing_metric_groups configured. " +
"Your experiment may run, but you will not have any " +
"evaluation metrics computed")
if "training_metric_groups" not in scoring_config:
logger.warning(
"Section: scoring - No training_metric_groups configured. " +
"If training set evaluation metrics are desired, they must be added"
)
metric_lookup = catwalk.evaluation.ModelEvaluator.available_metrics
available_metrics = set(metric_lookup.keys())
for group in ("testing_metric_groups", "training_metric_groups"):
for metric_group in scoring_config.get(group, {}):
given_metrics = set(metric_group["metrics"])
bad_metrics = given_metrics - available_metrics
if bad_metrics:
raise ValueError(
dedent("""Section: scoring -
The following given metrics '{}' are unavailable.
Available metrics are: '{}'
""".format(bad_metrics, available_metrics)))
for given_metric in given_metrics:
metric_function = metric_lookup[given_metric]
if not hasattr(metric_function, "greater_is_better"):
raise ValueError(
dedent("""Section: scoring -
The metric {} does not define the attribute
'greater_is_better'. This can only be fixed in the catwalk.metrics
module. If you still would like to use this metric, consider
submitting a pull request""".format(given_metric)))
if "subsets" in scoring_config:
for subset in scoring_config["subsets"]:
# 1. Validate that all required keys are present
if "query" not in subset:
raise ValueError(
dedent(f"""Section: subsets -
The subset {subset} does not have a query key.
To run evaluations on a subset, you must
include a query that returns a list of distinct
entity_ids and has a placeholder for an
as_of_date
"""))
if "name" not in subset:
raise ValueError(
dedent(f"""Section: subsets -
The subset {subset} does not have a name key.
Please give a name to your subset. This is used
in the namespacing of subset tables created by
triage.
"""))
if not string_is_tablesafe(subset['name']):
raise ValueError(
dedent(f"""Section: subsets -
The subset {subset} name should only contain
lowercase letters, numbers, and underscores
"""))
# 2. Validate that query conforms to the expectations
if "{as_of_date}" not in subset["query"]:
raise ValueError(
dedent(f"""Section: subsets -
The subset query {subset["query"]} must
include a placeholder for the as_of_date
"""))
if "entity_id" not in subset["query"]:
raise ValueError(
dedent(f"""The subset qeury {subset["query"]} must
return a list of distinct entity_ids
"""))
logger.debug("Validation of scoring configuration was successful")