def predict():
to_test = to_test()
matrix_storage_engine = ProjectStorage(
's3://<url-to-S3-bucket>').matrix_storage_engine()
model_storage_engine = ProjectStorage(
's3://<url-to-S3-bucket>').model_storage_engine()
matrix_uuid = '895a1b1dd28e1c9d974b8f684aec7b69'
matrix_store = matrix_storage_engine.get_store(matrix_uuid)
matrix = matrix_store.design_matrix
mat_index = matrix.reset_index()[['entity_id', 'as_of_date']]
matrix_list = []
for model_id, model_group_id, model_hash in to_test:
mat_pred = mat_index
mat_pred['model_id'] = model_id
mat_pred['model_group_id'] = model_group_id
model = model_storage_engine.load(model_hash)
mat_pred['prediction'] = model.pred_proba(matrix)
matrix_list.append(mat_pred)
predictions = pandas.concat(matrix_list)
predictions.pg_copy_to('preds_final_models', engine)
index = "CREATE INDEX entity_"
def predict_chunk(chunk_size):
pred_models = to_test()
pred_models.reverse()
print(pred_models)
model_storage_engine = ProjectStorage(
's3://<url-to-S3-bucket>').model_storage_engine()
matrix_uuid = '895a1b1dd28e1c9d974b8f684aec7b69'
for model_id, model_group_id, model_hash in pred_models:
print(model_id)
model = model_storage_engine.load(model_hash)
with open('{}_prediction.csv'.format(model_id), 'w') as f:
writer = csv.writer(f, delimiter=',')
s3 = boto3.client('s3')
obj = s3.get_object(Bucket='<bucket>',
Key='<key>' + matrix_uuid + '.csv.gz')
reader = pd.read_csv(io.BytesIO(obj['Body'].read()),
compression='gzip',
chunksize=chunk_size)
for chunk in reader:
chunk_feat = chunk.drop(
columns=['entity_id', 'as_of_date', 'dropout_dummy'])
for index, row in chunk_feat.iterrows():
row = np.asarray(row).reshape(1, 296)
prediction = model.predict_proba(row)
writer.writerow([
model_id, model_group_id, chunk['as_of_date'][index],
chunk['entity_id'][index], prediction[0][0],
prediction[0][1]
])
f.close()
def __call__(self, args):
if args.validate_only:
try:
logger.info(f"Validating experiment [config file: {self.args.config}]")
self.experiment.validate()
logger.success(
f"Experiment ({self.experiment.experiment_hash})'s configuration file is OK!"
)
except Exception:
logger.exception(f"Validation failed!")
logger.info(
f"Experiment [config file: {self.args.config}] configuration file is incorrect"
)
elif args.show_timechop:
experiment_name = os.path.splitext(os.path.basename(self.args.config))[0]
project_storage = ProjectStorage(self.args.project_path)
timechop_store = project_storage.get_store(
["images"], f"{experiment_name}.png"
)
with timechop_store.open("wb") as fd:
visualize_chops(self.experiment.chopper, save_target=fd)
else:
try:
logger.info(f"Running Experiment ({self.experiment.experiment_hash})")
self.experiment.run()
logger.success(
f"Experiment ({self.experiment.experiment_hash}) ran through completion"
)
except Exception:
logger.exception("Something went wrong")
logger.info(f"Experiment [config file: {self.args.config}] run failed!")
def sample_matrix_store(sample_df, sample_metadata):
with tempfile.TemporaryDirectory() as tempdir:
project_storage = ProjectStorage(tempdir)
store = project_storage.matrix_storage_engine().get_store("1234")
store.matrix = sample_df
store.metadata = sample_metadata
return store
def shared_project_storage():
"""Set up a temporary project storage engine on the filesystem at module scope
Yields (catwalk.storage.ProjectStorage)
"""
with tempfile.TemporaryDirectory() as temp_dir:
project_storage = ProjectStorage(temp_dir)
yield project_storage
def rig_engines():
"""Set up a db engine and project storage engine
Yields (tuple) (database engine, project storage engine)
"""
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
ensure_db(db_engine)
init_engine(db_engine)
with tempfile.TemporaryDirectory() as temp_dir:
project_storage = ProjectStorage(temp_dir)
yield db_engine, project_storage
def test_as_of_dates_entity_index(self):
data = {
"entity_id": [1, 2],
"feature_one": [0.5, 0.6],
"feature_two": [0.5, 0.6],
}
with tempfile.TemporaryDirectory() as tmpdir:
project_storage = ProjectStorage(tmpdir)
matrix_store = CSVMatrixStore(project_storage, [], "test")
matrix_store.matrix = pd.DataFrame.from_dict(data)
matrix_store.metadata = {"end_time": "2016-01-01", "indices": ["entity_id"]}
self.assertEqual(matrix_store.as_of_dates, ["2016-01-01"])
def __init__(
self,
config,
db_engine,
project_path=None,
matrix_storage_class=CSVMatrixStore,
replace=True,
cleanup=False,
cleanup_timeout=None,
materialize_subquery_fromobjs=True,
features_ignore_cohort=False,
profile=False,
save_predictions=True,
):
self._check_config_version(config)
self.config = config
self.project_storage = ProjectStorage(project_path)
self.model_storage_engine = ModelStorageEngine(self.project_storage)
self.matrix_storage_engine = MatrixStorageEngine(
self.project_storage, matrix_storage_class)
self.project_path = project_path
self.replace = replace
self.save_predictions = save_predictions
self.db_engine = db_engine
upgrade_db(db_engine=self.db_engine)
self.features_schema_name = "features"
self.materialize_subquery_fromobjs = materialize_subquery_fromobjs
self.features_ignore_cohort = features_ignore_cohort
self.experiment_hash = save_experiment_and_get_hash(
self.config, self.db_engine)
self.initialize_components()
self.cleanup = cleanup
if self.cleanup:
logging.info(
"cleanup is set to True, so intermediate tables (labels and cohort) "
"will be removed after matrix creation and subset tables will be "
"removed after model training and testing")
else:
logging.info(
"cleanup is set to False, so intermediate tables (labels, cohort, and subsets) "
"will not be removed")
self.cleanup_timeout = (self.cleanup_timeout if cleanup_timeout is None
else cleanup_timeout)
self.profile = profile
logging.info("Generate profiling stats? (profile option): %s",
self.profile)
def __call__(self, args):
if args.validate_only:
self.experiment.validate()
elif args.show_timechop:
experiment_name = os.path.splitext(
os.path.basename(self.args.config))[0]
project_storage = ProjectStorage(self.args.project_path)
timechop_store = project_storage.get_store(
["images"], f"{experiment_name}.png")
with timechop_store.open('wb') as fd:
visualize_chops(self.experiment.chopper, save_target=fd)
else:
self.experiment.run()
def test_as_of_dates_entity_index(self):
data = {
'entity_id': [1, 2],
'feature_one': [0.5, 0.6],
'feature_two': [0.5, 0.6],
}
with tempfile.TemporaryDirectory() as tmpdir:
project_storage = ProjectStorage(tmpdir)
matrix_store = CSVMatrixStore(project_storage, [], 'test')
matrix_store.matrix = pd.DataFrame.from_dict(data)
matrix_store.metadata = {
'end_time': '2016-01-01',
'indices': ['entity_id']
}
self.assertEqual(matrix_store.as_of_dates, ['2016-01-01'])
def test_s3_save(self):
with mock_s3():
client = boto3.client("s3")
client.create_bucket(Bucket="fake-matrix-bucket", ACL="public-read-write")
example = next(self.matrix_stores())
project_storage = ProjectStorage("s3://fake-matrix-bucket")
tosave = CSVMatrixStore(project_storage, [], "test")
tosave.matrix = example.matrix
tosave.metadata = example.metadata
tosave.save()
tocheck = CSVMatrixStore(project_storage, [], "test")
assert tocheck.metadata == example.metadata
assert tocheck.matrix.to_dict() == example.matrix.to_dict()
def matrix_stores(self):
df = pd.DataFrame.from_dict(self.data_dict).set_index(["entity_id"])
with tempfile.TemporaryDirectory() as tmpdir:
project_storage = ProjectStorage(tmpdir)
tmpcsv = os.path.join(tmpdir, "df.csv")
tmpyaml = os.path.join(tmpdir, "df.yaml")
tmphdf = os.path.join(tmpdir, "df.h5")
with open(tmpyaml, "w") as outfile:
yaml.dump(self.metadata, outfile, default_flow_style=False)
df.to_csv(tmpcsv)
df.to_hdf(tmphdf, "matrix")
csv = CSVMatrixStore(project_storage, [], "df")
hdf = HDFMatrixStore(project_storage, [], "df")
assert csv.matrix.equals(hdf.matrix)
yield from [csv, hdf]
def __init__(
self,
config,
db_engine,
project_path=None,
matrix_storage_class=CSVMatrixStore,
replace=True,
cleanup=False,
cleanup_timeout=None,
):
self._check_config_version(config)
self.config = config
if isinstance(db_engine, Engine):
logging.warning(
"Raw, unserializable SQLAlchemy engine passed. "
"URL will be used, other options may be lost in multi-process environments"
)
self.db_engine = create_engine(db_engine.url)
else:
self.db_engine = db_engine
self.project_storage = ProjectStorage(project_path)
self.model_storage_engine = ModelStorageEngine(self.project_storage)
self.matrix_storage_engine = MatrixStorageEngine(
self.project_storage, matrix_storage_class)
self.project_path = project_path
self.replace = replace
upgrade_db(db_engine=self.db_engine)
self.features_schema_name = "features"
self.experiment_hash = save_experiment_and_get_hash(
self.config, self.db_engine)
self.labels_table_name = "labels_{}".format(self.experiment_hash)
self.initialize_components()
self.cleanup = cleanup
if self.cleanup:
logging.info(
"cleanup is set to True, so intermediate tables (labels and states) "
"will be removed after matrix creation")
else:
logging.info(
"cleanup is set to False, so intermediate tables (labels and states) "
"will not be removed after matrix creation")
self.cleanup_timeout = (self.cleanup_timeout if cleanup_timeout is None
else cleanup_timeout)
def matrix_stores():
df = pd.DataFrame.from_dict(DATA_DICT).set_index(MatrixStore.indices)
with tempfile.TemporaryDirectory() as tmpdir:
project_storage = ProjectStorage(tmpdir)
tmpcsv = os.path.join(tmpdir, "df.csv.gz")
tmpyaml = os.path.join(tmpdir, "df.yaml")
with open(tmpyaml, "w") as outfile:
yaml.dump(METADATA, outfile, default_flow_style=False)
df.to_csv(tmpcsv, compression="gzip")
csv = CSVMatrixStore(project_storage, [], "df")
# first test with caching
with csv.cache():
yield csv
# with the caching out of scope they will be nuked
# and this last version will not have any cache
yield csv
def test_s3_save(self):
with mock_s3():
import boto3
client = boto3.client('s3')
client.create_bucket(Bucket='fake-matrix-bucket',
ACL='public-read-write')
example = next(self.matrix_stores())
project_storage = ProjectStorage('s3://fake-matrix-bucket')
tosave = CSVMatrixStore(project_storage, [], 'test')
tosave.matrix = example.matrix
tosave.metadata = example.metadata
tosave.save()
tocheck = CSVMatrixStore(project_storage, [], 'test')
assert tocheck.metadata == example.metadata
assert tocheck.matrix.to_dict() == example.matrix.to_dict()
def test_s3_save():
with mock_s3():
client = boto3.client("s3")
client.create_bucket(Bucket="fake-matrix-bucket", ACL="public-read-write")
for example in matrix_stores():
if not isinstance(example, CSVMatrixStore):
continue
project_storage = ProjectStorage("s3://fake-matrix-bucket")
tosave = CSVMatrixStore(project_storage, [], "test")
tosave.metadata = example.metadata
tosave.matrix_label_tuple = example.matrix_label_tuple
tosave.save()
tocheck = CSVMatrixStore(project_storage, [], "test")
assert tocheck.metadata == example.metadata
assert tocheck.design_matrix.to_dict() == example.design_matrix.to_dict()
def matrix_stores():
df = pd.DataFrame.from_dict(DATA_DICT).set_index(["entity_id"])
with tempfile.TemporaryDirectory() as tmpdir:
project_storage = ProjectStorage(tmpdir)
tmpcsv = os.path.join(tmpdir, "df.csv")
tmpyaml = os.path.join(tmpdir, "df.yaml")
tmphdf = os.path.join(tmpdir, "df.h5")
with open(tmpyaml, "w") as outfile:
yaml.dump(METADATA, outfile, default_flow_style=False)
df.to_csv(tmpcsv)
df.to_hdf(tmphdf, "matrix")
csv = CSVMatrixStore(project_storage, [], "df")
hdf = HDFMatrixStore(project_storage, [], "df")
assert csv.design_matrix.equals(hdf.design_matrix)
# first test with caching
with csv.cache(), hdf.cache():
yield csv
yield hdf
# with the caching out of scope they will be nuked
# and these last two versions will not have any cache
yield csv
yield hdf
def basic_integration_test(
cohort_names,
feature_group_create_rules,
feature_group_mix_rules,
expected_matrix_multiplier,
expected_group_lists,
):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
Base.metadata.create_all(db_engine)
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
chopper = Timechop(
feature_start_time=datetime(2010, 1, 1),
feature_end_time=datetime(2014, 1, 1),
label_start_time=datetime(2011, 1, 1),
label_end_time=datetime(2014, 1, 1),
model_update_frequency="1year",
training_label_timespans=["6months"],
test_label_timespans=["6months"],
training_as_of_date_frequencies="1day",
test_as_of_date_frequencies="3months",
max_training_histories=["1months"],
test_durations=["1months"],
)
entity_date_table_generator = EntityDateTableGenerator(
db_engine=db_engine,
entity_date_table_name="cohort_abcd",
query="select distinct(entity_id) from events")
label_generator = LabelGenerator(
db_engine=db_engine,
query=sample_config()["label_config"]["query"])
feature_generator = FeatureGenerator(
db_engine=db_engine,
features_schema_name="features",
replace=True)
feature_dictionary_creator = FeatureDictionaryCreator(
db_engine=db_engine, features_schema_name="features")
feature_group_creator = FeatureGroupCreator(
feature_group_create_rules)
feature_group_mixer = FeatureGroupMixer(feature_group_mix_rules)
project_storage = ProjectStorage(temp_dir)
planner = Planner(
feature_start_time=datetime(2010, 1, 1),
label_names=["outcome"],
label_types=["binary"],
cohort_names=cohort_names,
user_metadata={},
)
builder = MatrixBuilder(
engine=db_engine,
db_config={
"features_schema_name": "features",
"labels_schema_name": "public",
"labels_table_name": "labels",
"cohort_table_name": "cohort_abcd",
},
experiment_hash=None,
matrix_storage_engine=project_storage.matrix_storage_engine(),
replace=True,
)
# chop time
split_definitions = chopper.chop_time()
num_split_matrices = sum(1 + len(split["test_matrices"])
for split in split_definitions)
# generate as_of_times for feature/label/state generation
all_as_of_times = []
for split in split_definitions:
all_as_of_times.extend(split["train_matrix"]["as_of_times"])
for test_matrix in split["test_matrices"]:
all_as_of_times.extend(test_matrix["as_of_times"])
all_as_of_times = list(set(all_as_of_times))
# generate entity_date state table
entity_date_table_generator.generate_entity_date_table(
as_of_dates=all_as_of_times)
# create labels table
label_generator.generate_all_labels(
labels_table="labels",
as_of_dates=all_as_of_times,
label_timespans=["6months"],
)
# create feature table tasks
# we would use FeatureGenerator#create_all_tables but want to use
# the tasks dict directly to create a feature dict
aggregations = feature_generator.aggregations(
feature_aggregation_config=[
{
"prefix":
"cat",
"from_obj":
"cat_complaints",
"knowledge_date_column":
"as_of_date",
"aggregates": [{
"quantity": "cat_sightings",
"metrics": ["count", "avg"],
"imputation": {
"all": {
"type": "mean"
}
},
}],
"intervals": ["1y"],
"groups": ["entity_id"],
},
{
"prefix":
"dog",
"from_obj":
"dog_complaints",
"knowledge_date_column":
"as_of_date",
"aggregates_imputation": {
"count": {
"type": "constant",
"value": 7
},
"sum": {
"type": "mean"
},
"avg": {
"type": "zero"
},
},
"aggregates": [{
"quantity": "dog_sightings",
"metrics": ["count", "avg"]
}],
"intervals": ["1y"],
"groups": ["entity_id"],
},
],
feature_dates=all_as_of_times,
state_table=entity_date_table_generator.entity_date_table_name,
)
feature_table_agg_tasks = feature_generator.generate_all_table_tasks(
aggregations, task_type="aggregation")
# create feature aggregation tables
feature_generator.process_table_tasks(feature_table_agg_tasks)
feature_table_imp_tasks = feature_generator.generate_all_table_tasks(
aggregations, task_type="imputation")
# create feature imputation tables
feature_generator.process_table_tasks(feature_table_imp_tasks)
# build feature dictionaries from feature tables and
# subsetting config
master_feature_dict = feature_dictionary_creator.feature_dictionary(
feature_table_names=feature_table_imp_tasks.keys(),
index_column_lookup=feature_generator.index_column_lookup(
aggregations),
)
feature_dicts = feature_group_mixer.generate(
feature_group_creator.subsets(master_feature_dict))
# figure out what matrices need to be built
_, matrix_build_tasks = planner.generate_plans(
split_definitions, feature_dicts)
# go and build the matrices
builder.build_all_matrices(matrix_build_tasks)
# super basic assertion: did matrices we expect get created?
matrices_records = list(
db_engine.execute(
"""select matrix_uuid, num_observations, matrix_type
from triage_metadata.matrices
"""))
matrix_directory = os.path.join(temp_dir, "matrices")
matrices = [
path for path in os.listdir(matrix_directory) if ".csv" in path
]
metadatas = [
path for path in os.listdir(matrix_directory)
if ".yaml" in path
]
assert len(matrices) == num_split_matrices * \
expected_matrix_multiplier
assert len(metadatas) == num_split_matrices * \
expected_matrix_multiplier
assert len(matrices) == len(matrices_records)
feature_group_name_lists = []
for metadata_path in metadatas:
with open(os.path.join(matrix_directory, metadata_path)) as f:
metadata = yaml.full_load(f)
feature_group_name_lists.append(metadata["feature_groups"])
for matrix_uuid, num_observations, matrix_type in matrices_records:
assert matrix_uuid in matrix_build_tasks # the hashes of the matrices
assert type(num_observations) is int
assert matrix_type == matrix_build_tasks[matrix_uuid][
"matrix_type"]
def deep_unique_tuple(l):
return set([tuple(i) for i in l])
assert deep_unique_tuple(
feature_group_name_lists) == deep_unique_tuple(
expected_group_lists)
def predict_forward_with_existed_model(db_engine, project_path, model_id,
as_of_date):
"""Predict forward given model_id and as_of_date and store the prediction in database
Args:
db_engine (sqlalchemy.db.engine)
project_storage (catwalk.storage.ProjectStorage)
model_id (int) The id of a given model in the database
as_of_date (string) a date string like "YYYY-MM-DD"
"""
logger.spam("In PREDICT LIST................")
upgrade_db(db_engine=db_engine)
project_storage = ProjectStorage(project_path)
matrix_storage_engine = project_storage.matrix_storage_engine()
# 1. Get feature and cohort config from database
(train_matrix_uuid,
matrix_metadata) = train_matrix_info_from_model_id(db_engine, model_id)
experiment_config = experiment_config_from_model_id(db_engine, model_id)
# 2. Generate cohort
cohort_table_name = f"triage_production.cohort_{experiment_config['cohort_config']['name']}"
cohort_table_generator = EntityDateTableGenerator(
db_engine=db_engine,
query=experiment_config['cohort_config']['query'],
entity_date_table_name=cohort_table_name)
cohort_table_generator.generate_entity_date_table(
as_of_dates=[dt_from_str(as_of_date)])
# 3. Generate feature aggregations
feature_generator = FeatureGenerator(
db_engine=db_engine,
features_schema_name="triage_production",
feature_start_time=experiment_config['temporal_config']
['feature_start_time'],
)
collate_aggregations = feature_generator.aggregations(
feature_aggregation_config=experiment_config['feature_aggregations'],
feature_dates=[as_of_date],
state_table=cohort_table_name)
feature_generator.process_table_tasks(
feature_generator.generate_all_table_tasks(collate_aggregations,
task_type='aggregation'))
# 4. Reconstruct feature disctionary from feature_names and generate imputation
reconstructed_feature_dict = FeatureGroup()
imputation_table_tasks = OrderedDict()
for aggregation in collate_aggregations:
feature_group, feature_names = get_feature_names(
aggregation, matrix_metadata)
reconstructed_feature_dict[feature_group] = feature_names
# Make sure that the features imputed in training should also be imputed in production
features_imputed_in_train = get_feature_needs_imputation_in_train(
aggregation, feature_names)
features_imputed_in_production = get_feature_needs_imputation_in_production(
aggregation, db_engine)
total_impute_cols = set(features_imputed_in_production) | set(
features_imputed_in_train)
total_nonimpute_cols = set(f for f in set(feature_names)
if '_imp' not in f) - total_impute_cols
task_generator = feature_generator._generate_imp_table_tasks_for
imputation_table_tasks.update(
task_generator(aggregation,
impute_cols=list(total_impute_cols),
nonimpute_cols=list(total_nonimpute_cols)))
feature_generator.process_table_tasks(imputation_table_tasks)
# 5. Build matrix
db_config = {
"features_schema_name": "triage_production",
"labels_schema_name": "public",
"cohort_table_name": cohort_table_name,
}
matrix_builder = MatrixBuilder(
db_config=db_config,
matrix_storage_engine=matrix_storage_engine,
engine=db_engine,
experiment_hash=None,
replace=True,
)
feature_start_time = experiment_config['temporal_config'][
'feature_start_time']
label_name = experiment_config['label_config']['name']
label_type = 'binary'
cohort_name = experiment_config['cohort_config']['name']
user_metadata = experiment_config['user_metadata']
# Use timechop to get the time definition for production
temporal_config = experiment_config["temporal_config"]
temporal_config.update(
temporal_params_from_matrix_metadata(db_engine, model_id))
timechopper = Timechop(**temporal_config)
prod_definitions = timechopper.define_test_matrices(
train_test_split_time=dt_from_str(as_of_date),
test_duration=temporal_config['test_durations'][0],
test_label_timespan=temporal_config['test_label_timespans'][0])
matrix_metadata = Planner.make_metadata(
prod_definitions[-1],
reconstructed_feature_dict,
label_name,
label_type,
cohort_name,
'production',
feature_start_time,
user_metadata,
)
matrix_metadata['matrix_id'] = str(
as_of_date) + f'_model_id_{model_id}' + '_risklist'
matrix_uuid = filename_friendly_hash(matrix_metadata)
matrix_builder.build_matrix(
as_of_times=[as_of_date],
label_name=label_name,
label_type=label_type,
feature_dictionary=reconstructed_feature_dict,
matrix_metadata=matrix_metadata,
matrix_uuid=matrix_uuid,
matrix_type="production",
)
# 6. Predict the risk score for production
predictor = Predictor(
model_storage_engine=project_storage.model_storage_engine(),
db_engine=db_engine,
rank_order='best')
predictor.predict(
model_id=model_id,
matrix_store=matrix_storage_engine.get_store(matrix_uuid),
misc_db_parameters={},
train_matrix_columns=matrix_storage_engine.get_store(
train_matrix_uuid).columns())
def __init__(
self,
config,
db_engine,
project_path=None,
matrix_storage_class=CSVMatrixStore,
replace=True,
cleanup=False,
cleanup_timeout=None,
materialize_subquery_fromobjs=True,
features_ignore_cohort=False,
additional_bigtrain_classnames=None,
profile=False,
save_predictions=True,
skip_validation=False,
partial_run=False,
):
# For a partial run, skip validation and avoid cleaning up
# we'll also skip filling default config values below
if partial_run:
cleanup = False
skip_validation = True
experiment_kwargs = bind_kwargs(
self.__class__,
**{
key: value
for (key, value) in locals().items()
if key not in {"db_engine", "config", "self"}
},
)
self._check_config_version(config)
self.config = config
if self.config.get("cohort_config") is not None:
self.config["cohort_config"] = load_query_if_needed(
self.config["cohort_config"]
)
if self.config.get("label_config") is not None:
self.config["label_config"] = load_query_if_needed(
self.config["label_config"]
)
self.project_storage = ProjectStorage(project_path)
self.model_storage_engine = ModelStorageEngine(self.project_storage)
self.matrix_storage_engine = MatrixStorageEngine(
self.project_storage, matrix_storage_class
)
self.project_path = project_path
logger.verbose(
f"Matrices and trained models will be saved in {self.project_path}"
)
self.replace = replace
if self.replace:
logger.notice(
f"Replace flag is set to true. Matrices, models, "
"evaluations and predictions (if they exist) will be replaced"
)
self.save_predictions = save_predictions
if not self.save_predictions:
logger.notice(
f"Save predictions flag is set to false. "
"Individual predictions won't be stored in the predictions "
"table. This will decrease both the running time "
"of an experiment and also decrease the space needed in the db"
)
self.skip_validation = skip_validation
if self.skip_validation:
logger.notice(
f"Warning: Skip validation flag is set to true. "
"The experiment config file specified won't be validated. "
"This will reduce (a little) the running time of the experiment, "
"but has some potential risks, e.g. the experiment could fail"
"after some time due to some misconfiguration. Proceed with care."
)
self.db_engine = db_engine
results_schema.upgrade_if_clean(dburl=self.db_engine.url)
self.features_schema_name = "features"
self.materialize_subquery_fromobjs = materialize_subquery_fromobjs
if not self.materialize_subquery_fromobjs:
logger.notice(
"Materialize from_objs is set to false. "
"The from_objs will be calculated on the fly every time."
)
self.features_ignore_cohort = features_ignore_cohort
if self.features_ignore_cohort:
logger.notice(
"Features will be calculated for all the entities "
"(i.e. ignoring cohort) this setting will have the effect "
"that more db space will be used, but potentially could save "
"time if you are running several similar experiments with "
"different cohorts."
)
self.additional_bigtrain_classnames = additional_bigtrain_classnames
# only fill default values for full runs
if not partial_run:
## Defaults to sane values
self.config["temporal_config"] = fill_timechop_config_missing(
self.config, self.db_engine
)
## Defaults to all the entities found in the features_aggregation's from_obj
self.config["cohort_config"] = fill_cohort_config_missing(self.config)
## Defaults to all the feature_aggregation's prefixes
self.config["feature_group_definition"] = fill_feature_group_definition(
self.config
)
grid_config = fill_model_grid_presets(self.config)
self.config.pop("model_grid_preset", None)
if grid_config is not None:
self.config["grid_config"] = grid_config
if not self.config.get("random_seed", None):
logger.notice(
"Random seed not specified. A random seed will be provided. "
"This could have interesting side effects, "
"e.g. new models per model group are trained, "
"tested and evaluated everytime that you run this experiment configuration"
)
self.random_seed = self.config.pop("random_seed", random.randint(1, 1e7))
logger.verbose(
f"Using random seed [{self.random_seed}] for running the experiment"
)
random.seed(self.random_seed)
###################### RUBICON ######################
self.experiment_hash = save_experiment_and_get_hash(self.config, self.db_engine)
logger.debug(f"Experiment hash [{self.experiment_hash}] assigned")
self.run_id = initialize_tracking_and_get_run_id(
self.experiment_hash,
experiment_class_path=classpath(self.__class__),
random_seed=self.random_seed,
experiment_kwargs=experiment_kwargs,
db_engine=self.db_engine,
)
logger.debug(f"Experiment run id [{self.run_id}] assigned")
self.initialize_components()
self.cleanup = cleanup
if self.cleanup:
logger.notice(
"Cleanup is set to true, so intermediate tables (labels and cohort) "
"will be removed after matrix creation and subset tables will be "
"removed after model training and testing"
)
self.cleanup_timeout = (
self.cleanup_timeout if cleanup_timeout is None else cleanup_timeout
)
self.profile = profile
if self.profile:
logger.spam("Profiling will be stored using cProfile")
def add_predictions(db_engine,
model_groups,
project_path,
experiment_hashes=None,
train_end_times_range=None,
rank_order='worst',
replace=True):
""" For a set of modl_groups generate test predictions and write to DB
Args:
db_engine: Sqlalchemy engine
model_groups (list): The list of model group ids we are interested in (ideally, chosen through audition)
project_path (str): Path where the created matrices and trained model objects are stored for the experiment
experiment_hashes (List[str]): Optional. hash(es) of the experiments we are interested in. Can be used to narrow down the model_ids in the model groups specified
range_train_end_times (Dict): Optional. If provided, only the models with train_end_times that fall in the range are scored.
This too, helps narrow down model_ids in the model groups specified.
A dictionary with two possible keys 'range_start_date' and 'range_end_date'. Either or both could be set
rank_order (str) : How to deal with ties in the scores.
replace (bool) : Whether to overwrite the preditctions for a model_id, if already found in the DB.
Returns: None
This directly writes to the test_results.predictions table
"""
model_matrix_info = _fetch_relevant_model_matrix_info(
db_engine=db_engine,
model_groups=model_groups,
experiment_hashes=experiment_hashes)
# If we are only generating predictions for a specific time range
if train_end_times_range is not None:
if 'range_start_date' in train_end_times_range:
range_start = train_end_times_range['range_start_date']
msk = (model_matrix_info['train_end_time'] >= range_start)
logging.info(
'Filtering out models with a train_end_time before {}'.format(
range_start))
model_matrix_info = model_matrix_info[msk]
if 'range_end_date' in train_end_times_range:
range_end = train_end_times_range['range_end_date']
msk = (model_matrix_info['train_end_time'] <= range_end)
logging.info(
'Filtering out models with a train_end_time after {}'.format(
range_end))
model_matrix_info = model_matrix_info[msk]
if len(model_matrix_info) == 0:
raise ValueError('Configis not valid. No models were found!')
# Al the model groups specified in the config file should valid (even if the experiment_hashes and train_end_times are specified)
not_fetched_model_grps = [
x for x in model_groups
if not x in model_matrix_info['model_group_id'].unique()
]
if len(not_fetched_model_grps) > 0:
raise ValueError(
'The config is not valid. No models were found for the model group(s) {}. All specified model groups should be present'
.format(not_fetched_model_grps))
logging.info('Scoring {} model ids'.format(len(model_matrix_info)))
# summary of the models that we are scoring. To check any special things worth noting
_summary_of_models(model_matrix_info)
logging.info('Instantiating storage engines and the predictor')
# Storage objects to handle already stored models and matrices
project_storage = ProjectStorage(project_path)
model_storage_engine = project_storage.model_storage_engine()
matrix_storage_engine = project_storage.matrix_storage_engine()
# Prediction generation is handled by the Predictor class in catwalk
predictor = Predictor(model_storage_engine=model_storage_engine,
db_engine=db_engine,
rank_order=rank_order,
replace=replace,
save_predictions=True)
# Organizing prediction run over unique (train_mat, test_mat) pairs
# This is to reduce no. the times the matrices get loaded to memory
groupby_obj = model_matrix_info.groupby(
['train_matrix_uuid', 'test_matrix_uuid'])
for group, _ in groupby_obj:
train_uuid = group[0]
test_uuid = group[1]
df_grp = groupby_obj.get_group(group)
logging.info(
'Processing {} model_ids for train matrix {} and test matrix {}'.
format(len(df_grp), train_uuid, test_uuid))
train_matrix_store = matrix_storage_engine.get_store(
matrix_uuid=train_uuid)
# To ensure that the column order we use for predictions match the order we used in model training
train_matrix_columns = list(train_matrix_store.design_matrix.columns)
test_matrix_store = matrix_storage_engine.get_store(
matrix_uuid=test_uuid)
for model_id in df_grp['model_id'].tolist():
logging.info(
'Writing predictions for model_id {}'.format(model_id))
predictor.predict(model_id=model_id,
matrix_store=test_matrix_store,
train_matrix_columns=train_matrix_columns,
misc_db_parameters={})
logging.info('Successfully generated predictions for {} models!'.format(
len(model_matrix_info)))
def get_matrix_storage_engine():
with TemporaryDirectory() as temp_dir:
yield ProjectStorage(temp_dir).matrix_storage_engine()
def project_storage(project_path):
"""Set up a temporary project storage engine on the filesystem
Yields (catwalk.storage.ProjectStorage)
"""
yield ProjectStorage(project_path)
def __init__(
self,
config,
db_engine,
project_path=None,
matrix_storage_class=CSVMatrixStore,
replace=True,
cleanup=False,
cleanup_timeout=None,
materialize_subquery_fromobjs=True,
features_ignore_cohort=False,
profile=False,
save_predictions=True,
skip_validation=False,
):
experiment_kwargs = bind_kwargs(
self.__class__, **{
key: value
for (key, value) in locals().items()
if key not in {'db_engine', 'config', 'self'}
})
self._check_config_version(config)
self.config = config
random.seed(config['random_seed'])
self.project_storage = ProjectStorage(project_path)
self.model_storage_engine = ModelStorageEngine(self.project_storage)
self.matrix_storage_engine = MatrixStorageEngine(
self.project_storage, matrix_storage_class)
self.project_path = project_path
self.replace = replace
self.save_predictions = save_predictions
self.skip_validation = skip_validation
self.db_engine = db_engine
results_schema.upgrade_if_clean(dburl=self.db_engine.url)
self.features_schema_name = "features"
self.materialize_subquery_fromobjs = materialize_subquery_fromobjs
self.features_ignore_cohort = features_ignore_cohort
self.experiment_hash = save_experiment_and_get_hash(
self.config, self.db_engine)
self.run_id = initialize_tracking_and_get_run_id(
self.experiment_hash,
experiment_class_path=classpath(self.__class__),
experiment_kwargs=experiment_kwargs,
db_engine=self.db_engine)
self.initialize_components()
self.cleanup = cleanup
if self.cleanup:
logging.info(
"cleanup is set to True, so intermediate tables (labels and cohort) "
"will be removed after matrix creation and subset tables will be "
"removed after model training and testing")
else:
logging.info(
"cleanup is set to False, so intermediate tables (labels, cohort, and subsets) "
"will not be removed")
self.cleanup_timeout = (self.cleanup_timeout if cleanup_timeout is None
else cleanup_timeout)
self.profile = profile
logging.info("Generate profiling stats? (profile option): %s",
self.profile)
def __init__(
self,
config,
db_engine,
project_path=None,
matrix_storage_class=CSVMatrixStore,
replace=True,
cleanup=False,
cleanup_timeout=None,
materialize_subquery_fromobjs=True,
features_ignore_cohort=False,
profile=False,
save_predictions=True,
skip_validation=False,
partial_run=False,
):
# For a partial run, skip validation and avoid cleaning up
# we'll also skip filling default config values below
if partial_run:
cleanup = False
skip_validation = True
experiment_kwargs = bind_kwargs(
self.__class__, **{
key: value
for (key, value) in locals().items()
if key not in {'db_engine', 'config', 'self'}
})
self._check_config_version(config)
self.config = config
self.config['random_seed'] = self.config.get('random_seed',
random.randint(1, 1e7))
random.seed(self.config['random_seed'])
self.project_storage = ProjectStorage(project_path)
self.model_storage_engine = ModelStorageEngine(self.project_storage)
self.matrix_storage_engine = MatrixStorageEngine(
self.project_storage, matrix_storage_class)
self.project_path = project_path
self.replace = replace
self.save_predictions = save_predictions
self.skip_validation = skip_validation
self.db_engine = db_engine
results_schema.upgrade_if_clean(dburl=self.db_engine.url)
self.features_schema_name = "features"
self.materialize_subquery_fromobjs = materialize_subquery_fromobjs
self.features_ignore_cohort = features_ignore_cohort
# only fill default values for full runs
if not partial_run:
## Defaults to sane values
self.config['temporal_config'] = fill_timechop_config_missing(
self.config, self.db_engine)
## Defaults to all the entities found in the features_aggregation's from_obj
self.config['cohort_config'] = fill_cohort_config_missing(
self.config)
## Defaults to all the feature_aggregation's prefixes
self.config[
'feature_group_definition'] = fill_feature_group_definition(
self.config)
grid_config = fill_model_grid_presets(self.config)
self.config.pop('model_grid_preset', None)
if grid_config is not None:
self.config['grid_config'] = grid_config
###################### RUBICON ######################
self.experiment_hash = save_experiment_and_get_hash(
self.config, self.db_engine)
self.run_id = initialize_tracking_and_get_run_id(
self.experiment_hash,
experiment_class_path=classpath(self.__class__),
experiment_kwargs=experiment_kwargs,
db_engine=self.db_engine)
self.initialize_components()
self.cleanup = cleanup
if self.cleanup:
logging.info(
"cleanup is set to True, so intermediate tables (labels and cohort) "
"will be removed after matrix creation and subset tables will be "
"removed after model training and testing")
else:
logging.info(
"cleanup is set to False, so intermediate tables (labels, cohort, and subsets) "
"will not be removed")
self.cleanup_timeout = (self.cleanup_timeout if cleanup_timeout is None
else cleanup_timeout)
self.profile = profile
logging.info("Generate profiling stats? (profile option): %s",
self.profile)
def basic_integration_test(state_filters, feature_group_create_rules,
feature_group_mix_rules, expected_matrix_multiplier,
expected_group_lists):
with testing.postgresql.Postgresql() as postgresql:
db_engine = create_engine(postgresql.url())
Base.metadata.create_all(db_engine)
populate_source_data(db_engine)
with TemporaryDirectory() as temp_dir:
chopper = Timechop(
feature_start_time=datetime(2010, 1, 1),
feature_end_time=datetime(2014, 1, 1),
label_start_time=datetime(2011, 1, 1),
label_end_time=datetime(2014, 1, 1),
model_update_frequency='1year',
training_label_timespans=['6months'],
test_label_timespans=['6months'],
training_as_of_date_frequencies='1day',
test_as_of_date_frequencies='3months',
max_training_histories=['1months'],
test_durations=['1months'],
)
state_table_generator = StateTableGeneratorFromDense(
db_engine=db_engine,
experiment_hash='abcd',
dense_state_table='states',
)
label_generator = LabelGenerator(
db_engine=db_engine,
query=sample_config()['label_config']['query'])
feature_generator = FeatureGenerator(
db_engine=db_engine,
features_schema_name='features',
replace=True,
)
feature_dictionary_creator = FeatureDictionaryCreator(
db_engine=db_engine, features_schema_name='features')
feature_group_creator = FeatureGroupCreator(
feature_group_create_rules)
feature_group_mixer = FeatureGroupMixer(feature_group_mix_rules)
project_storage = ProjectStorage(temp_dir)
planner = Planner(
feature_start_time=datetime(2010, 1, 1),
label_names=['outcome'],
label_types=['binary'],
states=state_filters,
user_metadata={},
)
builder = MatrixBuilder(
engine=db_engine,
db_config={
'features_schema_name': 'features',
'labels_schema_name': 'public',
'labels_table_name': 'labels',
'sparse_state_table_name': 'tmp_sparse_states_abcd',
},
matrix_storage_engine=project_storage.matrix_storage_engine(),
replace=True)
# chop time
split_definitions = chopper.chop_time()
num_split_matrices = sum(1 + len(split['test_matrices'])
for split in split_definitions)
# generate as_of_times for feature/label/state generation
all_as_of_times = []
for split in split_definitions:
all_as_of_times.extend(split['train_matrix']['as_of_times'])
for test_matrix in split['test_matrices']:
all_as_of_times.extend(test_matrix['as_of_times'])
all_as_of_times = list(set(all_as_of_times))
# generate sparse state table
state_table_generator.generate_sparse_table(
as_of_dates=all_as_of_times)
# create labels table
label_generator.generate_all_labels(labels_table='labels',
as_of_dates=all_as_of_times,
label_timespans=['6months'])
# create feature table tasks
# we would use FeatureGenerator#create_all_tables but want to use
# the tasks dict directly to create a feature dict
aggregations = feature_generator.aggregations(
feature_aggregation_config=[{
'prefix':
'cat',
'from_obj':
'cat_complaints',
'knowledge_date_column':
'as_of_date',
'aggregates': [{
'quantity': 'cat_sightings',
'metrics': ['count', 'avg'],
'imputation': {
'all': {
'type': 'mean'
}
}
}],
'intervals': ['1y'],
'groups': ['entity_id']
}, {
'prefix':
'dog',
'from_obj':
'dog_complaints',
'knowledge_date_column':
'as_of_date',
'aggregates_imputation': {
'count': {
'type': 'constant',
'value': 7
},
'sum': {
'type': 'mean'
},
'avg': {
'type': 'zero'
}
},
'aggregates': [{
'quantity': 'dog_sightings',
'metrics': ['count', 'avg'],
}],
'intervals': ['1y'],
'groups': ['entity_id']
}],
feature_dates=all_as_of_times,
state_table=state_table_generator.sparse_table_name)
feature_table_agg_tasks = feature_generator.generate_all_table_tasks(
aggregations, task_type='aggregation')
# create feature aggregation tables
feature_generator.process_table_tasks(feature_table_agg_tasks)
feature_table_imp_tasks = feature_generator.generate_all_table_tasks(
aggregations, task_type='imputation')
# create feature imputation tables
feature_generator.process_table_tasks(feature_table_imp_tasks)
# build feature dictionaries from feature tables and
# subsetting config
master_feature_dict = feature_dictionary_creator.feature_dictionary(
feature_table_names=feature_table_imp_tasks.keys(),
index_column_lookup=feature_generator.index_column_lookup(
aggregations))
feature_dicts = feature_group_mixer.generate(
feature_group_creator.subsets(master_feature_dict))
# figure out what matrices need to be built
_, matrix_build_tasks =\
planner.generate_plans(
split_definitions,
feature_dicts
)
# go and build the matrices
builder.build_all_matrices(matrix_build_tasks)
# super basic assertion: did matrices we expect get created?
matrices_records = list(
db_engine.execute(
'''select matrix_uuid, num_observations, matrix_type
from model_metadata.matrices
'''))
matrix_directory = os.path.join(temp_dir, 'matrices')
matrices = [
path for path in os.listdir(matrix_directory) if '.csv' in path
]
metadatas = [
path for path in os.listdir(matrix_directory)
if '.yaml' in path
]
assert len(matrices) == num_split_matrices * \
expected_matrix_multiplier
assert len(metadatas) == num_split_matrices * \
expected_matrix_multiplier
assert len(matrices) == len(matrices_records)
feature_group_name_lists = []
for metadata_path in metadatas:
with open(os.path.join(matrix_directory, metadata_path)) as f:
metadata = yaml.load(f)
feature_group_name_lists.append(metadata['feature_groups'])
for matrix_uuid, num_observations, matrix_type in matrices_records:
assert matrix_uuid in matrix_build_tasks #the hashes of the matrices
assert type(num_observations) is int
assert matrix_type == matrix_build_tasks[matrix_uuid][
'matrix_type']
def deep_unique_tuple(l):
return set([tuple(i) for i in l])
assert deep_unique_tuple(
feature_group_name_lists) == deep_unique_tuple(
expected_group_lists)
def __init__(self, db_engine, project_path, model_group_id):
self.retrain_hash = None
self.db_engine = db_engine
upgrade_db(db_engine=self.db_engine)
self.project_storage = ProjectStorage(project_path)
self.model_group_id = model_group_id
self.model_group_info = get_model_group_info(self.db_engine,
self.model_group_id)
self.matrix_storage_engine = self.project_storage.matrix_storage_engine(
)
self.triage_run_id, self.experiment_config = experiment_config_from_model_group_id(
self.db_engine, self.model_group_id)
# This feels like it needs some refactoring since in some edge cases at least the test matrix temporal parameters
# might differ across models in the mdoel group (the training ones shouldn't), but this should probably work for
# the vast majorty of use cases...
self.experiment_config['temporal_config'].update(
temporal_params_from_matrix_metadata(
self.db_engine, self.model_group_info['model_id_last_split']))
# Since "testing" here is predicting forward to a single new date, the test_duration should always be '0day'
# (regardless of what it may have been before)
self.experiment_config['temporal_config']['test_durations'] = ['0day']
# These lists should now only contain one item (the value actually used for the last model in this group)
self.training_label_timespan = self.experiment_config[
'temporal_config']['training_label_timespans'][0]
self.test_label_timespan = self.experiment_config['temporal_config'][
'test_label_timespans'][0]
self.test_duration = self.experiment_config['temporal_config'][
'test_durations'][0]
self.feature_start_time = self.experiment_config['temporal_config'][
'feature_start_time']
self.label_name = self.experiment_config['label_config']['name']
self.cohort_name = self.experiment_config['cohort_config']['name']
self.user_metadata = self.experiment_config['user_metadata']
self.feature_dictionary_creator = FeatureDictionaryCreator(
features_schema_name='triage_production', db_engine=self.db_engine)
self.label_generator = LabelGenerator(
label_name=self.experiment_config['label_config'].get(
"name", None),
query=self.experiment_config['label_config']["query"],
replace=True,
db_engine=self.db_engine,
)
self.labels_table_name = "labels_{}_{}_production".format(
self.experiment_config['label_config'].get('name', 'default'),
filename_friendly_hash(
self.experiment_config['label_config']['query']))
self.feature_generator = FeatureGenerator(
db_engine=self.db_engine,
features_schema_name="triage_production",
feature_start_time=self.feature_start_time,
)
self.model_trainer = ModelTrainer(
experiment_hash=None,
model_storage_engine=ModelStorageEngine(self.project_storage),
db_engine=self.db_engine,
replace=True,
run_id=self.triage_run_id,
)