def __init__(self,
modellist,
eval_comparator,
datafile=None,
train_data_person=None,
silent=False,
split_ratio=0.5,
no_pretrain=False,
no_adapt=False):
"""
Parameters
----------
modellist : list(ccobra.ModelInfo)
List containing model paths and additional arguments for model initialization.
eval_comparator : Comparator
Comparator to be used for computing hits/misses.
test_datafile : str
Path to the test data file.
train_datafile : str
Path to the general training data file.
train_data_person : str
Path to the person training data file.
silent : bool
Indicates whether evaluation progress should be logged using print statements.
corresponding_data : bool
Indicates whether test and training data should contain the same
user ids.
"""
self.adapt = not no_adapt
self.modellist = modellist
self.silent = silent
self.domains = set()
self.response_types = set()
self.comparator = eval_comparator
# Load the data set
self.data = ccobra.CCobraData(pd.read_csv(datafile))
self.split_ratio = split_ratio
self.pretrain = not no_pretrain
# Load the personal training data
self.train_data_person = None
if train_data_person:
self.train_data_person = ccobra.CCobraData(
pd.read_csv(train_data_person))
def evaluate(self):
""" CCobra evaluation loop. Iterates over the models and performs training and evaluation.
Returns
-------
pd.DataFrame
DataFrame containing the CCOBRA evaluation results.
"""
assert self.data is not None
result_data = []
model_name_cache = set()
# Pre-compute the training data dictionaries
train_data_dict = None
train_data_dict = self.get_train_data_dict(self.data)
# Activate model context
for idx, modelinfo in enumerate(self.modellist):
# Print the progress
if not self.silent:
print("Evaluating '{}' ({}/{})...".format(
modelinfo.path, idx + 1, len(self.modellist)))
# Setup the model context
context = os.path.abspath(modelinfo.path)
if os.path.isfile(context):
context = os.path.dirname(context)
with dir_context(context):
# Dynamically import the CCOBRA model
importer = modelimporter.ModelImporter(
modelinfo.path,
ccobra.CCobraModel,
load_specific_class=modelinfo.load_specific_class)
# Instantiate and prepare the model for predictions
pre_model = importer.instantiate(modelinfo.args)
# Check if model is applicable to domains/response types
self.check_model_applicability(pre_model)
# Only use the model's name if no override is specified
model_name = modelinfo.override_name
if not model_name:
model_name = pre_model.name
# Ensure that names are unique and show a warning if duplicates are detected
original_model_name = model_name
changed = False
while model_name in model_name_cache:
model_name = model_name + '\''
changed = True
model_name_cache.add(model_name)
if changed:
warnings.warn(
'Duplicate model name detected ("{}"). Changed to "{}".'
.format(original_model_name, model_name))
# Iterate subject
for subj_id, subj_df in self.data.get().groupby('id'):
model = copy.deepcopy(pre_model)
# Perform pre-training for individual subjects
# Remove one participant
cur_train_data_dict = [
value for key, value in train_data_dict.items()
if key != subj_id
]
# Train on incomplete training data
if self.pretrain:
model.pre_train(cur_train_data_dict)
# Perform the personalized pre-training
if self.train_data_person is not None:
# Pick out the person training data for the current
# individual
subj_pre_train_data_person = self.train_data_person.get(
).loc[self.train_data_person.get()['id'] == subj_id]
person_train_data = self.get_train_data_dict(
ccobra.CCobraData(subj_pre_train_data_person))
model.person_train(person_train_data[subj_id])
# Extract the subject demographics
demographics = self.extract_demographics(subj_df)
# Set the models to new participant
model.start_participant(id=subj_id, **demographics)
# split the individuals data in train and test set
perm = np.random.permutation(np.arange(len(subj_df)))
split_id = int(self.split_ratio * len(subj_df))
train_ids, test_ids = perm[:split_id], perm[split_id:]
train_set, test_set = subj_df.iloc[
train_ids], subj_df.iloc[test_ids]
# adapt to train set
adapt_items_list = []
truths_list = []
optionals_list = []
# prepare all tasks from train set
for _, row in train_set.iterrows():
optionals = self.extract_optionals(row)
# Evaluation
sequence = row['sequence']
task = row['task']
choices = row['choices']
truth = row['response']
response_type = row['response_type']
domain = row['domain']
if isinstance(truth, str):
if response_type == 'multiple-choice':
truth = [
y.split(';') for y in
[x.split('/') for x in truth.split('|')]
]
else:
truth = [
x.split(';') for x in truth.split('/')
]
item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
# Adapt to true response
adapt_item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
adapt_items_list.append(adapt_item)
truths_list.append(truth)
optionals_list.append(optionals)
# give all items to models at once for data driven training
try:
model.batch_adapt(adapt_items_list, truths_list,
optionals_list)
except AttributeError: # regular adapt if other not implemented
for items, truth, optionals in zip(
adapt_items_list, truths_list, optionals_list):
model.adapt(item, truth, **optionals)
for _, row in test_set.iterrows():
optionals = self.extract_optionals(row)
# Evaluation
sequence = row['sequence']
task = row['task']
choices = row['choices']
truth = row['response']
response_type = row['response_type']
domain = row['domain']
if isinstance(truth, str):
if response_type == 'multiple-choice':
truth = [
y.split(';') for y in
[x.split('/') for x in truth.split('|')]
]
else:
truth = [
x.split(';') for x in truth.split('/')
]
item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
prediction = model.predict(item, **optionals)
hit = int(self.comparator.compare(prediction, truth))
# Collect the evaluation result data
result_data.append({
'model':
model_name,
'id':
subj_id,
'domain':
domain,
'sequence':
sequence,
'task':
task,
'choices':
choices,
'truth':
row['response'],
'prediction':
comparator.tuple_to_string(prediction),
'hit':
hit
})
# Call the end participant hook
model.end_participant(subj_id, **optionals)
# De-load the imported model and its dependencies. Might
# cause garbage collection issues.
importer.unimport()
return pd.DataFrame(result_data)
def __init__(self,
modellist,
eval_comparator,
test_datafile,
train_datafile=None,
train_data_person=None,
silent=False,
corresponding_data=False,
no_adapt=False):
"""
Parameters
----------
modellist : list(ccobra.ModelInfo)
List containing model paths and additional arguments for model initialization.
eval_comparator : Comparator
Comparator to be used for computing hits/misses.
test_datafile : str
Path to the test data file.
train_datafile : str
Path to the general training data file.
train_data_person : str
Path to the person training data file.
silent : bool
Indicates whether evaluation progress should be logged using print statements.
corresponding_data : bool
Indicates whether test and training data should contain the same
user ids.
"""
self.adapt = not no_adapt
self.modellist = modellist
self.silent = silent
self.domains = set()
self.response_types = set()
self.comparator = eval_comparator
self.corresponding_data = corresponding_data
# Load the test data
self.test_data = ccobra.CCobraData(pd.read_csv(test_datafile))
self.domains.update(self.test_data.get()['domain'].unique())
self.response_types.update(
self.test_data.get()['response_type'].unique())
# Load the general training data
self.train_data = None
if train_datafile:
# Load the data. Domains and response types are not updated,
# because training is considered optional information the models
# are not forced to use.
self.train_data = ccobra.CCobraData(pd.read_csv(train_datafile))
# If non-corresponding datasets, update with new identification
if not self.corresponding_data:
test_ids = self.test_data.get()['id'].unique()
# Identify the ID offset as the largest numerical index from
# the test dataset.
idx_offset = 0
for identifier in test_ids:
# Strings cause the float conversion to throw an exception
# Numberness is identified accordingly.
try:
if idx_offset < float(identifier):
idx_offset = float(identifier)
except ValueError:
pass
idx_offset = int(np.ceil(idx_offset)) + 1
# Update the training IDs accordingly
train_ids = self.train_data.get()['id'].unique()
new_train_ids = dict(
zip(train_ids,
range(idx_offset, idx_offset + len(train_ids))))
self.train_data.get()['id'].replace(new_train_ids,
inplace=True)
# Load the personal training data
self.train_data_person = None
if train_data_person:
self.train_data_person = ccobra.CCobraData(
pd.read_csv(train_data_person))
def evaluate(self):
""" CCobra evaluation loop. Iterates over the models and performs training and evaluation.
Returns
-------
pd.DataFrame
DataFrame containing the CCOBRA evaluation results.
"""
result_data = []
model_name_cache = set()
# Pre-compute the training data dictionaries
train_data_dict = None
if self.train_data is not None:
train_data_dict = self.get_train_data_dict(self.train_data)
# Activate model context
for idx, modelinfo in enumerate(self.modellist):
model_evaluations = {}
# Print the progress
if not self.silent:
print("Evaluating '{}' ({}/{})...".format(
modelinfo.path, idx + 1, len(self.modellist)))
# Setup the model context
context = os.path.abspath(modelinfo.path)
if os.path.isfile(context):
context = os.path.dirname(context)
with dir_context(context):
# Dynamically import the CCOBRA model
importer = modelimporter.ModelImporter(
modelinfo.path,
ccobra.CCobraModel,
load_specific_class=modelinfo.load_specific_class)
# Instantiate and prepare the model for predictions
pre_model = importer.instantiate(modelinfo.args)
# Check if model is applicable to domains/response types
self.check_model_applicability(pre_model)
# Only use the model's name if no override is specified
model_name = modelinfo.override_name
if not model_name:
model_name = pre_model.name
# Ensure that names are unique and show a warning if duplicates are detected
original_model_name = model_name
changed = False
while model_name in model_name_cache:
model_name = model_name + '\''
changed = True
model_name_cache.add(model_name)
if changed:
warnings.warn(
'Duplicate model name detected ("{}"). Changed to "{}".'
.format(original_model_name, model_name))
# Only perform general pre-training if training data is
# supplied and corresponding data is false. Otherwise, the
# model has to be re-trained for each subject.
if self.train_data is not None and not self.corresponding_data:
# Prepare training data
pre_model.pre_train(list(train_data_dict.values()))
# Iterate subject
for subj_id, subj_df in self.test_data.get().groupby('id'):
model = copy.deepcopy(pre_model)
model_evaluations[subj_id] = {}
# Perform pre-training for individual subjects only if
# corresponding data is set to true.
if self.train_data is not None and self.corresponding_data:
# Remove one participant
cur_train_data_dict = [
value for key, value in train_data_dict.items()
if key != subj_id
]
model_checkpoints = []
# Train on incomplete training data
model.pre_train(cur_train_data_dict,
checkpoints=model_checkpoints)
if pre_train_str in self.learning_curves_for:
model_evaluations[subj_id][
pre_train_str] = self.evaluate_checkpoints(
model_checkpoints, subj_id)
# Perform the personalized pre-training
if self.train_data_person is not None:
# Pick out the person training data for the current
# individual
subj_pre_train_data_person = self.train_data_person.get(
).loc[self.train_data_person.get()['id'] == subj_id]
model_checkpoints = []
person_train_data = self.get_train_data_dict(
ccobra.CCobraData(subj_pre_train_data_person))
model.person_train(person_train_data[subj_id],
checkpoints=model_checkpoints)
if person_train_str in self.learning_curves_for:
model_evaluations[subj_id][
person_train_str] = self.evaluate_checkpoints(
model_checkpoints, subj_id)
# Extract the subject demographics
demographics = self.extract_demographics(subj_df)
model_checkpoints = []
# Set the models to new participant
model.start_participant(id=subj_id,
checkpoints=model_checkpoints,
**demographics)
if start_participant_str in self.learning_curves_for:
model_evaluations[subj_id][
start_participant_str] = self.evaluate_checkpoints(
model_checkpoints, subj_id)
if main_train_str in self.learning_curves_for:
model_evaluations[subj_id][
main_train_str] = self.evaluate_checkpoints(
[copy.deepcopy(model)], subj_id)
if adapt_str in self.learning_curves_for:
model_evaluations[subj_id][adapt_str] = {}
for r, (_, row) in enumerate(
subj_df.sort_values('sequence').iterrows()):
optionals = self.extract_optionals(row)
# Evaluation
sequence = row['sequence']
task = row['task']
choices = row['choices']
truth = row['response']
response_type = row['response_type']
domain = row['domain']
if isinstance(truth, str):
if response_type == 'multiple-choice':
truth = [
y.split(';') for y in
[x.split('/') for x in truth.split('|')]
]
else:
truth = [
x.split(';') for x in truth.split('/')
]
item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
prediction = model.predict(item, **optionals)
hit = int(self.comparator.compare(prediction, truth))
model_checkpoints = []
# Adapt to true response
adapt_item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
if self.adapt:
model.adapt(adapt_item,
truth,
checkpoints=model_checkpoints,
**optionals)
if adapt_str in self.learning_curves_for:
model_evaluations[subj_id][adapt_str][
sequence] = self.evaluate_checkpoints(
model_checkpoints, subj_id)
if (main_train_str in self.learning_curves_for
and r % self.evaluation_frequency == 0):
model_evaluations[subj_id][main_train_str].append(
self.evaluate_checkpoints(
[copy.deepcopy(model)], subj_id)[0])
result_data.append({
'model':
model.name,
'id':
subj_id,
'domain':
domain,
'sequence':
sequence,
'task':
task,
'choices':
choices,
'truth':
row['response'],
'prediction':
comparator.tuple_to_string(prediction),
'hit':
hit,
})
# De-load the imported model and its dependencies. Might
# cause garbage collection issues.
importer.unimport()
if self.learning_curves_folder:
self.generate_learning_curves(model.name,
model_evaluations)
return pd.DataFrame(result_data)
def evaluate(self):
""" CCobra evaluation loop. Iterates over the models and performs training and evaluation.
Returns
-------
pd.DataFrame
DataFrame containing the CCOBRA evaluation results.
"""
result_data = []
# Pre-compute the training data dictionaries
train_data_dict = None
if self.train_data is not None:
train_data_dict = self.get_train_data_dict(self.train_data)
# Activate model context
for idx, modelitem in enumerate(self.modeldict.items()):
model, model_kwargs = modelitem
# Print the progress
if not self.silent:
print("Evaluating '{}' ({}/{})...".format(
model, idx + 1, len(self.modeldict)))
# Setup the model context
context = os.path.abspath(model)
if os.path.isfile(context):
context = os.path.dirname(context)
# Extract load_specific_class
specific_class = None
if 'load_specific_class' in model_kwargs:
specific_class = model_kwargs['load_specific_class']
del model_kwargs['load_specific_class']
with dir_context(context):
importer = modelimporter.ModelImporter(
model,
ccobra.CCobraModel,
load_specific_class=specific_class)
# Instantiate and prepare the model for predictions
pre_model = importer.instantiate(model_kwargs)
# Check if model is applicable to domains/response types
self.check_model_applicability(pre_model)
# Only perform general pre-training if training data is
# supplied and corresponding data is false. Otherwise, the
# model has to be re-trained for each subject.
if self.train_data is not None and not self.corresponding_data:
# Prepare training data
pre_model.pre_train(list(train_data_dict.values()))
# Iterate subject
for subj_id, subj_df in self.test_data.get().groupby('id'):
model = copy.deepcopy(pre_model)
# Perform pre-training for individual subjects only if
# corresponding data is set to true.
if self.train_data is not None and self.corresponding_data:
# Remove one participant
cur_train_data_dict = [
value for key, value in train_data_dict.items()
if key != subj_id
]
# Train on incomplete training data
model.pre_train(cur_train_data_dict)
# Perform the personalized pre-training
if self.train_data_person is not None:
# Pick out the person training data for the current
# individual
subj_pre_train_data_person = self.train_data_person.get(
).loc[self.train_data_person.get()['id'] == subj_id]
person_train_data = self.get_train_data_dict(
ccobra.CCobraData(subj_pre_train_data_person))
model.person_train(person_train_data[subj_id])
# Extract the subject demographics
demographics = self.extract_demographics(subj_df)
# Set the models to new participant
model.start_participant(id=subj_id, **demographics)
for _, row in subj_df.sort_values('sequence').iterrows():
optionals = self.extract_optionals(row)
# Evaluation
sequence = row['sequence']
task = row['task']
choices = row['choices']
truth = row['response']
response_type = row['response_type']
domain = row['domain']
if isinstance(truth, str):
if response_type == 'multiple-choice':
truth = [
y.split(';') for y in
[x.split('/') for x in truth.split('|')]
]
else:
truth = [
x.split(';') for x in truth.split('/')
]
item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
prediction = model.predict(item, **optionals)
hit = int(self.comparator.compare(prediction, truth))
# Adapt to true response
adapt_item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
model.adapt(adapt_item, truth, **optionals)
result_data.append({
'model':
model.name,
'id':
subj_id,
'domain':
domain,
'sequence':
sequence,
'task':
task,
'choices':
choices,
'truth':
row['response'],
'prediction':
comparator.tuple_to_string(prediction),
'hit':
hit,
})
# De-load the imported model and its dependencies. Might
# cause garbage collection issues.
importer.unimport()
return pd.DataFrame(result_data)
def evaluate(self):
""" CCobra evaluation loop. Iterates over the models and performs training and evaluation.
Returns
-------
pd.DataFrame
DataFrame containing the CCOBRA evaluation results.
"""
result_data = []
model_name_cache = set() if self.cache_df is None else set(
self.cache_df['model'].unique())
# Pre-compute the training data dictionaries
train_data_dict = None
if self.train_data is not None:
train_data_dict = self.get_train_data_dict(self.train_data)
# Activate model context
for idx, modelinfo in enumerate(self.modellist):
# Print the progress
if not self.silent:
print("Evaluating '{}' ({}/{})...".format(
modelinfo.path, idx + 1, len(self.modellist)))
# Setup the model context
with contextmanager.dir_context(modelinfo.path):
# Dynamically import the CCOBRA model
importer = modelimporter.ModelImporter(
modelinfo.path,
ccobra.CCobraModel,
load_specific_class=modelinfo.load_specific_class)
# Instantiate and prepare the model for predictions
pre_model = importer.instantiate(modelinfo.args)
# Check if model is applicable to domains/response types
self.check_model_applicability(pre_model)
# Only use the model's name if no override is specified
model_name = modelinfo.override_name
if not model_name:
model_name = pre_model.name
# Ensure that names are unique and show a warning if duplicates are detected
original_model_name = model_name
changed = False
while model_name in model_name_cache:
model_name = model_name + '\''
changed = True
model_name_cache.add(model_name)
if changed:
warnings.warn(
'Duplicate model name detected ("{}"). Changed to "{}".'
.format(original_model_name, model_name))
# Only perform general pre-training if training data is
# supplied and corresponding data is false. Otherwise, the
# model has to be re-trained for each subject.
if self.train_data is not None and not self.corresponding_data:
# Prepare training data
pre_model.pre_train(list(train_data_dict.values()))
# Iterate subject
for subj_id, subj_df in self.test_data.get().groupby('id'):
model = copy.deepcopy(pre_model)
# Perform pre-training for individual subjects only if
# corresponding data is set to true.
if self.train_data is not None and self.corresponding_data:
# Remove one participant
cur_train_data_dict = [
value for key, value in train_data_dict.items()
if key != subj_id
]
# Train on incomplete training data
model.pre_train(cur_train_data_dict)
# Perform the personalized pre-training
if self.train_data_person is not None:
# Pick out the person training data for the current
# individual
subj_pre_train_data_person = self.train_data_person.get(
).loc[self.train_data_person.get()['id'] == subj_id]
person_train_data = self.get_train_data_dict(
ccobra.CCobraData(subj_pre_train_data_person))
model.person_train(person_train_data[subj_id])
# Extract the subject demographics
demographics = self.extract_demographics(subj_df)
# Set the models to new participant
model.start_participant(id=subj_id, **demographics)
# Iterate over individual tasks
for _, row in subj_df.sort_values('sequence').iterrows():
optionals = self.extract_optionals(row)
# Evaluation
sequence = row['sequence']
task = row['task']
choices = row['choices']
truth = row['response']
response_type = row['response_type']
domain = row['domain']
if isinstance(truth, str):
if response_type == 'multiple-choice':
truth = [
y.split(';') for y in
[x.split('/') for x in truth.split('|')]
]
else:
truth = [
x.split(';') for x in truth.split('/')
]
item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
prediction = model.predict(copy.deepcopy(item),
**optionals)
hit = int(self.comparator.compare(prediction, truth))
# Adapt to true response
adapt_item = ccobra.data.Item(subj_id, domain, task,
response_type, choices,
sequence)
model.adapt(adapt_item, truth, **optionals)
# Collect the evaluation result data
prediction_data = {
'model': model_name,
'id': subj_id,
'domain': domain,
'response_type': response_type,
'sequence': sequence,
'task': task,
'choices': choices,
'truth': row['response'],
'prediction':
comparator.tuple_to_string(prediction),
'hit': hit,
'type': 'adaption'
}
# If domain encoders are specified, attach encodings to the result
task_enc = None
truth_enc = None
pred_enc = None
if domain in self.domain_encoders:
task_enc = self.domain_encoders[domain].encode_task(
item.task
) if domain in self.domain_encoders else np.nan
truth_enc = self.domain_encoders[
domain].encode_response(
truth, item.task
) if domain in self.domain_encoders else np.nan
pred_enc = self.domain_encoders[
domain].encode_response(
prediction, item.task
) if domain in self.domain_encoders else np.nan
prediction_data.update({
'task_enc': task_enc,
'truth_enc': truth_enc,
'prediction_enc': pred_enc
})
result_data.append(prediction_data)
# Call the end participant hook
model.end_participant(subj_id, **optionals)
# De-load the imported model and its dependencies. Might
# cause garbage collection issues.
importer.unimport()
res_df = pd.DataFrame(result_data)
if self.cache_df is None:
return res_df
if not result_data:
return self.cache_df
assert sorted(list(res_df)) == sorted(list(
self.cache_df)), 'Incompatible cache'
return pd.concat([res_df, self.cache_df])