def evaluate(model_path=os.path.join('/home/ubuntu/checkpoints',
'xception05-0.236.mod')):
# load test data
test_X = np.load(os.path.join('/home/ubuntu/training_data', 'test_X.npy'))
test_Y = np.load(os.path.join('/home/ubuntu/training_data', 'test_Y.npy'))
# load model
model = model = load_model(model_path)
# get predictions
test_predictions = model.predict(test_X, batch_size=16)
# get measures
test_measures = measures.get_measures(test_predictions, test_Y, .5)
# read genres
genre_file_path = os.path.join('/home/ubuntu/training_data', 'genres.txt')
with open(genre_file_path, 'r') as handler:
genres = handler.readlines()
genres = [genre[:-1] for genre in genres]
# print measures
print("Statistics on test data:")
measures.print_measures(test_measures, genres)
def evaluate(
model_path=os.path.join('model', 'mod'), cutoff_file='cutoffs.npy'):
# load test data
test_X = np.load(os.path.join('training_data', 'test_X.npy'))
test_Y = np.load(os.path.join('training_data', 'test_Y.npy'))
# load model
model = model = load_model(model_path)
# load cutoffs
cutoffs = np.load(os.path.join('cutoffs', cutoff_file))
# get predictions
test_predictions = model.predict(test_X, batch_size=16)
# get measures
test_measures = measures.get_measures(test_predictions, test_Y, cutoffs)
# read genres
genre_file_path = os.path.join('training_data', 'genres.txt')
with open(genre_file_path, 'r') as handler:
genres = handler.readlines()
genres = [genre[:-1] for genre in genres]
# print measures
print("Statistics on test data:")
measures.print_measures(test_measures, genres)
def pcc_experiment(dataset,
prediction_type,
classifier,
epsilon=0.0,
mc_iterations=0):
print("------- {0} -------".format(dataset['name']))
print("PCC: {0}, epsilon: {1}, MC iterations: {2}".format(
prediction_type, epsilon, mc_iterations))
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(
dataset)
pcc = ProbabilisticClassifierChain(classifier)
start_time = time.clock()
pcc.fit(x_train, y_train, number_of_labels)
learning_end_time = time.clock()
y_predicted = pcc.predict(x_test, prediction_type, epsilon, mc_iterations)
prediction_end_time = time.clock()
number_of_visited_classifiers = float(
pcc.get_number_of_visited_classifiers()) / len(x_test)
print("Number of used classifiers: {0}".format(
number_of_visited_classifiers))
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(
start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(
learning_time, prediction_time))
def hc_regression_experiment(dataset,
depth_of_search,
loss_function,
regression_h,
regression_c,
br=None):
print("------- {0} -------".format(dataset['name']))
print "HC: regression, depth: {0}, Loss function: {1}".format(
str(depth_of_search), loss_function)
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(
dataset)
hc_search = HCSearchRegression(regression_h, regression_c, loss_function,
depth_of_search, number_of_labels)
start_time = time.clock()
hc_search.fit(x_train, y_train)
learning_end_time = time.clock()
y_predicted = hc_search.predict(x_test)
prediction_end_time = time.clock()
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(
start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(
learning_time, prediction_time))
def fuzzy(timetable, loglevel=logging.INFO):
# init logger for this function
log = logging.getLogger("evaluation.fuzzy")
log.setLevel(loglevel)
# check if global variables are initialized
assert fs[
'scoring_ctrl'] is not None, "Fuzzy control system not initialized."
# compute score
scoring = skfuzzy.control.ControlSystemSimulation(fs['scoring_ctrl'])
scoring.input['overlaps'] = measures.count_overlaps(timetable)
scoring.input['slotdiff'] = measures.sum_up_slot_differences(timetable)
scoring.input['testdiff'] = measures.sum_up_testlength_differences(
timetable)
scoring.input['rchanges'] = measures.count_room_changes(timetable)
scoring.compute()
if loglevel == logging.DEBUG:
fs['score'].view(sim=scoring)
log.debug(measures.print_measures(timetable))
log.debug('score: %.2f' % (scoring.output['score']))
return scoring.output['score']
def hc_ranking_experiment(dataset, depth_of_search, loss_function,
classifier_h, classifier_c, parameter_grid=None,
br=None, reduction=1.0):
print("------- {0} -------".format(dataset['name']))
if reduction != 1.0:
print("DATASET REDUCED TO: {0}%".format(reduction*100))
print "HC: ranking, depth: {0}, Loss function: {1}".format(str(depth_of_search), loss_function)
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(dataset)
if dataset['name'] in ['bibtex']:
x_train, _, y_train, _ = train_test_split(x_train, y_train, test_size=0.90, random_state=42)
h = deepcopy(classifier_h)
c = deepcopy(classifier_c)
if parameter_grid is not None:
best_parameters_h = None
best_parameters_c = None
best_loss = 1.0
x_train_train, x_train_valid, y_train_train, y_train_valid = train_test_split(x_train, y_train, test_size=0.25, random_state=42)
for parameters_h in ParameterGrid(parameter_grid):
for parameters_c in ParameterGrid(parameter_grid):
print("------H params: {0}, C params: {1}------".format(parameters_h, parameters_c))
h = deepcopy(classifier_h).set_params(**parameters_h)
c = deepcopy(classifier_c).set_params(**parameters_c)
y_predicted = hc_learn_and_predict(br, c, depth_of_search, h, loss_function, number_of_labels,
x_train_train, x_train_valid, y_train_train, y_train_valid,
reduction)
calculated_loss = calculate_average_loss(loss_function, y_predicted, y_train_valid, number_of_labels)
print("Calculated loss: {0}".format(calculated_loss))
if calculated_loss < best_loss:
best_parameters_h = parameters_h
best_parameters_c = parameters_c
best_loss = calculated_loss
print {"Final H params: {0}, Final C params: {1}".format(best_parameters_h, best_parameters_c)}
h = h.set_params(**best_parameters_h)
c = c.set_params(**best_parameters_c)
y_predicted = hc_learn_and_predict(br, c, depth_of_search, h, loss_function, number_of_labels,
x_train, x_test, y_train, y_test, reduction)
print_measures(y_predicted, y_test, number_of_labels)
def br_experiment(dataset, base_classifier):
print("------- {0} -------".format(dataset['name']))
print("BR experiment")
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(dataset)
classifier = OneVsRestClassifier(base_classifier, n_jobs=-1)
start_time = time.clock()
classifier.fit(x_train, y_train)
learning_end_time = time.clock()
y_predicted = classifier.predict(x_test)
prediction_end_time = time.clock()
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(learning_time, prediction_time))
return classifier
def hc_regression_experiment(dataset, depth_of_search, loss_function, regression_h, regression_c, br=None):
print("------- {0} -------".format(dataset['name']))
print "HC: regression, depth: {0}, Loss function: {1}".format(str(depth_of_search), loss_function)
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(dataset)
hc_search = HCSearchRegression(regression_h, regression_c, loss_function, depth_of_search, number_of_labels)
start_time = time.clock()
hc_search.fit(x_train, y_train)
learning_end_time = time.clock()
y_predicted = hc_search.predict(x_test)
prediction_end_time = time.clock()
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(learning_time, prediction_time))
def pcc_experiment(dataset, prediction_type, classifier, epsilon=0.0, mc_iterations=0):
print("------- {0} -------".format(dataset['name']))
print("PCC: {0}, epsilon: {1}, MC iterations: {2}".format(prediction_type, epsilon, mc_iterations))
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(dataset)
pcc = ProbabilisticClassifierChain(classifier)
start_time = time.clock()
pcc.fit(x_train, y_train, number_of_labels)
learning_end_time = time.clock()
y_predicted = pcc.predict(x_test, prediction_type, epsilon, mc_iterations)
prediction_end_time = time.clock()
number_of_visited_classifiers = float(pcc.get_number_of_visited_classifiers()) / len(x_test)
print("Number of used classifiers: {0}".format(number_of_visited_classifiers))
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(learning_time, prediction_time))
def br_experiment(dataset, base_classifier):
print("------- {0} -------".format(dataset['name']))
print("BR experiment")
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(
dataset)
classifier = OneVsRestClassifier(base_classifier, n_jobs=-1)
start_time = time.clock()
classifier.fit(x_train, y_train)
learning_end_time = time.clock()
y_predicted = classifier.predict(x_test)
prediction_end_time = time.clock()
print_measures(y_predicted, y_test, number_of_labels)
learning_time, prediction_time, total_time = calculate_run_times(
start_time, learning_end_time, prediction_end_time)
print("Learning time: {0:.4f}, Prediction time: {1:.4f}".format(
learning_time, prediction_time))
return classifier
def fuzzy(timetable, loglevel=logging.INFO):
# init logger for this function
log = logging.getLogger("evaluation.fuzzy")
log.setLevel(loglevel)
# check if global variables are initialized
assert fs['scoring_ctrl'] is not None, "Fuzzy control system not initialized."
# compute score
scoring = skfuzzy.control.ControlSystemSimulation(fs['scoring_ctrl'])
scoring.input['overlaps'] = measures.count_overlaps(timetable)
scoring.input['slotdiff'] = measures.sum_up_slot_differences(timetable)
scoring.input['testdiff'] = measures.sum_up_testlength_differences(timetable)
scoring.input['rchanges'] = measures.count_room_changes(timetable)
scoring.compute()
if loglevel == logging.DEBUG:
fs['score'].view(sim=scoring)
log.debug(measures.print_measures(timetable))
log.debug('score: %.2f' %(scoring.output['score']))
return scoring.output['score']
def hc_ranking_experiment(dataset,
depth_of_search,
loss_function,
classifier_h,
classifier_c,
parameter_grid=None,
br=None,
reduction=1.0):
print("------- {0} -------".format(dataset['name']))
if reduction != 1.0:
print("DATASET REDUCED TO: {0}%".format(reduction * 100))
print "HC: ranking, depth: {0}, Loss function: {1}".format(
str(depth_of_search), loss_function)
x_train, y_train, x_test, y_test, number_of_labels = prepare_experiment(
dataset)
if dataset['name'] in ['bibtex']:
x_train, _, y_train, _ = train_test_split(x_train,
y_train,
test_size=0.90,
random_state=42)
h = deepcopy(classifier_h)
c = deepcopy(classifier_c)
if parameter_grid is not None:
best_parameters_h = None
best_parameters_c = None
best_loss = 1.0
x_train_train, x_train_valid, y_train_train, y_train_valid = train_test_split(
x_train, y_train, test_size=0.25, random_state=42)
for parameters_h in ParameterGrid(parameter_grid):
for parameters_c in ParameterGrid(parameter_grid):
print("------H params: {0}, C params: {1}------".format(
parameters_h, parameters_c))
h = deepcopy(classifier_h).set_params(**parameters_h)
c = deepcopy(classifier_c).set_params(**parameters_c)
y_predicted = hc_learn_and_predict(
br, c, depth_of_search, h, loss_function, number_of_labels,
x_train_train, x_train_valid, y_train_train, y_train_valid,
reduction)
calculated_loss = calculate_average_loss(
loss_function, y_predicted, y_train_valid,
number_of_labels)
print("Calculated loss: {0}".format(calculated_loss))
if calculated_loss < best_loss:
best_parameters_h = parameters_h
best_parameters_c = parameters_c
best_loss = calculated_loss
print {
"Final H params: {0}, Final C params: {1}".format(
best_parameters_h, best_parameters_c)
}
h = h.set_params(**best_parameters_h)
c = c.set_params(**best_parameters_c)
y_predicted = hc_learn_and_predict(br, c, depth_of_search, h,
loss_function, number_of_labels,
x_train, x_test, y_train, y_test,
reduction)
print_measures(y_predicted, y_test, number_of_labels)