def main():
print("+--------------------------------------+")
print("| Machine Learning Project, Part 2 |")
print("+--------------------------------------+")
print("| Author: Rodrigo Castiel |")
print("+--------------------------------------+")
# Set seed for deterministic execution.
random.seed(0)
np.random.seed(0)
# Load training and test data.
loader = DataLoader()
loader.load("data/segmentation")
x_train, w_train = loader.training_data()
# Build and train classifiers with the optimal hyper-parameters.
gaussian_mle = train_gaussian_mle(x_train, w_train)
knn_classifier = train_knn_classifier(x_train, w_train)
combined_max_classifier = train_combined_classifier(x_train, w_train)
classifiers = [gaussian_mle, knn_classifier, combined_max_classifier]
# Evaluate estimators.
evaluate_cross_validation(classifiers, x_train, w_train)
evaluate_accuracy_on_test_set(loader, classifiers)
# Perform Friedman's Test.
perform_friedman_test(classifiers, x_train, w_train)
def __init__(self, model):
super().__init__()
self.model = model
self.data_loader = DataLoader()
self.tranlator = Translator.get_translator('main')
self.logger = logging.getLogger(LOGGER_NAME)
CrossEvent.reload_scenarios_event += self.reload_scenarios
CrossEvent.locale_changed_event += self.reloads
def __init__(self, model):
self.model = model
self.data_loader = DataLoader()
self.tranlator = Translator.get_translator('main')
CrossWidgetEvents.start_editor_event += self.start_editor_handler
self.memento_manager = MementoManager()
self.memento_manager.can_undo_changed += self.model.can_undo_changed.emit
self.memento_manager.can_redo_changed += self.model.can_redo_changed.emit
self.memento_manager.index_changed += \
lambda x: self.model.can_save_changed.emit(self.can_save)
def fit(self, x_train, w_train):
# Train all committee classifiers with their corresponding views.
for (classifier, view) in self.committee:
classifier.fit(x_train[:, view], w_train)
# Estimate a prior probabilities p(wi) for each class wi.
self.p_w = DataLoader.compute_a_priori(w_train)
return self
def main():
print("+--------------------------------------+")
print("| Machine Learning Project, Part 1 |")
print("+--------------------------------------+")
print("| Author: Rodrigo Castiel |")
print("+--------------------------------------+")
print()
# Set seed for deterministic execution.
random.seed(42)
np.random.seed(42)
# Load training and test data.
loader = DataLoader()
loader.load("data/segmentation")
x_data, w_data = loader.test_data()
K = len(np.unique(w_data))
shape_view = [0, 1, 3, 5, 6, 7, 8]
rgb_view = [9, 10, 11, 12, 13, 14, 15, 16, 17, 18]
full_view = shape_view + rgb_view
if "FULL" in args.views:
## FULL VIEW ---------------------------------------------------------------
print("+--------------------+")
print("| FULL VIEW |")
print("+--------------------+")
evaluate_kcm_f_gh(K, num_times, x_data[:, full_view], w_data)
evaluate_k_means(K, x_data[:, full_view], w_data)
if "RGB" in args.views:
## RGB VIEW ----------------------------------------------------------------
print("+--------------------+")
print("| RGB VIEW |")
print("+--------------------+")
evaluate_kcm_f_gh(K, num_times, x_data[:, rgb_view], w_data)
evaluate_k_means(K, x_data[:, rgb_view], w_data)
if "SHAPE" in args.views:
## SHAPE VIEW --------------------------------------------------------------
print("+--------------------+")
print("| SHAPE VIEW |")
print("+--------------------+")
evaluate_kcm_f_gh(K, num_times, x_data[:, shape_view], w_data)
evaluate_k_means(K, x_data[:, shape_view], w_data)
def __init__(self):
self.data_loader = DataLoader()
self._is_options_enabled = NotifyProperty('is_options_enabled', False)
self._is_options_enabled += self.is_options_enabled_changed.emit
self._editor_mode = NotifyProperty('editor_mode', 0)
self.locales = []
self._is_applied_settings = NotifyProperty('is_applied_settings', True)
self._is_applied_settings += self.is_applied_settings_changed.emit
self._current_locale = NotifyProperty('current_locale')
self._current_locale += self.current_locale_changed.emit
self._modules_dir = ''
self._scenarios_dir = ''
def fit(self, x_train, w_train):
"""
Stores training points *x_train* and their correponsindg labels *w_train*,
and estimates the a prior probabilities p(w_i) for each class w_i.
"""
# Store examples.
self.x_train = x_train
self.w_train = w_train
# Estimate a prior probabilities p(wi) for each class wi.
self.p_w = DataLoader.compute_a_priori(w_train)
self.num_classes = len(self.p_w)
return self
def fit(self, x_train, w_train):
"""
Estimates a prior probabilities p(w_i), the mean i and the variance i for
each class in the training dataset. That is, from (x_train, w_train).
Once trained, you can call classify() to predict the class/label
for a given feature vector.
Note: all classes must have at least one sample.
"""
# Break down dataset into smaller groups sharing the same label.
x_groups = DataLoader.group_by_label(x_train, w_train)
# Estimate a prior probabilities p(wi) for each class wi.
self.p_w = DataLoader.compute_a_priori(w_train)
# Estimate mean and [diagonal] variances for each class w_i.
# Pattern Classification (Second Edition), Section 3.2.3.
self.mu = np.array(
list(map(
lambda x_train_i: np.mean(x_train_i, axis=0),
x_groups,
)))
self.sigma = np.array(
list(
map(
lambda i: np.mean((x_groups[i] - self.mu[i])**2, axis=0),
range(len(x_groups)),
)))
# For the sake of optimization, we may precompute some constants in the
# gaussian pdf equations - the amplitudes and the inverse of sigma.
n = len(self.mu)
pi_const = np.power(2 * np.pi, n / 2.0)
det_sigma = np.abs(np.product(self.sigma, axis=1))
self.inv_sigma = 1.0 / (self.sigma + epsilon)
self.amplitudes = 1.0 / (pi_const * np.sqrt(det_sigma + epsilon))
return self
def init_locale():
dataloader = DataLoader()
Translator.set_locale(dataloader.get_config_param(Constants.LOCALE))
Translator.register_translator('main', './')
def recurse(node, level, x):
plot_node(node, level, x)
if node.is_leaf:
return
x_lo, x_hi = plot_branches(level, x)
recurse(node.low, level + 1, x_lo)
recurse(node.high, level + 1, x_hi)
recurse(node=self.root, level=initial_level, x=initial_x)
plt.title("Decision tree")
plt.savefig(os.path.join(results_dir, "decision_tree.png"),
format='png')
plt.show()
if __name__ == "__main__":
data_loader = DataLoader()
data_handler = DataHandler(trips=data_loader.data,
config=data_loader.config,
saves_dir=data_loader.saves_dir,
results_dir=data_loader.results_dir)
train, test = data_handler.data_sets
judge = Judge(features_df=train)
training_features_df = data_handler.create_features(list(train.columns))
tree = DecisionTree(error_function=judge.find_total_abs_deviation)
tree.train(training_features=training_features_df)
tree.render(results_dir=data_loader.results_dir)
testing_score = data_handler.evaluate(tree=tree, data_set=test)
print(f"testing score is: {testing_score}")
"noveltyCount5D", "noveltyCount7D", "volumeCounts12H",
"volumeCounts24H", "volumeCounts3D", "volumeCounts5D",
"volumeCounts7D", "coverage"]
#Removing the target column for the prediction phase
test_cols = ["returnsClosePrevRaw1",
"returnsOpenPrevRaw1", "returnsClosePrevMktres1",
"returnsOpenPrevMktres1","returnsClosePrevMktres10",
"returnsOpenPrevMktres10", "dailychange","companyCount", "relevance",
"sentimentNegative", "sentimentNeutral", "sentimentPositive",
"noveltyCount12H", "noveltyCount24H", "noveltyCount3D",
"noveltyCount5D", "noveltyCount7D", "volumeCounts12H",
"volumeCounts24H", "volumeCounts3D", "volumeCounts5D",
"volumeCounts7D", "coverage"]
loader = DataLoader(cols, test_cols)
X, y = loader.get_train_data(train_df, assetName='Apple Inc', seq_len=5, normalize=True)
model = build_model()
model.fit(X, y, epochs=5, batch_size=50)
pred_df = predict_looper_lstm(model, test_df, loader)
evaluator = ReturnsEvaluator()
print("kaggle mean variance", evaluator.get_kaggle_mean_variance(pred_df))
metrics_dict = evaluator.get_returns(pred_df)
def data_loader(self, warehouse, source):
loader = DataLoader(source=source, warehouse=warehouse)
return loader
class MainController:
event_is_exec = False # Fix double messege shows
def __init__(self, model):
super().__init__()
self.model = model
self.data_loader = DataLoader()
self.tranlator = Translator.get_translator('main')
self.logger = logging.getLogger(LOGGER_NAME)
CrossEvent.reload_scenarios_event += self.reload_scenarios
CrossEvent.locale_changed_event += self.reloads
def exec(self):
self.reload_modules()
self.reload_scenarios()
self.model.set_locales(Locales.get_locales())
self.model.is_options_enabled = self.data_loader.get_config_param(
Constants.OPTIONS_ENABLED_PARAM) or False
self.reload_setting()
def reload_modules(self):
self.data_loader.load_modules()
self.model.update_modules()
def reload_scenarios(self):
self.data_loader.load_scenarios()
self.model.update_scenarios()
def start_scenario(self, scenario, opt_enb):
self.logger.debug(f'Start {scenario.name}')
check_mod = all(r_mod.is_enabled
for r_mod in scenario.required_modules)
if not check_mod:
title = self.tranlator.translate('WARNING_TEXT')
msg = self.tranlator.translate('REQ_MODS_NOT_ENB_TEXT')
self.logger.debug(msg)
CrossEvent.show_message_event.emit(MsgType.WARN, title, msg)
return
CrossEvent.load_scenario_event.emit(scenario, opt_enb)
CrossEvent.change_screen_event.emit(ScI.SCENARIO)
def set_editor_mode(self, value):
self.model.editor_mode = value
@staticmethod
def start_editor(mode, *args):
CrossEvent.start_editor_event.emit(mode, *args)
CrossEvent.change_screen_event.emit(ScI.EDITOR)
def delete_scenarios(self, scenarios):
def _action():
[self.data_loader.remove_scenario(sc) for sc in scenarios]
self.reload_scenarios()
title = self.tranlator.translate('QUESTION_TITLE')
msg = self.tranlator.translate('REMOVE_SELECTED_TEXT')
CrossEvent.show_question_event.emit(title, msg, _action)
def reloads(self):
self.reload_modules()
self.reload_scenarios()
def set_is_options_enabled(self, value):
self.model.is_options_enabled = value
self.data_loader.set_config_param(Constants.OPTIONS_ENABLED_PARAM,
value)
def select_locale(self, value):
self.model.current_locale = value
def show_dir_warning(self):
if self.event_is_exec:
return
self.event_is_exec = True
title = self.tranlator.translate('WARNING_TEXT')
msg = self.tranlator.translate('DIR_NOT_FOUND_TEXT')
CrossEvent.show_message_event.emit(MsgType.WARN, title, msg)
self.event_is_exec = False
def set_modules_dir(self, value):
if not os.path.isdir(value):
self.show_dir_warning()
return
self.model.modules_dir = value
def set_scenario_dir(self, value):
if not os.path.isdir(value):
self.show_dir_warning()
return
self.model.scenarios_dir = value
def apply_settings(self):
self.data_loader.set_config_param(Constants.MODULES_DIR,
self.model.modules_dir)
self.reload_modules()
self.data_loader.set_config_param(Constants.SCENARIOS_DIR,
self.model.scenarios_dir)
self.reload_scenarios()
self.data_loader.set_config_param(Constants.LOCALE,
self.model.current_locale)
Translator.set_locale(self.model.current_locale)
self.model.is_applied_settings = True
def reload_setting(self):
self.model.modules_dir = self.data_loader.modules_dir
self.model.scenarios_dir = self.data_loader.scenarios_dir
self.model.current_locale = Translator.get_locale()
self.model.is_applied_settings = True
from core.warehouse import FileWarehouse
from core import config
logger = logging.getLogger(__name__)
source = MovieLensSource('movielens',
ratings_file='external_data/ml-1m/ratings.dat',
products_file='external_data/ml-1m/movies.dat')
warehouse = FileWarehouse(partition=source.name)
transporter = Transporter(warehouse=warehouse, user_model=Users)
warehouse.cleanup()
dataloader = DataLoader(warehouse=warehouse, source=source)
dataloader.create_product_catalog_in_serving_layer()
dataloader.create_product_catalog_in_warehouse()
dataloader.create_ratings_data_in_warehouse()
engine = ALSRecommendationEngine(warehouse=warehouse)
engine.train_new_model(**config.als_opts)
#engine = ALSRecommendationEngine.import_from_path(config.MODELS_DIR + "/core_app_movielens")
transporter.send_users_to_warehouse()
class EditorController:
WIDGETS = {}
MODS = {}
_save_index = -1
def __init__(self, model):
self.model = model
self.data_loader = DataLoader()
self.tranlator = Translator.get_translator('main')
CrossWidgetEvents.start_editor_event += self.start_editor_handler
self.memento_manager = MementoManager()
self.memento_manager.can_undo_changed += self.model.can_undo_changed.emit
self.memento_manager.can_redo_changed += self.model.can_redo_changed.emit
self.memento_manager.index_changed += \
lambda x: self.model.can_save_changed.emit(self.can_save)
@property
def can_save(self):
return self._save_index != self.memento_manager.index
def load_modules(self):
self.model.blocks = [
m for m in self.data_loader.modules.values() if m.is_enabled
]
# region start editor methods
def start_editor_handler(self, mode, *args):
if mode == eMode.CREATE_NEW:
self.create_new_scenario()
elif mode == eMode.CREATE_FROM:
self.create_from_scenarios(*args)
elif mode == eMode.LOAD:
self.load_scenario(*args)
else:
raise ValueError('Inccorect editor mode')
self.memento_manager.set_subject(self.model.scenario)
self.load_modules()
self._save_index = -1
def create_new_scenario(self):
def_name = self.tranlator.translate('UNTITLED_TEXT')
self.model.scenario = Scenario(f'<{def_name}>')
def create_from_scenarios(self, scenarios):
names = []
req_mods = set()
sc_data = []
for sc in scenarios:
names.append(sc.name)
req_mods.update(sc.required_modules)
sc_data.extend(sc.scenario_data)
new_name = ', '.join(names)
self.model.scenario = Scenario(new_name,
required_modules=req_mods,
scenario_data=sc_data)
def load_scenario(self, scenario):
self.model.scenario = scenario
# endregion
# region UI handlers
def save_scenario(self):
old_sc_name = next((name
for name, sc in self.data_loader.scenarios.items()
if sc == self.model.scenario), None)
if old_sc_name:
self.data_loader.remove_scenario(old_sc_name)
self.data_loader.save_scenario(self.model.scenario)
self._save_index = self.memento_manager.index
self.model.can_save_changed.emit(self.can_save)
def undo(self):
self.memento_manager.undo()
def redo(self):
self.memento_manager.redo()
def back_to_menu(self, *args):
def _return_action():
self.WIDGETS.clear()
CrossWidgetEvents.reload_scenarios_event.emit()
CrossWidgetEvents.change_screen_event.emit(ScI.MAIN)
def _cancel_action():
self.memento_manager.cancel()
_return_action()
if self.can_save:
title = self.tranlator.translate('QUESTION_TITLE')
msg = self.tranlator.translate('CANCEL_UNSAVED_TEXT')
CrossWidgetEvents.show_question_event.emit(title, msg,
_cancel_action)
else:
_return_action()
def set_sc_block_index(self, value):
self.model.current_sc_block_index = value
self.apply_sc_block()
def apply_sc_block(self):
curr_sc_block = self.model.get_current_sc_block()
if curr_sc_block:
mod_init = curr_sc_block.module.init
q_types = mod_init.get_question_types()
li_wid = mod_init.get_editor_listitem_widget_cls()
bl_wid = mod_init.get_editor_block_widget()
else:
q_types, li_wid, bl_wid = None, None, None
self.model.quest_types = q_types
self.model.listitem_widget = li_wid
self.model.block_widget = bl_wid
self.model.send_sc_block()
def set_data_index(self, value):
self.model.current_data_index = value
# endregion
# region actions with record history
def add_block(self, block):
bl = ScenarioData.empty_init(block)
self.model.append_scenario_data(bl)
def remove_block(self, block):
self.model.remove_scenario_data(block)
def add_block_data(self):
mod_init = self.model.get_current_sc_block().module.init
self.model.append_block_data(mod_init.create_new_data_object())
def remove_block_data(self, value):
self.model.remove_block_data(value)
def set_quest_type(self, value):
if value:
self.model.current_quest_type = value
def change_scenarion_name(self, value):
@ChangeMemento('name', self.model.update_scenario_name_event)
def action():
self.model.scenario.name = value
action()
epochs = 50
# TF GPU memory graph
limit_gpu()
# gpus = tf.config.experimental.list_physical_devices('GPU')
# if gpus:
# try:
# # Currently, memory growth needs to be the same across GPUs
# for gpu in gpus:
# tf.config.experimental.set_memory_growth(gpu, True)
# logical_gpus = tf.config.experimental.list_logical_devices('GPU')
# print('[INFO]... ',len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
# except RuntimeError as e:
# # Memory growth must be set before GPUs have been initialized
# print(e)
dl = DataLoader()
train_generator, xtest, ytest = dl.from_common_dir(
directory='/home/akash/project/Dataset/cifar10 (1)/train/',
target_size=(32, 32),
batch_size=batch_size)
model = build_resnet_model(input_shape=(32, 32, 3),
depth=depth,
num_classes=dl.num_classes)
callbacks = callbacks(save_path='./assets/weights/exp2', depth=depth)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(amsgrad=True, decay=0.001 / epochs),
metrics=['accuracy'])