def main(): print("+--------------------------------------+") print("| Machine Learning Project, Part 2 |") print("+--------------------------------------+") print("| Author: Rodrigo Castiel |") print("+--------------------------------------+") # Set seed for deterministic execution. random.seed(0) np.random.seed(0) # Load training and test data. loader = DataLoader() loader.load("data/segmentation") x_train, w_train = loader.training_data() # Build and train classifiers with the optimal hyper-parameters. gaussian_mle = train_gaussian_mle(x_train, w_train) knn_classifier = train_knn_classifier(x_train, w_train) combined_max_classifier = train_combined_classifier(x_train, w_train) classifiers = [gaussian_mle, knn_classifier, combined_max_classifier] # Evaluate estimators. evaluate_cross_validation(classifiers, x_train, w_train) evaluate_accuracy_on_test_set(loader, classifiers) # Perform Friedman's Test. perform_friedman_test(classifiers, x_train, w_train)
def __init__(self, model): super().__init__() self.model = model self.data_loader = DataLoader() self.tranlator = Translator.get_translator('main') self.logger = logging.getLogger(LOGGER_NAME) CrossEvent.reload_scenarios_event += self.reload_scenarios CrossEvent.locale_changed_event += self.reloads
def __init__(self, model): self.model = model self.data_loader = DataLoader() self.tranlator = Translator.get_translator('main') CrossWidgetEvents.start_editor_event += self.start_editor_handler self.memento_manager = MementoManager() self.memento_manager.can_undo_changed += self.model.can_undo_changed.emit self.memento_manager.can_redo_changed += self.model.can_redo_changed.emit self.memento_manager.index_changed += \ lambda x: self.model.can_save_changed.emit(self.can_save)
def fit(self, x_train, w_train): # Train all committee classifiers with their corresponding views. for (classifier, view) in self.committee: classifier.fit(x_train[:, view], w_train) # Estimate a prior probabilities p(wi) for each class wi. self.p_w = DataLoader.compute_a_priori(w_train) return self
def main(): print("+--------------------------------------+") print("| Machine Learning Project, Part 1 |") print("+--------------------------------------+") print("| Author: Rodrigo Castiel |") print("+--------------------------------------+") print() # Set seed for deterministic execution. random.seed(42) np.random.seed(42) # Load training and test data. loader = DataLoader() loader.load("data/segmentation") x_data, w_data = loader.test_data() K = len(np.unique(w_data)) shape_view = [0, 1, 3, 5, 6, 7, 8] rgb_view = [9, 10, 11, 12, 13, 14, 15, 16, 17, 18] full_view = shape_view + rgb_view if "FULL" in args.views: ## FULL VIEW --------------------------------------------------------------- print("+--------------------+") print("| FULL VIEW |") print("+--------------------+") evaluate_kcm_f_gh(K, num_times, x_data[:, full_view], w_data) evaluate_k_means(K, x_data[:, full_view], w_data) if "RGB" in args.views: ## RGB VIEW ---------------------------------------------------------------- print("+--------------------+") print("| RGB VIEW |") print("+--------------------+") evaluate_kcm_f_gh(K, num_times, x_data[:, rgb_view], w_data) evaluate_k_means(K, x_data[:, rgb_view], w_data) if "SHAPE" in args.views: ## SHAPE VIEW -------------------------------------------------------------- print("+--------------------+") print("| SHAPE VIEW |") print("+--------------------+") evaluate_kcm_f_gh(K, num_times, x_data[:, shape_view], w_data) evaluate_k_means(K, x_data[:, shape_view], w_data)
def __init__(self): self.data_loader = DataLoader() self._is_options_enabled = NotifyProperty('is_options_enabled', False) self._is_options_enabled += self.is_options_enabled_changed.emit self._editor_mode = NotifyProperty('editor_mode', 0) self.locales = [] self._is_applied_settings = NotifyProperty('is_applied_settings', True) self._is_applied_settings += self.is_applied_settings_changed.emit self._current_locale = NotifyProperty('current_locale') self._current_locale += self.current_locale_changed.emit self._modules_dir = '' self._scenarios_dir = ''
def fit(self, x_train, w_train): """ Stores training points *x_train* and their correponsindg labels *w_train*, and estimates the a prior probabilities p(w_i) for each class w_i. """ # Store examples. self.x_train = x_train self.w_train = w_train # Estimate a prior probabilities p(wi) for each class wi. self.p_w = DataLoader.compute_a_priori(w_train) self.num_classes = len(self.p_w) return self
def fit(self, x_train, w_train): """ Estimates a prior probabilities p(w_i), the mean i and the variance i for each class in the training dataset. That is, from (x_train, w_train). Once trained, you can call classify() to predict the class/label for a given feature vector. Note: all classes must have at least one sample. """ # Break down dataset into smaller groups sharing the same label. x_groups = DataLoader.group_by_label(x_train, w_train) # Estimate a prior probabilities p(wi) for each class wi. self.p_w = DataLoader.compute_a_priori(w_train) # Estimate mean and [diagonal] variances for each class w_i. # Pattern Classification (Second Edition), Section 3.2.3. self.mu = np.array( list(map( lambda x_train_i: np.mean(x_train_i, axis=0), x_groups, ))) self.sigma = np.array( list( map( lambda i: np.mean((x_groups[i] - self.mu[i])**2, axis=0), range(len(x_groups)), ))) # For the sake of optimization, we may precompute some constants in the # gaussian pdf equations - the amplitudes and the inverse of sigma. n = len(self.mu) pi_const = np.power(2 * np.pi, n / 2.0) det_sigma = np.abs(np.product(self.sigma, axis=1)) self.inv_sigma = 1.0 / (self.sigma + epsilon) self.amplitudes = 1.0 / (pi_const * np.sqrt(det_sigma + epsilon)) return self
def init_locale(): dataloader = DataLoader() Translator.set_locale(dataloader.get_config_param(Constants.LOCALE)) Translator.register_translator('main', './')
def recurse(node, level, x): plot_node(node, level, x) if node.is_leaf: return x_lo, x_hi = plot_branches(level, x) recurse(node.low, level + 1, x_lo) recurse(node.high, level + 1, x_hi) recurse(node=self.root, level=initial_level, x=initial_x) plt.title("Decision tree") plt.savefig(os.path.join(results_dir, "decision_tree.png"), format='png') plt.show() if __name__ == "__main__": data_loader = DataLoader() data_handler = DataHandler(trips=data_loader.data, config=data_loader.config, saves_dir=data_loader.saves_dir, results_dir=data_loader.results_dir) train, test = data_handler.data_sets judge = Judge(features_df=train) training_features_df = data_handler.create_features(list(train.columns)) tree = DecisionTree(error_function=judge.find_total_abs_deviation) tree.train(training_features=training_features_df) tree.render(results_dir=data_loader.results_dir) testing_score = data_handler.evaluate(tree=tree, data_set=test) print(f"testing score is: {testing_score}")
"noveltyCount5D", "noveltyCount7D", "volumeCounts12H", "volumeCounts24H", "volumeCounts3D", "volumeCounts5D", "volumeCounts7D", "coverage"] #Removing the target column for the prediction phase test_cols = ["returnsClosePrevRaw1", "returnsOpenPrevRaw1", "returnsClosePrevMktres1", "returnsOpenPrevMktres1","returnsClosePrevMktres10", "returnsOpenPrevMktres10", "dailychange","companyCount", "relevance", "sentimentNegative", "sentimentNeutral", "sentimentPositive", "noveltyCount12H", "noveltyCount24H", "noveltyCount3D", "noveltyCount5D", "noveltyCount7D", "volumeCounts12H", "volumeCounts24H", "volumeCounts3D", "volumeCounts5D", "volumeCounts7D", "coverage"] loader = DataLoader(cols, test_cols) X, y = loader.get_train_data(train_df, assetName='Apple Inc', seq_len=5, normalize=True) model = build_model() model.fit(X, y, epochs=5, batch_size=50) pred_df = predict_looper_lstm(model, test_df, loader) evaluator = ReturnsEvaluator() print("kaggle mean variance", evaluator.get_kaggle_mean_variance(pred_df)) metrics_dict = evaluator.get_returns(pred_df)
def data_loader(self, warehouse, source): loader = DataLoader(source=source, warehouse=warehouse) return loader
class MainController: event_is_exec = False # Fix double messege shows def __init__(self, model): super().__init__() self.model = model self.data_loader = DataLoader() self.tranlator = Translator.get_translator('main') self.logger = logging.getLogger(LOGGER_NAME) CrossEvent.reload_scenarios_event += self.reload_scenarios CrossEvent.locale_changed_event += self.reloads def exec(self): self.reload_modules() self.reload_scenarios() self.model.set_locales(Locales.get_locales()) self.model.is_options_enabled = self.data_loader.get_config_param( Constants.OPTIONS_ENABLED_PARAM) or False self.reload_setting() def reload_modules(self): self.data_loader.load_modules() self.model.update_modules() def reload_scenarios(self): self.data_loader.load_scenarios() self.model.update_scenarios() def start_scenario(self, scenario, opt_enb): self.logger.debug(f'Start {scenario.name}') check_mod = all(r_mod.is_enabled for r_mod in scenario.required_modules) if not check_mod: title = self.tranlator.translate('WARNING_TEXT') msg = self.tranlator.translate('REQ_MODS_NOT_ENB_TEXT') self.logger.debug(msg) CrossEvent.show_message_event.emit(MsgType.WARN, title, msg) return CrossEvent.load_scenario_event.emit(scenario, opt_enb) CrossEvent.change_screen_event.emit(ScI.SCENARIO) def set_editor_mode(self, value): self.model.editor_mode = value @staticmethod def start_editor(mode, *args): CrossEvent.start_editor_event.emit(mode, *args) CrossEvent.change_screen_event.emit(ScI.EDITOR) def delete_scenarios(self, scenarios): def _action(): [self.data_loader.remove_scenario(sc) for sc in scenarios] self.reload_scenarios() title = self.tranlator.translate('QUESTION_TITLE') msg = self.tranlator.translate('REMOVE_SELECTED_TEXT') CrossEvent.show_question_event.emit(title, msg, _action) def reloads(self): self.reload_modules() self.reload_scenarios() def set_is_options_enabled(self, value): self.model.is_options_enabled = value self.data_loader.set_config_param(Constants.OPTIONS_ENABLED_PARAM, value) def select_locale(self, value): self.model.current_locale = value def show_dir_warning(self): if self.event_is_exec: return self.event_is_exec = True title = self.tranlator.translate('WARNING_TEXT') msg = self.tranlator.translate('DIR_NOT_FOUND_TEXT') CrossEvent.show_message_event.emit(MsgType.WARN, title, msg) self.event_is_exec = False def set_modules_dir(self, value): if not os.path.isdir(value): self.show_dir_warning() return self.model.modules_dir = value def set_scenario_dir(self, value): if not os.path.isdir(value): self.show_dir_warning() return self.model.scenarios_dir = value def apply_settings(self): self.data_loader.set_config_param(Constants.MODULES_DIR, self.model.modules_dir) self.reload_modules() self.data_loader.set_config_param(Constants.SCENARIOS_DIR, self.model.scenarios_dir) self.reload_scenarios() self.data_loader.set_config_param(Constants.LOCALE, self.model.current_locale) Translator.set_locale(self.model.current_locale) self.model.is_applied_settings = True def reload_setting(self): self.model.modules_dir = self.data_loader.modules_dir self.model.scenarios_dir = self.data_loader.scenarios_dir self.model.current_locale = Translator.get_locale() self.model.is_applied_settings = True
from core.warehouse import FileWarehouse from core import config logger = logging.getLogger(__name__) source = MovieLensSource('movielens', ratings_file='external_data/ml-1m/ratings.dat', products_file='external_data/ml-1m/movies.dat') warehouse = FileWarehouse(partition=source.name) transporter = Transporter(warehouse=warehouse, user_model=Users) warehouse.cleanup() dataloader = DataLoader(warehouse=warehouse, source=source) dataloader.create_product_catalog_in_serving_layer() dataloader.create_product_catalog_in_warehouse() dataloader.create_ratings_data_in_warehouse() engine = ALSRecommendationEngine(warehouse=warehouse) engine.train_new_model(**config.als_opts) #engine = ALSRecommendationEngine.import_from_path(config.MODELS_DIR + "/core_app_movielens") transporter.send_users_to_warehouse()
class EditorController: WIDGETS = {} MODS = {} _save_index = -1 def __init__(self, model): self.model = model self.data_loader = DataLoader() self.tranlator = Translator.get_translator('main') CrossWidgetEvents.start_editor_event += self.start_editor_handler self.memento_manager = MementoManager() self.memento_manager.can_undo_changed += self.model.can_undo_changed.emit self.memento_manager.can_redo_changed += self.model.can_redo_changed.emit self.memento_manager.index_changed += \ lambda x: self.model.can_save_changed.emit(self.can_save) @property def can_save(self): return self._save_index != self.memento_manager.index def load_modules(self): self.model.blocks = [ m for m in self.data_loader.modules.values() if m.is_enabled ] # region start editor methods def start_editor_handler(self, mode, *args): if mode == eMode.CREATE_NEW: self.create_new_scenario() elif mode == eMode.CREATE_FROM: self.create_from_scenarios(*args) elif mode == eMode.LOAD: self.load_scenario(*args) else: raise ValueError('Inccorect editor mode') self.memento_manager.set_subject(self.model.scenario) self.load_modules() self._save_index = -1 def create_new_scenario(self): def_name = self.tranlator.translate('UNTITLED_TEXT') self.model.scenario = Scenario(f'<{def_name}>') def create_from_scenarios(self, scenarios): names = [] req_mods = set() sc_data = [] for sc in scenarios: names.append(sc.name) req_mods.update(sc.required_modules) sc_data.extend(sc.scenario_data) new_name = ', '.join(names) self.model.scenario = Scenario(new_name, required_modules=req_mods, scenario_data=sc_data) def load_scenario(self, scenario): self.model.scenario = scenario # endregion # region UI handlers def save_scenario(self): old_sc_name = next((name for name, sc in self.data_loader.scenarios.items() if sc == self.model.scenario), None) if old_sc_name: self.data_loader.remove_scenario(old_sc_name) self.data_loader.save_scenario(self.model.scenario) self._save_index = self.memento_manager.index self.model.can_save_changed.emit(self.can_save) def undo(self): self.memento_manager.undo() def redo(self): self.memento_manager.redo() def back_to_menu(self, *args): def _return_action(): self.WIDGETS.clear() CrossWidgetEvents.reload_scenarios_event.emit() CrossWidgetEvents.change_screen_event.emit(ScI.MAIN) def _cancel_action(): self.memento_manager.cancel() _return_action() if self.can_save: title = self.tranlator.translate('QUESTION_TITLE') msg = self.tranlator.translate('CANCEL_UNSAVED_TEXT') CrossWidgetEvents.show_question_event.emit(title, msg, _cancel_action) else: _return_action() def set_sc_block_index(self, value): self.model.current_sc_block_index = value self.apply_sc_block() def apply_sc_block(self): curr_sc_block = self.model.get_current_sc_block() if curr_sc_block: mod_init = curr_sc_block.module.init q_types = mod_init.get_question_types() li_wid = mod_init.get_editor_listitem_widget_cls() bl_wid = mod_init.get_editor_block_widget() else: q_types, li_wid, bl_wid = None, None, None self.model.quest_types = q_types self.model.listitem_widget = li_wid self.model.block_widget = bl_wid self.model.send_sc_block() def set_data_index(self, value): self.model.current_data_index = value # endregion # region actions with record history def add_block(self, block): bl = ScenarioData.empty_init(block) self.model.append_scenario_data(bl) def remove_block(self, block): self.model.remove_scenario_data(block) def add_block_data(self): mod_init = self.model.get_current_sc_block().module.init self.model.append_block_data(mod_init.create_new_data_object()) def remove_block_data(self, value): self.model.remove_block_data(value) def set_quest_type(self, value): if value: self.model.current_quest_type = value def change_scenarion_name(self, value): @ChangeMemento('name', self.model.update_scenario_name_event) def action(): self.model.scenario.name = value action()
epochs = 50 # TF GPU memory graph limit_gpu() # gpus = tf.config.experimental.list_physical_devices('GPU') # if gpus: # try: # # Currently, memory growth needs to be the same across GPUs # for gpu in gpus: # tf.config.experimental.set_memory_growth(gpu, True) # logical_gpus = tf.config.experimental.list_logical_devices('GPU') # print('[INFO]... ',len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") # except RuntimeError as e: # # Memory growth must be set before GPUs have been initialized # print(e) dl = DataLoader() train_generator, xtest, ytest = dl.from_common_dir( directory='/home/akash/project/Dataset/cifar10 (1)/train/', target_size=(32, 32), batch_size=batch_size) model = build_resnet_model(input_shape=(32, 32, 3), depth=depth, num_classes=dl.num_classes) callbacks = callbacks(save_path='./assets/weights/exp2', depth=depth) model.compile(loss='categorical_crossentropy', optimizer=Adam(amsgrad=True, decay=0.001 / epochs), metrics=['accuracy'])