Exemple #1
0
    def train(self):
        training_dataframe, model_definition, timeseries_cols = self._create_ludwig_dataframe(
            'train')

        if len(timeseries_cols) > 0:
            training_dataframe, model_definition = self._translate_df_to_timeseries_format(
                training_dataframe, model_definition, timeseries_cols, 'train')

        with disable_console_output(True):
            # <---- Ludwig currently broken, since mode can't be initialized without train_set_metadata and train_set_metadata can't be obtained without running train... see this issue for any updates on the matter: https://github.com/uber/ludwig/issues/295
            #model.initialize_model(train_set_metadata={})
            #train_stats = model.train_online(data_df=training_dataframe) # ??Where to add model_name?? ----> model_name=self.transaction.lmd['name']

            ludwig_save_is_working = False

            if not ludwig_save_is_working:
                shutil.rmtree('results', ignore_errors=True)

            if self.transaction.lmd['rebuild_model'] is True:
                model = LudwigModel(model_definition)
                merged_model_definition = model.model_definition
                train_set_metadata = build_metadata(
                    training_dataframe,
                    (merged_model_definition['input_features'] +
                     merged_model_definition['output_features']),
                    merged_model_definition['preprocessing'])
                model.initialize_model(train_set_metadata=train_set_metadata,
                                       gpus=self.get_useable_gpus())

                train_stats = model.train(
                    data_df=training_dataframe,
                    model_name=self.transaction.lmd['name'],
                    skip_save_model=ludwig_save_is_working,
                    skip_save_progress=True,
                    gpus=self.get_useable_gpus())
            else:
                model = LudwigModel.load(model_dir=self.get_model_dir())
                train_stats = model.train(
                    data_df=training_dataframe,
                    model_name=self.transaction.lmd['name'],
                    skip_save_model=ludwig_save_is_working,
                    skip_save_progress=True,
                    gpus=self.get_useable_gpus())

            for k in train_stats['train']:
                if k not in self.transaction.lmd['model_accuracy']['train']:
                    self.transaction.lmd['model_accuracy']['train'][k] = []
                    self.transaction.lmd['model_accuracy']['test'][k] = []
                elif k is not 'combined':
                    # We should be adding the accuracy here but we only have it for combined, so, for now use that, will only affect multi-output scenarios anyway
                    pass
                else:
                    self.transaction.lmd['model_accuracy']['train'][k].extend(
                        train_stats['train'][k]['accuracy'])
                    self.transaction.lmd['model_accuracy']['test'][k].extend(
                        train_stats['test'][k]['accuracy'])
            '''
            @ TRAIN ONLINE BIT That's not working
            model = LudwigModel.load(self.transaction.lmd['ludwig_data']['ludwig_save_path'])
            for i in range(0,100):
                train_stats = model.train_online(data_df=training_dataframe)
                # The resulting train_stats are "None"... wonderful -_-
            '''

        ludwig_model_savepath = os.path.join(
            CONFIG.MINDSDB_STORAGE_PATH,
            self.transaction.lmd['name'] + '_ludwig_data')
        if ludwig_save_is_working:
            model.save(ludwig_model_savepath)
            model.close()
        else:
            shutil.rmtree(ludwig_model_savepath, ignore_errors=True)
            shutil.move(os.path.join('results',
                                     os.listdir('results')[0]),
                        ludwig_model_savepath)
        self.transaction.lmd['ludwig_data'] = {
            'ludwig_save_path': ludwig_model_savepath
        }
        self.transaction.hmd['ludwig_data'] = {
            'model_definition': model_definition
        }
Exemple #2
0
    def train(self):
        training_dataframe, model_definition, timeseries_cols, has_heavy_data, self.transaction.lmd[
            'ludwig_tf_self_col_map'] = self._create_ludwig_dataframe('train')

        if len(timeseries_cols) > 0:
            training_dataframe, model_definition = self._translate_df_to_timeseries_format(
                training_dataframe, model_definition, timeseries_cols, 'train')

        with disable_console_output(True):
            # <---- Ludwig currently broken, since mode can't be initialized without train_set_metadata and train_set_metadata can't be obtained without running train... see this issue for any updates on the matter: https://github.com/uber/ludwig/issues/295
            #model.initialize_model(train_set_metadata={})
            #train_stats = model.train_online(data_df=training_dataframe) # ??Where to add model_name?? ----> model_name=self.transaction.lmd['name']

            ludwig_save_is_working = False

            if not ludwig_save_is_working:
                shutil.rmtree('results', ignore_errors=True)

            if self.transaction.lmd['rebuild_model'] is True:
                model = LudwigModel(model_definition)
                merged_model_definition = model.model_definition
                train_set_metadata = build_metadata(
                    training_dataframe,
                    (merged_model_definition['input_features'] +
                     merged_model_definition['output_features']),
                    merged_model_definition['preprocessing'])
                model.initialize_model(train_set_metadata=train_set_metadata,
                                       gpus=self._get_useable_gpus())
            else:
                model = LudwigModel.load(model_dir=self._get_model_dir())

            split_by = int(20 * pow(10, 6))
            if has_heavy_data:
                split_by = 40
            df_len = len(training_dataframe[training_dataframe.columns[0]])
            if df_len > split_by:
                i = 0
                while i < df_len:
                    end = i + split_by
                    self.transaction.log.info(
                        f'Training with batch from index {i} to index {end}')
                    training_sample = training_dataframe.iloc[i:end]
                    training_sample = training_sample.reset_index()

                    if len(training_sample) < 1:
                        continue

                    train_stats = model.train(
                        data_df=training_sample,
                        model_name=self.transaction.lmd['name'],
                        skip_save_model=ludwig_save_is_working,
                        skip_save_progress=True,
                        gpus=self._get_useable_gpus())
                    i = end
            else:
                train_stats = model.train(
                    data_df=training_dataframe,
                    model_name=self.transaction.lmd['name'],
                    skip_save_model=ludwig_save_is_working,
                    skip_save_progress=True,
                    gpus=self._get_useable_gpus())

            for k in train_stats['train']:
                if k not in self.transaction.lmd['model_accuracy']['train']:
                    self.transaction.lmd['model_accuracy']['train'][k] = []
                    self.transaction.lmd['model_accuracy']['test'][k] = []
                elif k != 'combined':
                    # We should be adding the accuracy here but we only have it for combined, so, for now use that, will only affect multi-output scenarios anyway
                    pass
                else:
                    self.transaction.lmd['model_accuracy']['train'][k].extend(
                        train_stats['train'][k]['accuracy'])
                    self.transaction.lmd['model_accuracy']['test'][k].extend(
                        train_stats['test'][k]['accuracy'])
            '''
            @ TRAIN ONLINE BIT That's not working
            model = LudwigModel.load(self.transaction.lmd['ludwig_data']['ludwig_save_path'])
            for i in range(0,100):
                train_stats = model.train_online(data_df=training_dataframe)
                # The resulting train_stats are "None"... wonderful -_-
            '''

        ludwig_model_savepath = os.path.join(CONFIG.MINDSDB_STORAGE_PATH,
                                             self.transaction.lmd['name'],
                                             'ludwig_data')
        Path(CONFIG.MINDSDB_STORAGE_PATH).joinpath(
            self.transaction.lmd['name']).mkdir(mode=0o777,
                                                exist_ok=True,
                                                parents=True)
        if ludwig_save_is_working:
            model.save(ludwig_model_savepath)
            model.close()
        else:
            shutil.rmtree(ludwig_model_savepath, ignore_errors=True)
            shutil.move(os.path.join('results',
                                     os.listdir('results')[0]),
                        ludwig_model_savepath)
        self.transaction.lmd['ludwig_data'] = {
            'ludwig_save_path': ludwig_model_savepath
        }
        self.transaction.hmd['ludwig_data'] = {
            'model_definition': model_definition
        }