def test_init(self):
		helper = Helper()

		model_name = 'rf:classifier'
		model = helper.getDefaultModel(model_name)
		self.assertEqual( model, RandomForestClassifier ) 
		
		args = helper.getDefaultArgs(model_name)
		self.assertEqual( args, {'random_state':0} )
		
		model = model(**args)
		self.assertEqual( type(model), RandomForestClassifier )

		
		helper.setRandomState(1)

		model_name = 'dt:regressor'
		model = helper.getDefaultModel(model_name)
		self.assertEqual( model, DecisionTreeRegressor ) 
		
		args = helper.getDefaultArgs(model_name)
		self.assertEqual( args, {'random_state':1} )
		
		model = model(**args)
		self.assertEqual( type(model), DecisionTreeRegressor )
class Experiment:
    def __init__(self, name='', models=[], exp_type=None, metrics=[]):
        self.helper = Helper()
        self.unnamed_model_count = 0
        self.random_state = 0
        self._valid_experiment_types = ['classification', 'regression']

        self.metrics_manager = MetricsManager()

        self.type_of_experiment = None
        if exp_type:
            if type(exp_type) != str:
                raise ValueError(
                    'Experiment type attribute must be string, not {}'.format(
                        str(type(exp_type))))

            if self.isValidExperimentType(exp_type):
                self.type_of_experiment = exp_type
            else:
                raise ValueError(
                    'The provided experiment type is not supported: \'{}\'\nOnly these experiment types are supported: {}'
                    .format(exp_type, self.getValidExperimentTypes()))

        if type(name) is str:
            if name != '':
                self.name = name
            else:
                self.name = 'unnamed_experiment'
        else:
            raise ValueError(
                'Experiment name attribute must be string, not {}'.format(
                    str(type(name))))

        # initialize the dictionary of models within the experiment
        self.models_dict = {}

        # Add the models (if any) that the user wants to use
        if type(models) is list:
            if len(models) > 1:
                # Then add all of the model objects to the experiment
                for model in models:
                    self.addModel(model)
            elif len(models) == 1:
                # Add the only model to the experiment list
                self.addModel(models)
        elif type(models) is dict:
            # Add all of the models with their corresponding names provided from the user
            for name in models.keys():
                self.addModel(model, name)
        elif type(models) is str:
            self.addModel(models)
        else:
            raise ValueError(
                'Models must be in list format if more than one is provided. Ex. models=[\'rf\', \'Decision Tree\', RandomForestClassifer()... ]'
            )

        # initialize the metrics list
        self.metrics = []

        if type(metrics) is list:
            self.metrics = metrics
        elif type(metrics) is str:
            self.metrics = [metrics]
        else:
            raise ValueError(
                '\'metrics\' argument must be string or list of strings. Ex: [\'accuracy\', \'prec\']. Cannot be {}'
                .format(str(type(metrics))))

    def getDefaultModelName(self):
        return 'model_' + str(len(self.models_dict))

    def getValidExperimentTypes(self):
        return self._valid_experiment_types

    def isValidExperimentType(self, exp_type):
        if type(exp_type) == str:
            if exp_type in self._valid_experiment_types:
                return True
            else:
                return False
        else:
            raise ValueError('Experiment type must be string: {}'.format(
                str(exp_type)))

    def setExperimentType(self, exp_type):
        if self.isValidExperimentType(exp_type):
            self.type_of_experiment = exp_type
        else:
            raise ValueError(
                'Experiment must be \'regression\' or \'classification\', cannot be {}'
                .format(str(exp_type)))

    def getExperimentType(self):
        return self.type_of_experiment

    def getDefaultVersionOf(self, model_name):
        # TO-DO: Contact state object to get the object!
        # Do we know what type of experiment we're dealing with here?
        if self.type_of_experiment:
            # Figure out what type of model we need
            if self.type_of_experiment == 'regression':
                regularized_model_name = model_name + ':regressor'
            elif self.type_of_experiment == 'classification':
                regularized_model_name = model_name + ':classifier'
            return Model(model_name,
                         self.helper.getDefaultModel(regularized_model_name),
                         self.helper.getDefaultArgs(regularized_model_name))
        else:
            return Model(model_name)

    def addModel(self, model, name=''):
        # TO-DO: What are valid objects to pass here? String? Yes, but gives default. Actual model object (such as RandomForestClassifier)? Yes!
        if type(model) is str:
            # Add the default version of the model they are requesting
            if model != '':
                model_name = model
                model = self.getDefaultVersionOf(model_name)
                self.models_dict[model_name] = model
            else:
                if self.name:
                    raise ValueError(
                        'Experiment object {} cannot add model default model: {}'
                        .format(str(self.name), model))
                else:
                    raise ValueError(
                        'Experiment object cannot add model default model: {}'.
                        format(model))
        elif type(model) is Model:
            model_name = model.getName()
            if model_name:
                self.models_dict[model_name] = model
            elif type(name) == str and name != '':
                self.models_dict[name] = model
            else:
                self.models_dict[self.getDefaultModelName()] = model
        elif type(model) is object:
            # TO-DO: What type of object is it? Scikit learn?
            print('the model is some kind of object!')

        # TO-DO: Find a way to generate automatic names for these models in a way that sounds smarter than model_1, model_2, ...
        elif type(name) is str:
            if name == '':
                # Default name will be assigned
                name = self.getDefaultModelName()
            elif self.hasModelByName(name):
                raise ValueError(
                    'Experiment object already has model by name: {}'.format(
                        name))
        else:
            raise ValueError('Model name must be string: {}'.format(str(name)))

    def train(self, X, y, cv=False, n_folds=0, shuffle=False, metrics=[]):
        if len(X) == len(y):
            if cv:
                if n_folds > 0:
                    # If they want a ROC curve, start that
                    if 'roc' in metrics or 'ROC' in metrics or 'roc' in self.metrics or 'ROC' in self.metrics:
                        self.metrics_manager.startROCCurve()
                        print('startedROCCurve')

                    # Cross validation
                    sss = None
                    if shuffle:
                        sss = StratifiedKFold(n_splits=n_folds,
                                              shuffle=True,
                                              random_state=self.random_state)
                    else:
                        sss = StratifiedKFold(n_splits=n_folds)
                    fold_num = 0
                    for train_idx, val_idx in sss.split(X, y):
                        fold_num += 1

                        # Organize the data
                        X_train, y_train = [], []
                        X_val, y_val = [], []
                        if type(X) == np.ndarray and type(y) == np.ndarray:
                            X_train, y_train = X[train_idx], y[train_idx]
                            X_val, y_val = X[val_idx], y[val_idx]
                        elif type(X) == pd.DataFrame and (type(y) == pd.Series
                                                          or type(y) == list):
                            X_train, y_train = X.iloc[train_idx], y[train_idx]
                            X_val, y_val = X.iloc[val_idx], y[val_idx]
                        else:
                            raise ValueError(
                                'unrecognized datatypes:\n\ttype(X) => {}\n\ttype(y) => {}'
                                .format(str(type(X)), str(type(y))))

                        for model_name in self.models_dict.keys():
                            model = self.models_dict[model_name]
                            model.run(X_train, y_train)

                        if metrics:
                            # Evaluate the performance of this model and keep track of it
                            self.eval(X_val,
                                      y_val,
                                      metrics=metrics,
                                      fold_num=fold_num)
                        elif len(self.metrics) > 0:
                            # Evaluate the performance of this model and keep track of it
                            self.eval(X_val,
                                      y_val,
                                      metrics=self.metrics,
                                      fold_num=fold_num)
                    if ('roc' in metrics
                            or 'ROC' in metrics) and fold_num == n_folds:
                        print('reached')
                        self.metrics_manager.showROCCurve()
                else:
                    raise ValueError(
                        'Number of folds in cross validation (n_folds) must be larger than zero!'
                    )
            else:
                for model_name in self.models_dict.keys():
                    model = self.models_dict[model_name]
                    model.run(X, y)
            self.completed = True
        else:
            if self.name:
                raise ValueError(
                    'Data and target provided to \'{}\' must be same length:\n\tlen of data: {}\n\tlen of target: {}'
                    .format(self.name, str(len(X)), str(len(y))))
            else:
                raise ValueError(
                    'Provided data and target must be same length:\n\tlen of data: {}\n\tlen of target: {}'
                    .format(str(len(X)), str(len(y))))

    def predict(self, X):
        predicted_values = {}
        for model_name in self.models_dict.keys():
            model = self.models_dict[model_name]
            predicted_values[model_name] = model.predict(X)
        return predicted_values

    def eval(self, X, y, metrics=[], fold_num=None):
        if len(X) == len(y):
            # If we aren't running cross validation, then clear out the metrics
            if not fold_num:
                self.metrics_manager.clearMetrics()
            for model_name in self.models_dict.keys():
                model = self.models_dict[model_name]
                if fold_num and type(fold_num) == int:
                    metric_object = Metric(model_name, fold=fold_num)
                else:
                    metric_object = Metric(model_name)

                # If a ROC curve is requested, graph it!
                if 'roc' in metrics or 'ROC' in metrics:
                    fpr, tpr, roc_auc = model.calcROC(X, y,
                                                      self.type_of_experiment)
                    self.metrics_manager.addToROCCurve(model_name, fpr, tpr,
                                                       roc_auc)

                model_metrics = model.eval(X, y, metrics)
                for measure_name in model_metrics.keys():
                    metric_object.addValue(measure_name,
                                           model_metrics[measure_name])
                self.metrics_manager.addMetric(metric_object)

            if not fold_num:
                return self.metrics_manager.getMetrics()
        else:
            if self.name:
                raise ValueError(
                    'Data and target provided to \'{}\' must be same length:\n\tlen of data: {}\n\tlen of target: {}'
                    .format(self.name, str(len(X)), str(len(y))))
            else:
                raise ValueError(
                    'Provided data and target must be same length:\n\tlen of data: {}\n\tlen of target: {}'
                    .format(str(len(X)), str(len(y))))

    def getMetrics(self):
        return self.metrics_manager.getMetrics()

    def getSummarizedMetrics(self):
        return self.metrics_manager.printMeasures()

    def summarizeMetrics(self):
        metrics = self.metrics_manager.printMeasures()
        metrics_list_format = []

        headers = ['model']

        # Combine statistics where possible (such as acc_avg and acc_std => acc_avg \pm acc_std)
        metrics_add = {}
        for model_name in metrics.keys():
            measures = metrics[model_name].getMeasures()
            metrics_add[model_name] = metrics[model_name].copy()
            for measure in metrics[model_name].getMeasures():
                measure_arr = measure.split('_')
                if len(measure_arr) >= 2 and '_'.join(
                    [measure_arr[0], 'avg']) in measures and '_'.join(
                        [measure_arr[0], 'std']) in measures:
                    avg = metrics[model_name].getValue('_'.join(
                        [measure_arr[0], 'avg']))
                    std = metrics[model_name].getValue('_'.join(
                        [measure_arr[0], 'std']))
                    metrics_add[model_name].addValue(
                        measure_arr[0], '{:.4f}\u00B1{:.4f}'.format(avg, std))
                    metrics_add[model_name].removeValue('_'.join(
                        [measure_arr[0], 'avg']))
                    metrics_add[model_name].removeValue('_'.join(
                        [measure_arr[0], 'std']))
        metrics = metrics_add

        # Create the objecs to hand over to tabulate
        for model_name in metrics.keys():
            new_row = [model_name]
            for measure in metrics[model_name].getMeasures():
                if measure not in headers:
                    headers.append(measure)
                new_row.append(metrics[model_name].getValue(measure))
            metrics_list_format.append(new_row)

        print(tabulate(metrics_list_format, headers=headers))

    def setRandomState(self, rand_state):
        if type(rand_state) == int:
            self.random_state = rand_state
            self.helper.setRandomState(rand_state)

            for model_name in self.models_dict.keys():
                model = self.models_dict[model_name]
                if model.hasConstructorArg('random_state'):
                    model.setConstructorArg('random_state', rand_state)
        else:
            raise ValueError(
                'random state must be integer: {}'.format(rand_state))

    def getName(self):
        return self.name

    def getModels(self):
        return self.models_dict

    def compareModels_2x5cv(self, models=[], X=None, y=None, a=0.05):
        comparison_rows = []
        headers = [
            'models', 'model1', 'mean1', 'std1', 'model2', 'mean2', 'std2',
            'sig/notsig'
        ]

        comparisons_ran = dict()

        for model1 in self.models_dict.keys():
            if not model1 in comparisons_ran:
                comparisons_ran[model1] = []
            for model2 in self.models_dict.keys():
                if not model2 in comparisons_ran:
                    comparisons_ran[model2] = []
                if model1 != model2 and (model1
                                         not in comparisons_ran[model2]) and (
                                             model2
                                             not in comparisons_ran[model1]):
                    row = ['{} & {}'.format(model1, model2)]

                    row.append(model1)
                    cv1 = RepeatedStratifiedKFold(
                        n_splits=10,
                        n_repeats=2,
                        random_state=self.random_state)
                    scores1 = cross_val_score(
                        self.models_dict[model1].getBuiltModel(),
                        X,
                        y,
                        scoring='accuracy',
                        cv=cv1)

                    row.append(model2)
                    cv2 = RepeatedStratifiedKFold(
                        n_splits=10,
                        n_repeats=2,
                        random_state=self.random_state)
                    scores2 = cross_val_score(
                        self.models_dict[model2].getBuiltModel(),
                        X,
                        y,
                        scoring='accuracy',
                        cv=cv1)

                    meanScore1 = np.mean(scores1)
                    meanScore2 = np.mean(scores2)
                    row.append(f'{np.std(scores1):.5f}')
                    row.append(f'{np.std(scores2):.5f}')

                    if meanScore1 > meanScore2:
                        row.append(f'*{meanScore1:.5f}')
                    else:
                        row.append(f'*{meanScore2:.5f}')

                    t, p = paired_ttest_5x2cv(
                        estimator1=self.models_dict[model1].getBuiltModel(),
                        estimator2=self.models_dict[model2].getBuiltModel(),
                        X=X,
                        y=y,
                        scoring='accuracy')
                    if p <= a:
                        row.append('sig')
                    else:
                        row.append('notsig')
                    comparisons_ran[model1].append(model2)
                    comparisons_ran[model2].append(model1)
                    comparison_rows.append(row)

        print(tabulate(comparison_rows, headers=headers))

    def compareModels_ttest(self, models=[], X=None, y=None, a=0.05):
        comparison_rows = []
        headers = [
            'model pairs', 'model1', 'mean1', 'std1', 'model2', 'mean2',
            'std2', 't-stat', 'p-val', 'sig/notsig'
        ]
        print(*headers)
        models = list(self.models_dict.keys())
        assert len(
            models
        ) >= 2, 'there must be at least 2 models to run ttest statistical comparison'
        betterClassifier = models[0]
        betterCV = RepeatedStratifiedKFold(n_splits=10,
                                           n_repeats=2,
                                           random_state=self.random_state)
        betterScore = cross_val_score(
            self.models_dict[betterClassifier].getBuiltModel(),
            X,
            y,
            scoring='accuracy',
            cv=betterCV)
        betterMeanScore = np.mean(betterScore)
        betterStdDev = np.std(betterScore)
        comparison_rows = []
        for model in models[1:]:
            row = [f'{betterClassifier} v {model}']
            cv = RepeatedStratifiedKFold(n_splits=10,
                                         n_repeats=2,
                                         random_state=self.random_state)
            score = cross_val_score(self.models_dict[model].getBuiltModel(),
                                    X,
                                    y,
                                    scoring='accuracy',
                                    cv=cv)
            meanScore = np.mean(score)
            stdDev = np.std(score)
            if meanScore > betterMeanScore:
                # this is better classifier
                row.extend([model, f'{meanScore:.5f}*', f'{stdDev:.5f}'])
                row.extend([
                    betterClassifier, f'{betterMeanScore:.5f}',
                    f'{betterStdDev:.5f}'
                ])
                betterClassifier = model
                betterMeanScore = meanScore
                betterStdDev = stdDev
            else:
                row.extend([
                    betterClassifier, f'{betterMeanScore:.5f}*',
                    f'{betterStdDev:.5f}'
                ])
                row.extend([model, f'{meanScore:.5f}', f'{stdDev:.5f}'])

            t, p = paired_ttest_5x2cv(
                estimator1=self.models_dict[betterClassifier].getBuiltModel(),
                estimator2=self.models_dict[model].getBuiltModel(),
                X=X,
                y=y,
                scoring='accuracy')
            row.extend([f'{t:.3f}', f'{p:.3f}'])
            if p <= a:
                row.append('sig')
            else:
                row.append('notsig')
            comparison_rows.append(row)
            print(*row)
        print(tabulate(comparison_rows, headers=headers))