def test_arff(): arff_path = realpath(pjoin(dirname(dirname(dirname(__file__))), # 3 levels up 'example_datasets', 'arff', 'iris.arff')) sys.argv = shlex.split('neuropredict -a {} -t {} -n {} -c {} -g {} -o {} -e {} -fs {}'.format(arff_path, train_perc, num_repetitions, num_procs, gs_level, out_dir, classifier, fs_method)) cli()
def test_separable_100perc(): """Test to ensure fully separable classes lead to close to perfect prediction! """ separable_ds = make_fully_separable_classes(max_class_size=100, max_dim=np.random.randint( 2, max_dim)) separable_ds.description = 'fully_separable_dataset' out_path_sep = os.path.join(out_dir, 'two_separable_classes.pkl') out_dir_sep = os.path.join(out_dir, 'fully_separable_test') os.makedirs(out_dir_sep, exist_ok=True) separable_ds.save(out_path_sep) nrep = 10 gsl = 'none' # to speed up the process for clf_name in cfg.classifier_choices: for fs_name in cfg.all_dim_red_methods: cli_str = 'neuropredict -y {} -t {} -n {} -c {} -g {} -o {} -e {} -fs {}' \ ''.format(out_path_sep, train_perc, nrep, 1, gsl, out_dir_sep, clf_name, fs_name) sys.argv = shlex.split(cli_str) cli() cv_results = rhst.load_results_from_folder(out_dir_sep) for sg, result in cv_results.items(): raise_if_mean_differs_from( result['accuracy_balanced'], result['class_sizes'], reference_level=1.0, #comparing to perfect eps_chance_acc=0.5, method_descr='{} {}'.format(fs_name, clf_name))
def test_print_options(): " ensures the CLI works. " known_out_dir = '/Users/Reddy/rotman/CANBIND/results/base_vs_network_thk_curv_area_HistInt_many_subgroups' with raises(SystemExit): sys.argv = shlex.split( 'neuropredict --print_options {}'.format(known_out_dir)) cli()
def test_vis(): " ensures the CLI works. " res_path = pjoin(out_dir, 'rhst_results.pkl') if pexists(res_path): with raises(SystemExit): sys.argv = shlex.split('neuropredict --make_vis {}'.format(out_dir)) cli() expected_results = ['balanced_accuracy.pdf', 'compare_misclf_rates.pdf', 'feature_importance.pdf'] for rpath in expected_results: if not pexists(rpath): raise ValueError('an expected result {} not produced'.format(rpath)) else: print('previously computed results not found in \n {}'.format(out_dir))
def test_chance_clf_binary_svm(): global ds_path_list, method_names, out_dir, num_repetitions, gs_level, train_perc, num_procs sys.argv = shlex.split('neuropredict -y {} {} -t {} -n {} -c {} -g {} -o {} -e {} -fs {}'.format(out_path, out_path2, train_perc, min_rep_per_class*rand_two_class.num_classes, num_procs, gs_level, out_dir, classifier, fs_method)) cli() cv_results = rhst.load_results_from_folder(out_dir) for sg, result in cv_results.items(): raise_if_mean_differs_from_chance(result['accuracy_balanced'], result['class_sizes'], eps_chance_acc_binary)
def test_each_combination_works(): "Ensures each of combination of feature selection and classifier works." nrep = 10 gsl = 'none' # to speed up the process for clf_name in cfg.classifier_choices: for fs_name in cfg.feature_selection_choices: try: cli_str = 'neuropredict -y {} -t {} -n {} -c {} -o {} -e {} -fs {} -g {} '.format(out_path, train_perc, nrep, num_procs, out_dir, clf_name, fs_name, gsl) sys.argv = shlex.split(cli_str) cli() except: print(' ---> combination failed: {} {}'.format(clf_name, fs_name)) raise
def test_chance_multiclass(): global ds_path_list, method_names, out_dir, num_repetitions, gs_level, train_perc, num_procs clf = 'randomforestclassifier' fs_method = 'variancethreshold' nrep = total_num_classes*min_rep_per_class gsl = 'none' # to speed up the process sys.argv = shlex.split('neuropredict -y {} -t {} -n {} -c {} -g {} -o {} -e {} -fs {}'.format(out_path_multiclass, train_perc, nrep, num_procs, gsl, out_dir, clf, fs_method)) cli() cv_results = rhst.load_results_from_folder(out_dir) for sg, result in cv_results.items(): raise_if_mean_differs_from_chance(result['accuracy_balanced'], result['class_sizes'], eps_chance_acc)
def test_print_options(): " ensures the CLI works. " known_out_dir = out_dir options_path = pjoin(out_dir, cfg.file_name_options) if pexists(options_path): with raises(SystemExit): sys.argv = shlex.split('neuropredict --print_options {}' ''.format(known_out_dir)) cli() known_nonexisting_dir = known_out_dir + '_43_34563$#*$@)' with raises(IOError): sys.argv = shlex.split('neuropredict --po {}' ''.format(known_nonexisting_dir)) cli()
def test_versioning(): " ensures the CLI works. " with raises(SystemExit): sys.argv = shlex.split('neuropredict -v') cli()