def run_experiments(data_loader, transformer, dataset_name, class_name, n_runs): save_path = os.path.join(RESULTS_DIR, dataset_name) utils.check_paths(save_path) # Transformations for _ in range(n_runs): _transformations_experiment(data_loader, transformer, dataset_name, class_name, save_path)
def __init__(self, data_loader: ZTFOutlierLoader, transformer: AbstractTransformer, input_shape, results_folder_name='', name='Ensemble_OVA_Transformer_OD_Simple_Model', **kwargs): super(TransformODModel, self).__init__(name=name) self.date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") self.main_model_path = self.create_main_model_paths( results_folder_name, self.name) utils.check_paths(self.main_model_path) self.data_loader = data_loader self.transformer = transformer self.models_list = self._get_model_list(input_shape, **kwargs)
def __init__(self, data_loader: ZTFOutlierLoader, transformer: AbstractTransformer, input_shape, results_folder_name='', name='Transformer_OD_Simple_Model', **kwargs): super(TransformODModel, self).__init__(name=name) self.date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") self.main_model_path = self.create_main_model_paths( results_folder_name, self.name) utils.check_paths(self.main_model_path) self.data_loader = data_loader self.transformer = transformer self.network = self.get_network( input_shape=input_shape, n_classes=self.transformer.n_transforms, **kwargs)
def save_normal_and_transformed_data( transformer, normal_data_name='tf2_normal.pkl', transformed_data_name='tf2_old_transformed.pkl'): save_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_data') utils.check_paths(save_dir) hits_params = { loader_keys.DATA_PATH: os.path.join(PROJECT_PATH, '../datasets/HiTS2013_300k_samples.pkl'), loader_keys.N_SAMPLES_BY_CLASS: 10000, loader_keys.TEST_PERCENTAGE: 0.2, loader_keys.VAL_SET_INLIER_PERCENTAGE: 0.1, loader_keys.USED_CHANNELS: [0, 1, 2, 3], loader_keys.CROP_SIZE: 21, general_keys.RANDOM_SEED: 42, loader_keys.TRANSFORMATION_INLIER_CLASS_VALUE: 1 } hits_outlier_dataset = HiTSOutlierLoader(hits_params) (x_train, y_train), (x_val, y_val), ( x_test, y_test) = hits_outlier_dataset.get_outlier_detection_datasets() x_train_transform, y_train_transform = transformer.apply_all_transforms( x=x_train) x_val_transform, y_val_transform = transformer.apply_all_transforms( x=x_val) x_test_transform, y_test_transform = transformer.apply_all_transforms( x=x_test) normal_data = ((x_train, y_train), (x_val, y_val), (x_test, y_test)) transformed_data = ((x_train_transform, y_train_transform), (x_val_transform, y_val_transform), (x_test_transform, y_test_transform)) utils.save_pickle(normal_data, os.path.join(save_dir, normal_data_name)) utils.save_pickle(transformed_data, os.path.join(save_dir, transformed_data_name))
def __init__(self, data_loader: ZTFOutlierLoader, transformer: AbstractTransformer, input_shape, depth=10, widen_factor=4, results_folder_name='', name='Ensemble_OVO_Transformer_OD_Model', **kwargs): super(TransformODModel, self).__init__(name=name) self.builder_input_shape = input_shape self.depth = 10 self.widen_factor = widen_factor self.date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") self.main_model_path = self.create_main_model_paths( results_folder_name, self.name) utils.check_paths(self.main_model_path) self.data_loader = data_loader self.transformer = transformer self.models_list = self._get_model_list(input_shape, depth=depth, widen_factor=widen_factor, **kwargs) self.models_index_tuples = self._get_models_index_tuples()
if path: fig.savefig(os.path.join(path, '%s_hist_thr_acc.png' % x_label_name), bbox_inches='tight') if show: plt.show() else: plt.close() if __name__ == "__main__": config = tf.ConfigProto() config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU sess = tf.Session(config=config) set_session(sess) save_path = '../results/Transforms_hits' check_paths(save_path) single_class_ind = 1 (x_train, y_train), (x_val, y_val), (x_test, y_test) = load_hits(n_samples_by_class=10000, test_size=0.20, val_size=0.10, return_val=True, channels_to_get=[2]) print(x_train.shape) print(x_val.shape) print(x_test.shape) transformer = Transformer(8, 8)
import tensorflow as tf from tqdm import tqdm from scripts.detached_transformer_od_hits import \ plot_histogram_disc_loss_acc_thr, \ dirichlet_normality_score, fixed_point_dirichlet_mle, calc_approx_alpha_sum from scripts.ensemble_transform_vs_all_od_hits import get_entropy, \ plot_matrix_score import torch import torch.nn as nn from modules.utils import check_paths EXPERIMENT_NAME = 'ZTF_v1_TransTransformations' if __name__ == "__main__": results_folder = os.path.join(PROJECT_PATH, 'results', EXPERIMENT_NAME) check_paths(results_folder) config = tf.ConfigProto() config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU sess = tf.Session(config=config) set_session(sess) single_class_ind = 1 (x_train, y_train), (x_val, y_val), (x_test, y_test) = load_ztf_real_bog(return_val=True)#, data_file_name='ztf_v1_bogus_added.pkl') print(x_train.shape) print(x_val.shape) print(x_test.shape) # (x_train, y_train) = (x_train[:1000], y_train[:1000]) # (x_val, y_val) = (x_val[:1000], y_val[:1000])