def test_parse_training_labels(self): parsed = ingest.parse_training_labels( train_box_df=pd.read_csv( GlobalConfig.get('EXAMPLE_TRAIN_BOX_PATH')), train_image_dirpath=GlobalConfig.get( 'EXAMPLE_STAGE1_TRAIN_IMAGE_DIR')) # Negative Case self.assertEquals( parsed['0004cfab-14fd-4e49-80ba-63a80b6bddd6'], { 'dicom': 'data/example/stage_1_train_images/0004cfab-14fd-4e49-80ba-63a80b6bddd6.dcm', 'label': 0, 'boxes': [] }) # Positive Case self.assertEquals( parsed['00436515-870c-4b36-a041-de91049b9ab4'], { 'dicom': 'data/example/stage_1_train_images/00436515-870c-4b36-a041-de91049b9ab4.dcm', 'label': 1, 'boxes': [[264.0, 152.0, 213.0, 379.0], [562.0, 152.0, 256.0, 453.0]] })
def _retrieve_training_box_labels(self): if self.data_source == 'local': train_box_df = pd.read_csv(GlobalConfig.get('LOCAL_TRAIN_BOX_PATH')) elif self.data_source == 'example': train_box_df = pd.read_csv(GlobalConfig.get('EXAMPLE_TRAIN_BOX_PATH')) elif self.data_source == 's3': train_box_df = ingest.read_s3_df( bucket=GlobalConfig.get('S3_BUCKET_NAME'), file_key=GlobalConfig.get('S3_TRAIN_BOX_KEY')) return train_box_df
def get_cross_validator(): """ Generates and returns the ShuffleSplit cross validator. This uses a fixed random_state=0, so that every time each cross fold contains the same subset of data. :return: sklearn.model_selection.ShuffleSplit with relevant parameters. """ return StratifiedShuffleSplit( n_splits=GlobalConfig.get('folds'), train_size=GlobalConfig.get('train_ratio'), random_state=0)
def _retrieve_annotation_dict(self, train_box_df): if self.data_source == 'local': dirpath = GlobalConfig.get('LOCAL_STAGE1_TRAIN_IMAGE_DIR') if self.data_source == 'example': dirpath = GlobalConfig.get('EXAMPLE_STAGE1_TRAIN_IMAGE_DIR') elif self.data_source == 's3': dirpath = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR') annotation_dict = ingest.parse_training_labels( train_box_df=train_box_df, train_image_dirpath=dirpath) return annotation_dict
def __init__(self, num_classes: int, data_ratio: float): self.num_categories = num_classes self.data_ratio = data_ratio self.classes = [] self.train = [] self.test = [] if GlobalConfig.get('rotate'): print("Using rotated textures") if GlobalConfig.get('ECS'): self.KYLBERG_DIR = os.path.join('C:/', 'Local', 'data', 'kylberg-rotated') else: self.KYLBERG_DIR = os.path.join(GlobalConfig.get('CWD'), 'data', 'kylberg-rotated') else: print("Using non-rotated textures") if GlobalConfig.get('ECS'): self.KYLBERG_DIR = os.path.join('C:/', 'Local', 'data', 'kylberg') else: self.KYLBERG_DIR = os.path.join(GlobalConfig.get('CWD'), 'data', 'kylberg') print('Using {} scale'.format(GlobalConfig.get('scale'))) if GlobalConfig.get('noise') is not None: print('Applying {} noise'.format(GlobalConfig.get('noise')))
def make_classification_report(y_true, y_pred, classes, out_dir): # Get the full report rpt = classification_report(y_true, y_pred, labels=classes, output_dict=True) # Get the average results from all classes rpt_accuracy = rpt['accuracy'] rpt_macro_avg = rpt['macro avg'] rpt_weighted_avg = rpt['weighted avg'] # Get the per-class results only del rpt['accuracy'] del rpt['macro avg'] del rpt['weighted avg'] pandas_rpt = pd.DataFrame.from_dict(rpt, orient='index') pandas_rpt.rename(columns={'support': 'N Predictions'}, inplace=True) pandas_macro_avg = pd.DataFrame.from_dict(rpt_macro_avg, orient='index', columns=['Macro Average']) pandas_macro_avg.rename(columns={'support': 'N Predictions'}, inplace=True) pandas_weighted_avg = pd.DataFrame.from_dict(rpt_weighted_avg, orient='index', columns=['Weighted Average']) pandas_weighted_avg.rename(columns={'support': 'N Predictions'}, inplace=True) pandas_accuracy = pd.DataFrame({'': rpt_accuracy}, index=['overall accuracy']) if GlobalConfig.get('algorithm') == 'MRLBP': if GlobalConfig.get('mrlbp_classifier') == 'knn': out_file = os.path.join( out_dir, 'Classification Report{} - KNN.csv'.format( describe_test_setup())) else: out_file = os.path.join( out_dir, 'Classification Report{} - SVM.csv'.format( describe_test_setup())) else: out_file = os.path.join( out_dir, 'Classification Report{}.csv'.format(describe_test_setup())) pandas_rpt.to_csv(out_file) pandas_macro_avg.to_csv(out_file, mode='a', header=True) pandas_weighted_avg.to_csv(out_file, mode='a', header=True) pandas_accuracy.to_csv(out_file, mode='a', header=True)
def split_dataset_by_class(annotation_dict, subset_size, validation_split): """Split dataset for training and validation, preserving outcome class distribution.""" n_positive = sum(1 for v in annotation_dict.values() if v['label'] == 1) n_total = len(annotation_dict) p_positive = n_positive / n_total positive_ids = [k for k, v in annotation_dict.items() if v['label'] == 1] negative_ids = [k for k, v in annotation_dict.items() if v['label'] == 0] random.seed(GlobalConfig.get('RANDOM_SEED')) random.shuffle(positive_ids) random.shuffle(negative_ids) n_positive_subset = int(math.ceil(subset_size * p_positive)) n_negative_subset = int(subset_size - n_positive_subset) # ceilings & floors to ensure that we get exactly subset_size*validation_split train_pos_index = int(math.ceil(n_positive_subset * (1 - validation_split))) train_neg_index = int( math.floor(n_negative_subset * (1 - validation_split))) train_ids = positive_ids[:train_pos_index] + negative_ids[:train_neg_index] valid_ids = positive_ids[train_pos_index:n_positive_subset] + \ negative_ids[train_neg_index:n_negative_subset] print('Training count: %s' % len(train_ids)) print('Validation instance count: %s' % len(valid_ids)) return {'train_ids': train_ids, 'valid_ids': valid_ids}
def get_outdir(self, noisy_image: bool, scaled_image: bool): if noisy_image: noise_type = GlobalConfig.get('noise') noise_val = GlobalConfig.get('noise_val') else: noise_type = 'None' noise_val = 'None' if scaled_image: image_scale = int(GlobalConfig.get('test_scale') * 100) else: image_scale = int(GlobalConfig.get('scale') * 100) if GlobalConfig.get('train_noise'): return "scale-{}_noise-{}_noiseval-{}-trainnoise".format(image_scale, noise_type, noise_val) else: return "scale-{}_noise-{}_noiseval-{}".format(image_scale, noise_type, noise_val)
def get_dataset_train(self): dataset_train = DetectorDataset( patient_ids=self.patient_id_train, annotation_dict=self.annotation_dict, orig_height=self.DICOM_HEIGHT, orig_width=self.DICOM_WIDTH, data_source=self.data_source, s3_bucket=(GlobalConfig.get('S3_BUCKET_NAME') if self.data_source == 's3' else None)) dataset_train.prepare() assert len(dataset_train.image_ids) == len(self.patient_id_train) return dataset_train
def train(self, train: List[DatasetManager.Image]): X_train = [img.featurevector for img in train] # Convert List[narray] to ndarray X_train = np.stack(X_train, axis=0) X_train = X_train.astype(np.float64) y_train = [img.label for img in train] if GlobalConfig.get('mrlbp_classifier') == 'knn': # Calculate transposed covariance matrix as described here: https://stackoverflow.com/a/55623162/6008271 X_train_cov = np.linalg.inv(np.cov(X_train.transpose())).transpose() self.classifier = KNeighborsClassifier(n_neighbors=3, algorithm='brute', metric='mahalanobis', metric_params={'VI': X_train_cov}) self.classifier.fit(X_train, y_train) elif GlobalConfig.get('mrlbp_classifier') == 'svm': self.classifier = SVC() self.classifier.fit(X_train, y_train) else: raise ValueError('Invalid Classifier Type specified as --mrlbp-classifier.')
def describe_test_setup(): test_str = "" if GlobalConfig.get('noise') is not None: test_str += ' Noise Invariance {}-{}'.format( GlobalConfig.get('noise'), GlobalConfig.get('noise_val')) if GlobalConfig.get('rotate') is not False: test_str += ' Rotation Invariance' if GlobalConfig.get('test_scale') is not None: test_str += ' Scale Invariance Train {} Test {}'.format( int(GlobalConfig.get('scale') * 100), int(GlobalConfig.get('test_scale') * 100)) return test_str
def get_outdir(self, noisy_image: bool, scaled_image: bool): """ Gets a string name for the read/write directory depending on the configuration :param noisy_image: Whether the algorithm is being applied to images containing noise :param scaled_image: Whether the algorithm is being applied to a scaled image :return: String output directory name """ if noisy_image: noise_type = GlobalConfig.get('noise') noise_val = GlobalConfig.get('noise_val') else: noise_type = 'None' noise_val = 'None' if scaled_image: image_scale = int(GlobalConfig.get('test_scale') * 100) else: image_scale = int(GlobalConfig.get('scale') * 100) if GlobalConfig.get('train_noise'): return "scale-{}_noise-{}_noiseval-{}_p-{}_r-{}-trainnoise".format(image_scale, noise_type, noise_val, self.p, self.r) else: return "scale-{}_noise-{}_noiseval-{}_p-{}_r-{}".format(image_scale, noise_type, noise_val, self.p, self.r)
def write_examples(): """ Generates example images for use in report :return: None """ print("Generating algorithm example images") ex = GenerateExamples.GenerateExamples( os.path.join(GlobalConfig.get('CWD'), 'data', 'kylberg', 'grass1', 'grass1-a-p001.png')) ex.write_noise_examples() ex.write_RLBP_example() ex.write_MRLBP_example() ex.write_MRELBP_example() ex.write_BM3DELBP_example() print("Finished generating examples")
def __init__(self, path): """ Generate Example images for dissertation write-up :param path: Image to produce example images with """ self.image_path = path image_name = path.split(os.sep)[-1].partition('.')[0] #image_uint8 = cv2.imread(path, cv2.IMREAD_GRAYSCALE) image_uint8 = cv2.resize(cv2.imread(path, cv2.IMREAD_GRAYSCALE), (0, 0), fx=0.5, fy=0.5) # Convert from uint8 to float32 without normalizing to zero mean image_unscaled = ImageUtils.convert_uint8_image_float32(image_uint8) # Convert from uint8 to float32 while normalizing to zero mean image_scaled = ImageUtils.scale_uint8_image_float32(image_uint8) image_gauss_10 = ImageUtils.add_gaussian_noise_skimage(image_scaled, 10) image_gauss_25 = ImageUtils.add_gaussian_noise_skimage(image_scaled, 25) image_speckle_002 = ImageUtils.add_speckle_noise_skimage(image_scaled, 0.02) image_speckle_004 = ImageUtils.add_speckle_noise_skimage(image_scaled, 0.04) image_salt_pepper_002 = ImageUtils.add_salt_pepper_noise_skimage(image_scaled, 0.02) image_salt_pepper_004 = ImageUtils.add_salt_pepper_noise_skimage(image_scaled, 0.04) image_label = path.split(os.sep)[-1].partition('-')[0] # Generate different permutations of this sample image self.image_uint8 = DatasetManager.Image(image_uint8, image_name, image_label) self.image_unscaled = DatasetManager.Image(image_unscaled, image_name, image_label) self.image_scaled = DatasetManager.Image(image_scaled, image_name, image_label) self.image_gauss_10 = DatasetManager.Image(image_gauss_10, image_name, image_label) self.image_gauss_10.test_noise='gaussian'; self.image_gauss_10.test_noise_val=10 self.image_gauss_25 = DatasetManager.Image(image_gauss_25, image_name, image_label) self.image_gauss_25.test_noise = 'gaussian'; self.image_gauss_25.test_noise_val = 25 self.image_speckle_002 = DatasetManager.Image(image_speckle_002, image_name, image_label) self.image_speckle_002.test_noise = 'speckle'; self.image_speckle_002.test_noise_val = 0.02 self.image_speckle_004 = DatasetManager.Image(image_speckle_004, image_name, image_label) self.image_speckle_004.test_noise = 'speckle'; self.image_speckle_004.test_noise_val = 0.04 self.image_salt_pepper_002 = DatasetManager.Image(image_salt_pepper_002, image_name, image_label) self.image_salt_pepper_002.test_noise = 'salt-pepper'; self.image_salt_pepper_002.noise_val = 0.02 self.image_salt_pepper_004 = DatasetManager.Image(image_salt_pepper_004, image_name, image_label) self.image_salt_pepper_004.test_noise = 'salt-pepper'; self.image_salt_pepper_004.noise_val = 0.04 self.path = os.path.join(GlobalConfig.get('CWD'), 'example') write_image(ImageUtils.convert_float32_image_uint8(self.image_unscaled.data), None, image_name + '-unedited.png') write_image(ImageUtils.convert_float32_image_uint8(self.image_scaled.data), None, image_name + '-scaled.png')
def __init__(self, subset_size, validation_split, data_source='example', subdir='train'): random.seed(GlobalConfig.get('RANDOM_SEED')) self.subset_size = subset_size self.validation_split = validation_split self.data_source = data_source self.subdir = subdir assert self.data_source in ["local", "s3", "example"] assert self.subdir in ["train", "test"] self.train_box_df = self._retrieve_training_box_labels() self.annotation_dict = self._retrieve_annotation_dict(self.train_box_df) self.image_df = self._retrieve_dicom_image_list() # Split train/test sets, preserving outcome class distribution id_split = utils.split_dataset_by_class( annotation_dict=self.annotation_dict, subset_size=subset_size, validation_split=validation_split) self.patient_id_train = id_split['train_ids'] self.patient_id_valid = id_split['valid_ids']
def _retrieve_dicom_image_list(self): if self.data_source == 'local': cache_path = GlobalConfig.get('LOCAL_DICOM_IMAGE_LIST_PATH') if self.subdir == 'train': datadir = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR') elif self.subdir == 'test': datadir = GlobalConfig.get('S3_STAGE1_TEST_IMAGE_DIR') elif self.data_source == 'example': cache_path = GlobalConfig.get('EXAMPLE_DICOM_IMAGE_LIST_PATH') datadir = GlobalConfig.get('EXAMPLE_STAGE1_TRAIN_IMAGE_DIR') elif self.data_source == 's3': print('S3 data source selected. Caching functionality not yet available.') return ingest.parse_s3_dicom_image_list( bucket=GlobalConfig.get('S3_BUCKET_NAME'), subdir=self.subdir) image_df = ingest.parse_dicom_image_list( datadir=datadir, subdir=self.subdir, cache_path=cache_path, verbose=False) return image_df
import json from utils import shell, log from config import GlobalConfig # get config singleton config = GlobalConfig.get() def parse_artillery_output(report_file: str) -> bool: report = json.load(report_file) import pprint pprint(report)
def describe(self, image, test_image: bool, sigma_psd=70/255, cutoff_freq=10, a=0.75, b=1.0): if isinstance(image, DatasetManager.Image): if test_image: image_data = image.test_data else: image_data = image.data else: image_data = image.copy() # Perform BM3D Filter image_bm3d_filtered = SharedFunctions.bm3d_filter(image_data, 70/255) # Perform Homomorphic filter, Note: This requires images to be normalised in range [0, 255] image_scaled_255 = ImageUtils.convert_float32_image_uint8(image_data) cutoff, a, b = 10, 0.75, 0.1 image_homomorphic_filtered = SharedFunctions.homomorphic_filter(image_scaled_255, cutoff, a, b) image_homomorphic_filtered = ImageUtils.convert_uint8_image_float32(image_homomorphic_filtered) # Perform Median filter. Padding is required for median filter. image_padded = pad(array=image_data, pad_width=1, mode='constant', constant_values=0) image_median_filtered = np.zeros(image_padded.shape, dtype=np.float32) SharedFunctions.median_filter(image_padded, 3, 1, image_median_filtered) image_median_filtered = image_median_filtered[1:-1, 1:-1] # Remove padding now median filter done # Subtract original image from filtered image to get noise only bm3d_noise = image_data - image_bm3d_filtered homomorphic_noise = image_data - image_homomorphic_filtered median_noise = image_data - image_median_filtered if self.save_img: if isinstance(image, DatasetManager.Image): GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_bm3d_filtered), os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'), '{}-{}-{}-BM3D-filtered.png'.format(image.name, image.test_noise, image.test_noise_val)) GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_homomorphic_filtered), os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'), '{}-{}-{}-homomorphic-filtered-cutoff_{}-a_{}-b_{}.png'.format(image.name, image.test_noise, image.test_noise_val, cutoff, a, b)) GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_median_filtered), os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'), '{}-{}-{}-median-filtered.png'.format(image.name, image.test_noise, image.test_noise_val)) GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(bm3d_noise), os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'), '{}-{}-{}-BM3D-noise-estimate.png'.format(image.name, image.test_noise, image.test_noise_val)) GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(homomorphic_noise), os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'), '{}-{}-{}-homomorphic-filtered-cutoff_{}-a_{}-b_{}.png'.format(image.name, image.test_noise, image.test_noise_val, cutoff, a, b)) GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(median_noise), os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'), '{}-{}-{}-median-noise-estimate.png'.format(image.name, image.test_noise, image.test_noise_val)) else: raise ValueError('save_img set but not passed as DatasetManager.Image or BM3DELBPImage') kurtosis_bm3d = kurtosis(a=bm3d_noise, axis=None, fisher=False, nan_policy='raise') skewness_bm3d = skew(a=bm3d_noise, axis=None, nan_policy='raise') kurtosis_homomorphic = kurtosis(a=homomorphic_noise, axis=None, fisher=False, nan_policy='raise') skewness_homomorphic = skew(a=homomorphic_noise, axis=None, nan_policy='raise') kurtosis_median = kurtosis(a=median_noise, axis=None, fisher=False, nan_policy='raise') skewness_median = skew(a=median_noise, axis=None, nan_policy='raise') if GlobalConfig.get('debug'): print("BM3D Filtered Kurtosis:", kurtosis_bm3d, ", Skewness: ", skewness_bm3d) print("Homomorphic Filtered Kurtosis:", kurtosis_homomorphic, ", Skewness: ", skewness_homomorphic) print("Median Filtered Kurtosis:", kurtosis_median, ", Skewness: ", skewness_median) # Generate image featurevector of 6 characteristics featurevector = np.array([kurtosis_bm3d, skewness_bm3d, kurtosis_homomorphic, skewness_homomorphic, kurtosis_median, skewness_median]) return featurevector
def begin_cross_validation(self) -> Tuple[List[np.array], List[str]]: print("Training MRLBP using", GlobalConfig.get('mrlbp_classifier'), "classifier.") return super().begin_cross_validation()
def main(): # Parse Args. # 'scale' allows the image_scaled scale to be set. Eg: 0.25, 0.5, 1.0 argList = sys.argv[1:] shortArg = 'a:d:t:s:S:k:rn:i:me' longArg = [ 'algorithm=', 'dataset=', 'train-ratio=', 'scale=', 'test-scale=', 'folds=', 'rotations', 'noise=', 'noise-intensity=', 'multiprocess', 'example', 'data-ratio=', 'mrlbp-classifier=', 'noise-train', 'ecs', 'debug' ] valid_algorithms = [ 'RLBP', 'MRLBP', 'MRELBP', 'BM3DELBP', 'NoiseClassifier' ] valid_datasets = ['kylberg'] valid_noise = ['gaussian', 'speckle', 'salt-pepper'] valid_mrlbp_classifiers = ['svm', 'knn'] try: args, vals = getopt.getopt(argList, shortArg, longArg) for arg, val in args: if arg in ('-a', '--algorithm'): if val in valid_algorithms: print('Using algorithm:', val) GlobalConfig.set("algorithm", val) else: raise ValueError( 'Invalid algorithm configured, choose one of the following:', valid_algorithms) elif arg in ('-d', '--dataset'): if val in valid_datasets: print("Using dataset:", val) GlobalConfig.set("dataset", val) else: raise ValueError( 'Invalid dataset configured, choose one of the following:', valid_datasets) elif arg in ('-t', '--train-test'): if 0 < float(val) <= 1.0: print('Using train-ratio ratio of', val) GlobalConfig.set('train_ratio', float(val)) else: raise ValueError( 'Train-test ratio must be 0 < train-test <= 1.0') elif arg in ('-s', '--scale'): if 0 < float(val) <= 1.0: print('Using training image scale:', val) GlobalConfig.set('scale', float(val)) else: raise ValueError('Scale must be 0 < scale <= 1.0') elif arg in ('-S', '--test-scale'): if 0 < float(val) <= 1.0: print('Using testing image scale:', val) GlobalConfig.set('test_scale', float(val)) else: raise ValueError('Test scale must be 0 < scale <= 1.0') elif arg in ('-k', '--folds'): print('Doing {} folds'.format(val)) GlobalConfig.set("folds", int(val)) elif arg in ('-r', '--rotations'): print('Using rotated image_scaled sources') GlobalConfig.set("rotate", True) elif arg in ('-n', '--noise'): if val in valid_noise: print('Applying noise:', val) GlobalConfig.set("noise", val) else: raise ValueError( 'Invalid noise type, choose one of the following:', valid_noise) elif arg in ('-i', '--noise-intensity'): print('Using noise intensity (sigma / ratio) of:', val) GlobalConfig.set("noise_val", float(val)) elif arg in ('-m', '--multiprocess'): cores = psutil.cpu_count() print('Using {} processor cores for computing featurevectors'. format(cores)) GlobalConfig.set('multiprocess', True) GlobalConfig.set('cpu_count', cores) elif arg in ('-e', '--example'): print('Generating algorithm example image_scaled') GlobalConfig.set('examples', True) elif arg == '--data-ratio': if 0 < float(val) <= 1.0: print('Using dataset ratio:', val) GlobalConfig.set('data_ratio', float(val)) else: raise ValueError('Data ratio must be 0 < ratio <= 1.0') elif arg == '--mrlbp-classifier': if val in valid_mrlbp_classifiers: print( "MRLBP algorithm (if configured) will use {} classifier" .format(val)) GlobalConfig.set('mrlbp_classifier', val) else: raise ValueError( 'Invalid classifier chosen for mrlbp, choose one of the following:', valid_mrlbp_classifiers) elif arg == '--noise-train': print( "Applying noise to the training dataset as well as the test dataset" ) GlobalConfig.set('train_noise', True) elif arg == '--ecs': print("Loading dataset from C:\Local") GlobalConfig.set('ECS', True) elif arg == '--debug': print("Running in debug mode") GlobalConfig.set('debug', True) else: raise ValueError('Unhandled argument provided:', arg) except getopt.error as err: print(str(err)) if GlobalConfig.get('ECS'): GlobalConfig.set( 'CWD', r'\\filestore.soton.ac.uk\users\ojvl1g17\mydocuments\COMP3200-Texture-Classification' ) else: GlobalConfig.set('CWD', os.getcwd()) if GlobalConfig.get('examples'): write_examples() # Load configured Dataset if GlobalConfig.get('dataset') == 'kylberg': if GlobalConfig.get('debug'): # To save time in debug mode, only load one class and load a smaller proportion of it (25% of samples) kylberg = DatasetManager.KylbergTextures( num_classes=2, data_ratio=GlobalConfig.get('data_ratio')) else: kylberg = DatasetManager.KylbergTextures( num_classes=28, data_ratio=GlobalConfig.get('data_ratio')) # Load Dataset & Cross Validator dataset = kylberg.load_data() cross_validator = kylberg.get_cross_validator() print("Dataset loaded") elif GlobalConfig.get('dataset') is None: raise ValueError('No Dataset configured') else: raise ValueError('Invalid dataset') if GlobalConfig.get('rotate'): dataset_folder = GlobalConfig.get('dataset') + '-rotated' else: dataset_folder = GlobalConfig.get('dataset') out_folder = os.path.join(GlobalConfig.get('CWD'), 'out', GlobalConfig.get('algorithm'), dataset_folder) # Initialise algorithm if GlobalConfig.get('algorithm') == 'RLBP': print("Applying RLBP algorithm") algorithm = RLBP.RobustLBP() elif GlobalConfig.get('algorithm') == 'MRLBP': print("Applying MRLBP algorithm") algorithm = RLBP.MultiresolutionLBP(p=[8, 16, 24], r=[1, 2, 3]) elif GlobalConfig.get('algorithm') == 'MRELBP': print("Applying MRELBP algorithm") algorithm = MRELBP.MedianRobustExtendedLBP(r1=[2, 4, 6, 8], p=8, w_center=3, w_r1=[3, 5, 7, 9]) elif GlobalConfig.get('algorithm') == 'BM3DELBP': print("Applying BM3DELBP algorithm") algorithm = BM3DELBP.BM3DELBP() elif GlobalConfig.get('algorithm') == 'NoiseClassifier': # Noise Classifier is used in BM3DELBP algorithm usually, this allows for benchmarking of the classifier alone algorithm = NoiseClassifier.NoiseClassifier() pass else: raise ValueError('Invalid algorithm choice') # Get the Training out directory (i.e. Images without scaling/rotation/noise) train_out_dir = os.path.join( out_folder, algorithm.get_outdir(noisy_image=False, scaled_image=False)) # Get the Testing out directory (i.e. Images with scaling/rotation/noise) if GlobalConfig.get('noise') is not None: noisy_image = True else: noisy_image = False if GlobalConfig.get('test_scale') is not None: scaled_image = True else: scaled_image = False test_out_dir = os.path.join( out_folder, algorithm.get_outdir(noisy_image, scaled_image)) # Out path for noise classifier noise_out_dir = os.path.join( GlobalConfig.get('CWD'), 'out', 'NoiseClassifier', dataset_folder, "scale-{}".format(int(GlobalConfig.get('scale') * 100))) test_noise_out_dir = os.path.join( GlobalConfig.get('CWD'), 'out', 'NoiseClassifier', dataset_folder, algorithm.get_outdir(noisy_image, scaled_image)) print("Replacing DatasetManager.Image with BM3DELBPImages") # Convert DatasetManager.Image into BM3DELBP.BM3DELBPImage if GlobalConfig.get('algorithm') == 'NoiseClassifier' or GlobalConfig.get( 'algorithm') == 'BM3DELBP': for index, img in enumerate(dataset): dataset[index] = BM3DELBP.BM3DELBPImage(img) # Also convert rotated images if necessary if img.test_rotations is not None: for index, rotated_img in enumerate(img.test_rotations): img.test_rotations[index] = BM3DELBP.BM3DELBPImage( rotated_img) if GlobalConfig.get('multiprocess'): for index, img in enumerate(dataset): dataset[index] = (index, img) if GlobalConfig.get('rotate'): maxtasks = 50 else: maxtasks = None if GlobalConfig.get( 'algorithm') == 'NoiseClassifier' or GlobalConfig.get( 'algorithm') == 'BM3DELBP': with Pool(processes=GlobalConfig.get('cpu_count'), maxtasksperchild=maxtasks) as pool: # Generate image noise featurevectors for index, image in tqdm.tqdm(pool.istarmap( describe_noise_pool, zip(dataset, repeat(noise_out_dir), repeat(test_noise_out_dir))), total=len(dataset), desc='Noise Featurevectors'): dataset[index] = image else: with Pool(processes=GlobalConfig.get('cpu_count'), maxtasksperchild=maxtasks) as pool: # Generate featurevectors for index, image in tqdm.tqdm(pool.istarmap( describe_image_pool, zip(repeat(algorithm), dataset, repeat(train_out_dir), repeat(test_out_dir))), total=len(dataset), desc='Texture Featurevectors'): dataset[index] = image else: # Process the images without using multiprocessing Pools if GlobalConfig.get( 'algorithm') == 'NoiseClassifier' or GlobalConfig.get( 'algorithm') == 'BM3DELBP': for index, img in enumerate(dataset): # Generate image noise featurevectors describe_noise(img, noise_out_dir, test_noise_out_dir) else: print("BEGINNING TIMER:") start = timer() for index, img in enumerate(dataset): # Generate featurevetors describe_image(algorithm, img, train_out_dir, test_out_dir) end = timer() print("TIME TAKEN:", end - start) # Train models and perform predictions if GlobalConfig.get('algorithm') == 'RLBP': predictor = RLBP.RobustLBPPredictor(dataset, cross_validator) elif GlobalConfig.get('algorithm') == 'MRLBP': print("Performing MRLBP Classification") predictor = RLBP.MultiresolutionLBPPredictor(dataset, cross_validator) elif GlobalConfig.get('algorithm') == 'MRELBP': print("Performing MRELBP Classification") predictor = MRELBP.MedianRobustExtendedLBPPredictor( dataset, cross_validator) elif GlobalConfig.get('algorithm') == 'BM3DELBP': print("Performing BM3DELBP Classification") predictor = BM3DELBP.BM3DELBPPredictor(dataset, cross_validator) elif GlobalConfig.get('algorithm') == 'NoiseClassifier': print("Applying noise classifier") predictor = BM3DELBP.NoiseTypePredictor(dataset, cross_validator) else: raise ValueError('Invalid algorithm choice') # Get the test label & test prediction for every fold of cross validation y_test, y_predicted = predictor.begin_cross_validation() if GlobalConfig.get('algorithm') == 'NoiseClassifier': if GlobalConfig.get('noise') is None: classes = ['no-noise', 'gaussian', 'speckle', 'salt-pepper'] else: classes = ['gaussian', 'speckle', 'salt-pepper'] else: classes = kylberg.classes # Display confusion matrix ClassificationUtils.pretty_print_conf_matrix( y_test, y_predicted, classes, title='{} Confusion Matrix'.format(GlobalConfig.get('algorithm')), out_dir=test_out_dir) # Display classification report ClassificationUtils.make_classification_report(y_test, y_predicted, classes, test_out_dir)
import logging import logging.config import yaml with open('config/logging.yml', 'rt') as f: config = yaml.safe_load(f.read()) logging.config.dictConfig(config) logger = logging.getLogger('__name__') # Import modeling utils import Mask_RCNN.mrcnn.model as modellib from config import GlobalConfig from config import DetectorConfig from data import TrainingData # Set manually since not properly picked up from system config os.environ['AWS_REGION'] = GlobalConfig.get('AWS_REGION') if __name__ == '__main__': import argparse parser = argparse.ArgumentParser( description='Runs Mask R-CNN for lung opacity detection in pneumonia.') parser.add_argument( '--data_source', required=False, help= "One of ['example', 'local', or 's3']. Requires configuration in config.py. Default 'example'." ) parser.add_argument( '--subset_size', required=True, default=1000,
def describe_image(algorithm: ImageProcessorInterface, image: DatasetManager.Image, train_out_dir: str, test_out_dir: str): """ Applies an Algorithm to an image and writes the serialised featurevector to a directory. If a noise type is configured, a featurevector is computed for that noisy image too :param algorithm: Algorithm to apply :param image: Image :param train_out_dir: Directory to write serialised featurevectors of the image with no noise / scaling applied :param test_out_dir: Directory to read/write serialised featurevetors of the image with noise / scaling applied :return: Image after attribute changes """ train_out_cat = os.path.join(train_out_dir, image.label) train_out_file = os.path.join(train_out_cat, '{}.npy'.format(image.name)) test_out_cat = os.path.join(test_out_dir, image.label) test_out_file = os.path.join(test_out_cat, '{}.npy'.format(image.name)) # Apply algorithm to normal image (or read from file if already applied before). if GlobalConfig.get('debug'): print("Read/Write train file to", train_out_file) try: if GlobalConfig.get('train_noise'): image.featurevector = algorithm.describe(image, test_image=False) image.data = None else: image.featurevector = np.load(train_out_file, allow_pickle=True) image.data = None if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.featurevector = algorithm.describe(image, test_image=False) image.data = None # Make output folder if it doesn't exist try: os.makedirs(train_out_cat) except FileExistsError: pass np.save(train_out_file, image.featurevector) # If relevant, apply algorithm to test image (or read from file if already applied before). if (image.test_noise is not None) or (image.test_scale is not None): if GlobalConfig.get('debug'): print("Read/Write test file to", test_out_file) try: image.test_featurevector = np.load(test_out_file, allow_pickle=True) # Remove test image data from memory, we don't need it anymore. image.test_data = None if GlobalConfig.get('debug'): print("Noisy Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.test_featurevector = algorithm.describe(image, test_image=True) # Remove test image data from memory, we don't need it anymore. image.test_data = None # Make output folder if it doesn't exist try: os.makedirs(test_out_cat) except FileExistsError: pass np.save(test_out_file, image.test_featurevector) # Ensure rotated variants of the image also have featurevectors generated/loaded if image.test_rotations is not None: for image in image.test_rotations: describe_image(algorithm, image, train_out_dir, test_out_dir) return image
def describe_noise(image: BM3DELBP.BM3DELBPImage, out_dir: str, test_out_dir: str): """ Applies BM3DELBP's Noise Classifier to generate featurevectors for every image, for each type of noise applied. :param image: BM3DELBPImage to apply noise to :param out_dir: Directory to write serialised featurevectors :param test_out_dir: Directory to write serialised featurevector generated on test image :return: BM3DELBPImage after attribute changes """ noise_classifier = NoiseClassifier.NoiseClassifier() if GlobalConfig.get('noise') is None: # Generate non-noisy image noise featurevector out_cat = os.path.join(out_dir, 'no-noise', image.label) out_featurevector = os.path.join( out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: # Try loading serialised featurevectors if it's ran before already image.no_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing iamge", image.name) image.generate_normal_featurevector(noise_classifier) # Make output folder if it doesn't exist try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.no_noise_featurevector) # Use a smaller NoiseClassifier training dataset if testing rotated images if not GlobalConfig.get('rotate'): # Load / generate Gussian sigma 10 noise featurevector out_cat = os.path.join(out_dir, 'gaussian-10', image.label) out_featurevector = os.path.join( out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.gauss_10_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_gauss_10(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.gauss_10_noise_featurevector) # Load / generate Speckle var=0.02 noise featurevector out_cat = os.path.join(out_dir, 'speckle-002', image.label) out_featurevector = os.path.join( out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.speckle_002_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_speckle_002(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.speckle_002_noise_featurevector) # Load / generate Salt and Pepper 2% noise featurevector out_cat = os.path.join(out_dir, 'salt-pepper-002', image.label) out_featurevector = os.path.join( out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.salt_pepper_002_noise_featurevector = np.load( out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_salt_pepper_002(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.salt_pepper_002_noise_featurevector) # Load / generate Gaussian sigma 25 noise featurevector out_cat = os.path.join(out_dir, 'gaussian-25', image.label) out_featurevector = os.path.join(out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.gauss_25_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_gauss_25(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.gauss_25_noise_featurevector) # Load / generate Speckle var=0.04 noise featurevector out_cat = os.path.join(out_dir, 'speckle-004', image.label) out_featurevector = os.path.join(out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.speckle_004_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_speckle_004(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.speckle_004_noise_featurevector) # Load / generate Salt and Pepper 4% noise featurevector out_cat = os.path.join(out_dir, 'salt-pepper-004', image.label) out_featurevector = os.path.join(out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.salt_pepper_004_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) image.generate_salt_pepper_004(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.salt_pepper_004_noise_featurevector) # Load / generate featurevector on test image out_cat = os.path.join(test_out_dir, image.label) out_featurevector = os.path.join(out_cat, '{}-featurevector.npy'.format(image.name)) if GlobalConfig.get('debug'): print("Read/Write to", out_featurevector) try: image.test_noise_featurevector = np.load(out_featurevector, allow_pickle=True) if GlobalConfig.get('debug'): print("Image featurevector loaded from file") except (IOError, ValueError): if GlobalConfig.get('debug'): print("Processing image", image.name) if image.test_data is None: raise ValueError('Image.test_data has not been assigned') image.generate_noise_featurevector(noise_classifier) try: os.makedirs(out_cat) except FileExistsError: pass np.save(out_featurevector, image.test_noise_featurevector) # Ensure rotated variants of the image also have noise featurevectors generated/loaded if image.test_rotations is not None: for image_rotated in image.test_rotations: describe_noise(image_rotated, out_dir, test_out_dir) return image
def getCurrentCluster(self): tc = GlobalConfig(TEMP_FILE) return tc.get("TEMP", "cluster_index")
def getCurrentCluster(self): if os.path.exists(TEMP_FILE): tc = GlobalConfig(TEMP_FILE) return tc.get("TEMP", "cluster_index") else: return "1"
def test_get_s3_dcm(self): bucket = GlobalConfig.get('S3_BUCKET_NAME') imgdir = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR') test_dcm_path = imgdir + '/0004cfab-14fd-4e49-80ba-63a80b6bddd6.dcm' ds = ingest.get_s3_dcm(bucket=bucket, file_key=test_dcm_path) self.assertEqual(ds.pixel_array.shape, (1024, 1024))
def pretty_print_conf_matrix(y_true, y_pred, classes, normalize=False, title='{} Confusion matrix'.format( describe_test_setup()), cmap=plt.cm.Blues, out_dir=None): """ Code adapted from: http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py """ fig, ax = plt.subplots(figsize=(15, 15)) cm = confusion_matrix(y_true, y_pred, labels=classes) # Configure Confusion Matrix Plot Aesthetics (no text yet) cax = ax.imshow(cm, interpolation='nearest', cmap=cmap) ax.set_title(title, fontsize=16) ax.set_xticks(np.arange(len(classes))) ax.set_yticks(np.arange(len(classes))) ax.set_xticklabels(classes) ax.set_yticklabels(classes) ax.tick_params(axis='x', labelsize=14) ax.tick_params(axis='y', labelsize=14) plt.setp(ax.get_xticklabels(), rotation=45, ha='right', rotation_mode='anchor') plt.colorbar(cax) ax.set_ylabel('True label', fontsize=16) ax.set_xlabel('Predicted label', fontsize=16, rotation='horizontal') # Calculate normalized values (so all cells sum to 1) if desired if normalize: cm = np.round(cm.astype('float') / cm.sum(), 2) # (axis=1)[:, np.newaxis] thresh = cm.max() / 1.5 if normalize else cm.max() / 2 # Place Numbers as Text on Confusion Matrix Plot for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): ax.text(j, i, cm[i, j], ha="center", va="center", color="white" if cm[i, j] > thresh else "black", fontsize=12) #fig.tight_layout() plt.show(block=False) if out_dir is not None: if GlobalConfig.get('algorithm') == 'MRLBP': if GlobalConfig.get('mrlbp_classifier') == 'knn': out_file = os.path.join( out_dir, 'Confusion Matrix{} - KNN.png'.format( describe_test_setup())) else: out_file = os.path.join( out_dir, 'Confusion Matrix{} - SVM.png'.format( describe_test_setup())) else: out_file = os.path.join( out_dir, 'Confusion Matrix{}.png'.format(describe_test_setup())) fig.savefig(out_file, dpi=300)
class MainWindow(QMainWindow): def __init__(self, *args, **kwargs): super(MainWindow, self).__init__(*args, **kwargs) self.setupUi() self.initData() def setupUi(self): self.setWindowTitle("饥荒联机版服务器管理工具") # 设置初始化的窗口大小 self.setFixedSize(1085, 700) # 最开始窗口要居中显示 self.center() # 设置整体布局 左右显示 pagelayout = QGridLayout() ''' 开始左上侧布局 ''' # 创建左上侧部件 top_left_frame = QFrame(self) top_left_frame.setFrameShape(QFrame.StyledPanel) # 按钮垂直布局 top_button_layout = QVBoxLayout(top_left_frame) # 五个存档槽按钮 self.cluster_btns = {} for b_index in range(5): self.cluster_btns[b_index] = QPushButton(top_left_frame) self.cluster_btns[b_index].setFixedSize(180, 30) self.cluster_btns[b_index].setText("存档槽 " + str(b_index + 1)) # cluster_btns[b_index].setEnabled(False) self.cluster_btns[b_index].index = b_index + 1 self.cluster_btns[b_index].clicked.connect(self.set_cluster) top_button_layout.addWidget(self.cluster_btns[b_index]) # 删除存档按钮 delete_cluster_btn = QPushButton(top_left_frame) delete_cluster_btn.setFixedSize(180, 30) delete_cluster_btn.setText("删除存档") top_button_layout.addWidget(delete_cluster_btn) delete_cluster_btn.clicked.connect(self.deleteCluster) # 导出远程存档按钮 export_remote_cluster_btn = QPushButton(top_left_frame) export_remote_cluster_btn.setText("导出远程存档") export_remote_cluster_btn.setFixedSize(180, 30) top_button_layout.addWidget(export_remote_cluster_btn) # 导入本地存档按钮 import_local_cluster_btn = QPushButton(top_left_frame) import_local_cluster_btn.setText("导入本地存档") import_local_cluster_btn.setFixedSize(180, 30) top_button_layout.addWidget(import_local_cluster_btn) ''' 开始左下侧布局 ''' # 创建左下侧部件 bottom_left_frame = QFrame(self) bottom_left_frame.setFrameShape(QFrame.StyledPanel) # 按钮垂直布局 bottom_button_layout = QVBoxLayout(bottom_left_frame) # 控制台按钮 console_btn = QPushButton(bottom_left_frame) console_btn.setText("控制台") console_btn.setFixedSize(180, 30) bottom_button_layout.addWidget(console_btn) # 软件设置按钮 settings_btn = QPushButton(bottom_left_frame) settings_btn.setText("软件设置") settings_btn.setFixedSize(180, 30) settings_btn.clicked.connect(self.soft_settings) bottom_button_layout.addWidget(settings_btn) # 导出远程存档按钮 browser_online_server_btn = QPushButton(bottom_left_frame) browser_online_server_btn.setText("浏览在线服务器") browser_online_server_btn.setFixedSize(180, 30) bottom_button_layout.addWidget(browser_online_server_btn) # 导出远程存档按钮 help_and_about_btn = QPushButton(bottom_left_frame) help_and_about_btn.setText("使用帮助和关于") help_and_about_btn.setFixedSize(180, 30) bottom_button_layout.addWidget(help_and_about_btn) # 弹簧控件 spacerItem = QSpacerItem(20, 20, QSizePolicy.Minimum, QSizePolicy.Expanding) bottom_button_layout.addItem(spacerItem) ''' 开始右侧布局 ''' # 右侧开始布局 对应按钮布局 right_frame = QFrame(self) right_frame.setFrameShape(QFrame.StyledPanel) # 右边显示为stack布局 self.right_layout = QStackedLayout(right_frame) self.settings_widget = SettingsWidget() self.cluster_tab = MainTab() self.right_layout.addWidget(self.cluster_tab) self.right_layout.addWidget(self.settings_widget) # 划分界面 splitterV = QSplitter(Qt.Vertical) splitterV.addWidget(top_left_frame) splitterV.addWidget(bottom_left_frame) # 固定左上高度 top_left_frame.setFixedHeight(330) splitterH = QSplitter(Qt.Horizontal) splitterH.addWidget(splitterV) splitterH.addWidget(right_frame) # 固定左侧宽度 top_left_frame.setFixedWidth(200) # 窗口部件添加布局 widget = QWidget() pagelayout.addWidget(splitterH) widget.setLayout(pagelayout) self.setCentralWidget(widget) # 按钮函数绑定 def soft_settings(self): self.right_layout.setCurrentIndex(1) # 存档按钮状态刷新 def refresh_cluster_btn_state(self, index): for i in self.cluster_btns: if index == self.cluster_btns[i].index: self.cluster_btns[i].setStyleSheet("") else: self.cluster_btns[i].setStyleSheet("color:gray") # 存档设置 def set_cluster(self): self.current_cluster_index = self.sender().index self.tempconfig.set("TEMP", "cluster_index", str(self.current_cluster_index)) self.tempconfig.save(TEMP_FILE) self.refresh_cluster_btn_state(self.current_cluster_index) self.right_layout.setCurrentIndex(0) self.mk_cluster_dir() self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join( self.current_cluster_folder, "cluster.ini") self.cluster_tab.cluster_settings_tab.read_cluster_data( self.cluster_tab.cluster_settings_tab.current_cluster_file) self.cluster_tab.cluster_settings_tab.setServerIP( self.cluster_tab.cluster_settings_tab.masterip, self.cluster_tab.cluster_settings_tab.getServerIP()) self.cluster_tab.shard_settings_tab.initShardTab() def deleteCluster(self): cindex = self.current_cluster_index delm = QMessageBox.warning(self, "删除警告", "你确定要删除存档槽" + str(cindex) + "?", QMessageBox.Yes | QMessageBox.No, QMessageBox.No) if delm == QMessageBox.Yes: sdir = os.path.join(CLUSTER_DIR, "Cluster_" + str(cindex)) if os.path.exists(sdir): shutil.rmtree(sdir) self.right_layout.setCurrentIndex(0) self.mk_cluster_dir() self.cluster_tab.setCurrentIndex(0) self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join( self.current_cluster_folder, "cluster.ini") self.cluster_tab.cluster_settings_tab.read_cluster_data( self.cluster_tab.cluster_settings_tab.current_cluster_file) self.cluster_tab.cluster_settings_tab.setServerIP( self.cluster_tab.cluster_settings_tab.masterip, self.cluster_tab.cluster_settings_tab.getServerIP()) self.cluster_tab.shard_settings_tab.initShardTab() QMessageBox.information(self, "删除完毕", "存档槽" + str(cindex) + "已删除并重置!", QMessageBox.Yes) def mk_cluster_dir(self): self.current_cluster_folder = os.path.join( CLUSTER_DIR, "Cluster_" + str(self.current_cluster_index)) if not os.path.exists(self.current_cluster_folder): os.mkdir(self.current_cluster_folder) def init_cluster_data(self, index): self.mk_cluster_dir() self.right_layout.setCurrentIndex(0) self.refresh_cluster_btn_state(self.current_cluster_index) self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join( self.current_cluster_folder, "cluster.ini") self.cluster_tab.cluster_settings_tab.read_cluster_data( self.cluster_tab.cluster_settings_tab.current_cluster_file) def initDir(self): if not os.path.exists(ROOT_DIR): os.mkdir(ROOT_DIR) if not os.path.exists(CLUSTER_DIR): os.mkdir(CLUSTER_DIR) if not os.path.exists(TEMP_FILE): self.tempconfig = GlobalConfig(TEMP_FILE) self.tempconfig.add_section("TEMP") self.tempconfig.set("TEMP", "cluster_index", "1") self.tempconfig.save(TEMP_FILE) def initData(self): self.initDir() self.tempconfig = GlobalConfig(TEMP_FILE) if os.path.exists(TEMP_FILE): self.current_cluster_index = int( self.tempconfig.get("TEMP", "cluster_index")) else: self.current_cluster_index = 1 self.init_cluster_data(self.current_cluster_index) # 设置窗口居中 def center(self): screen = QDesktopWidget().screenGeometry() size = self.geometry() self.move((screen.width() - size.width()) / 2, (screen.height() - size.height()) / 2)
def tune_noise_classifier(): GlobalConfig.set('dataset', 'kylberg') GlobalConfig.set('ECS', True) GlobalConfig.set('algorithm', 'NoiseClassifier') GlobalConfig.set('scale', 0.5) GlobalConfig.set( 'CWD', r'\\filestore.soton.ac.uk\users\ojvl1g17\mydocuments\COMP3200-Texture-Classification' ) #GlobalConfig.set('CWD', os.getcwd()) GlobalConfig.set('folds', 10) cores = psutil.cpu_count() GlobalConfig.set('cpu_count', cores) dataset = DatasetManager.KylbergTextures(num_classes=28, data_ratio=0.5) images = dataset.load_data() gc.collect() bm3d_images = [] # Convert to BM3D images for image in images: new_image = BM3DELBP.BM3DELBPImage(image) bm3d_images.append(new_image) print("Image dataset loaded, loaded {} images".format(len(images))) noise_classifier = NoiseClassifier() cross_validator = dataset.get_cross_validator() bm3d_sigma = [10, 30, 40, 50] homomorphic_cutoff = [0.1, 0.5, 5, 10] homomorphic_a = [0.5, 0.75, 1.0] homomorphic_b = [0.1, 0.5, 1.0, 1.25] settings_jobs = [] # List of all configuration tuples for sigma_val in bm3d_sigma: for cutoff in homomorphic_cutoff: for a in homomorphic_a: for b in homomorphic_b: settings_jobs.append((sigma_val, cutoff, a, b)) results = [] # List of tuples (F1, sigma_val, cutoff, a, b) out_csv = os.path.join(GlobalConfig.get('CWD'), 'NoiseClassifierTuning', 'Results.txt') with Pool(processes=GlobalConfig.get('cpu_count'), maxtasksperchild=50) as pool: # Generate image featurevectors and replace DatasetManager.Image with BM3DELBP.BM3DELBPImage for result in tqdm.tqdm(pool.istarmap( do_classification, zip(settings_jobs, repeat(noise_classifier), repeat(cross_validator), repeat(bm3d_images))), total=len(settings_jobs), desc='NoiseClassifier tuning'): f1, sigma, cutoff, a, b = result # Log to CSV file if os.path.isfile(out_csv): # CSV exists, append to end of file with open(out_csv, 'a', encoding="utf-8", newline='') as resultsfile: writer = csv.writer(resultsfile) writer.writerow([f1, sigma, cutoff, a, b]) else: # CSV does not exist. Write the headings with open(out_csv, 'w', encoding="utf-8", newline='') as resultsfile: writer = csv.writer(resultsfile) writer.writerow(['f1', 'sigma_psd', 'cutoff', 'a', 'b']) writer.writerow([f1, sigma, cutoff, a, b]) results.append(result) # Sort largest to smallest, by F1 score results.sort(key=lambda tup: tup[0], reverse=True) print("Finished tuning parameters.") print("The top 3 results were:") f1, sigma, cutoff, a, b = results[0] print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format( f1, sigma, cutoff, a, b)) f1, sigma, cutoff, a, b = results[1] print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format( f1, sigma, cutoff, a, b)) f1, sigma, cutoff, a, b = results[2] print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format( f1, sigma, cutoff, a, b))
class ClusterWidget(QWidget): def __init__(self, parent=None): super(ClusterWidget, self).__init__(parent) layout = QVBoxLayout() layout1 = QHBoxLayout() layout2 = QHBoxLayout() self.cluster_name = QLineEdit() name_label = QLabel() name_label.setText('房间名称:') description_label = QLabel() description_label.setText('房间简介:') self.cluster_description = QLineEdit() self.cluster_name.setText("南风颂的饥荒世界") self.cluster_description.setText("由饥荒联机版服务器管理工具开设!") layout1.addWidget(name_label) layout1.addWidget(self.cluster_name) layout2.addWidget(description_label) layout2.addWidget(self.cluster_description) layout3 = QHBoxLayout() layout4 = QHBoxLayout() layout5 = QHBoxLayout() self.cluster_intention = QComboBox() self.cluster_intention.setItemDelegate(QStyledItemDelegate()) self.cluster_intention.setStyleSheet( "QComboBox QAbstractItemView::item { min-height: 30px; min-width: 60px; }" ) self.cluster_intention_cn = ['休闲', '合作', '竞赛', '疯狂'] self.cluster_intention_value = [ 'social', 'cooperative', 'competitive', 'madness' ] self.cluster_intention.addItems(self.cluster_intention_cn) self.game_mode = QComboBox() self.game_mode.setStyleSheet( "QComboBox QAbstractItemView::item { min-height: 30px; min-width: 60px; }" ) self.game_mode.setItemDelegate(QStyledItemDelegate()) self.game_mode_cn = ['无尽', '生存', '荒野', '熔炉', '暴食'] self.game_mode_value = [ 'endless', 'survival', 'wilderness', 'lavaarena', 'quagmire' ] self.game_mode.addItems(self.game_mode_cn) label3 = QLabel() label3.setText('游戏风格:') label3.setFixedWidth(70) label4 = QLabel() label4.setFixedWidth(70) label4.setText('游戏模式:') layout3.addWidget(label3) layout3.addWidget(self.cluster_intention) layout4.addWidget(label4) layout4.addWidget(self.game_mode) layout6 = QHBoxLayout() label5 = QLabel() label5.setText("游戏语言:") label5.setFixedWidth(70) self.game_language = QButtonGroup() self.en_rbtn = QRadioButton('英语') self.en_rbtn.setFixedWidth(70) self.zh_rbtn = QRadioButton('简体中文') self.game_language.addButton(self.en_rbtn, 1) self.game_language.addButton(self.zh_rbtn, 2) self.zh_rbtn.setChecked(True) layout6.addWidget(label5) layout6.addWidget(self.en_rbtn) layout6.addWidget(self.zh_rbtn) layout7 = QHBoxLayout() label6 = QLabel() label6.setText("Steam群组ID:") self.steam_group_id = QLineEdit() self.steam_group_admin = QCheckBox('群组官员设为管理员') self.steam_group_only = QCheckBox('仅群组成员可进') layout7.addWidget(label6) layout7.addWidget(self.steam_group_id) layout7.addWidget(self.steam_group_admin) layout7.addWidget(self.steam_group_only) layout8 = QHBoxLayout() self.pvp = QCheckBox("开启PVP竞技") self.pause_when_empty = QCheckBox("开启无人暂停") self.pause_when_empty.setChecked(True) self.vote = QCheckBox('开启玩家投票') self.vote.setChecked(True) layout8.addWidget(self.pvp) layout8.addWidget(self.pause_when_empty) layout8.addWidget(self.vote) layout9 = QHBoxLayout() label7 = QLabel() label7.setText('最大玩家人数:') self.max_players = QLineEdit() self.max_players.setText("0") label8 = QLabel() label8.setText('房间预留位置个数:') self.white_players = QLineEdit() self.white_players.setText("0") layout9.addWidget(label7) layout9.addWidget(self.max_players) layout9.addWidget(label8) layout9.addWidget(self.white_players) layout10 = QHBoxLayout() label9 = QLabel() label9.setText("房间密码:") self.password = QLineEdit() layout10.addWidget(label9) layout10.addWidget(self.password) layout11 = QHBoxLayout() label10 = QLabel() label10.setText("主世界服务器:") label10.setFixedWidth(100) self.masterip = QComboBox() self.masterip.setStyleSheet( "QComboBox QAbstractItemView::item { min-height: 25px; min-width: 100px; }" ) self.masterip.setItemDelegate(QStyledItemDelegate()) layout11.addWidget(label10) layout11.addWidget(self.masterip) layout12 = QHBoxLayout() self.load_default_cluster_settings = QPushButton() self.load_default_cluster_settings.setText("载入默认设置") self.set_default_cluster_settings = QPushButton() self.set_default_cluster_settings.setText("保存为默认设置") self.save_cluster_setttings = QPushButton() self.save_cluster_setttings.setText("保存房间设置") layout12.addWidget(self.load_default_cluster_settings) layout12.addWidget(self.set_default_cluster_settings) layout12.addWidget(self.save_cluster_setttings) layout.addLayout(layout1) layout.addLayout(layout2) layout5.addLayout(layout3) layout5.addLayout(layout4) layout.addLayout(layout5) layout.addLayout(layout6) layout.addLayout(layout7) layout.addLayout(layout8) layout.addLayout(layout9) layout.addLayout(layout10) layout.addLayout(layout11) layout.addLayout(layout12) self.setLayout(layout) self.load_default_cluster_settings.clicked.connect( self.read_default_cluster_data) self.set_default_cluster_settings.clicked.connect( self.write_to_default_cluster_data) self.save_cluster_setttings.clicked.connect( self.write_curret_cluster_data) self.setServerIP(self.masterip, ip="127.0.0.1") def setServerIP(self, combox, ip): if ip == "": oldvalue = self.getServerIP() else: oldvalue = ip combox.clear() self.serverlist = SettingsWidget().get_server_list() oldindex = 0 index = 0 for slist in self.serverlist: if slist[0] != "": combox.addItem(slist[0] + "@" + slist[1]) else: combox.addItem(slist[1]) if slist[1] == oldvalue: oldindex = index index += 1 combox.setCurrentIndex(oldindex) def getServerIP(self): iparr = self.masterip.currentText().split('@') if len(iparr) > 1: ip = iparr[1] else: ip = iparr[0] if ip == "": ip = "127.0.0.1" return ip def read_default_cluster_data(self): file = os.path.join(CONFIG_DIR, "cluster.ini") self.read_cluster_data(file) def write_to_default_cluster_data(self): file = os.path.join(CONFIG_DIR, "cluster.ini") self.write_cluster_data(file) def write_curret_cluster_data(self): self.write_cluster_data(self.current_cluster_file) def write_cluster_data(self, file): self.cluster_config.set("STEAM", "steam_group_id", self.steam_group_id.text()) self.cluster_config.setboolean("STEAM", "steam_group_only", self.steam_group_only.isChecked()) self.cluster_config.setboolean("STEAM", "steam_group_admins", self.steam_group_admin.isChecked()) self.cluster_config.setboolean("GAMEPLAY", "pvp", self.pvp.isChecked()) self.cluster_config.set( "GAMEPLAY", "game_mode", self.game_mode_value[self.game_mode.currentIndex()]) self.cluster_config.setboolean("GAMEPLAY", "pause_when_empty", self.pause_when_empty.isChecked()) self.cluster_config.setboolean("GAMEPLAY", "vote_enabled", self.vote.isChecked()) self.cluster_config.set("GAMEPLAY", "max_players", self.max_players.text()) self.cluster_config.set("NETWORK", "cluster_name", self.cluster_name.text()) self.cluster_config.set("NETWORK", "cluster_description", self.cluster_description.text()) self.cluster_config.set( "NETWORK", "cluster_intention", self.cluster_intention_value[ self.cluster_intention.currentIndex()]) if self.zh_rbtn.isChecked(): lang = "zh" else: lang = "en" self.cluster_config.set("NETWORK", "cluster_language", lang) self.cluster_config.set("NETWORK", "whitelist_slots", self.white_players.text()) self.cluster_config.set("NETWORK", "cluster_password", self.password.text()) self.cluster_config.set("SHARD", "master_ip", self.getServerIP()) self.cluster_config.save(file) def read_current_cluster_data(self): self.read_cluster_data(self.current_cluster_file) def read_cluster_data(self, file): if not os.path.exists(file): file = os.path.join(CONFIG_DIR, "cluster.ini") self.cluster_config = GlobalConfig(file) self.steam_group_id.setText( self.cluster_config.get("STEAM", "steam_group_id")) self.steam_group_only.setChecked( self.cluster_config.getboolean("STEAM", "steam_group_only")) self.steam_group_admin.setChecked( self.cluster_config.getboolean("STEAM", "steam_group_admins")) self.pvp.setChecked(self.cluster_config.getboolean("GAMEPLAY", "pvp")) self.game_mode.setCurrentIndex( self.game_mode_value.index( self.cluster_config.get("GAMEPLAY", "game_mode"))) self.pause_when_empty.setChecked( self.cluster_config.getboolean("GAMEPLAY", "pause_when_empty")) self.vote.setChecked( self.cluster_config.getboolean("GAMEPLAY", "vote_enabled")) self.max_players.setText( self.cluster_config.get("GAMEPLAY", "max_players")) self.cluster_name.setText( self.cluster_config.get("NETWORK", "cluster_name")) self.cluster_description.setText( self.cluster_config.get("NETWORK", "cluster_description")) self.cluster_intention.setCurrentIndex( self.cluster_intention_value.index( self.cluster_config.get("NETWORK", "cluster_intention"))) if self.cluster_config.get("NETWORK", "cluster_language") == "zh": self.zh_rbtn.setChecked(True) else: self.en_rbtn.setChecked(True) self.white_players.setText( self.cluster_config.get("NETWORK", "whitelist_slots")) self.password.setText( self.cluster_config.get("NETWORK", "cluster_password")) self.setServerIP(self.masterip, self.cluster_config.get("SHARD", "master_ip"))