def test_parse_training_labels(self):
parsed = ingest.parse_training_labels(
train_box_df=pd.read_csv(
GlobalConfig.get('EXAMPLE_TRAIN_BOX_PATH')),
train_image_dirpath=GlobalConfig.get(
'EXAMPLE_STAGE1_TRAIN_IMAGE_DIR'))
# Negative Case
self.assertEquals(
parsed['0004cfab-14fd-4e49-80ba-63a80b6bddd6'], {
'dicom':
'data/example/stage_1_train_images/0004cfab-14fd-4e49-80ba-63a80b6bddd6.dcm',
'label': 0,
'boxes': []
})
# Positive Case
self.assertEquals(
parsed['00436515-870c-4b36-a041-de91049b9ab4'], {
'dicom':
'data/example/stage_1_train_images/00436515-870c-4b36-a041-de91049b9ab4.dcm',
'label':
1,
'boxes': [[264.0, 152.0, 213.0, 379.0],
[562.0, 152.0, 256.0, 453.0]]
})
def _retrieve_training_box_labels(self):
if self.data_source == 'local':
train_box_df = pd.read_csv(GlobalConfig.get('LOCAL_TRAIN_BOX_PATH'))
elif self.data_source == 'example':
train_box_df = pd.read_csv(GlobalConfig.get('EXAMPLE_TRAIN_BOX_PATH'))
elif self.data_source == 's3':
train_box_df = ingest.read_s3_df(
bucket=GlobalConfig.get('S3_BUCKET_NAME'),
file_key=GlobalConfig.get('S3_TRAIN_BOX_KEY'))
return train_box_df
def get_cross_validator():
"""
Generates and returns the ShuffleSplit cross validator.
This uses a fixed random_state=0, so that every time each cross fold contains the same subset of data.
:return: sklearn.model_selection.ShuffleSplit with relevant parameters.
"""
return StratifiedShuffleSplit(
n_splits=GlobalConfig.get('folds'),
train_size=GlobalConfig.get('train_ratio'),
random_state=0)
def _retrieve_annotation_dict(self, train_box_df):
if self.data_source == 'local':
dirpath = GlobalConfig.get('LOCAL_STAGE1_TRAIN_IMAGE_DIR')
if self.data_source == 'example':
dirpath = GlobalConfig.get('EXAMPLE_STAGE1_TRAIN_IMAGE_DIR')
elif self.data_source == 's3':
dirpath = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR')
annotation_dict = ingest.parse_training_labels(
train_box_df=train_box_df,
train_image_dirpath=dirpath)
return annotation_dict
def __init__(self, num_classes: int, data_ratio: float):
self.num_categories = num_classes
self.data_ratio = data_ratio
self.classes = []
self.train = []
self.test = []
if GlobalConfig.get('rotate'):
print("Using rotated textures")
if GlobalConfig.get('ECS'):
self.KYLBERG_DIR = os.path.join('C:/', 'Local', 'data',
'kylberg-rotated')
else:
self.KYLBERG_DIR = os.path.join(GlobalConfig.get('CWD'),
'data', 'kylberg-rotated')
else:
print("Using non-rotated textures")
if GlobalConfig.get('ECS'):
self.KYLBERG_DIR = os.path.join('C:/', 'Local', 'data',
'kylberg')
else:
self.KYLBERG_DIR = os.path.join(GlobalConfig.get('CWD'),
'data', 'kylberg')
print('Using {} scale'.format(GlobalConfig.get('scale')))
if GlobalConfig.get('noise') is not None:
print('Applying {} noise'.format(GlobalConfig.get('noise')))
def make_classification_report(y_true, y_pred, classes, out_dir):
# Get the full report
rpt = classification_report(y_true,
y_pred,
labels=classes,
output_dict=True)
# Get the average results from all classes
rpt_accuracy = rpt['accuracy']
rpt_macro_avg = rpt['macro avg']
rpt_weighted_avg = rpt['weighted avg']
# Get the per-class results only
del rpt['accuracy']
del rpt['macro avg']
del rpt['weighted avg']
pandas_rpt = pd.DataFrame.from_dict(rpt, orient='index')
pandas_rpt.rename(columns={'support': 'N Predictions'}, inplace=True)
pandas_macro_avg = pd.DataFrame.from_dict(rpt_macro_avg,
orient='index',
columns=['Macro Average'])
pandas_macro_avg.rename(columns={'support': 'N Predictions'}, inplace=True)
pandas_weighted_avg = pd.DataFrame.from_dict(rpt_weighted_avg,
orient='index',
columns=['Weighted Average'])
pandas_weighted_avg.rename(columns={'support': 'N Predictions'},
inplace=True)
pandas_accuracy = pd.DataFrame({'': rpt_accuracy},
index=['overall accuracy'])
if GlobalConfig.get('algorithm') == 'MRLBP':
if GlobalConfig.get('mrlbp_classifier') == 'knn':
out_file = os.path.join(
out_dir, 'Classification Report{} - KNN.csv'.format(
describe_test_setup()))
else:
out_file = os.path.join(
out_dir, 'Classification Report{} - SVM.csv'.format(
describe_test_setup()))
else:
out_file = os.path.join(
out_dir,
'Classification Report{}.csv'.format(describe_test_setup()))
pandas_rpt.to_csv(out_file)
pandas_macro_avg.to_csv(out_file, mode='a', header=True)
pandas_weighted_avg.to_csv(out_file, mode='a', header=True)
pandas_accuracy.to_csv(out_file, mode='a', header=True)
def split_dataset_by_class(annotation_dict, subset_size, validation_split):
"""Split dataset for training and validation, preserving outcome class distribution."""
n_positive = sum(1 for v in annotation_dict.values() if v['label'] == 1)
n_total = len(annotation_dict)
p_positive = n_positive / n_total
positive_ids = [k for k, v in annotation_dict.items() if v['label'] == 1]
negative_ids = [k for k, v in annotation_dict.items() if v['label'] == 0]
random.seed(GlobalConfig.get('RANDOM_SEED'))
random.shuffle(positive_ids)
random.shuffle(negative_ids)
n_positive_subset = int(math.ceil(subset_size * p_positive))
n_negative_subset = int(subset_size - n_positive_subset)
# ceilings & floors to ensure that we get exactly subset_size*validation_split
train_pos_index = int(math.ceil(n_positive_subset *
(1 - validation_split)))
train_neg_index = int(
math.floor(n_negative_subset * (1 - validation_split)))
train_ids = positive_ids[:train_pos_index] + negative_ids[:train_neg_index]
valid_ids = positive_ids[train_pos_index:n_positive_subset] + \
negative_ids[train_neg_index:n_negative_subset]
print('Training count: %s' % len(train_ids))
print('Validation instance count: %s' % len(valid_ids))
return {'train_ids': train_ids, 'valid_ids': valid_ids}
def get_outdir(self, noisy_image: bool, scaled_image: bool):
if noisy_image:
noise_type = GlobalConfig.get('noise')
noise_val = GlobalConfig.get('noise_val')
else:
noise_type = 'None'
noise_val = 'None'
if scaled_image:
image_scale = int(GlobalConfig.get('test_scale') * 100)
else:
image_scale = int(GlobalConfig.get('scale') * 100)
if GlobalConfig.get('train_noise'):
return "scale-{}_noise-{}_noiseval-{}-trainnoise".format(image_scale, noise_type, noise_val)
else:
return "scale-{}_noise-{}_noiseval-{}".format(image_scale, noise_type, noise_val)
def get_dataset_train(self):
dataset_train = DetectorDataset(
patient_ids=self.patient_id_train,
annotation_dict=self.annotation_dict,
orig_height=self.DICOM_HEIGHT,
orig_width=self.DICOM_WIDTH,
data_source=self.data_source,
s3_bucket=(GlobalConfig.get('S3_BUCKET_NAME') if self.data_source == 's3' else None))
dataset_train.prepare()
assert len(dataset_train.image_ids) == len(self.patient_id_train)
return dataset_train
def train(self, train: List[DatasetManager.Image]):
X_train = [img.featurevector for img in train]
# Convert List[narray] to ndarray
X_train = np.stack(X_train, axis=0)
X_train = X_train.astype(np.float64)
y_train = [img.label for img in train]
if GlobalConfig.get('mrlbp_classifier') == 'knn':
# Calculate transposed covariance matrix as described here: https://stackoverflow.com/a/55623162/6008271
X_train_cov = np.linalg.inv(np.cov(X_train.transpose())).transpose()
self.classifier = KNeighborsClassifier(n_neighbors=3,
algorithm='brute',
metric='mahalanobis',
metric_params={'VI': X_train_cov})
self.classifier.fit(X_train, y_train)
elif GlobalConfig.get('mrlbp_classifier') == 'svm':
self.classifier = SVC()
self.classifier.fit(X_train, y_train)
else:
raise ValueError('Invalid Classifier Type specified as --mrlbp-classifier.')
def describe_test_setup():
test_str = ""
if GlobalConfig.get('noise') is not None:
test_str += ' Noise Invariance {}-{}'.format(
GlobalConfig.get('noise'), GlobalConfig.get('noise_val'))
if GlobalConfig.get('rotate') is not False:
test_str += ' Rotation Invariance'
if GlobalConfig.get('test_scale') is not None:
test_str += ' Scale Invariance Train {} Test {}'.format(
int(GlobalConfig.get('scale') * 100),
int(GlobalConfig.get('test_scale') * 100))
return test_str
def get_outdir(self, noisy_image: bool, scaled_image: bool):
"""
Gets a string name for the read/write directory depending on the configuration
:param noisy_image: Whether the algorithm is being applied to images containing noise
:param scaled_image: Whether the algorithm is being applied to a scaled image
:return: String output directory name
"""
if noisy_image:
noise_type = GlobalConfig.get('noise')
noise_val = GlobalConfig.get('noise_val')
else:
noise_type = 'None'
noise_val = 'None'
if scaled_image:
image_scale = int(GlobalConfig.get('test_scale') * 100)
else:
image_scale = int(GlobalConfig.get('scale') * 100)
if GlobalConfig.get('train_noise'):
return "scale-{}_noise-{}_noiseval-{}_p-{}_r-{}-trainnoise".format(image_scale, noise_type, noise_val, self.p, self.r)
else:
return "scale-{}_noise-{}_noiseval-{}_p-{}_r-{}".format(image_scale, noise_type, noise_val, self.p, self.r)
def write_examples():
"""
Generates example images for use in report
:return: None
"""
print("Generating algorithm example images")
ex = GenerateExamples.GenerateExamples(
os.path.join(GlobalConfig.get('CWD'), 'data', 'kylberg', 'grass1',
'grass1-a-p001.png'))
ex.write_noise_examples()
ex.write_RLBP_example()
ex.write_MRLBP_example()
ex.write_MRELBP_example()
ex.write_BM3DELBP_example()
print("Finished generating examples")
def __init__(self, path):
"""
Generate Example images for dissertation write-up
:param path: Image to produce example images with
"""
self.image_path = path
image_name = path.split(os.sep)[-1].partition('.')[0]
#image_uint8 = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
image_uint8 = cv2.resize(cv2.imread(path, cv2.IMREAD_GRAYSCALE), (0, 0),
fx=0.5,
fy=0.5)
# Convert from uint8 to float32 without normalizing to zero mean
image_unscaled = ImageUtils.convert_uint8_image_float32(image_uint8)
# Convert from uint8 to float32 while normalizing to zero mean
image_scaled = ImageUtils.scale_uint8_image_float32(image_uint8)
image_gauss_10 = ImageUtils.add_gaussian_noise_skimage(image_scaled, 10)
image_gauss_25 = ImageUtils.add_gaussian_noise_skimage(image_scaled, 25)
image_speckle_002 = ImageUtils.add_speckle_noise_skimage(image_scaled, 0.02)
image_speckle_004 = ImageUtils.add_speckle_noise_skimage(image_scaled, 0.04)
image_salt_pepper_002 = ImageUtils.add_salt_pepper_noise_skimage(image_scaled, 0.02)
image_salt_pepper_004 = ImageUtils.add_salt_pepper_noise_skimage(image_scaled, 0.04)
image_label = path.split(os.sep)[-1].partition('-')[0]
# Generate different permutations of this sample image
self.image_uint8 = DatasetManager.Image(image_uint8, image_name, image_label)
self.image_unscaled = DatasetManager.Image(image_unscaled, image_name, image_label)
self.image_scaled = DatasetManager.Image(image_scaled, image_name, image_label)
self.image_gauss_10 = DatasetManager.Image(image_gauss_10, image_name, image_label)
self.image_gauss_10.test_noise='gaussian'; self.image_gauss_10.test_noise_val=10
self.image_gauss_25 = DatasetManager.Image(image_gauss_25, image_name, image_label)
self.image_gauss_25.test_noise = 'gaussian'; self.image_gauss_25.test_noise_val = 25
self.image_speckle_002 = DatasetManager.Image(image_speckle_002, image_name, image_label)
self.image_speckle_002.test_noise = 'speckle'; self.image_speckle_002.test_noise_val = 0.02
self.image_speckle_004 = DatasetManager.Image(image_speckle_004, image_name, image_label)
self.image_speckle_004.test_noise = 'speckle'; self.image_speckle_004.test_noise_val = 0.04
self.image_salt_pepper_002 = DatasetManager.Image(image_salt_pepper_002, image_name, image_label)
self.image_salt_pepper_002.test_noise = 'salt-pepper'; self.image_salt_pepper_002.noise_val = 0.02
self.image_salt_pepper_004 = DatasetManager.Image(image_salt_pepper_004, image_name, image_label)
self.image_salt_pepper_004.test_noise = 'salt-pepper'; self.image_salt_pepper_004.noise_val = 0.04
self.path = os.path.join(GlobalConfig.get('CWD'), 'example')
write_image(ImageUtils.convert_float32_image_uint8(self.image_unscaled.data), None, image_name + '-unedited.png')
write_image(ImageUtils.convert_float32_image_uint8(self.image_scaled.data), None, image_name + '-scaled.png')
def __init__(self, subset_size, validation_split, data_source='example', subdir='train'):
random.seed(GlobalConfig.get('RANDOM_SEED'))
self.subset_size = subset_size
self.validation_split = validation_split
self.data_source = data_source
self.subdir = subdir
assert self.data_source in ["local", "s3", "example"]
assert self.subdir in ["train", "test"]
self.train_box_df = self._retrieve_training_box_labels()
self.annotation_dict = self._retrieve_annotation_dict(self.train_box_df)
self.image_df = self._retrieve_dicom_image_list()
# Split train/test sets, preserving outcome class distribution
id_split = utils.split_dataset_by_class(
annotation_dict=self.annotation_dict,
subset_size=subset_size,
validation_split=validation_split)
self.patient_id_train = id_split['train_ids']
self.patient_id_valid = id_split['valid_ids']
def _retrieve_dicom_image_list(self):
if self.data_source == 'local':
cache_path = GlobalConfig.get('LOCAL_DICOM_IMAGE_LIST_PATH')
if self.subdir == 'train':
datadir = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR')
elif self.subdir == 'test':
datadir = GlobalConfig.get('S3_STAGE1_TEST_IMAGE_DIR')
elif self.data_source == 'example':
cache_path = GlobalConfig.get('EXAMPLE_DICOM_IMAGE_LIST_PATH')
datadir = GlobalConfig.get('EXAMPLE_STAGE1_TRAIN_IMAGE_DIR')
elif self.data_source == 's3':
print('S3 data source selected. Caching functionality not yet available.')
return ingest.parse_s3_dicom_image_list(
bucket=GlobalConfig.get('S3_BUCKET_NAME'),
subdir=self.subdir)
image_df = ingest.parse_dicom_image_list(
datadir=datadir, subdir=self.subdir,
cache_path=cache_path, verbose=False)
return image_df
import json
from utils import shell, log
from config import GlobalConfig
# get config singleton
config = GlobalConfig.get()
def parse_artillery_output(report_file: str) -> bool:
report = json.load(report_file)
import pprint
pprint(report)
def describe(self, image, test_image: bool, sigma_psd=70/255, cutoff_freq=10, a=0.75, b=1.0):
if isinstance(image, DatasetManager.Image):
if test_image:
image_data = image.test_data
else:
image_data = image.data
else:
image_data = image.copy()
# Perform BM3D Filter
image_bm3d_filtered = SharedFunctions.bm3d_filter(image_data, 70/255)
# Perform Homomorphic filter, Note: This requires images to be normalised in range [0, 255]
image_scaled_255 = ImageUtils.convert_float32_image_uint8(image_data)
cutoff, a, b = 10, 0.75, 0.1
image_homomorphic_filtered = SharedFunctions.homomorphic_filter(image_scaled_255, cutoff, a, b)
image_homomorphic_filtered = ImageUtils.convert_uint8_image_float32(image_homomorphic_filtered)
# Perform Median filter. Padding is required for median filter.
image_padded = pad(array=image_data, pad_width=1, mode='constant', constant_values=0)
image_median_filtered = np.zeros(image_padded.shape, dtype=np.float32)
SharedFunctions.median_filter(image_padded, 3, 1, image_median_filtered)
image_median_filtered = image_median_filtered[1:-1, 1:-1] # Remove padding now median filter done
# Subtract original image from filtered image to get noise only
bm3d_noise = image_data - image_bm3d_filtered
homomorphic_noise = image_data - image_homomorphic_filtered
median_noise = image_data - image_median_filtered
if self.save_img:
if isinstance(image, DatasetManager.Image):
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_bm3d_filtered),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'),
'{}-{}-{}-BM3D-filtered.png'.format(image.name, image.test_noise,
image.test_noise_val))
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_homomorphic_filtered),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'),
'{}-{}-{}-homomorphic-filtered-cutoff_{}-a_{}-b_{}.png'.format(image.name,
image.test_noise,
image.test_noise_val,
cutoff, a,
b))
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(image_median_filtered),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Filtered Images'),
'{}-{}-{}-median-filtered.png'.format(image.name, image.test_noise,
image.test_noise_val))
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(bm3d_noise),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'),
'{}-{}-{}-BM3D-noise-estimate.png'.format(image.name, image.test_noise,
image.test_noise_val))
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(homomorphic_noise),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'),
'{}-{}-{}-homomorphic-filtered-cutoff_{}-a_{}-b_{}.png'.format(image.name,
image.test_noise,
image.test_noise_val,
cutoff, a,
b))
GenerateExamples.write_image(ImageUtils.convert_float32_image_uint8(median_noise),
os.path.join('BM3DELBP', 'NoiseClassifier', 'Noise Estimates'),
'{}-{}-{}-median-noise-estimate.png'.format(image.name, image.test_noise,
image.test_noise_val))
else:
raise ValueError('save_img set but not passed as DatasetManager.Image or BM3DELBPImage')
kurtosis_bm3d = kurtosis(a=bm3d_noise, axis=None, fisher=False, nan_policy='raise')
skewness_bm3d = skew(a=bm3d_noise, axis=None, nan_policy='raise')
kurtosis_homomorphic = kurtosis(a=homomorphic_noise, axis=None, fisher=False, nan_policy='raise')
skewness_homomorphic = skew(a=homomorphic_noise, axis=None, nan_policy='raise')
kurtosis_median = kurtosis(a=median_noise, axis=None, fisher=False, nan_policy='raise')
skewness_median = skew(a=median_noise, axis=None, nan_policy='raise')
if GlobalConfig.get('debug'):
print("BM3D Filtered Kurtosis:", kurtosis_bm3d, ", Skewness: ", skewness_bm3d)
print("Homomorphic Filtered Kurtosis:", kurtosis_homomorphic, ", Skewness: ", skewness_homomorphic)
print("Median Filtered Kurtosis:", kurtosis_median, ", Skewness: ", skewness_median)
# Generate image featurevector of 6 characteristics
featurevector = np.array([kurtosis_bm3d, skewness_bm3d,
kurtosis_homomorphic, skewness_homomorphic,
kurtosis_median, skewness_median])
return featurevector
def begin_cross_validation(self) -> Tuple[List[np.array], List[str]]:
print("Training MRLBP using", GlobalConfig.get('mrlbp_classifier'), "classifier.")
return super().begin_cross_validation()
def main():
# Parse Args.
# 'scale' allows the image_scaled scale to be set. Eg: 0.25, 0.5, 1.0
argList = sys.argv[1:]
shortArg = 'a:d:t:s:S:k:rn:i:me'
longArg = [
'algorithm=', 'dataset=', 'train-ratio=', 'scale=', 'test-scale=',
'folds=', 'rotations', 'noise=', 'noise-intensity=', 'multiprocess',
'example', 'data-ratio=', 'mrlbp-classifier=', 'noise-train', 'ecs',
'debug'
]
valid_algorithms = [
'RLBP', 'MRLBP', 'MRELBP', 'BM3DELBP', 'NoiseClassifier'
]
valid_datasets = ['kylberg']
valid_noise = ['gaussian', 'speckle', 'salt-pepper']
valid_mrlbp_classifiers = ['svm', 'knn']
try:
args, vals = getopt.getopt(argList, shortArg, longArg)
for arg, val in args:
if arg in ('-a', '--algorithm'):
if val in valid_algorithms:
print('Using algorithm:', val)
GlobalConfig.set("algorithm", val)
else:
raise ValueError(
'Invalid algorithm configured, choose one of the following:',
valid_algorithms)
elif arg in ('-d', '--dataset'):
if val in valid_datasets:
print("Using dataset:", val)
GlobalConfig.set("dataset", val)
else:
raise ValueError(
'Invalid dataset configured, choose one of the following:',
valid_datasets)
elif arg in ('-t', '--train-test'):
if 0 < float(val) <= 1.0:
print('Using train-ratio ratio of', val)
GlobalConfig.set('train_ratio', float(val))
else:
raise ValueError(
'Train-test ratio must be 0 < train-test <= 1.0')
elif arg in ('-s', '--scale'):
if 0 < float(val) <= 1.0:
print('Using training image scale:', val)
GlobalConfig.set('scale', float(val))
else:
raise ValueError('Scale must be 0 < scale <= 1.0')
elif arg in ('-S', '--test-scale'):
if 0 < float(val) <= 1.0:
print('Using testing image scale:', val)
GlobalConfig.set('test_scale', float(val))
else:
raise ValueError('Test scale must be 0 < scale <= 1.0')
elif arg in ('-k', '--folds'):
print('Doing {} folds'.format(val))
GlobalConfig.set("folds", int(val))
elif arg in ('-r', '--rotations'):
print('Using rotated image_scaled sources')
GlobalConfig.set("rotate", True)
elif arg in ('-n', '--noise'):
if val in valid_noise:
print('Applying noise:', val)
GlobalConfig.set("noise", val)
else:
raise ValueError(
'Invalid noise type, choose one of the following:',
valid_noise)
elif arg in ('-i', '--noise-intensity'):
print('Using noise intensity (sigma / ratio) of:', val)
GlobalConfig.set("noise_val", float(val))
elif arg in ('-m', '--multiprocess'):
cores = psutil.cpu_count()
print('Using {} processor cores for computing featurevectors'.
format(cores))
GlobalConfig.set('multiprocess', True)
GlobalConfig.set('cpu_count', cores)
elif arg in ('-e', '--example'):
print('Generating algorithm example image_scaled')
GlobalConfig.set('examples', True)
elif arg == '--data-ratio':
if 0 < float(val) <= 1.0:
print('Using dataset ratio:', val)
GlobalConfig.set('data_ratio', float(val))
else:
raise ValueError('Data ratio must be 0 < ratio <= 1.0')
elif arg == '--mrlbp-classifier':
if val in valid_mrlbp_classifiers:
print(
"MRLBP algorithm (if configured) will use {} classifier"
.format(val))
GlobalConfig.set('mrlbp_classifier', val)
else:
raise ValueError(
'Invalid classifier chosen for mrlbp, choose one of the following:',
valid_mrlbp_classifiers)
elif arg == '--noise-train':
print(
"Applying noise to the training dataset as well as the test dataset"
)
GlobalConfig.set('train_noise', True)
elif arg == '--ecs':
print("Loading dataset from C:\Local")
GlobalConfig.set('ECS', True)
elif arg == '--debug':
print("Running in debug mode")
GlobalConfig.set('debug', True)
else:
raise ValueError('Unhandled argument provided:', arg)
except getopt.error as err:
print(str(err))
if GlobalConfig.get('ECS'):
GlobalConfig.set(
'CWD',
r'\\filestore.soton.ac.uk\users\ojvl1g17\mydocuments\COMP3200-Texture-Classification'
)
else:
GlobalConfig.set('CWD', os.getcwd())
if GlobalConfig.get('examples'):
write_examples()
# Load configured Dataset
if GlobalConfig.get('dataset') == 'kylberg':
if GlobalConfig.get('debug'):
# To save time in debug mode, only load one class and load a smaller proportion of it (25% of samples)
kylberg = DatasetManager.KylbergTextures(
num_classes=2, data_ratio=GlobalConfig.get('data_ratio'))
else:
kylberg = DatasetManager.KylbergTextures(
num_classes=28, data_ratio=GlobalConfig.get('data_ratio'))
# Load Dataset & Cross Validator
dataset = kylberg.load_data()
cross_validator = kylberg.get_cross_validator()
print("Dataset loaded")
elif GlobalConfig.get('dataset') is None:
raise ValueError('No Dataset configured')
else:
raise ValueError('Invalid dataset')
if GlobalConfig.get('rotate'):
dataset_folder = GlobalConfig.get('dataset') + '-rotated'
else:
dataset_folder = GlobalConfig.get('dataset')
out_folder = os.path.join(GlobalConfig.get('CWD'), 'out',
GlobalConfig.get('algorithm'), dataset_folder)
# Initialise algorithm
if GlobalConfig.get('algorithm') == 'RLBP':
print("Applying RLBP algorithm")
algorithm = RLBP.RobustLBP()
elif GlobalConfig.get('algorithm') == 'MRLBP':
print("Applying MRLBP algorithm")
algorithm = RLBP.MultiresolutionLBP(p=[8, 16, 24], r=[1, 2, 3])
elif GlobalConfig.get('algorithm') == 'MRELBP':
print("Applying MRELBP algorithm")
algorithm = MRELBP.MedianRobustExtendedLBP(r1=[2, 4, 6, 8],
p=8,
w_center=3,
w_r1=[3, 5, 7, 9])
elif GlobalConfig.get('algorithm') == 'BM3DELBP':
print("Applying BM3DELBP algorithm")
algorithm = BM3DELBP.BM3DELBP()
elif GlobalConfig.get('algorithm') == 'NoiseClassifier':
# Noise Classifier is used in BM3DELBP algorithm usually, this allows for benchmarking of the classifier alone
algorithm = NoiseClassifier.NoiseClassifier()
pass
else:
raise ValueError('Invalid algorithm choice')
# Get the Training out directory (i.e. Images without scaling/rotation/noise)
train_out_dir = os.path.join(
out_folder, algorithm.get_outdir(noisy_image=False,
scaled_image=False))
# Get the Testing out directory (i.e. Images with scaling/rotation/noise)
if GlobalConfig.get('noise') is not None:
noisy_image = True
else:
noisy_image = False
if GlobalConfig.get('test_scale') is not None:
scaled_image = True
else:
scaled_image = False
test_out_dir = os.path.join(
out_folder, algorithm.get_outdir(noisy_image, scaled_image))
# Out path for noise classifier
noise_out_dir = os.path.join(
GlobalConfig.get('CWD'), 'out', 'NoiseClassifier', dataset_folder,
"scale-{}".format(int(GlobalConfig.get('scale') * 100)))
test_noise_out_dir = os.path.join(
GlobalConfig.get('CWD'), 'out', 'NoiseClassifier', dataset_folder,
algorithm.get_outdir(noisy_image, scaled_image))
print("Replacing DatasetManager.Image with BM3DELBPImages")
# Convert DatasetManager.Image into BM3DELBP.BM3DELBPImage
if GlobalConfig.get('algorithm') == 'NoiseClassifier' or GlobalConfig.get(
'algorithm') == 'BM3DELBP':
for index, img in enumerate(dataset):
dataset[index] = BM3DELBP.BM3DELBPImage(img)
# Also convert rotated images if necessary
if img.test_rotations is not None:
for index, rotated_img in enumerate(img.test_rotations):
img.test_rotations[index] = BM3DELBP.BM3DELBPImage(
rotated_img)
if GlobalConfig.get('multiprocess'):
for index, img in enumerate(dataset):
dataset[index] = (index, img)
if GlobalConfig.get('rotate'):
maxtasks = 50
else:
maxtasks = None
if GlobalConfig.get(
'algorithm') == 'NoiseClassifier' or GlobalConfig.get(
'algorithm') == 'BM3DELBP':
with Pool(processes=GlobalConfig.get('cpu_count'),
maxtasksperchild=maxtasks) as pool:
# Generate image noise featurevectors
for index, image in tqdm.tqdm(pool.istarmap(
describe_noise_pool,
zip(dataset, repeat(noise_out_dir),
repeat(test_noise_out_dir))),
total=len(dataset),
desc='Noise Featurevectors'):
dataset[index] = image
else:
with Pool(processes=GlobalConfig.get('cpu_count'),
maxtasksperchild=maxtasks) as pool:
# Generate featurevectors
for index, image in tqdm.tqdm(pool.istarmap(
describe_image_pool,
zip(repeat(algorithm), dataset, repeat(train_out_dir),
repeat(test_out_dir))),
total=len(dataset),
desc='Texture Featurevectors'):
dataset[index] = image
else:
# Process the images without using multiprocessing Pools
if GlobalConfig.get(
'algorithm') == 'NoiseClassifier' or GlobalConfig.get(
'algorithm') == 'BM3DELBP':
for index, img in enumerate(dataset):
# Generate image noise featurevectors
describe_noise(img, noise_out_dir, test_noise_out_dir)
else:
print("BEGINNING TIMER:")
start = timer()
for index, img in enumerate(dataset):
# Generate featurevetors
describe_image(algorithm, img, train_out_dir, test_out_dir)
end = timer()
print("TIME TAKEN:", end - start)
# Train models and perform predictions
if GlobalConfig.get('algorithm') == 'RLBP':
predictor = RLBP.RobustLBPPredictor(dataset, cross_validator)
elif GlobalConfig.get('algorithm') == 'MRLBP':
print("Performing MRLBP Classification")
predictor = RLBP.MultiresolutionLBPPredictor(dataset, cross_validator)
elif GlobalConfig.get('algorithm') == 'MRELBP':
print("Performing MRELBP Classification")
predictor = MRELBP.MedianRobustExtendedLBPPredictor(
dataset, cross_validator)
elif GlobalConfig.get('algorithm') == 'BM3DELBP':
print("Performing BM3DELBP Classification")
predictor = BM3DELBP.BM3DELBPPredictor(dataset, cross_validator)
elif GlobalConfig.get('algorithm') == 'NoiseClassifier':
print("Applying noise classifier")
predictor = BM3DELBP.NoiseTypePredictor(dataset, cross_validator)
else:
raise ValueError('Invalid algorithm choice')
# Get the test label & test prediction for every fold of cross validation
y_test, y_predicted = predictor.begin_cross_validation()
if GlobalConfig.get('algorithm') == 'NoiseClassifier':
if GlobalConfig.get('noise') is None:
classes = ['no-noise', 'gaussian', 'speckle', 'salt-pepper']
else:
classes = ['gaussian', 'speckle', 'salt-pepper']
else:
classes = kylberg.classes
# Display confusion matrix
ClassificationUtils.pretty_print_conf_matrix(
y_test,
y_predicted,
classes,
title='{} Confusion Matrix'.format(GlobalConfig.get('algorithm')),
out_dir=test_out_dir)
# Display classification report
ClassificationUtils.make_classification_report(y_test, y_predicted,
classes, test_out_dir)
import logging
import logging.config
import yaml
with open('config/logging.yml', 'rt') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
logger = logging.getLogger('__name__')
# Import modeling utils
import Mask_RCNN.mrcnn.model as modellib
from config import GlobalConfig
from config import DetectorConfig
from data import TrainingData
# Set manually since not properly picked up from system config
os.environ['AWS_REGION'] = GlobalConfig.get('AWS_REGION')
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(
description='Runs Mask R-CNN for lung opacity detection in pneumonia.')
parser.add_argument(
'--data_source',
required=False,
help=
"One of ['example', 'local', or 's3']. Requires configuration in config.py. Default 'example'."
)
parser.add_argument(
'--subset_size',
required=True,
default=1000,
def describe_image(algorithm: ImageProcessorInterface,
image: DatasetManager.Image, train_out_dir: str,
test_out_dir: str):
"""
Applies an Algorithm to an image and writes the serialised featurevector to a directory.
If a noise type is configured, a featurevector is computed for that noisy image too
:param algorithm: Algorithm to apply
:param image: Image
:param train_out_dir: Directory to write serialised featurevectors of the image with no noise / scaling applied
:param test_out_dir: Directory to read/write serialised featurevetors of the image with noise / scaling applied
:return: Image after attribute changes
"""
train_out_cat = os.path.join(train_out_dir, image.label)
train_out_file = os.path.join(train_out_cat, '{}.npy'.format(image.name))
test_out_cat = os.path.join(test_out_dir, image.label)
test_out_file = os.path.join(test_out_cat, '{}.npy'.format(image.name))
# Apply algorithm to normal image (or read from file if already applied before).
if GlobalConfig.get('debug'):
print("Read/Write train file to", train_out_file)
try:
if GlobalConfig.get('train_noise'):
image.featurevector = algorithm.describe(image, test_image=False)
image.data = None
else:
image.featurevector = np.load(train_out_file, allow_pickle=True)
image.data = None
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.featurevector = algorithm.describe(image, test_image=False)
image.data = None
# Make output folder if it doesn't exist
try:
os.makedirs(train_out_cat)
except FileExistsError:
pass
np.save(train_out_file, image.featurevector)
# If relevant, apply algorithm to test image (or read from file if already applied before).
if (image.test_noise is not None) or (image.test_scale is not None):
if GlobalConfig.get('debug'):
print("Read/Write test file to", test_out_file)
try:
image.test_featurevector = np.load(test_out_file,
allow_pickle=True)
# Remove test image data from memory, we don't need it anymore.
image.test_data = None
if GlobalConfig.get('debug'):
print("Noisy Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.test_featurevector = algorithm.describe(image,
test_image=True)
# Remove test image data from memory, we don't need it anymore.
image.test_data = None
# Make output folder if it doesn't exist
try:
os.makedirs(test_out_cat)
except FileExistsError:
pass
np.save(test_out_file, image.test_featurevector)
# Ensure rotated variants of the image also have featurevectors generated/loaded
if image.test_rotations is not None:
for image in image.test_rotations:
describe_image(algorithm, image, train_out_dir, test_out_dir)
return image
def describe_noise(image: BM3DELBP.BM3DELBPImage, out_dir: str,
test_out_dir: str):
"""
Applies BM3DELBP's Noise Classifier to generate featurevectors for every image, for each type of noise applied.
:param image: BM3DELBPImage to apply noise to
:param out_dir: Directory to write serialised featurevectors
:param test_out_dir: Directory to write serialised featurevector generated on test image
:return: BM3DELBPImage after attribute changes
"""
noise_classifier = NoiseClassifier.NoiseClassifier()
if GlobalConfig.get('noise') is None:
# Generate non-noisy image noise featurevector
out_cat = os.path.join(out_dir, 'no-noise', image.label)
out_featurevector = os.path.join(
out_cat, '{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
# Try loading serialised featurevectors if it's ran before already
image.no_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing iamge", image.name)
image.generate_normal_featurevector(noise_classifier)
# Make output folder if it doesn't exist
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.no_noise_featurevector)
# Use a smaller NoiseClassifier training dataset if testing rotated images
if not GlobalConfig.get('rotate'):
# Load / generate Gussian sigma 10 noise featurevector
out_cat = os.path.join(out_dir, 'gaussian-10', image.label)
out_featurevector = os.path.join(
out_cat, '{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.gauss_10_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_gauss_10(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.gauss_10_noise_featurevector)
# Load / generate Speckle var=0.02 noise featurevector
out_cat = os.path.join(out_dir, 'speckle-002', image.label)
out_featurevector = os.path.join(
out_cat, '{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.speckle_002_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_speckle_002(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.speckle_002_noise_featurevector)
# Load / generate Salt and Pepper 2% noise featurevector
out_cat = os.path.join(out_dir, 'salt-pepper-002', image.label)
out_featurevector = os.path.join(
out_cat, '{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.salt_pepper_002_noise_featurevector = np.load(
out_featurevector, allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_salt_pepper_002(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector,
image.salt_pepper_002_noise_featurevector)
# Load / generate Gaussian sigma 25 noise featurevector
out_cat = os.path.join(out_dir, 'gaussian-25', image.label)
out_featurevector = os.path.join(out_cat,
'{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.gauss_25_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_gauss_25(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.gauss_25_noise_featurevector)
# Load / generate Speckle var=0.04 noise featurevector
out_cat = os.path.join(out_dir, 'speckle-004', image.label)
out_featurevector = os.path.join(out_cat,
'{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.speckle_004_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_speckle_004(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.speckle_004_noise_featurevector)
# Load / generate Salt and Pepper 4% noise featurevector
out_cat = os.path.join(out_dir, 'salt-pepper-004', image.label)
out_featurevector = os.path.join(out_cat,
'{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.salt_pepper_004_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
image.generate_salt_pepper_004(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.salt_pepper_004_noise_featurevector)
# Load / generate featurevector on test image
out_cat = os.path.join(test_out_dir, image.label)
out_featurevector = os.path.join(out_cat,
'{}-featurevector.npy'.format(image.name))
if GlobalConfig.get('debug'):
print("Read/Write to", out_featurevector)
try:
image.test_noise_featurevector = np.load(out_featurevector,
allow_pickle=True)
if GlobalConfig.get('debug'):
print("Image featurevector loaded from file")
except (IOError, ValueError):
if GlobalConfig.get('debug'):
print("Processing image", image.name)
if image.test_data is None:
raise ValueError('Image.test_data has not been assigned')
image.generate_noise_featurevector(noise_classifier)
try:
os.makedirs(out_cat)
except FileExistsError:
pass
np.save(out_featurevector, image.test_noise_featurevector)
# Ensure rotated variants of the image also have noise featurevectors generated/loaded
if image.test_rotations is not None:
for image_rotated in image.test_rotations:
describe_noise(image_rotated, out_dir, test_out_dir)
return image
def getCurrentCluster(self):
tc = GlobalConfig(TEMP_FILE)
return tc.get("TEMP", "cluster_index")
def getCurrentCluster(self):
if os.path.exists(TEMP_FILE):
tc = GlobalConfig(TEMP_FILE)
return tc.get("TEMP", "cluster_index")
else:
return "1"
def test_get_s3_dcm(self):
bucket = GlobalConfig.get('S3_BUCKET_NAME')
imgdir = GlobalConfig.get('S3_STAGE1_TRAIN_IMAGE_DIR')
test_dcm_path = imgdir + '/0004cfab-14fd-4e49-80ba-63a80b6bddd6.dcm'
ds = ingest.get_s3_dcm(bucket=bucket, file_key=test_dcm_path)
self.assertEqual(ds.pixel_array.shape, (1024, 1024))
def pretty_print_conf_matrix(y_true,
y_pred,
classes,
normalize=False,
title='{} Confusion matrix'.format(
describe_test_setup()),
cmap=plt.cm.Blues,
out_dir=None):
"""
Code adapted from: http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py
"""
fig, ax = plt.subplots(figsize=(15, 15))
cm = confusion_matrix(y_true, y_pred, labels=classes)
# Configure Confusion Matrix Plot Aesthetics (no text yet)
cax = ax.imshow(cm, interpolation='nearest', cmap=cmap)
ax.set_title(title, fontsize=16)
ax.set_xticks(np.arange(len(classes)))
ax.set_yticks(np.arange(len(classes)))
ax.set_xticklabels(classes)
ax.set_yticklabels(classes)
ax.tick_params(axis='x', labelsize=14)
ax.tick_params(axis='y', labelsize=14)
plt.setp(ax.get_xticklabels(),
rotation=45,
ha='right',
rotation_mode='anchor')
plt.colorbar(cax)
ax.set_ylabel('True label', fontsize=16)
ax.set_xlabel('Predicted label', fontsize=16, rotation='horizontal')
# Calculate normalized values (so all cells sum to 1) if desired
if normalize:
cm = np.round(cm.astype('float') / cm.sum(),
2) # (axis=1)[:, np.newaxis]
thresh = cm.max() / 1.5 if normalize else cm.max() / 2
# Place Numbers as Text on Confusion Matrix Plot
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
ax.text(j,
i,
cm[i, j],
ha="center",
va="center",
color="white" if cm[i, j] > thresh else "black",
fontsize=12)
#fig.tight_layout()
plt.show(block=False)
if out_dir is not None:
if GlobalConfig.get('algorithm') == 'MRLBP':
if GlobalConfig.get('mrlbp_classifier') == 'knn':
out_file = os.path.join(
out_dir, 'Confusion Matrix{} - KNN.png'.format(
describe_test_setup()))
else:
out_file = os.path.join(
out_dir, 'Confusion Matrix{} - SVM.png'.format(
describe_test_setup()))
else:
out_file = os.path.join(
out_dir,
'Confusion Matrix{}.png'.format(describe_test_setup()))
fig.savefig(out_file, dpi=300)
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.setupUi()
self.initData()
def setupUi(self):
self.setWindowTitle("饥荒联机版服务器管理工具")
# 设置初始化的窗口大小
self.setFixedSize(1085, 700)
# 最开始窗口要居中显示
self.center()
# 设置整体布局 左右显示
pagelayout = QGridLayout()
'''
开始左上侧布局
'''
# 创建左上侧部件
top_left_frame = QFrame(self)
top_left_frame.setFrameShape(QFrame.StyledPanel)
# 按钮垂直布局
top_button_layout = QVBoxLayout(top_left_frame)
# 五个存档槽按钮
self.cluster_btns = {}
for b_index in range(5):
self.cluster_btns[b_index] = QPushButton(top_left_frame)
self.cluster_btns[b_index].setFixedSize(180, 30)
self.cluster_btns[b_index].setText("存档槽 " + str(b_index + 1))
# cluster_btns[b_index].setEnabled(False)
self.cluster_btns[b_index].index = b_index + 1
self.cluster_btns[b_index].clicked.connect(self.set_cluster)
top_button_layout.addWidget(self.cluster_btns[b_index])
# 删除存档按钮
delete_cluster_btn = QPushButton(top_left_frame)
delete_cluster_btn.setFixedSize(180, 30)
delete_cluster_btn.setText("删除存档")
top_button_layout.addWidget(delete_cluster_btn)
delete_cluster_btn.clicked.connect(self.deleteCluster)
# 导出远程存档按钮
export_remote_cluster_btn = QPushButton(top_left_frame)
export_remote_cluster_btn.setText("导出远程存档")
export_remote_cluster_btn.setFixedSize(180, 30)
top_button_layout.addWidget(export_remote_cluster_btn)
# 导入本地存档按钮
import_local_cluster_btn = QPushButton(top_left_frame)
import_local_cluster_btn.setText("导入本地存档")
import_local_cluster_btn.setFixedSize(180, 30)
top_button_layout.addWidget(import_local_cluster_btn)
'''
开始左下侧布局
'''
# 创建左下侧部件
bottom_left_frame = QFrame(self)
bottom_left_frame.setFrameShape(QFrame.StyledPanel)
# 按钮垂直布局
bottom_button_layout = QVBoxLayout(bottom_left_frame)
# 控制台按钮
console_btn = QPushButton(bottom_left_frame)
console_btn.setText("控制台")
console_btn.setFixedSize(180, 30)
bottom_button_layout.addWidget(console_btn)
# 软件设置按钮
settings_btn = QPushButton(bottom_left_frame)
settings_btn.setText("软件设置")
settings_btn.setFixedSize(180, 30)
settings_btn.clicked.connect(self.soft_settings)
bottom_button_layout.addWidget(settings_btn)
# 导出远程存档按钮
browser_online_server_btn = QPushButton(bottom_left_frame)
browser_online_server_btn.setText("浏览在线服务器")
browser_online_server_btn.setFixedSize(180, 30)
bottom_button_layout.addWidget(browser_online_server_btn)
# 导出远程存档按钮
help_and_about_btn = QPushButton(bottom_left_frame)
help_and_about_btn.setText("使用帮助和关于")
help_and_about_btn.setFixedSize(180, 30)
bottom_button_layout.addWidget(help_and_about_btn)
# 弹簧控件
spacerItem = QSpacerItem(20, 20, QSizePolicy.Minimum,
QSizePolicy.Expanding)
bottom_button_layout.addItem(spacerItem)
'''
开始右侧布局
'''
# 右侧开始布局 对应按钮布局
right_frame = QFrame(self)
right_frame.setFrameShape(QFrame.StyledPanel)
# 右边显示为stack布局
self.right_layout = QStackedLayout(right_frame)
self.settings_widget = SettingsWidget()
self.cluster_tab = MainTab()
self.right_layout.addWidget(self.cluster_tab)
self.right_layout.addWidget(self.settings_widget)
# 划分界面
splitterV = QSplitter(Qt.Vertical)
splitterV.addWidget(top_left_frame)
splitterV.addWidget(bottom_left_frame)
# 固定左上高度
top_left_frame.setFixedHeight(330)
splitterH = QSplitter(Qt.Horizontal)
splitterH.addWidget(splitterV)
splitterH.addWidget(right_frame)
# 固定左侧宽度
top_left_frame.setFixedWidth(200)
# 窗口部件添加布局
widget = QWidget()
pagelayout.addWidget(splitterH)
widget.setLayout(pagelayout)
self.setCentralWidget(widget)
# 按钮函数绑定
def soft_settings(self):
self.right_layout.setCurrentIndex(1)
# 存档按钮状态刷新
def refresh_cluster_btn_state(self, index):
for i in self.cluster_btns:
if index == self.cluster_btns[i].index:
self.cluster_btns[i].setStyleSheet("")
else:
self.cluster_btns[i].setStyleSheet("color:gray")
# 存档设置
def set_cluster(self):
self.current_cluster_index = self.sender().index
self.tempconfig.set("TEMP", "cluster_index",
str(self.current_cluster_index))
self.tempconfig.save(TEMP_FILE)
self.refresh_cluster_btn_state(self.current_cluster_index)
self.right_layout.setCurrentIndex(0)
self.mk_cluster_dir()
self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join(
self.current_cluster_folder, "cluster.ini")
self.cluster_tab.cluster_settings_tab.read_cluster_data(
self.cluster_tab.cluster_settings_tab.current_cluster_file)
self.cluster_tab.cluster_settings_tab.setServerIP(
self.cluster_tab.cluster_settings_tab.masterip,
self.cluster_tab.cluster_settings_tab.getServerIP())
self.cluster_tab.shard_settings_tab.initShardTab()
def deleteCluster(self):
cindex = self.current_cluster_index
delm = QMessageBox.warning(self, "删除警告",
"你确定要删除存档槽" + str(cindex) + "?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if delm == QMessageBox.Yes:
sdir = os.path.join(CLUSTER_DIR, "Cluster_" + str(cindex))
if os.path.exists(sdir):
shutil.rmtree(sdir)
self.right_layout.setCurrentIndex(0)
self.mk_cluster_dir()
self.cluster_tab.setCurrentIndex(0)
self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join(
self.current_cluster_folder, "cluster.ini")
self.cluster_tab.cluster_settings_tab.read_cluster_data(
self.cluster_tab.cluster_settings_tab.current_cluster_file)
self.cluster_tab.cluster_settings_tab.setServerIP(
self.cluster_tab.cluster_settings_tab.masterip,
self.cluster_tab.cluster_settings_tab.getServerIP())
self.cluster_tab.shard_settings_tab.initShardTab()
QMessageBox.information(self, "删除完毕",
"存档槽" + str(cindex) + "已删除并重置！",
QMessageBox.Yes)
def mk_cluster_dir(self):
self.current_cluster_folder = os.path.join(
CLUSTER_DIR, "Cluster_" + str(self.current_cluster_index))
if not os.path.exists(self.current_cluster_folder):
os.mkdir(self.current_cluster_folder)
def init_cluster_data(self, index):
self.mk_cluster_dir()
self.right_layout.setCurrentIndex(0)
self.refresh_cluster_btn_state(self.current_cluster_index)
self.cluster_tab.cluster_settings_tab.current_cluster_file = os.path.join(
self.current_cluster_folder, "cluster.ini")
self.cluster_tab.cluster_settings_tab.read_cluster_data(
self.cluster_tab.cluster_settings_tab.current_cluster_file)
def initDir(self):
if not os.path.exists(ROOT_DIR):
os.mkdir(ROOT_DIR)
if not os.path.exists(CLUSTER_DIR):
os.mkdir(CLUSTER_DIR)
if not os.path.exists(TEMP_FILE):
self.tempconfig = GlobalConfig(TEMP_FILE)
self.tempconfig.add_section("TEMP")
self.tempconfig.set("TEMP", "cluster_index", "1")
self.tempconfig.save(TEMP_FILE)
def initData(self):
self.initDir()
self.tempconfig = GlobalConfig(TEMP_FILE)
if os.path.exists(TEMP_FILE):
self.current_cluster_index = int(
self.tempconfig.get("TEMP", "cluster_index"))
else:
self.current_cluster_index = 1
self.init_cluster_data(self.current_cluster_index)
# 设置窗口居中
def center(self):
screen = QDesktopWidget().screenGeometry()
size = self.geometry()
self.move((screen.width() - size.width()) / 2,
(screen.height() - size.height()) / 2)
def tune_noise_classifier():
GlobalConfig.set('dataset', 'kylberg')
GlobalConfig.set('ECS', True)
GlobalConfig.set('algorithm', 'NoiseClassifier')
GlobalConfig.set('scale', 0.5)
GlobalConfig.set(
'CWD',
r'\\filestore.soton.ac.uk\users\ojvl1g17\mydocuments\COMP3200-Texture-Classification'
)
#GlobalConfig.set('CWD', os.getcwd())
GlobalConfig.set('folds', 10)
cores = psutil.cpu_count()
GlobalConfig.set('cpu_count', cores)
dataset = DatasetManager.KylbergTextures(num_classes=28, data_ratio=0.5)
images = dataset.load_data()
gc.collect()
bm3d_images = []
# Convert to BM3D images
for image in images:
new_image = BM3DELBP.BM3DELBPImage(image)
bm3d_images.append(new_image)
print("Image dataset loaded, loaded {} images".format(len(images)))
noise_classifier = NoiseClassifier()
cross_validator = dataset.get_cross_validator()
bm3d_sigma = [10, 30, 40, 50]
homomorphic_cutoff = [0.1, 0.5, 5, 10]
homomorphic_a = [0.5, 0.75, 1.0]
homomorphic_b = [0.1, 0.5, 1.0, 1.25]
settings_jobs = [] # List of all configuration tuples
for sigma_val in bm3d_sigma:
for cutoff in homomorphic_cutoff:
for a in homomorphic_a:
for b in homomorphic_b:
settings_jobs.append((sigma_val, cutoff, a, b))
results = [] # List of tuples (F1, sigma_val, cutoff, a, b)
out_csv = os.path.join(GlobalConfig.get('CWD'), 'NoiseClassifierTuning',
'Results.txt')
with Pool(processes=GlobalConfig.get('cpu_count'),
maxtasksperchild=50) as pool:
# Generate image featurevectors and replace DatasetManager.Image with BM3DELBP.BM3DELBPImage
for result in tqdm.tqdm(pool.istarmap(
do_classification,
zip(settings_jobs, repeat(noise_classifier),
repeat(cross_validator), repeat(bm3d_images))),
total=len(settings_jobs),
desc='NoiseClassifier tuning'):
f1, sigma, cutoff, a, b = result
# Log to CSV file
if os.path.isfile(out_csv):
# CSV exists, append to end of file
with open(out_csv, 'a', encoding="utf-8",
newline='') as resultsfile:
writer = csv.writer(resultsfile)
writer.writerow([f1, sigma, cutoff, a, b])
else:
# CSV does not exist. Write the headings
with open(out_csv, 'w', encoding="utf-8",
newline='') as resultsfile:
writer = csv.writer(resultsfile)
writer.writerow(['f1', 'sigma_psd', 'cutoff', 'a', 'b'])
writer.writerow([f1, sigma, cutoff, a, b])
results.append(result)
# Sort largest to smallest, by F1 score
results.sort(key=lambda tup: tup[0], reverse=True)
print("Finished tuning parameters.")
print("The top 3 results were:")
f1, sigma, cutoff, a, b = results[0]
print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format(
f1, sigma, cutoff, a, b))
f1, sigma, cutoff, a, b = results[1]
print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format(
f1, sigma, cutoff, a, b))
f1, sigma, cutoff, a, b = results[2]
print("F1: {}, sigma_val: {}, cutoff_freq: {}, a: {}, b: {}".format(
f1, sigma, cutoff, a, b))
class ClusterWidget(QWidget):
def __init__(self, parent=None):
super(ClusterWidget, self).__init__(parent)
layout = QVBoxLayout()
layout1 = QHBoxLayout()
layout2 = QHBoxLayout()
self.cluster_name = QLineEdit()
name_label = QLabel()
name_label.setText('房间名称:')
description_label = QLabel()
description_label.setText('房间简介:')
self.cluster_description = QLineEdit()
self.cluster_name.setText("南风颂的饥荒世界")
self.cluster_description.setText("由饥荒联机版服务器管理工具开设！")
layout1.addWidget(name_label)
layout1.addWidget(self.cluster_name)
layout2.addWidget(description_label)
layout2.addWidget(self.cluster_description)
layout3 = QHBoxLayout()
layout4 = QHBoxLayout()
layout5 = QHBoxLayout()
self.cluster_intention = QComboBox()
self.cluster_intention.setItemDelegate(QStyledItemDelegate())
self.cluster_intention.setStyleSheet(
"QComboBox QAbstractItemView::item { min-height: 30px; min-width: 60px; }"
)
self.cluster_intention_cn = ['休闲', '合作', '竞赛', '疯狂']
self.cluster_intention_value = [
'social', 'cooperative', 'competitive', 'madness'
]
self.cluster_intention.addItems(self.cluster_intention_cn)
self.game_mode = QComboBox()
self.game_mode.setStyleSheet(
"QComboBox QAbstractItemView::item { min-height: 30px; min-width: 60px; }"
)
self.game_mode.setItemDelegate(QStyledItemDelegate())
self.game_mode_cn = ['无尽', '生存', '荒野', '熔炉', '暴食']
self.game_mode_value = [
'endless', 'survival', 'wilderness', 'lavaarena', 'quagmire'
]
self.game_mode.addItems(self.game_mode_cn)
label3 = QLabel()
label3.setText('游戏风格:')
label3.setFixedWidth(70)
label4 = QLabel()
label4.setFixedWidth(70)
label4.setText('游戏模式:')
layout3.addWidget(label3)
layout3.addWidget(self.cluster_intention)
layout4.addWidget(label4)
layout4.addWidget(self.game_mode)
layout6 = QHBoxLayout()
label5 = QLabel()
label5.setText("游戏语言:")
label5.setFixedWidth(70)
self.game_language = QButtonGroup()
self.en_rbtn = QRadioButton('英语')
self.en_rbtn.setFixedWidth(70)
self.zh_rbtn = QRadioButton('简体中文')
self.game_language.addButton(self.en_rbtn, 1)
self.game_language.addButton(self.zh_rbtn, 2)
self.zh_rbtn.setChecked(True)
layout6.addWidget(label5)
layout6.addWidget(self.en_rbtn)
layout6.addWidget(self.zh_rbtn)
layout7 = QHBoxLayout()
label6 = QLabel()
label6.setText("Steam群组ID:")
self.steam_group_id = QLineEdit()
self.steam_group_admin = QCheckBox('群组官员设为管理员')
self.steam_group_only = QCheckBox('仅群组成员可进')
layout7.addWidget(label6)
layout7.addWidget(self.steam_group_id)
layout7.addWidget(self.steam_group_admin)
layout7.addWidget(self.steam_group_only)
layout8 = QHBoxLayout()
self.pvp = QCheckBox("开启PVP竞技")
self.pause_when_empty = QCheckBox("开启无人暂停")
self.pause_when_empty.setChecked(True)
self.vote = QCheckBox('开启玩家投票')
self.vote.setChecked(True)
layout8.addWidget(self.pvp)
layout8.addWidget(self.pause_when_empty)
layout8.addWidget(self.vote)
layout9 = QHBoxLayout()
label7 = QLabel()
label7.setText('最大玩家人数:')
self.max_players = QLineEdit()
self.max_players.setText("0")
label8 = QLabel()
label8.setText('房间预留位置个数:')
self.white_players = QLineEdit()
self.white_players.setText("0")
layout9.addWidget(label7)
layout9.addWidget(self.max_players)
layout9.addWidget(label8)
layout9.addWidget(self.white_players)
layout10 = QHBoxLayout()
label9 = QLabel()
label9.setText("房间密码:")
self.password = QLineEdit()
layout10.addWidget(label9)
layout10.addWidget(self.password)
layout11 = QHBoxLayout()
label10 = QLabel()
label10.setText("主世界服务器:")
label10.setFixedWidth(100)
self.masterip = QComboBox()
self.masterip.setStyleSheet(
"QComboBox QAbstractItemView::item { min-height: 25px; min-width: 100px; }"
)
self.masterip.setItemDelegate(QStyledItemDelegate())
layout11.addWidget(label10)
layout11.addWidget(self.masterip)
layout12 = QHBoxLayout()
self.load_default_cluster_settings = QPushButton()
self.load_default_cluster_settings.setText("载入默认设置")
self.set_default_cluster_settings = QPushButton()
self.set_default_cluster_settings.setText("保存为默认设置")
self.save_cluster_setttings = QPushButton()
self.save_cluster_setttings.setText("保存房间设置")
layout12.addWidget(self.load_default_cluster_settings)
layout12.addWidget(self.set_default_cluster_settings)
layout12.addWidget(self.save_cluster_setttings)
layout.addLayout(layout1)
layout.addLayout(layout2)
layout5.addLayout(layout3)
layout5.addLayout(layout4)
layout.addLayout(layout5)
layout.addLayout(layout6)
layout.addLayout(layout7)
layout.addLayout(layout8)
layout.addLayout(layout9)
layout.addLayout(layout10)
layout.addLayout(layout11)
layout.addLayout(layout12)
self.setLayout(layout)
self.load_default_cluster_settings.clicked.connect(
self.read_default_cluster_data)
self.set_default_cluster_settings.clicked.connect(
self.write_to_default_cluster_data)
self.save_cluster_setttings.clicked.connect(
self.write_curret_cluster_data)
self.setServerIP(self.masterip, ip="127.0.0.1")
def setServerIP(self, combox, ip):
if ip == "":
oldvalue = self.getServerIP()
else:
oldvalue = ip
combox.clear()
self.serverlist = SettingsWidget().get_server_list()
oldindex = 0
index = 0
for slist in self.serverlist:
if slist[0] != "":
combox.addItem(slist[0] + "@" + slist[1])
else:
combox.addItem(slist[1])
if slist[1] == oldvalue:
oldindex = index
index += 1
combox.setCurrentIndex(oldindex)
def getServerIP(self):
iparr = self.masterip.currentText().split('@')
if len(iparr) > 1:
ip = iparr[1]
else:
ip = iparr[0]
if ip == "":
ip = "127.0.0.1"
return ip
def read_default_cluster_data(self):
file = os.path.join(CONFIG_DIR, "cluster.ini")
self.read_cluster_data(file)
def write_to_default_cluster_data(self):
file = os.path.join(CONFIG_DIR, "cluster.ini")
self.write_cluster_data(file)
def write_curret_cluster_data(self):
self.write_cluster_data(self.current_cluster_file)
def write_cluster_data(self, file):
self.cluster_config.set("STEAM", "steam_group_id",
self.steam_group_id.text())
self.cluster_config.setboolean("STEAM", "steam_group_only",
self.steam_group_only.isChecked())
self.cluster_config.setboolean("STEAM", "steam_group_admins",
self.steam_group_admin.isChecked())
self.cluster_config.setboolean("GAMEPLAY", "pvp", self.pvp.isChecked())
self.cluster_config.set(
"GAMEPLAY", "game_mode",
self.game_mode_value[self.game_mode.currentIndex()])
self.cluster_config.setboolean("GAMEPLAY", "pause_when_empty",
self.pause_when_empty.isChecked())
self.cluster_config.setboolean("GAMEPLAY", "vote_enabled",
self.vote.isChecked())
self.cluster_config.set("GAMEPLAY", "max_players",
self.max_players.text())
self.cluster_config.set("NETWORK", "cluster_name",
self.cluster_name.text())
self.cluster_config.set("NETWORK", "cluster_description",
self.cluster_description.text())
self.cluster_config.set(
"NETWORK", "cluster_intention", self.cluster_intention_value[
self.cluster_intention.currentIndex()])
if self.zh_rbtn.isChecked():
lang = "zh"
else:
lang = "en"
self.cluster_config.set("NETWORK", "cluster_language", lang)
self.cluster_config.set("NETWORK", "whitelist_slots",
self.white_players.text())
self.cluster_config.set("NETWORK", "cluster_password",
self.password.text())
self.cluster_config.set("SHARD", "master_ip", self.getServerIP())
self.cluster_config.save(file)
def read_current_cluster_data(self):
self.read_cluster_data(self.current_cluster_file)
def read_cluster_data(self, file):
if not os.path.exists(file):
file = os.path.join(CONFIG_DIR, "cluster.ini")
self.cluster_config = GlobalConfig(file)
self.steam_group_id.setText(
self.cluster_config.get("STEAM", "steam_group_id"))
self.steam_group_only.setChecked(
self.cluster_config.getboolean("STEAM", "steam_group_only"))
self.steam_group_admin.setChecked(
self.cluster_config.getboolean("STEAM", "steam_group_admins"))
self.pvp.setChecked(self.cluster_config.getboolean("GAMEPLAY", "pvp"))
self.game_mode.setCurrentIndex(
self.game_mode_value.index(
self.cluster_config.get("GAMEPLAY", "game_mode")))
self.pause_when_empty.setChecked(
self.cluster_config.getboolean("GAMEPLAY", "pause_when_empty"))
self.vote.setChecked(
self.cluster_config.getboolean("GAMEPLAY", "vote_enabled"))
self.max_players.setText(
self.cluster_config.get("GAMEPLAY", "max_players"))
self.cluster_name.setText(
self.cluster_config.get("NETWORK", "cluster_name"))
self.cluster_description.setText(
self.cluster_config.get("NETWORK", "cluster_description"))
self.cluster_intention.setCurrentIndex(
self.cluster_intention_value.index(
self.cluster_config.get("NETWORK", "cluster_intention")))
if self.cluster_config.get("NETWORK", "cluster_language") == "zh":
self.zh_rbtn.setChecked(True)
else:
self.en_rbtn.setChecked(True)
self.white_players.setText(
self.cluster_config.get("NETWORK", "whitelist_slots"))
self.password.setText(
self.cluster_config.get("NETWORK", "cluster_password"))
self.setServerIP(self.masterip,
self.cluster_config.get("SHARD", "master_ip"))
