def plot_prediction(valGen, model, dataGenerator):
#create instance of Plotter class
plotter = Plotter()
imgNum = 0
while imgNum < config1["VALSIZE"]:
imgBatch, label = next(valGen)
labelBatch = label["organ_output"]
#denormalize image for plotting
imgBatchDeNorm = dataGenerator.denormalize(imgBatch)
#output of image is one-hot encoded, so take argmax
#labelTrue = dataGenerator.morphological_operation(labelBatch.argmax(axis=-1).astype('uint8'),'open')
labelTrue = labelBatch.argmax(axis=-1)
#predict image and normalize first
labelPred = model.predict(imgBatch)
#get selected index of predictions
labelPred = dataGenerator.morphological_operation(
labelPred.argmax(axis=-1).astype('uint8'), 'open')
k = plotter.plot_label_prediction(imgBatchDeNorm, labelTrue, labelPred)
if k == 27: break
if k == ord('s'):
imsave("true.png", labelTrue[0].astype('uint8'))
imsave("pred.png", labelPred[0].astype('uint8'))
imgNum += imgBatch.shape[0]
def plot_prediction(valGen, model, dataGenerator):
ax = None
idx = 0
imgNum = 0
#create instance of Plotter class
plotter = Plotter()
while imgNum < config1["VALSIZE"]:
imgBatch, label = next(valGen)
labelBatch = label["organ_output"]
#denormalize image for plotting
imgBatchDeNorm = dataGenerator.denormalize(imgBatch)
#apply morphological tranformation
#true_organ = morphological_closing(true_organ.astype('uint8'))
#predict image and normalize first
labelPred = model.predict(imgBatch)
#get selected index of predictions
labelPred = dataGenerator.morphological_operation(
labelPred.argmax(axis=-1).astype('uint8'), 'open')
k = plotter.plot_label_prediction(imgBatchDeNorm, labelBatch,
labelPred)
if k == 27: break
#ax = overlay_contours_plot(pred_organ,true_organ,image,imgNum,ax);
#overlay_contours_save(pred_organ,true_organ,image,imgNum);
imgNum += 1
def test_random_images(fileName, batchSize=6):
#impor Image Processor
processor = ImageProcessor()
#create instance of Plotter class
plotter = Plotter()
with h5py.File(fileName, "r") as organFile:
data = organFile["features"]
labels = organFile["labels"]
idx, n = 0, labels.shape[0]
while idx < n:
imgBatch = data[idx:idx + batchSize]
labelBacth = labels[idx:idx + batchSize]
#plot label and data
k = plotter.plot_slice_label(imgBatch, labelBacth)
# k = processor_unit_tester(imgBatch,processor=processor,plotter=plotter)
if k == ord("d"): idx += batchSize
elif k == ord("a"): idx -= batchSize
elif k == 27: break
def fuse_labels(valGen, model, dataGenerator, outFile="TRAIN_MERGED.h5"):
'''
Method to create dataset that contains labels from both datasets (SegTHOR & LCTSC)
'''
#create instance of Plotter class
plotter = Plotter()
#create instance of FeatureLabelReader object, I will use "save_image_mask" method
# reader = FeatureLabelReader()
imgNum = 0
# while imgNum<config1["VALSIZE"]:
while imgNum < 5981: #total number of slices in LCTSC file
imgBatch, label = next(valGen)
labelBatch = label["organ_output"]
#denormalize image for plotting
# imgBatchDeNorm = dataGenerator.denormalize(imgBatch)
imgOriginal = dataGenerator.images
#output of image is one-hot encoded, so take argmax
#labelTrue = dataGenerator.morphological_operation(labelBatch.argmax(axis=-1).astype('uint8'),'open')
labelOriginal = dataGenerator.labels
#predict labels and apply inverse transformation
labelPred = model.predict(imgBatch)
labelPredUnZoom = dataGenerator.unzoom_label(
labelPred,
numClasses=5) #model predicts 5 labels, true labels contain 6
labelPredDeCrop = dataGenerator.uncrop_label(labelPredUnZoom,
numClasses=5)
#get selected index of predictions
labelPredMorph = dataGenerator.morphological_operation(
labelPredDeCrop.argmax(axis=-1).astype('uint8'), 'open')
#create merged labels
labelMerged = label_merger(labelOriginal, labelPredMorph)
k = plotter.plot_label_prediction(imgOriginal, labelMerged,
labelPredMorph)
if k == 27: break
elif k == ord('s'):
print("Saving ", imgNum)
FeatureLabelReader.save_image_mask(imgOriginal,
labelMerged,
fileName=outFile)
#print("Processed ",imgNum)
imgNum += imgBatch.shape[0]
def load_weights(self,
results_dir,
set_weights_folder,
plot_stats,
set_weights_epoch,
simulation_num=None):
if not set_weights_folder:
self._create_dir_if_not_exists(results_dir)
np.random.seed(simulation_num if not None else
18) # set number of simulation as the seed
stats = defaultdict(list)
for layer in self.layers:
if not layer.in_size:
continue
# weights folder should contain the same num of simulations (or more)
if set_weights_folder:
w_dir = os.path.join(set_weights_folder, "weights")
weights_fname = "weights_%s_%s.npz" % (layer.name,
set_weights_epoch)
layer.in_weights = np.load(os.path.join(
w_dir, weights_fname))['arr_0']
else:
layer.sd = 0.05 # or calculate according to input size: self.input_sd(layer.in_size)
# Using random weights with μ = 0 and low variance is CRUCIAL.
# np.random.standard_normal has variance of 1 (too high) and np.random.uniform doesn't always have μ = 0
layer.in_weights = np.random.normal(
0,
layer.sd,
size=[layer.in_size + int(layer.has_bias), layer.size])
stats['means'].append(np.mean(layer.in_weights))
stats['std'].append(np.std(layer.in_weights))
stats['labels'].append(layer.name)
np.savez_compressed(
"%s/weights/weights_%s_0.npz" % (results_dir, layer.name),
layer.in_weights)
if self.debug_messages:
with open('%s/weights/weight_stats.out' % results_dir,
'a') as f:
f.write("name, max, min, mean, std\n"
"%s,%g,%g,%g,%g\n" %
(layer.name, np.max(layer.in_weights),
np.min(layer.in_weights), stats['means'][-1],
stats['std'][-1]))
if plot_stats:
plt = Plotter(results_dir=results_dir)
plt.plot_layer_stats(stats)
self.reset_context_delta_and_crole()
self._complete_initialization()
def test_dicriminator(myGen, dataGenerator):
#create instance of Plotter class
plotter = Plotter()
while True:
#images are already processed in
imgBatch, label = next(myGen)
#denormalize image for plotting
imgBatch = dataGenerator.denormalize(imgBatch)
#gen output is dictionary
labelBatch = label["out"]
print(imgBatch.shape, labelBatch)
#plot label and data
k = plotter.plot_slices(imgBatch)
if k == 27: break
def merged_datasets(valGen, model, dataGenerator, outFile="TRAIN_MERGED.h5"):
'''
Method used to merge SegTHOR and LCTSC dataset
SegTHOR has {'esophagus':1,'heart':2,'trachea':3,'aorta'}
LCTSC has {'esophagus':1, 'heart':2, 'spinal cord':3 ,'L-R Lung':4,5}
'''
#create instance of Plotter class
plotter = Plotter()
imgNum = 0
while imgNum < 5981: #total number of slices in LCTSC file
imgBatch, label = next(valGen)
imgOriginal = dataGenerator.images
labelOriginal = dataGenerator.labels
#I am assigning different value to spinal cord
#labelOriginal[labelOriginal==3] = 6
labelOriginal[labelOriginal == 4] = 0
labelOriginal[labelOriginal == 5] = 0
labelOriginal[labelOriginal == 3] = 0
#look for slices that has non-zero label
zSlices = np.array([
np.max(labelOriginal[idx, ...]) > 0
for idx in range(imgBatch.shape[0])
],
dtype=np.bool)
print(imgOriginal.shape)
imgSave = imgOriginal[zSlices, ...]
labelSave = labelOriginal[zSlices, ...]
print(imgSave.shape)
if imgSave.shape[0] > 0:
print("Saving ", imgNum)
FeatureLabelReader.save_image_mask(imgSave,
labelSave,
fileName=outFile)
#k = plotter.plot_slice_label(imgOriginal,labelOriginal)
#if k==27: break
imgNum += imgBatch.shape[0]
simulations_with_pron_err = len([
simulation for simulation in valid_results
if sum(simulation['pronoun_errors_flex']['test']) > 0
])
eval_sets = set()
if args.eval_test:
eval_sets.add('test')
if args.eval_training:
eval_sets.add('training')
results_mean_and_std = compute_mean_and_std(
valid_results, evaluated_sets=eval_sets, epochs=args.epochs)
with open("%s/summary_results.pickled" % results_dir,
'wb') as pckl:
pickle.dump(results_mean_and_std, pckl)
plot = Plotter(results_dir=results_dir,
summary_sim=num_valid_simulations,
title=args.title,
epochs=args.epochs)
plot.plot_results(
results_mean_and_std,
cognate_experiment=args.cognate_experiment,
test_sentences_with_pronoun=inputs.test_sentences_with_pronoun,
num_test=inputs.num_test,
num_train=inputs.num_train,
test_df=inputs.testlines_df,
auxiliary_experiment=args.auxiliary_experiment,
evaluated_datasets=eval_sets)
if not isinstance(
results_mean_and_std['correct_code_switches']['test'],
int):
simulation_logger.info(
"Code-switched percentage (test set): %s" %
arg = sys.argv[1]
if arg == 'train':
myModel = DCGAN(numLabels=config1["NUMCLASSES"])
myModel.train_discriminator(trainGen, valGen)
elif arg == 'train_gen':
myModel = DCGAN(numLabels=config1["NUMCLASSES"])
myModel.train_generator(trainGen, valGen)
elif arg == 'loss':
plot_losses(gpu)
elif arg == 'test':
generator = DCGAN.load_json_model('Generator_290')
plotter = Plotter()
while True:
noise = np.random.normal(0, 1, (4, config1["LATENTDIM"]))
generated = generator.predict(noise)
imgNorm = np.zeros_like(generated)
for i in range(imgNorm.shape[0]):
imgNorm[i] = (generated[i] - generated[i].min()) / (
generated[i].max() - generated[i].min())
k = plotter.plot_slices(imgNorm)
if k == 27: break
# plot_prediction(trainGen,model,dataGenerator)
elif arg == 'plot':
import os
import pandas as pd
from utils.constants import Armoire_MERGE,Int_MERGE,PL_MERGE
import sklearn.feature_selection
from sklearn.feature_selection import f_regression, mutual_info_regression
from scipy.cluster.hierarchy import linkage
# the date of saving the data
date_str='0723'
analyzer = Analyzer(datestr=date_str)
cleaner = Cleaner()
loader = Loader(datestr=date_str)
saver = Saver(datestr=date_str)
processor = Processor(datestr=date_str)
plotter = Plotter(datestr=date_str)
cluster = Cluster(datestr=date_str)
modeler = Modeler(datestr=date_str)
data_downloadtime = dt.datetime(2018, 5, 15, 0, 0, 0, 0)
data_starttime = dt.datetime(2013, 1, 1, 0, 0, 0, 0)
day_difference = (data_downloadtime - data_starttime).days
CURRENT_TIME_AP = '2018-05-15'
CURRENT_TIME_INT = '2018_05_15'
Intfilename_lst = ["BDDExportInterventions-{} du 01_01_2013 au 15_05_2018.xlsx".format(CURRENT_TIME_INT)]
"""Attention: for this project, I dupmp the data of BOGOR
"""
"""
Merge the observations of all the cities and save to excel, this will be done by the api later
# data = cleaner.stemming_dataframe(data)
# data.to_excel(savepath+'tp_rv_stemming.xlsx')
data = cleaner.unidecode_dataframe(data,var='Problem')
data.to_excel(savepath+'tp_rv_unidecode.xlsx')
# =======================tf-idf analysis======================
analyzer = Analyzer()
df_count, df_idf, df_freqidf = analyzer.get_freq_idf(data,var='Problem')
df_count.to_excel(savepath+'without_stopwords/freq_all.xlsx')
df_idf.to_excel(savepath+'without_stopwords/idf_all.xlsx')
df_freqidf.to_excel(savepath+'without_stopwords/freq_IDF.xlsx')
# ===================apply hierarchical clustering====================
plotter = Plotter()
tfidfvectorizer = TfidfVectorizer(max_features=100,
ngram_range=(1,2))
termdoc_matrix = tfidfvectorizer.fit_transform(data['Problem'])
hcluster_bi = linkage(termdoc_matrix.toarray(), 'ward')
plotter.plot_dendro(hcluster_bi, 'Dendrogram_stem_bigram')
# choose the number of clusters according to dendrogram
n_cluster = 7
labels = fcluster(hcluster_bi, n_cluster , criterion='maxclust')
clustersize_lst,df_merge = processor.merge_cluster(data=raw_data,labels=labels,n_cluster=n_cluster )
df_merge.to_excel(savepath+'/HC_cluster{}.xlsx'.format(n_cluster))