num_classes)
create_new_raster_from_base(local_img, inference_image,
sem_seg_results)
print(f"Semantic segmentation of image {img_name} completed")
if bucket:
bucket.upload_file(
inference_image,
os.path.join(params['inference']['working_folder'],
f"{img_name.split('.')[0]}_inference.tif"))
else:
raise ValueError(
f"The task should be either classification or segmentation. The provided value is {params['global']['task']}"
)
time_elapsed = time.time() - since
print('Inference completed in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
if __name__ == '__main__':
print('Start: ')
parser = argparse.ArgumentParser(
description='Inference on images using trained model')
parser.add_argument('param_file',
metavar='file',
help='Path to training parameters stored in yaml')
args = parser.parse_args()
params = read_parameters(args.param_file)
main(params)
from utils import read_parameters from plots import ParameterPlot parameters = read_parameters() param_plot = ParameterPlot(parameters) param_plot.plot()
parser.add_argument('-p',
'--param',
metavar='yaml_file',
nargs=1,
help='Path to parameters stored in yaml')
parser.add_argument('-i',
'--input',
metavar='model_pth img_dir',
nargs=2,
help='model_path and image_dir')
args = parser.parse_args()
# if a yaml is inputted, get those parameters and get model state_dict to overwrite global parameters afterwards
if args.param:
input_params = read_parameters(args.param[0])
model_ckpt = get_key_def('state_dict_path',
input_params['inference'],
expected_type=str)
# load checkpoint
checkpoint = load_checkpoint(model_ckpt)
if 'params' not in checkpoint.keys():
warnings.warn(
'No parameters found in checkpoint. Use GDL version 1.3 or more.'
)
else:
params = checkpoint['params']
# overwrite with inputted parameters
compare_config_yamls(yaml1=params,
yaml2=input_params,
update_yaml1=True)
'''
'''
from osgeo import gdal
import numpy
from json import load
from utils import read_parameters, load_last_config, create_folder, get_paths_for_files_in_folder, write_output_to_json
PARAMS = read_parameters()
ARGS = PARAMS["args"]
OUT_VOLUME = "/job_data"
LAST_FOLDER = "{0}/{1}".format(OUT_VOLUME, PARAMS["last"])
LAST_CONFIG = load_last_config(LAST_FOLDER)
OUT_FOLDER = create_folder(OUT_VOLUME, PARAMS["template_id"])
def perform_min_time():
''' Performs min time '''
print("-> Start calculating minimal NDVI...")
# Create Out folders
folder_stack_time_vrt = create_folder(OUT_FOLDER, "stack_time_vrt")
folder_stack_time_tif = create_folder(OUT_FOLDER, "stack_time_tif")
# Define output paths
filename_part = "-time_epsg-{0}".format(
LAST_CONFIG["data_srs"].split(":")[-1])
path_time_stack_vrt = "{0}/stack{1}.vrt".format(folder_stack_time_vrt,
filename_part)
def main(params):
"""
Training and validation datasets preparation.
:param params: (dict) Parameters found in the yaml config file.
"""
gpkg_file = []
bucket_name = params['global']['bucket_name']
data_path = params['global']['data_path']
metadata_file = params['global']['metadata_file']
csv_file = params['sample']['prep_csv_file']
if metadata_file:
image_metadata = read_parameters(metadata_file)
else:
image_metadata = None
final_samples_folder = None
if bucket_name:
s3 = boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
bucket.download_file(csv_file, 'samples_prep.csv')
list_data_prep = read_csv('samples_prep.csv')
if data_path:
final_samples_folder = os.path.join(data_path, "samples")
else:
final_samples_folder = "samples"
samples_folder = "samples"
out_label_folder = "label"
else:
list_data_prep = read_csv(csv_file)
samples_folder = os.path.join(data_path, "samples")
out_label_folder = os.path.join(data_path, "label")
create_or_empty_folder(samples_folder)
create_or_empty_folder(out_label_folder)
number_samples = {'trn': 0, 'val': 0, 'tst': 0}
number_classes = 0
trn_hdf5, val_hdf5, tst_hdf5 = create_files_and_datasets(
params, samples_folder)
with tqdm(list_data_prep) as _tqdm:
for info in _tqdm:
if bucket_name:
bucket.download_file(info['tif'],
"Images/" + info['tif'].split('/')[-1])
info['tif'] = "Images/" + info['tif'].split('/')[-1]
if info['gpkg'] not in gpkg_file:
gpkg_file.append(info['gpkg'])
bucket.download_file(info['gpkg'],
info['gpkg'].split('/')[-1])
info['gpkg'] = info['gpkg'].split('/')[-1]
assert_band_number(info['tif'],
params['global']['number_of_bands'])
_tqdm.set_postfix(
OrderedDict(file=f'{info["tif"]}',
sample_size=params['global']['samples_size']))
# Read the input raster image
np_input_image = image_reader_as_array(info['tif'])
# Validate the number of class in the vector file
validate_num_classes(info['gpkg'], params['global']['num_classes'],
info['attribute_name'])
# Burn vector file in a raster file
np_label_raster = vector_to_raster(info['gpkg'], info['tif'],
info['attribute_name'])
# Guidelines for pre-processing: http://cs231n.github.io/neural-networks-2/#datapre
# Scale arrays to values [0,1]. Default: will scale. Useful if dealing with 8 bit *and* 16 bit images.
scale = params['global']['scale_data'] if params['global'][
'scale_data'] else True
if scale:
sc_min, sc_max = params['global']['scale_data']
np_input_image = minmax_scale(
np_input_image,
orig_range=(np.min(np_input_image),
np.max(np_input_image)),
scale_range=(sc_min, sc_max))
# Mask the zeros from input image into label raster.
if params['sample']['mask_reference']:
np_label_raster = mask_image(np_input_image, np_label_raster)
if info['dataset'] == 'trn':
out_file = trn_hdf5
elif info['dataset'] == 'val':
out_file = val_hdf5
elif info['dataset'] == 'tst':
out_file = tst_hdf5
else:
raise ValueError(
f"Dataset value must be trn or val or tst. Provided value is {info['dataset']}"
)
np_label_raster = np.reshape(
np_label_raster,
(np_label_raster.shape[0], np_label_raster.shape[1], 1))
number_samples, number_classes = samples_preparation(
np_input_image, np_label_raster,
params['global']['samples_size'],
params['sample']['samples_dist'], number_samples,
number_classes, out_file, info['dataset'],
params['sample']['min_annotated_percent'], image_metadata)
_tqdm.set_postfix(OrderedDict(number_samples=number_samples))
out_file.flush()
trn_hdf5.close()
val_hdf5.close()
tst_hdf5.close()
print("Number of samples created: ", number_samples)
if bucket_name and final_samples_folder:
print('Transfering Samples to the bucket')
bucket.upload_file(samples_folder + "/trn_samples.hdf5",
final_samples_folder + '/trn_samples.hdf5')
bucket.upload_file(samples_folder + "/val_samples.hdf5",
final_samples_folder + '/val_samples.hdf5')
bucket.upload_file(samples_folder + "/tst_samples.hdf5",
final_samples_folder + '/tst_samples.hdf5')
print("End of process")
print("Number of samples created: ", number_samples)
if bucket_name and final_samples_folder:
print('Transfering Samples to the bucket')
bucket.upload_file(samples_folder + "/trn_samples.hdf5",
final_samples_folder + '/trn_samples.hdf5')
bucket.upload_file(samples_folder + "/val_samples.hdf5",
final_samples_folder + '/val_samples.hdf5')
bucket.upload_file(samples_folder + "/tst_samples.hdf5",
final_samples_folder + '/tst_samples.hdf5')
print("End of process")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Sample preparation')
parser.add_argument('ParamFile',
metavar='DIR',
help='Path to training parameters stored in yaml')
args = parser.parse_args()
params = read_parameters(args.ParamFile)
start_time = time.time()
debug = True if params['global']['debug_mode'] else False
main(params)
print("Elapsed time:{}".format(time.time() - start_time))
import numpy as np
import os
import random
import scipy.io as sio
from sklearn.preprocessing import normalize
import networkx as nx
# import matlab.engine
from utils import read_parameters, load_data, calculate_normalized_laplacian, calculate_quality_measures
from nonlinear_diffusion import calculate_nonlinear_diffusion, calculate_two_nonlinear_diffusions, self_learning, CalculateDiffusionGraph, CalculateSSPTree, find_Min_rank, Process_Result, calculate_nonlinear_diffusion_embeddings, Calculate_Embedding_Based_Graph, calculate_SSPTree_embedding, Calculate_Directed_Diffusion_Graph
from predict import predict_cv, predict_cv_fixed
from sklearn.neighbors import KDTree
import matplotlib.pyplot as plt
parser = read_parameters()
args = parser.parse_args()
random.seed(args.seed)
np.random.seed(args.seed)
np.random.RandomState(args.seed)
# Load data
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, labels, graph = load_data(
args.dataset, args.normalize)
print("Done loading data of" + " " + args.dataset)
idx_train = [i for i, x in enumerate(train_mask) if x]
idx_val = [i for i, x in enumerate(val_mask) if x]
idx_test = [i for i, x in enumerate(test_mask) if x]
n = adj.shape[0]
if args.generate_knn == 1:
dirname = os.path.dirname(os.path.realpath(__file__))
runtime_layers_gpu_path = os.path.join(dirname, "data_results", "current_layers_time_using_COLAB_gpu.csv")
runtime_layers_cpu_path = os.path.join(dirname, "data_results", "current_layers_time_using_COLAB_cpu.csv")
output_layers_size_path = os.path.join(dirname, "data_results", "b_alexNet_bytes_layers.csv")
save_path = os.path.join(dirname, "data_results", "COLAB_WEIGHTED_AFTER_optim_box_surgery_with_CPU.json")
accuracies_path = os.path.join(dirname, "data_results", "FINAL_FINAL_save_eval_thesholds_cifar_10.json")
runtime_cloud_gpu_data = read_parameters(runtime_layers_gpu_path)
runtime_cloud_cpu_data = read_parameters(runtime_layers_cpu_path)
output_size_data = 8*read_parameters(output_layers_size_path)/(10**6)
"""
accuracies_data columns:
['thresholds', 'overall_acc', 'overall_final', 'totaltime', 'num_exits',
'numexits_keep', 'acc_dual_models', 'accbreakdowns', 'accuracies',
'diff_branches', 'exit_ratio']
"""
branchyNet = alex_cifar10.get_network()
accuracies_data = pd.read_json(accuracies_path).thresholds
