num_classes) create_new_raster_from_base(local_img, inference_image, sem_seg_results) print(f"Semantic segmentation of image {img_name} completed") if bucket: bucket.upload_file( inference_image, os.path.join(params['inference']['working_folder'], f"{img_name.split('.')[0]}_inference.tif")) else: raise ValueError( f"The task should be either classification or segmentation. The provided value is {params['global']['task']}" ) time_elapsed = time.time() - since print('Inference completed in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) if __name__ == '__main__': print('Start: ') parser = argparse.ArgumentParser( description='Inference on images using trained model') parser.add_argument('param_file', metavar='file', help='Path to training parameters stored in yaml') args = parser.parse_args() params = read_parameters(args.param_file) main(params)
from utils import read_parameters from plots import ParameterPlot parameters = read_parameters() param_plot = ParameterPlot(parameters) param_plot.plot()
parser.add_argument('-p', '--param', metavar='yaml_file', nargs=1, help='Path to parameters stored in yaml') parser.add_argument('-i', '--input', metavar='model_pth img_dir', nargs=2, help='model_path and image_dir') args = parser.parse_args() # if a yaml is inputted, get those parameters and get model state_dict to overwrite global parameters afterwards if args.param: input_params = read_parameters(args.param[0]) model_ckpt = get_key_def('state_dict_path', input_params['inference'], expected_type=str) # load checkpoint checkpoint = load_checkpoint(model_ckpt) if 'params' not in checkpoint.keys(): warnings.warn( 'No parameters found in checkpoint. Use GDL version 1.3 or more.' ) else: params = checkpoint['params'] # overwrite with inputted parameters compare_config_yamls(yaml1=params, yaml2=input_params, update_yaml1=True)
''' ''' from osgeo import gdal import numpy from json import load from utils import read_parameters, load_last_config, create_folder, get_paths_for_files_in_folder, write_output_to_json PARAMS = read_parameters() ARGS = PARAMS["args"] OUT_VOLUME = "/job_data" LAST_FOLDER = "{0}/{1}".format(OUT_VOLUME, PARAMS["last"]) LAST_CONFIG = load_last_config(LAST_FOLDER) OUT_FOLDER = create_folder(OUT_VOLUME, PARAMS["template_id"]) def perform_min_time(): ''' Performs min time ''' print("-> Start calculating minimal NDVI...") # Create Out folders folder_stack_time_vrt = create_folder(OUT_FOLDER, "stack_time_vrt") folder_stack_time_tif = create_folder(OUT_FOLDER, "stack_time_tif") # Define output paths filename_part = "-time_epsg-{0}".format( LAST_CONFIG["data_srs"].split(":")[-1]) path_time_stack_vrt = "{0}/stack{1}.vrt".format(folder_stack_time_vrt, filename_part)
def main(params): """ Training and validation datasets preparation. :param params: (dict) Parameters found in the yaml config file. """ gpkg_file = [] bucket_name = params['global']['bucket_name'] data_path = params['global']['data_path'] metadata_file = params['global']['metadata_file'] csv_file = params['sample']['prep_csv_file'] if metadata_file: image_metadata = read_parameters(metadata_file) else: image_metadata = None final_samples_folder = None if bucket_name: s3 = boto3.resource('s3') bucket = s3.Bucket(bucket_name) bucket.download_file(csv_file, 'samples_prep.csv') list_data_prep = read_csv('samples_prep.csv') if data_path: final_samples_folder = os.path.join(data_path, "samples") else: final_samples_folder = "samples" samples_folder = "samples" out_label_folder = "label" else: list_data_prep = read_csv(csv_file) samples_folder = os.path.join(data_path, "samples") out_label_folder = os.path.join(data_path, "label") create_or_empty_folder(samples_folder) create_or_empty_folder(out_label_folder) number_samples = {'trn': 0, 'val': 0, 'tst': 0} number_classes = 0 trn_hdf5, val_hdf5, tst_hdf5 = create_files_and_datasets( params, samples_folder) with tqdm(list_data_prep) as _tqdm: for info in _tqdm: if bucket_name: bucket.download_file(info['tif'], "Images/" + info['tif'].split('/')[-1]) info['tif'] = "Images/" + info['tif'].split('/')[-1] if info['gpkg'] not in gpkg_file: gpkg_file.append(info['gpkg']) bucket.download_file(info['gpkg'], info['gpkg'].split('/')[-1]) info['gpkg'] = info['gpkg'].split('/')[-1] assert_band_number(info['tif'], params['global']['number_of_bands']) _tqdm.set_postfix( OrderedDict(file=f'{info["tif"]}', sample_size=params['global']['samples_size'])) # Read the input raster image np_input_image = image_reader_as_array(info['tif']) # Validate the number of class in the vector file validate_num_classes(info['gpkg'], params['global']['num_classes'], info['attribute_name']) # Burn vector file in a raster file np_label_raster = vector_to_raster(info['gpkg'], info['tif'], info['attribute_name']) # Guidelines for pre-processing: http://cs231n.github.io/neural-networks-2/#datapre # Scale arrays to values [0,1]. Default: will scale. Useful if dealing with 8 bit *and* 16 bit images. scale = params['global']['scale_data'] if params['global'][ 'scale_data'] else True if scale: sc_min, sc_max = params['global']['scale_data'] np_input_image = minmax_scale( np_input_image, orig_range=(np.min(np_input_image), np.max(np_input_image)), scale_range=(sc_min, sc_max)) # Mask the zeros from input image into label raster. if params['sample']['mask_reference']: np_label_raster = mask_image(np_input_image, np_label_raster) if info['dataset'] == 'trn': out_file = trn_hdf5 elif info['dataset'] == 'val': out_file = val_hdf5 elif info['dataset'] == 'tst': out_file = tst_hdf5 else: raise ValueError( f"Dataset value must be trn or val or tst. Provided value is {info['dataset']}" ) np_label_raster = np.reshape( np_label_raster, (np_label_raster.shape[0], np_label_raster.shape[1], 1)) number_samples, number_classes = samples_preparation( np_input_image, np_label_raster, params['global']['samples_size'], params['sample']['samples_dist'], number_samples, number_classes, out_file, info['dataset'], params['sample']['min_annotated_percent'], image_metadata) _tqdm.set_postfix(OrderedDict(number_samples=number_samples)) out_file.flush() trn_hdf5.close() val_hdf5.close() tst_hdf5.close() print("Number of samples created: ", number_samples) if bucket_name and final_samples_folder: print('Transfering Samples to the bucket') bucket.upload_file(samples_folder + "/trn_samples.hdf5", final_samples_folder + '/trn_samples.hdf5') bucket.upload_file(samples_folder + "/val_samples.hdf5", final_samples_folder + '/val_samples.hdf5') bucket.upload_file(samples_folder + "/tst_samples.hdf5", final_samples_folder + '/tst_samples.hdf5') print("End of process")
print("Number of samples created: ", number_samples) if bucket_name and final_samples_folder: print('Transfering Samples to the bucket') bucket.upload_file(samples_folder + "/trn_samples.hdf5", final_samples_folder + '/trn_samples.hdf5') bucket.upload_file(samples_folder + "/val_samples.hdf5", final_samples_folder + '/val_samples.hdf5') bucket.upload_file(samples_folder + "/tst_samples.hdf5", final_samples_folder + '/tst_samples.hdf5') print("End of process") if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sample preparation') parser.add_argument('ParamFile', metavar='DIR', help='Path to training parameters stored in yaml') args = parser.parse_args() params = read_parameters(args.ParamFile) start_time = time.time() debug = True if params['global']['debug_mode'] else False main(params) print("Elapsed time:{}".format(time.time() - start_time))
import numpy as np import os import random import scipy.io as sio from sklearn.preprocessing import normalize import networkx as nx # import matlab.engine from utils import read_parameters, load_data, calculate_normalized_laplacian, calculate_quality_measures from nonlinear_diffusion import calculate_nonlinear_diffusion, calculate_two_nonlinear_diffusions, self_learning, CalculateDiffusionGraph, CalculateSSPTree, find_Min_rank, Process_Result, calculate_nonlinear_diffusion_embeddings, Calculate_Embedding_Based_Graph, calculate_SSPTree_embedding, Calculate_Directed_Diffusion_Graph from predict import predict_cv, predict_cv_fixed from sklearn.neighbors import KDTree import matplotlib.pyplot as plt parser = read_parameters() args = parser.parse_args() random.seed(args.seed) np.random.seed(args.seed) np.random.RandomState(args.seed) # Load data adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask, labels, graph = load_data( args.dataset, args.normalize) print("Done loading data of" + " " + args.dataset) idx_train = [i for i, x in enumerate(train_mask) if x] idx_val = [i for i, x in enumerate(val_mask) if x] idx_test = [i for i, x in enumerate(test_mask) if x] n = adj.shape[0] if args.generate_knn == 1:
dirname = os.path.dirname(os.path.realpath(__file__)) runtime_layers_gpu_path = os.path.join(dirname, "data_results", "current_layers_time_using_COLAB_gpu.csv") runtime_layers_cpu_path = os.path.join(dirname, "data_results", "current_layers_time_using_COLAB_cpu.csv") output_layers_size_path = os.path.join(dirname, "data_results", "b_alexNet_bytes_layers.csv") save_path = os.path.join(dirname, "data_results", "COLAB_WEIGHTED_AFTER_optim_box_surgery_with_CPU.json") accuracies_path = os.path.join(dirname, "data_results", "FINAL_FINAL_save_eval_thesholds_cifar_10.json") runtime_cloud_gpu_data = read_parameters(runtime_layers_gpu_path) runtime_cloud_cpu_data = read_parameters(runtime_layers_cpu_path) output_size_data = 8*read_parameters(output_layers_size_path)/(10**6) """ accuracies_data columns: ['thresholds', 'overall_acc', 'overall_final', 'totaltime', 'num_exits', 'numexits_keep', 'acc_dual_models', 'accbreakdowns', 'accuracies', 'diff_branches', 'exit_ratio'] """ branchyNet = alex_cifar10.get_network() accuracies_data = pd.read_json(accuracies_path).thresholds