def slot_ok(self): self.setWindowModality(Qt.ApplicationModal) input_dir = self.lineEdit_input.text() if not os.path.isdir(input_dir): QMessageBox.warning(self, "Prompt", self.tr("Please check input directory!")) sys.exit(-1) output_dir = self.lineEdit_output.text() if not os.path.isdir(output_dir): QMessageBox.warning(self, "Prompt", self.tr("Output directory is not existed!")) os.mkdir(output_dir) try: files, nb = get_file(input_dir) if nb == 0: QMessageBox.warning(self, "Prompt", self.tr("No image found!")) sys.exit(-2) for file in tqdm(files): abs_filename = os.path.split(file)[1] abs_filename = abs_filename.split('.')[0] shp_file = ''.join([output_dir, '/', abs_filename, '.shp']) polygonize(file, shp_file) except: QMessageBox.warning(self, "Prompt", self.tr("Failed!")) else: QMessageBox.information(self, "Prompt", self.tr("successfully!")) self.setWindowModality(Qt.NonModal)
def batchbinarize_masks(inputdict): threshold = inputdict['threshold'] inputdir = inputdict['inputdir'] outputdir = inputdict['outputdir'] if not os.path.isdir(inputdir): print("Warning: ") return -1 files, num = get_file(inputdir) for file in tqdm(files): img = cv2.imread(file, cv2.IMREAD_GRAYSCALE) result = np.zeros(img.shape, np.uint8) ind_foreground = np.where(img > threshold) result[ind_foreground] = 1 # plt.imshow(result) # plt.show() absname = os.path.split(file)[1] # absname = absname.split('.')[0] # absname = 'shuidao.png' # absname = ''.join([absname, '.png']) mask_saving_path = os.path.join(outputdir, absname) cv2.imwrite(mask_saving_path, result) return 0
def creat_dataset_multiclass(labelfile, srcfile, base_path, image_num=5000, mode='original'): print('\ncreating dataset...') target_dir = os.path.join(base_path, 'label') if not os.path.isdir(target_dir): print("samples save path does not exist: {}".format(target_dir)) sys.exit(-3) _, baseNO = get_file(target_dir) g_count = baseNO + 1 count = 0 src_img = cv2.imread(src_file) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) """Check image size and invalid labels""" check_src_label_size(src_img, label_img) # check_invalid_labels(label_img, valid_labels) X_height, X_width, _ = src_img.shape while count < image_num: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = label_img[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_roi) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore whole background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((base_path + '/visualize/%d.png' % g_count), visualize) cv2.imwrite((base_path + '/src/%d.png' % g_count), src_roi) cv2.imwrite((base_path + '/label/%d.png' % g_count), label_roi) count += 1 g_count += 1
def save_hist_to_csv(in_dir, csv_file, bands, scale): input_files, _ = get_file(in_dir) Hist = get_hist(input_files, bands, scale) # Data = {'band_1':Hist[:,0], 'band_2':Hist[:,1],'band_3':Hist[:,2],'band_4':Hist[:,3]} df = pd.DataFrame(Hist) df.to_csv(csv_file)
def creat_dataset(image_num=50000, mode='original', in_path=input_path, out_path=output_path): print('\ncreating dataset...') src_files, tt = get_file(os.path.join(in_path, 'src/')) assert (tt != 0) image_each = image_num / len(src_files) g_count = 0 for scr_file in tqdm(src_files): count = 0 src_img = cv2.imread(scr_file) label_file = os.path.join(in_path, 'label/') + os.path.split(scr_file)[1] if not os.path.isfile(label_file): print("Have no file:".format(label_file)) sys.exit(-1) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) check_src_label_size(src_img, label_img) X_height, X_width, _ = src_img.shape while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = label_img[random_height:random_height + img_h, random_width:random_width + img_w] """check some invalid labels or NoData values""" check_invalid_labels(label_roi) """Cut down the pure background image with 80% probability""" if len(np.unique(label_roi)) < 2: if np.unique(label_roi)[0] == 0: if np.random.random() < 0.8: continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((out_path + 'visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + 'src/%d.png' % g_count), src_roi) cv2.imwrite((out_path + 'label/%d.png' % g_count), label_roi) count += 1 g_count += 1
def select_invalid_values(self, filepath): files, num = get_file(filepath) assert (num != 0) for label_file in tqdm(files): # label_file = input_label_path + os.path.split(src_file)[1] # # ret,src_img = load_img(src_file) # assert(ret==0) label_img = load_img_by_gdal(label_file, grayscale=True) label_img = np.array(label_img) local_labels = np.unique(label_img) invalid_labels = [] self.HAS_INVALID_VALUE = False for tmp in local_labels: if tmp not in self.valid_values: invalid_labels.append(tmp) print("\nWarning: some label is not valid value") print("\nFile: {}".format(label_file)) self.HAS_INVALID_VALUE = True if self.HAS_INVALID_VALUE == True: new_label_img = self.make_invalid_to_zeros( label_img, invalid_labels) new_label_file = os.path.split( label_file)[0] + '/new_' + os.path.split(label_file)[1] cv2.imwrite(new_label_file, new_label_img) self.HAS_INVALID_VALUE = False label_img = new_label_img plt.imshow(label_img, cmap='gray') plt.show() print("Check completely!\n")
return tp_img # # for i in range(height): # for j in range(width): # tmp = img[i,j] # if not tmp in true_values: # print("img[{},{}]: {}".format(i,j,tmp)) # img[i,j]=0 # return img if __name__ == '__main__': files,num = get_file(input_label_path) assert (num!=0) # valid_labels = [] # if FLAG_USING_UNET: # valid_labels = unet_labels # else: # valid_labels = segnet_labels for label_file in tqdm(files): # label_file = input_label_path + os.path.split(src_file)[1] # # ret,src_img = load_img(src_file) # assert(ret==0) ret,label_img = load_img_by_cv2(label_file, grayscale=True)
def predict_multiclass_for_batch_image(input_dict={}): gup_id = input_dict['GPUID'] os.environ["CUDA_VISIBLE_DEVICES"] = gup_id window_size = input_dict['windsize'] im_bands = input_dict['im_bands'] im_type = input_dict['dtype'] FLAG_APPROACH_PREDICT = input_dict['strategy'] input_path = input_dict['image_dir'] model_file = input_dict['model_file'] output_path = input_dict['mask_dir'] out_bands = input_dict['target_num'] all_files, num = get_file(input_path) if num == 0: print("There is no file in path:{}".format(input_path)) sys.exit(-1) for img_file in all_files: print("[INFO] opening image...".format(img_file)) input_img = load_img_by_gdal(img_file) if im_type == UINT8: input_img = input_img / 255.0 elif im_type == UINT10: input_img = input_img / 1024.0 elif im_type == UINT16: input_img = input_img / 65535.0 input_img = np.clip(input_img, 0.0, 1.0) input_img = input_img.astype(np.float16) model = load_model(model_file) print(model.summary()) layer_dict = dict([(layer.name, layer) for layer in model.layers]) layer_name = 'softmax' #sigmoid, softmax nb_classes = layer_dict[layer_name].output.shape[-1] if out_bands != nb_classes - 1: out_bands = nb_classes - 1 abs_filename = os.path.split(img_file)[1] abs_filename = abs_filename.split(".")[0] if FLAG_APPROACH_PREDICT == 0: print("[INFO] predict image by orignal approach\n") result = orignal_predict_onehot(input_img, im_bands, model, window_size) output_file = ''.join([output_path, '/', abs_filename, '.png']) print("result save as to: {}".format(output_file)) cv2.imwrite(output_file, result * 128) elif FLAG_APPROACH_PREDICT == 1: print("[INFO] predict image by smooth approach\n") result = predict_img_with_smooth_windowing_multiclassbands( input_img, model, window_size=window_size, subdivisions=2, real_classes=out_bands, # output channels = 是真的类别,总类别-背景 pred_func=smooth_predict_for_multiclass, PLOT_PROGRESS=False) H, W, C = np.array(input_img).shape output_file = ''.join( [output_path, '/', abs_filename, '_smooth_pred.png']) output_mask = np.zeros((H, W), np.uint8) for i in range(out_bands): indx = np.where(result[:, :, i] >= 127) output_mask[indx] = i + 1 print(np.unique(result)) cv2.imwrite(output_file, output_mask) print("Saved to:{}".format(output_file)) # for b in range(out_bands): # output_file = ''.join([output_path, '/', abs_filename, '_', dict_target[b],'smooth.png']) # cv2.imwrite(output_file, result[:,:,b]) # print("Saved to: {}".format(output_file)) gc.collect() return 0
FLAG_APPROACH_PREDICT = 1 else: pass date_time = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) output_dir = ''.join([config.mask_dir, '/', date_time]) os.mkdir(output_dir) if __name__ == '__main__': input_files = [] if os.path.isfile(config.img_input): print("[INFO] input is one file...") input_files.append(config.img_input) elif os.path.isdir(config.img_input): print("[INFO] input is a directory...") in_files, _ = get_file(config.img_input) for file in in_files: input_files.append(file) print("{} images will be classified".format(len(input_files))) # sys.exit(-1) out_bands = config.mask_classes model = load_model(config.model_path) print(model.summary()) layer_dict = dict([(layer.name, layer) for layer in model.layers]) layer_name = config.activation # sigmoid, softmax nb_classes = layer_dict[layer_name].output.shape[-1] if "sigmoid" in config.activation: if target_class != nb_classes: print(
def image_normalize(input_dict): input_dir = input_dict["input_dir"] output_dir = input_dict["output_dir"] nodata = input_dict["NoData"] result_bits = input_dict["OutBits"] valid_range = input_dict["StretchRange"] cut_value = input_dict["CutValue"] src_files, tt = get_file(input_dir) assert (tt != 0) factor = 4.0 if '8' in result_bits: assert (valid_range < 256) factor = 6.0 elif '16' in result_bits: assert (valid_range < 65536) factor = 4.0 else: pass for file in tqdm(src_files): absname = os.path.split(file)[1] absname = absname.split('.')[0] absname = ''.join([absname, '.tif']) print(absname) if not os.path.isfile(file): print("input file dose not exist:{}\n".format(file)) # sys.exit(-1) continue dataset = gdal.Open(file) if dataset == None: print("Open file failed: {}".format(file)) continue height = dataset.RasterYSize width = dataset.RasterXSize im_bands = dataset.RasterCount im_type = dataset.GetRasterBand(1).DataType img = dataset.ReadAsArray(0, 0, width, height) geotransform = dataset.GetGeoTransform() del dataset # img = np.array(img, np.uint16) img = np.array(img, np.float32) result = [] for i in range(im_bands): data = np.array(img[i]) maxium = data.max() minm = data.min() mean = data.mean() std = data.std() print("\nOriginal max, min, mean,std:[{},{},{},{}]".format( maxium, minm, mean, std)) data = data.reshape(height * width) ind = np.where((data > 0) & (data < nodata)) ind = np.array(ind) a, b = ind.shape print("valid value number: {}".format(b)) # tmp = np.zeros(b, np.uint16) tmp = np.zeros(b, np.float32) for j in range(b): tmp[j] = data[ind[0, j]] tmaxium = tmp.max() tminm = tmp.min() tmean = tmp.mean() tstd = tmp.std() # print(tmaxium, tminm, tmean, tstd) tt = (data - tmean) / tstd # first Z-score normalization tt = (tt + factor) * valid_range / (2 * factor) - cut_value tind = np.where(data == 0) tt = np.array(tt) # tt = tt.astype(np.uint8) tt = tt.astype(np.uint16) tt[tind] = 0 smaxium = tt.max() sminm = tt.min() smean = tt.mean() sstd = tt.std() # print(smaxium, sminm, smean, sstd) print("New max, min, mean,std:[{},{},{},{}]".format( smaxium, sminm, smean, sstd)) out = tt.reshape((height, width)) result.append(out) outputfile = os.path.join(output_dir, absname) driver = gdal.GetDriverByName("GTiff") if '8' in result_bits: outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_Byte) outdataset.SetGeoTransform(geotransform) elif '16' in result_bits: outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) outdataset.SetGeoTransform(geotransform) # outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(result[i]) del outdataset return 0
def predict_binary_for_batch_image(input_dict={}): gup_id = input_dict['GPUID'] os.environ["CUDA_VISIBLE_DEVICES"] = gup_id window_size = input_dict['windsize'] im_bands = input_dict['im_bands'] im_type = input_dict['dtype'] FLAG_APPROACH_PREDICT = input_dict['strategy'] input_path = input_dict['image_dir'] model_file = input_dict['model_file'] output_path = input_dict['mask_dir'] out_bands = 1 FLAG_ONEHOT = 0 if input_dict['onehot']: FLAG_ONEHOT = 1 all_files, num = get_file(input_path) if num == 0: print("There is no file in path:{}".format(input_path)) sys.exit(-1) for img_file in all_files: print("[INFO] opening image...") print("FileName:{}".format(img_file)) input_img = load_img_by_gdal(img_file) if im_type == UINT8: input_img = input_img / 255.0 elif im_type == UINT10: input_img = input_img / 1024.0 elif im_type == UINT16: input_img = input_img / 65535.0 input_img = np.clip(input_img, 0.0, 1.0) input_img = input_img.astype(np.float16) model = load_model(model_file) print(model.summary()) layer_dict = dict([(layer.name, layer) for layer in model.layers]) layer_name = 'sigmoid' # sigmoid, softmax nb_classes = layer_dict[layer_name].output.shape[-1] abs_filename = os.path.split(img_file)[1] abs_filename = abs_filename.split(".")[0] if FLAG_APPROACH_PREDICT == 0: print("[INFO] predict image by orignal approach\n") if FLAG_ONEHOT: result = orignal_predict_onehot(input_img, im_bands, model, window_size) else: result = orignal_predict_notonehot(input_img, im_bands, model, window_size) output_file = ''.join([output_path, '/', abs_filename, '.png']) print("result save as to: {}".format(output_file)) cv2.imwrite(output_file, result * 128) elif FLAG_APPROACH_PREDICT == 1: print("[INFO] predict image by smooth approach\n") if FLAG_ONEHOT: result = predict_img_with_smooth_windowing_multiclassbands( input_img, model, window_size=window_size, subdivisions=2, real_classes=out_bands, # output channels = 是真的类别,总类别-背景 pred_func=smooth_predict_for_binary_onehot, PLOT_PROGRESS=False) else: result = predict_img_with_smooth_windowing_multiclassbands( input_img, model, window_size=window_size, subdivisions=2, real_classes=out_bands, # output channels = 是真的类别,总类别-背景 pred_func=smooth_predict_for_binary_notonehot, PLOT_PROGRESS=False) output_file = ''.join( [output_path, '/', abs_filename, 'smooth.png']) cv2.imwrite(output_file, result) print("Saved to: {}".format(output_file)) gc.collect() return 0
def creat_dataset_binary(labelfile, srcfile, base_path, image_num=5000, mode='original'): print('\ncreating dataset...') target_dir = os.path.join(base_path, 'roads', 'label') if not os.path.isdir(target_dir): print("samples save path does not exist: {}".format(target_dir)) sys.exit(-3) _, baseNO = get_file(target_dir) src_img = cv2.imread(srcfile) label_img = cv2.imread(labelfile, cv2.IMREAD_GRAYSCALE) """Check image size and invalid labels""" check_src_label_size(src_img, label_img) # check_invalid_labels(label_img, valid_labels) X_height, X_width, _ = src_img.shape print("\n1: produce road labels---------------------") index = np.where(label_img == 1) # 1: roads road_label = np.zeros((X_height, X_width), np.uint8) road_label[index] = 1 print(np.unique(road_label)) g_count = baseNO + 1 count = 0 while count < image_num: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = road_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore whole background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((base_path + '/roads/visualize/%d.png' % g_count), visualize) cv2.imwrite((base_path + '/roads/src/%d.png' % g_count), src_roi) cv2.imwrite((base_path + '/roads/label/%d.png' % g_count), label_roi) count += 1 g_count += 1 print("\n2: produce buildings labels---------------------") index = np.where(label_img == 2) # 1: buildings building_label = np.zeros((X_height, X_width), np.uint8) building_label[index] = 1 target_dir = os.path.join(base_path, 'buildings', 'label') if not os.path.isdir(target_dir): print("samples save path does not exist: {}".format(target_dir)) sys.exit(-3) _, baseNO = get_file(target_dir) g_count = baseNO + 1 count = 0 while count < image_num: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = building_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore whole background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((base_path + '/buildings/visualize/%d.png' % g_count), visualize) cv2.imwrite((base_path + '/buildings/src/%d.png' % g_count), src_roi) cv2.imwrite((base_path + '/buildings/label/%d.png' % g_count), label_roi) count += 1 g_count += 1
def creat_dataset_binary(in_path, out_path, image_num=50000, mode='original'): print('\ncreating dataset...') label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_each = image_num / len(label_files) g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue # sys.exit(-1) print("src file:{}".format(os.path.split(src_file)[1])) src_img = cv2.imread(src_file) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) """Check image size and invalid labels""" check_src_label_size(src_img, label_img) # check_invalid_labels(label_img, valid_labels) X_height, X_width, _ = src_img.shape print("\n1: produce road labels---------------------") index = np.where(label_img == 1) # 1: roads road_label = np.zeros((X_height, X_width), np.uint8) road_label[index] = 1 print(np.unique(road_label)) count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = road_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue # """Cut down the pure background image with 80% probability""" # if len(np.unique(label_roi)) < 2: # if np.unique(label_roi)[0] ==0: # if np.random.random()< 0.8: # continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((out_path + '/roads/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/roads/src/%d.png' % g_count), src_roi) cv2.imwrite((out_path + '/roads/label/%d.png' % g_count), label_roi) count += 1 g_count += 1 print("\n2: produce buildings labels---------------------") index = np.where(label_img == 2) # 1: buildings building_label = np.zeros((X_height, X_width), np.uint8) building_label[index] = 1 count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[random_height:random_height + img_h, random_width:random_width + img_w, :] label_roi = building_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """Cut down the pure background image with 80% probability""" # if len(np.unique(label_roi)) < 2: # if np.unique(label_roi)[0] == 0: # if np.random.random() < 0.8: # continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi) visualize = label_roi * 50 cv2.imwrite((out_path + '/buildings/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/buildings/src/%d.png' % g_count), src_roi) cv2.imwrite((out_path + '/buildings/label/%d.png' % g_count), label_roi) count += 1 g_count += 1
def produce_training_samples_multiclass(in_path, out_path, image_num=50000, mode='original'): print('\ncreating dataset...') label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_each = image_num / len(label_files) g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue # sys.exit(-1) print("src file:{}".format(os.path.split(src_file)[1])) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue X_height = dataset.RasterYSize X_width = dataset.RasterXSize im_bands = dataset.RasterCount data_type = dataset.GetRasterBand(1).DataType # check size of label and src images x, y = label_img.shape print("Heigh, width of label is :{}, {}".format(x, y)) print("Heigh, width of src is :{}, {}".format(X_height, X_width)) if x != X_height or y != X_width: print("Warning: src and label have different size!") continue src_img = dataset.ReadAsArray(0, 0, X_width, X_height) src_img = np.array(src_img) del dataset count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[:, random_height:random_height + img_h, random_width:random_width + img_w] label_roi = label_img[random_height:random_height + img_h, random_width:random_width + img_w] # try: # label_roi = label_img[random_height: random_height + img_h, random_width: random_width + img_w] # except SelfDefinedExceptions as e_result: # print(e_result) """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue # print(np.unique(label_roi)) if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi, data_type) visualize = label_roi * 50 cv2.imwrite((out_path + '/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/label/%d.png' % g_count), label_roi) src_sample_file = out_path + '/src/%d.png' % g_count driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1
HAS_INVALID_VALUE = False def make_label_valid(img, true_values): height, width = img.shape for i in range(height): for j in range(width): tmp = img[i,j] if not tmp in true_values: print("img[{},{}]: {}".format(i,j,tmp)) img[i,j]=0 return img if __name__ == '__main__': src_files,num = get_file(input_src_path) assert (num!=0) valid_labels = [] if FLAG_USING_UNET: valid_labels = unet_labels else: valid_labels = segnet_labels for src_file in tqdm(src_files): label_file = input_label_path + os.path.split(src_file)[1] ret,src_img = load_img(src_file) assert(ret==0) ret,label_img = load_img(label_file, grayscale=True)
def predict_backend(input_dict): gpu_id = input_dict['gpu'] print("gpu_id:{}".format(gpu_id)) os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id config_file = input_dict['config'] print("cofig file:{}".format(config_file)) with open(config_file, 'r') as f: cfgl = json.load(f) config = Config(**cfgl) print(config) input = input_dict['input'] output = input_dict['output'] im_type = UINT8 if "10" in config.im_type: im_type = UINT10 elif "16" in config.im_type: im_type = UINT16 else: pass target_class = config.nb_classes if target_class > 1: # multiclass, target class = total class -1 if target_class == 2: print("Warning: target classes should not be 2, this must be binary classification!") target_class = 1 else: target_class -= 1 FLAG_APPROACH_PREDICT = 0 # 0: original predict, 1: smooth predict if "smooth" in config.strategy: FLAG_APPROACH_PREDICT = 1 else: pass date_time = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) output_dir = ''.join([output, '/', date_time]) os.mkdir(output_dir) block_size = config.block_size nodata = config.nodata input_files = [] if os.path.isfile(input): print("[INFO] input is one file...") input_files.append(input) elif os.path.isdir(input): print("[INFO] input is a directory...") in_files, _ = get_file(input) for file in in_files: input_files.append(file) if len(input_files) == 0: print("no input images") sys.exit(-1) print("{} images will be classified".format(len(input_files))) csv_file = os.path.join(output_dir, 'readme.csv') df = pd.DataFrame(list(config)) df.to_csv(csv_file) out_bands = target_class try: model = load_model(input_dict['model'], compile=False) """load model using customer loss""" # lossName = 'cce_jaccard_loss' # model = load_model(config.model_path, custom_objects={'closure':self_define_loss(lossName)}) except ValueError: print("Warning: there are several custom objects in model") print("For deeplab V3+, load model with parameters of custom_objects\n") model = load_model(input_dict['model'], custom_objects={'relu6': relu6, 'BilinearUpsampling': BilinearUpsampling}, compile=False) except Exception: print("Error: failde to load model!\n") sys.exit(-1) else: print("model is not deeplab V3+!\n") # print(model.summary()) for img_file in tqdm(input_files): print("\n[INFO] opening image:{}...".format(img_file)) abs_filename = os.path.split(img_file)[1] # abs_filename = abs_filename.split(".")[0] H, W, C, geoinf = load_img_by_gdal_info(img_file) if H==0: print("Open failed:{}".format(abs_filename)) continue gc.collect() nb_blocks = int(H*W/block_size) if H*W>nb_blocks*block_size: nb_blocks +=1 block_h = int(block_size/W) print("single block size :[{},{}]".format(block_h,W)) result_mask = np.zeros((H, W), np.uint8) for i in tqdm(list(range(nb_blocks))): print("[INFO] predict image for {} block".format(i)) start =block_h*i this_h = block_h if (i+1)*block_h>H: this_h = H-i*block_h end = start+this_h # b_img = load_img_by_gdal_blocks(img_file,0,start,W,this_h) b_img = load_img_by_gdal_blocks(img_file, 0, start, W, this_h+config.window_size) if i ==nb_blocks-1: tmp_img = np.zeros((this_h+config.window_size, W, C), np.uint16) tmp_img[:this_h,:,:] = b_img else: tmp_img = b_img # exp_img = np.zeros((this_h+config.window_size, W, C), np.uint16) # exp_img[:, :, :] = b_img[:,:,:] # b_img = whole_img[start:end,:,:] # plt.imshow(b_img[:,:,1]) # plt.show() # sys.exit(-3) """get data in bands of band_list""" band_list = config.band_list if len(band_list) == 0: band_list = range(C) if len(band_list) > C or max(band_list) >= C: print("input bands should not be bigger than image bands!") sys.exit(-2) a,b,c = tmp_img.shape input_img = np.zeros((a,b,len(band_list)), np.float16) for i in range(len(band_list)): input_img[:,:,i] = tmp_img[:,:,band_list[i]] if im_type == UINT8: input_img = input_img / 255.0 elif im_type == UINT10: input_img = input_img / 1024.0 elif im_type == UINT16: input_img = input_img / 65535.0 input_img = np.clip(input_img, 0.0, 1.0) input_img = input_img.astype(np.float16) if FLAG_APPROACH_PREDICT == 0: print("[INFO] predict image by orignal approach ...") a,b,c=input_img.shape num_of_bands = min(a,b,c) result = core_orignal_predict(input_img, num_of_bands, model, config.window_size, config.img_w, mask_bands=config.nb_classes) result_mask[start:end,:]=result[:this_h,:] elif FLAG_APPROACH_PREDICT == 1: print("[INFO] predict image by smooth approach... ") output_mask = np.zeros((this_h+config.window_size, W), np.uint8) if out_bands > 1: result = predict_img_with_smooth_windowing( input_img, model, window_size=config.window_size, subdivisions=config.subdivisions, slices= config.slices, real_classes=target_class, # output channels = 是真的类别,总类别-背景 pred_func=core_smooth_predict_multiclass, PLOT_PROGRESS=False ) for i in range(target_class): indx = np.where(result[:, :, i] >= 127) output_mask[indx] = i + 1 del result gc.collect() else: result = predict_img_with_smooth_windowing( input_img, model, window_size=config.window_size, subdivisions=config.subdivisions, slices=config.slices, real_classes=target_class, pred_func=core_smooth_predict_binary, PLOT_PROGRESS=False ) indx = np.where(result[:, :, 0] >= 127) output_mask[indx] = 1 # del result gc.collect() result_mask[start:end, :] = output_mask[:this_h, :] # del output_mask gc.collect() del b_img # del tmp_img # del input_img gc.collect() print(np.unique(result_mask)) # result_mask[nodata_indx]=255 # output_file = ''.join([output_dir, '/', abs_filename, config.suffix]) output_file = ''.join([output_dir, '/', abs_filename]) driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(output_file, W, H, 1, gdal.GDT_Byte) outdataset.SetGeoTransform(geoinf) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) outdataset.GetRasterBand(1).WriteArray(result_mask) del outdataset # result_mask[nodata_indx] = 255 del result_mask gc.collect() print("Saved to:{}".format(output_file)) # output vector file from raster file if config.tovector: shp_file= ''.join([output_dir, '/', abs_filename, '.shp']) polygonize(output_file, shp_file) return 0
def convert_all_image_to_16bits(): # src_files, tt = get_file(input_path,file_type='.tif') src_files, tt = get_file(input_path) assert (tt != 0) for file in tqdm(src_files): absname = os.path.split(file)[1] absname = absname.split('.')[0] # absname = 'shuidao.png' absname = ''.join([absname, '.png']) print(absname) if not os.path.isfile(file): print("input file dose not exist:{}\n".format(file)) # sys.exit(-1) continue dataset = gdal.Open(file) if dataset == None: print("Open file failed: {}".format(file)) continue height = dataset.RasterYSize width = dataset.RasterXSize im_bands = dataset.RasterCount im_type = dataset.GetRasterBand(1).DataType img = dataset.ReadAsArray(0, 0, width, height) del dataset # img = np.array(img, np.uint16) img = np.array(img, np.float32) result = [] for i in range(im_bands): if im_bands == 1: data = img else: data = np.array(img[i]) maxium = data.max() minm = data.min() mean = data.mean() std = data.std() print(maxium, minm, mean, std) data = data.reshape(height * width) ind = np.where((data > 0) & (data < NoData)) ind = np.array(ind) a, b = ind.shape print("valid value number: {}\n".format(b)) # tmp = np.zeros(b, np.uint16) tmp = np.zeros(b, np.float32) for j in range(b): tmp[j] = data[ind[0, j]] tmaxium = tmp.max() tminm = tmp.min() tmean = tmp.mean() tstd = tmp.std() print(tmaxium, tminm, tmean, tstd) tt = (data - tmean) / tstd # first Z-score normalization tt = (tt + 4) * 1024 / 8.0 - 100 tind = np.where(data == 0) tt = np.array(tt) # tt = tt.astype(np.uint8) tt = tt.astype(np.uint16) tt[tind] = 0 smaxium = tt.max() sminm = tt.min() smean = tt.mean() sstd = tt.std() print(smaxium, sminm, smean, sstd) out = tt.reshape((height, width)) result.append(out) outputfile = os.path.join(output_path, absname) driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(result[i]) del outdataset
cmd.append('{}'.format(bmin)) cmd.append('{}'.format(bmax)) cmd.append('{}'.format(0)) cmd.append('{}'.format(255)) cmd.append(inputRaster) cmd.append(outputRaster) print("Conversin command:", cmd) subprocess.call(cmd) if __name__ == '__main__': if not os.path.isdir(inputdir): print("Please check input directory:{}".format(inputdir)) sys.exit(-1) if not os.path.isdir(outputdir): print('Warning: output directory is not existed') os.mkdir(outputdir) files, _ = get_file(inputdir) for file in files: absname = os.path.split(file)[1] outputfile = os.path.join(outputdir, absname) convert_to_8Bit2(file, outputfile, outputDataType='Byte', stretch_type='rescale', nodata=65535, percentiles=[1, 99.9])
def creat_dataset_binary(in_path, out_path, image_num=50000, mode='original'): print('\ncreating dataset...') label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_each = image_num / len(label_files) print("\n1: produce road labels---------------------") g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue print("src file:{}".format(os.path.split(src_file)[1])) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue X_height = dataset.RasterYSize X_width = dataset.RasterXSize im_bands = dataset.RasterCount data_type = dataset.GetRasterBand(1).DataType src_img = dataset.ReadAsArray(0, 0, X_width, X_height) src_img = np.array(src_img) del dataset index = np.where(label_img == 1) # 1: roads road_label = np.zeros((X_height, X_width), np.uint8) road_label[index] = 1 print(np.unique(road_label)) count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[:, random_height:random_height + img_h, random_width:random_width + img_w] label_roi = road_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi, data_type) visualize = label_roi * 50 # cv2.imwrite((out_path + '/roads/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/roads/label/%d.png' % g_count), label_roi) src_sample_file = out_path + '/roads/src/%d.png' % g_count driver = gdal.GetDriverByName("GTiff") # driver = gdal.GetDriverByName("PNG") # outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, gdal.GDT_UInt16) outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1 print("\n2: produce buildings labels---------------------") g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue # sys.exit(-1) print("src file:{}".format(os.path.split(src_file)[1])) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue X_height = dataset.RasterYSize X_width = dataset.RasterXSize im_bands = dataset.RasterCount src_img = dataset.ReadAsArray(0, 0, X_width, X_height) src_img = np.array(src_img) data_type = dataset.GetRasterBand(1).DataType del dataset index = np.where(label_img == 2) # 1: buildings building_label = np.zeros((X_height, X_width), np.uint8) building_label[index] = 1 count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, X_height - img_h - 1) src_roi = src_img[:, random_height:random_height + img_h, random_width:random_width + img_w] label_roi = building_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi, data_type) visualize = label_roi * 50 # cv2.imwrite((out_path + '/buildings/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/buildings/label/%d.png' % g_count), label_roi) src_sample_file = out_path + '/buildings/src/%d.png' % g_count driver = gdal.GetDriverByName("GTiff") # outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, gdal.GDT_UInt16) outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1
def convert_all_image_to_8bits(): src_files, tt = get_file(input_path) assert (tt != 0) for file in tqdm(src_files): absname = os.path.split(file)[1] print(absname) if not os.path.isfile(file): print("input file dose not exist:{}\n".format(file)) continue # sys.exit(-1) dataset = gdal.Open(file) height = dataset.RasterYSize width = dataset.RasterXSize im_bands = dataset.RasterCount img = dataset.ReadAsArray(0, 0, width, height) del dataset img = np.array(img, np.uint16) result = [] for i in range(im_bands): data = np.array(img[i]) maxium = data.max() minm = data.min() mean = data.mean() std = data.std() data = data.reshape(height * width) ind = np.where((data > 0) & (data < NoData)) ind = np.array(ind) # ind = ind.sort() # ind = np.sort(ind) a, b = ind.shape print(b) tmp = np.zeros(b, np.uint16) for j in range(b): tmp[j] = data[ind[0, j]] tmaxium = tmp.max() tminm = tmp.min() tmean = tmp.mean() tstd = tmp.std() tt = (data - tmean) / tstd # first Z-score normalization # tt = (tt + 4) * 200 / 8.0 # second min-max normalization to 255 tt = (tt + 4) * 255 / 8.0 - 255 # second min-max normalization to 255 tind = np.where(data == 0) tt = np.array(tt) tt = tt.astype(np.uint8) # tt = tt.astype(np.uint16) tt[tind] = 0 out = tt.reshape((height, width)) result.append(out) # plt.imshow(out) # plt.show() # cv2.imwrite((output_path + '%d.png' % i),out) # absname = os.path.split(file)[1] outputfile = os.path.join(output_path, absname) driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_Byte) # outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) # if im_bands ==1: # outdataset.GetRasterBand(1).WriteArray(result[0]) # else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(result[i]) del outdataset
def produce_training_samples_binary(in_path, out_path, image_num=50000, mode='original'): print('\ncreating dataset...') label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_each = image_num / len(label_files) print("\n[INFO] produce samples---------------------") g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue print("src file:{}".format(os.path.split(src_file)[1])) # label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) label_img = load_img_by_gdal(label_file, grayscale=True) # print("label_img: {}".format(np.unique(label_img))) label_img = label_img.astype(np.uint8) y, x = label_img.shape # print("label_img: {}".format(np.unique(label_img))) dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue Y_height = dataset.RasterYSize X_width = dataset.RasterXSize # check size of label and src images x, y = label_img.shape if (X_width != x and Y_height != y): print("label and source image have different size:".format( label_file)) continue im_bands = dataset.RasterCount data_type = dataset.GetRasterBand(1).DataType src_img = dataset.ReadAsArray(0, 0, X_width, Y_height) src_img = np.array(src_img) del dataset index = np.where(label_img == target_label) all_label = np.zeros((Y_height, X_width), np.uint8) all_label[index] = 1 print(np.unique(all_label)) count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, Y_height - img_h - 1) src_roi = src_img[:, random_height:random_height + img_h, random_width:random_width + img_w] label_roi = all_label[random_height:random_height + img_h, random_width:random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height:random_height + img_h, random_width:random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if mode == 'augment': src_roi, label_roi = data_augment(src_roi, label_roi, data_type) visualize = label_roi * 50 cv2.imwrite((out_path + '/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/label/%d.png' % g_count), label_roi) src_sample_file = out_path + '/src/%d.png' % g_count driver = gdal.GetDriverByName("GTiff") # driver = gdal.GetDriverByName("PNG") # outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, gdal.GDT_UInt16) outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1
import cv2 from tqdm import tqdm from ulitities.base_functions import load_img, get_file input_path = '../../data//traindata/segnet/label_old/' output_path = '../../data//traindata/segnet/label/' label_class=[0,1,2] # remain only road, buildings height=256 width=256 if __name__=='__main__': files = get_file(input_path) for filename in tqdm(files): ret, img = load_img(filename, grayscale=True) assert(ret == 0) for i in range(height): for j in range(width): if img[i,j]==4: img[i,j] =1 elif img[i,j]==2: img[i,j]=2 else: print ("\n img[{},{}]:{}".format(i,j,img[i,j])) img[i,j]=0 cv2.imwrite(output_path+os.path.split(filename)[1],img)
import cv2 import sys, os from ulitities.base_functions import load_img, get_file input_path = '../../data/traindata/unet/buildings/label/' output_path = '../../data/traindata/unet/buildings/visulize/' if __name__ == '__main__': if not os.path.isdir(input_path): print("No input directory:{}".format(input_path)) sys.exit(-1) if not os.path.isdir(output_path): print("No output directory:{}".format(output_path)) os.mkdir(output_path) srcfiles, tt = get_file(input_path) assert (tt != 0) for index, file in enumerate(srcfiles): ret, img = load_img(file, grayscale=True) assert (ret == 0) img = img * 100 filename = os.path.split(file)[1] outfile = os.path.join(output_path, filename) print(outfile) cv2.imwrite(outfile, img)
def produce_training_samples_multiclass_selfAdapt(self): print('\ncreating dataset...') in_path = self.input_dict['input_dir'] out_path = self.input_dict['output_dir'] valid_labels = list(range(int(self.input_dict['min']), int(self.input_dict['max'] + 1))) target_label = int(self.input_dict['target_label']) label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_num_rate = int(self.input_dict['sample_scaleRate']) # image_each = image_num / len(label_files) img_w = int(self.input_dict['window_size']) img_h = int(self.input_dict['window_size']) g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue # sys.exit(-1) print("src file:{}".format(os.path.split(src_file)[1])) label_img = cv2.imread(label_file, cv2.IMREAD_GRAYSCALE) absname = os.path.split(label_file)[1] absname = absname.split('.')[0] dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue Y_height = dataset.RasterYSize X_width = dataset.RasterXSize im_bands = dataset.RasterCount data_type = dataset.GetRasterBand(1).DataType # check size of label and src images x, y = label_img.shape print("Heigh, width of label is :{}, {}".format(x, y)) print("Heigh, width of src is :{}, {}".format(Y_height, X_width)) if x != Y_height or y != X_width: print("Warning: src and label have different size!") continue src_img = dataset.ReadAsArray(0, 0, X_width, Y_height) src_img = np.array(src_img) del dataset # Evaluate samples numbers according to the image_size, window_size and sample_scaleRate samples_num_of_current_image = int((X_width * Y_height * image_num_rate) / (img_w * img_h) + 0.5) if 'augment' in self.input_dict['mode']: samples_num_of_current_image = 6*samples_num_of_current_image print("Extract {} samples from {}".format(samples_num_of_current_image, os.path.split(label_file)[1])) count = 0 while count < samples_num_of_current_image: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, Y_height - img_h - 1) src_roi = src_img[:, random_height: random_height + img_h, random_width: random_width + img_w] label_roi = label_img[random_height: random_height + img_h, random_width: random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height: random_height + img_h, random_width: random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue # print(np.unique(label_roi)) if 'augment' in self.input_dict['mode']: src_roi, label_roi = self.data_augment(src_roi, label_roi, data_type) visualize = label_roi * 50 cv2.imwrite((out_path + '/visualize/%d_%s.png' % (g_count,absname)), visualize) cv2.imwrite((out_path + '/label/%d_%s.png' % (g_count, absname)), label_roi) src_sample_file = out_path + '/src/%d_%s.png' % (g_count,absname) driver = gdal.GetDriverByName("GTiff") outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1
def produce_training_samples_binary(self): print('\ncreating dataset...') in_path = self.input_dict['input_dir'] out_path = self.input_dict['output_dir'] valid_labels = list(range(int(self.input_dict['min']), int(self.input_dict['max']+1))) target_label = int(self.input_dict['target_label']) label_files, tt = get_file(os.path.join(in_path, 'label/')) assert (tt != 0) image_num = int(self.input_dict['sample_num']) image_each = image_num / len(label_files) img_w = int(self.input_dict['window_size']) img_h = int(self.input_dict['window_size']) print("\n[INFO] produce samples---------------------") g_count = 0 for label_file in tqdm(label_files): src_file = os.path.join(in_path, 'src/') + os.path.split(label_file)[1] if not os.path.isfile(src_file): print("Have no file:".format(src_file)) continue print("src file:{}".format(os.path.split(src_file)[1])) label_img = load_img_by_gdal(label_file, grayscale=True) # print("label_img: {}".format(np.unique(label_img))) label_img = label_img.astype(np.uint8) y, x = label_img.shape # print("label_img: {}".format(np.unique(label_img))) dataset = gdal.Open(src_file) if dataset == None: print("open failed!\n") continue Y_height = dataset.RasterYSize X_width = dataset.RasterXSize if (X_width != x and Y_height != y): print("label and source image have different size:".format(label_file)) continue im_bands = dataset.RasterCount data_type = dataset.GetRasterBand(1).DataType src_img = dataset.ReadAsArray(0, 0, X_width, Y_height) src_img = np.array(src_img) del dataset index = np.where(label_img == target_label) all_label = np.zeros((Y_height, X_width), np.uint8) all_label[index] = 1 print(np.unique(all_label)) # if no pixel in target value, ignore this label file tp = np.unique(all_label) # if tp[0]==0: # print("no target value in {}".format(label_file)) # continue # if len(tp) < 2: print("Only one value {} in {}".format(tp, label_file)) if tp[0] == 0: print("no target value in {}".format(label_file)) continue count = 0 while count < image_each: random_width = random.randint(0, X_width - img_w - 1) random_height = random.randint(0, Y_height - img_h - 1) src_roi = src_img[:, random_height: random_height + img_h, random_width: random_width + img_w] label_roi = all_label[random_height: random_height + img_h, random_width: random_width + img_w] """ignore nodata area""" FLAG_HAS_NODATA = False tmp = np.unique(label_img[random_height: random_height + img_h, random_width: random_width + img_w]) for tt in tmp: if tt not in valid_labels: FLAG_HAS_NODATA = True continue if FLAG_HAS_NODATA == True: continue """ignore pure background area""" if len(np.unique(label_roi)) < 2: if 0 in np.unique(label_roi): continue if 'augment' in self.input_dict['mode']: src_roi, label_roi = self.data_augment(src_roi, label_roi, img_w, img_h, data_type) visualize = label_roi * 50 cv2.imwrite((out_path + '/visualize/%d.png' % g_count), visualize) cv2.imwrite((out_path + '/label/%d.png' % g_count), label_roi) src_sample_file = out_path + '/src/%d.png' % g_count driver = gdal.GetDriverByName("GTiff") # driver = gdal.GetDriverByName("PNG") # outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, gdal.GDT_UInt16) outdataset = driver.Create(src_sample_file, img_w, img_h, im_bands, data_type) if outdataset == None: print("create dataset failed!\n") sys.exit(-2) if im_bands == 1: outdataset.GetRasterBand(1).WriteArray(src_roi) else: for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(src_roi[i]) del outdataset count += 1 g_count += 1
else: for i in range(bands): outdataset.GetRasterBand(i + 1).WriteArray(data[i]) del outdataset input_dir = '/media/omnisky/e0331d4a-a3ea-4c31-90ab-41f5b0ee2663/rice/original_label/' output_dir = '/media/omnisky/e0331d4a-a3ea-4c31-90ab-41f5b0ee2663/rice/label/' if __name__ == '__main__': if not os.path.isdir(input_dir): print("input dir is not existed: {}".format(input_dir)) sys.exit(-1) files, numb = get_file(input_dir) for file in tqdm(files): absname = os.path.split(file)[1] absname = absname.split('.')[0] tmp_file = ''.join([output_dir, absname, '.png']) # img = load_img_by_gdal(file, grayscale=True) img = cv2.imread(file, 0) # img = np.array(img, np.uint8) print("original value: {}".format(np.unique(img))) size = img.shape result = np.zeros(size, np.uint8) ind_targt = np.where(img == 1) result[ind_targt] = 1
def stretch_all_image_from_dict(self): if None == self.in_dict: QMessageBox.warning(self, "Warning", self.tr("input dict errors!")) sys.exit(-1) src_files, tt = get_file(self.in_dict['input_dir']) assert (tt != 0) NoData = int(self.in_dict['NoData']) valid_range = float(self.in_dict['StretchRange']) print(valid_range) cut_value = float(self.in_dict['CutValue']) if '8' in self.in_dict['OutBits']: assert(valid_range < 256) elif '16' in self.in_dict['OutBits']: assert (valid_range < 65536) for file in tqdm(src_files): absname = os.path.split(file)[1] absname = absname.split('.')[0] # absname = 'shuidao.png' absname = ''.join([absname, '.png']) print(absname) if not os.path.isfile(file): print("input file dose not exist:{}\n".format(file)) # sys.exit(-1) continue dataset = gdal.Open(file) if dataset == None: print("Open file failed: {}".format(file)) continue height = dataset.RasterYSize width = dataset.RasterXSize im_bands = dataset.RasterCount im_type = dataset.GetRasterBand(1).DataType img = dataset.ReadAsArray(0, 0, width, height) del dataset # img = np.array(img, np.uint16) img = np.array(img, np.float32) result = [] for i in range(im_bands): data = np.array(img[i]) maxium = data.max() minm = data.min() mean = data.mean() std = data.std() print(maxium, minm, mean, std) data = data.reshape(height * width) ind = np.where((data > 0) & (data < NoData)) ind = np.array(ind) a, b = ind.shape print("valid value number: {}\n".format(b)) # tmp = np.zeros(b, np.uint16) tmp = np.zeros(b, np.float32) for j in range(b): tmp[j] = data[ind[0, j]] tmaxium = tmp.max() tminm = tmp.min() tmean = tmp.mean() tstd = tmp.std() print(tmaxium, tminm, tmean, tstd) tt = (data - tmean) / tstd # first Z-score normalization tt = (tt + 4) * valid_range / 8.0 - cut_value tind = np.where(data == 0) tt = np.array(tt) # tt = tt.astype(np.uint8) tt = tt.astype(np.uint16) tt[tind] = 0 smaxium = tt.max() sminm = tt.min() smean = tt.mean() sstd = tt.std() print(smaxium, sminm, smean, sstd) out = tt.reshape((height, width)) result.append(out) outputfile = os.path.join(self.in_dict['output_dir'], absname) driver = gdal.GetDriverByName("GTiff") if '8' in self.in_dict['OutBits']: outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_Byte) elif '16' in self.in_dict['OutBits']: outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) # outdataset = driver.Create(outputfile, width, height, im_bands, gdal.GDT_UInt16) for i in range(im_bands): outdataset.GetRasterBand(i + 1).WriteArray(result[i]) del outdataset
outputdir = '/home/omnisky/PycharmProjects/data/samples/global/miandian/train_10bits_2000' patch_size=2000 if __name__=='__main__': if not os.path.isdir(inputdir): print("Error: input directory is not existed") sys.exit(-1) if not os.path.isdir(outputdir): print("Warning: output directory is not existed") os.mkdir(outputdir) out_label_dir=outputdir+'/label/' out_src_dir = outputdir + '/src/' label_list, img_list =[], [] label_files, _=get_file(inputdir+'/label') img_files =[] for file in label_files: absname = os.path.split(file)[1] absname = absname.split('.')[0] img_f = find_file(inputdir+'/src',absname) img_files.append(img_f) # img_files = list() # img_files, _=get_file(inputdir+'/src') assert(len(label_files)==len(img_files)) name_list =[] for i,file in enumerate(label_files): l_img = load_img_by_gdal(file, grayscale=True) if len(l_img)==0: continue label_list.append(l_img)
model, window_size=window_size, subdivisions=2, real_classes=target_class, # output channels = 是真的类别,总类别-背景 pred_func=smooth_predict_for_binary_notonehot, PLOT_PROGRESS=False) cv2.imwrite(output_file, result) print("Saved to: {}".format(output_file)) gc.collect() if __name__ == '__main__': all_files, num = get_file(input_path) if num == 0: print("There is no file in path:{}".format(input_path)) sys.exit(-1) """checke model file""" print("model file: {}".format(model_file)) if not os.path.isfile(model_file): print("model does not exist:{}".format(model_file)) sys.exit(-2) for in_file in all_files: abs_filename = os.path.split(in_file)[1] abs_filename = abs_filename.split(".")[0] print(abs_filename) out_file = ''.join([ output_path, '/mask_binary_', abs_filename, '_',
date_time = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime()) output_dir = ''.join([config.mask_dir, '/', date_time]) os.mkdir(output_dir) block_size = config.block_size nodata = config.nodata if __name__ == '__main__': input_files = [] if os.path.isfile(config.img_input): print("[INFO] input is one file...") input_files.append(config.img_input) elif os.path.isdir(config.img_input): print("[INFO] input is a directory...") in_files, _ = get_file(config.img_input, config.suffix) for file in in_files: input_files.append(file) print("{} images will be classified".format(len(input_files))) # sys.exit(-1) csv_file = os.path.join(output_dir, 'readme.csv') df = pd.DataFrame(list(config)) df.to_csv(csv_file) out_bands = config.mask_classes model = load_model(config.model_path) # print(model.summary()) for img_file in tqdm(input_files): print("\n[INFO] opening image:{}...".format(img_file))