def main():
global logger1
# min_subgraph_length_pix = 300
min_spur_length_m = 0.001 # default = 5
local = False #True
verbose = True
super_verbose = False
make_plots = False #True
save_shapefiles = True #False
pickle_protocol = 4 # 4 is most recent, python 2.7 can't read 4
# local
if local:
albu_path = '/Users/avanetten/Documents/cosmiq/apls/albu_inference_mod'
path_images = '/Users/avanetten/Documents/cosmiq/spacenet/data/spacenetv2/AOI_2_Vegas_Test/400m/RGB-PanSharpen'
res_root_dir = os.path.join(albu_path, 'results/2m_4fold_512_30e_d0.2_g0.2_AOI_2_Vegas_Test')
csv_file = os.path.join(res_root_dir, 'wkt_submission.csv')
graph_dir = os.path.join(res_root_dir, 'graphs')
log_file = os.path.join(res_root_dir, 'wkt_to_G.log')
#os.makedirs(graph_dir, exist_ok=True)
try:
os.makedirs(graph_dir)
except:
pass
# deployed on dev box
else:
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
# outut files
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
csv_file = os.path.join(res_root_dir, config.wkt_submission)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
log_file = os.path.join(res_root_dir, 'wkt_to_G.log')
os.makedirs(graph_dir, exist_ok=True)
min_subgraph_length_pix = config.min_subgraph_length_pix
min_spur_length_m = config.min_spur_length_m
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# ###############################################################################
# # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations
# # set up logging to file - see previous section for more details
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
# datefmt='%m-%d %H:%M',
# filename=log_file,
# filemode='w')
# # define a Handler which writes INFO messages or higher to the sys.stderr
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# # set a format which is simpler for console use
# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# # tell the handler to use this format
# console.setFormatter(formatter)
# # add the handler to the root logger
# logging.getLogger('').addHandler(console)
# logger1 = logging.getLogger('log')
# logger1.info("log file: {x}".format(x=log_file))
# ###############################################################################
# csv_file = os.path.join(res_root_dir, 'merged_wkt_list.csv')
# graph_dir = os.path.join(res_root_dir, 'graphs')
# #os.makedirs(graph_dir, exist_ok=True)
# try:
# os.makedirs(graph_dir)
# except:
# pass
# read in wkt list
logger1.info("df_wkt at: {}".format(csv_file))
#print ("df_wkt at:", csv_file)
df_wkt = pd.read_csv(csv_file)
# columns=['ImageId', 'WKT_Pix'])
# iterate through image ids and create graphs
t0 = time.time()
image_ids = np.sort(np.unique(df_wkt['ImageId']))
print("image_ids:", image_ids)
print("len image_ids:", len(image_ids))
for i,image_id in enumerate(image_ids):
#if image_id != 'AOI_2_Vegas_img586':
# continue
out_file = os.path.join(graph_dir, image_id.split('.')[0] + '.gpickle')
logger1.info("\n{x} / {y}, {z}".format(x=i+1, y=len(image_ids), z=image_id))
#print ("\n")
#print (i, "/", len(image_ids), image_id)
# for geo referencing, im_file should be the raw image
if config.num_channels == 3:
im_file = os.path.join(path_images, 'RGB-PanSharpen_' + image_id + '.tif')
else:
im_file = os.path.join(path_images, 'MUL-PanSharpen_' + image_id + '.tif')
#im_file = os.path.join(path_images, image_id)
if not os.path.exists(im_file):
im_file = os.path.join(path_images, image_id + '.tif')
# filter
df_filt = df_wkt['WKT_Pix'][df_wkt['ImageId'] == image_id]
wkt_list = df_filt.values
#wkt_list = [z[1] for z in df_filt_vals]
# print a few values
logger1.info("\n{x} / {y}, num linestrings: {z}".format(x=i+1, y=len(image_ids), z=len(wkt_list)))
#print ("\n", i, "/", len(image_ids), "num linestrings:", len(wkt_list))
if verbose:
print ("image_file:", im_file)
print (" wkt_list[:2]", wkt_list[:2])
if (len(wkt_list) == 0) or (wkt_list[0] == 'LINESTRING EMPTY'):
G = nx.MultiDiGraph()
nx.write_gpickle(G, out_file, protocol=pickle_protocol)
continue
# create graph
t1 = time.time()
G = wkt_to_G(wkt_list, im_file=im_file,
min_subgraph_length_pix=min_subgraph_length_pix,
min_spur_length_m=min_spur_length_m,
verbose=super_verbose)
t2 = time.time()
if verbose:
logger1.info("Time to create graph: {} seconds".format(t2-t1))
#print ("Time to create graph:", t2-t1, "seconds")
if len(G.nodes()) == 0:
nx.write_gpickle(G, out_file, protocol=pickle_protocol)
continue
# print a node
node = list(G.nodes())[-1]
print (node, "random node props:", G.nodes[node])
# print an edge
edge_tmp = list(G.edges())[-1]
#print (edge_tmp, "random edge props:", G.edges([edge_tmp[0], edge_tmp[1]])) #G.edge[edge_tmp[0]][edge_tmp[1]])
print (edge_tmp, "random edge props:", G.get_edge_data(edge_tmp[0], edge_tmp[1]))
# save graph
logger1.info("Saving graph to directory: {}".format(graph_dir))
#print ("Saving graph to directory:", graph_dir)
nx.write_gpickle(G, out_file, protocol=pickle_protocol)
# save shapefile as well?
if save_shapefiles:
logger1.info("Saving shapefile to directory: {}".format(graph_dir))
try:
ox.save_graph_shapefile(G, filename=image_id.split('.')[0] , folder=graph_dir, encoding='utf-8')
except:
print("Cannot save shapefile...")
#out_file2 = os.path.join(graph_dir, image_id.split('.')[0] + '.graphml')
#ox.save_graphml(G, image_id.split('.')[0] + '.graphml', folder=graph_dir)
# plot, if desired
if make_plots:
print ("Plotting graph...")
outfile_plot = os.path.join(graph_dir, image_id)
print ("outfile_plot:", outfile_plot)
ox.plot_graph(G, fig_height=9, fig_width=9,
#save=True, filename=outfile_plot, margin=0.01)
)
#plt.tight_layout()
plt.savefig(outfile_plot, dpi=400)
#if i > 30:
# break
tf = time.time()
logger1.info("Time to run wkt_to_G.py: {} seconds".format(tf - t0))
fn_mapping = {
'masks': lambda name: os.path.splitext(name)[0] + '.tif' #'.png'
}
image_suffix = ''#'img'
# set folds
skip_folds = []
if args.fold is not None:
skip_folds = [i for i in range(4) if i != int(args.fold)]
print("paths:", paths)
print("fn_mapping:", fn_mapping)
print("image_suffix:", image_suffix)
###################
# set up logging
console, logger = make_logger.make_logger(log_file, logger_name='log',
write_to_console=bool(config.log_to_console))
logger.info("Testing: weight_dir: {x}".format(x=weight_dir))
# execute
t0 = time.time()
logging.info("Saving eval outputs to: {x}".format(x=save_dir))
folds = eval_cresi(config, paths, fn_mapping, image_suffix, save_dir,
test=True, weight_dir=weight_dir,
num_channels=config.num_channels,
nfolds=config.num_folds,
save_im_gdal_format=save_im_gdal_format)
t1 = time.time()
logger.info("Time to run {x} folds for {y} = {z} seconds".format(x=len(folds),
y=len(os.listdir(path_images)), z=t1-t0))
print("Time to run", len(folds), "folds for",
len(os.listdir(path_images)), "=", t1 - t0, "seconds")
def main():
skimage_compress = 6
# 0-9, https://scikit-image.org/docs/stable/api/skimage.external.tifffile.html#skimage.external.tifffile.TiffWriter
# if using config instead of argparse
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
# nothing to do if only one fold
if config.slice_x <= 0 or config.slice_y <=0 or config.stride_x <= 0 \
or config.stride_y <= 0:
print("no need to stitch")
return
print("Running stitch.py...")
save_overlay_and_raw = False # switch to save the stitchin overlay and
# non-normalized image
# compression 0 to 9 (most compressed)
compression_params = [cv2.IMWRITE_PNG_COMPRESSION, 5]
folds_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.merged_dir)
if config.num_folds > 1:
im_dir = merge_dir
im_prefix = ''
else:
im_dir = folds_dir
im_prefix = 'fold0_'
# output dirs
out_dir_mask_raw = os.path.join(config.path_results_root, config.test_results_dir, config.stitched_dir_raw)
out_dir_count = os.path.join(config.path_results_root, config.test_results_dir, config.stitched_dir_count)
out_dir_mask_norm = os.path.join(config.path_results_root, config.test_results_dir, config.stitched_dir_norm)
# assume tile csv is in data dir, not root dir
path_tile_df_csv = os.path.join(config.path_results_root, config.test_results_dir, config.tile_df_csv)
# make dirs
os.makedirs(out_dir_mask_norm, exist_ok=True)
os.makedirs(out_dir_mask_raw, exist_ok=True)
os.makedirs(out_dir_count, exist_ok=True)
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
log_file = os.path.join(res_root_dir, 'stitch.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log',
write_to_console=bool(config.log_to_console))
# read in df_pos
#df_file = os.path.join(out_dir_root, 'tile_df.csv')
df_pos_tot = pd.read_csv(path_tile_df_csv)
logger1.info("len df_pos_tot: {x}".format(x=len(df_pos_tot)))
#print("len df_pos_tot:", len(df_pos_tot))
t0 = time.time()
ttot = 0
# save for each individual image
idxs = np.sort(np.unique(df_pos_tot['idx']))
logger1.info("image idxs: {x}".format(x=idxs))
#print("image idxs:", idxs)
for idx in idxs:
logger1.info("idx: {x} / {y}".format(x=idx+1, y=len(idxs)))
# print("\nidx:", idx, "/", len(idxs))
# filter by idx
df_pos = df_pos_tot.loc[df_pos_tot['idx'] == idx]
logger1.info("len df_pos: {x}".format(x=len(df_pos)))
# print("len df_pos:", len(df_pos))
# execute
t1 = time.time()
name, mask_norm, mask_raw, overlay_count = \
post_process_image(df_pos, im_dir,
im_prefix=im_prefix,
num_classes=config.num_classes,
super_verbose=False)
t2 = time.time()
ttot += t2-t1
logger1.info("Time to run stitch for idx: {x} = {y} seconds".format(x=idx, y=t2-t1))
# print("Time to run stitch for idx:", idx, "=", t2 - t1, "seconds")
logger1.info("mask_norm.shape: {x}".format(x=mask_norm.shape))
print("mask_norm.dtype:", mask_norm.dtype)
print("mask_raw.dtype:", mask_raw.dtype)
print("overlay_count.dtype:", overlay_count.dtype)
print("np.max(overlay_count):", np.max(overlay_count))
print("np.min(overlay_count):", np.min(overlay_count))
# write to files (cv2 can't handle reading enormous files, can write large ones)
print("Saving to files...")
# remove prefix, if required
if len(im_prefix) > 0:
out_file_root = name.split(im_prefix)[-1] + '.tif'
else:
out_file_root = name + '.tif'
logger1.info("out_file_root {x}".format(x=out_file_root))
#print("out_file_root:", out_file_root)
out_file_mask_norm = os.path.join(out_dir_mask_norm, out_file_root)
out_file_mask_raw = os.path.join(out_dir_mask_raw, out_file_root)
out_file_count = os.path.join(out_dir_count, out_file_root)
if config.num_classes == 1:
cv2.imwrite(out_file_mask_norm, mask_norm.astype(np.uint8), compression_params)
del mask_norm
if save_overlay_and_raw:
cv2.imwrite(out_file_mask_raw, mask_raw.astype(np.uint8), compression_params)
del mask_raw
else:
mask_norm = np.moveaxis(mask_norm, -1, 0).astype(np.uint8)
print("mask_norm.shape:", mask_norm.shape)
print("mask_norm.dtype:", mask_norm.dtype)
skimage.io.imsave(out_file_mask_norm, mask_norm,
check_contrast=False,
compress=skimage_compress)
del mask_norm
if save_overlay_and_raw:
mask_raw = np.moveaxis(mask_raw, -1, 0).astype(np.uint8)
skimage.io.imsave(out_file_mask_raw, mask_raw,
check_contrast=False,
compress=skimage_compress)
del mask_raw
if save_overlay_and_raw:
cv2.imwrite(out_file_count, overlay_count, compression_params)
#cv2.imwrite(out_file_count, overlay_count.astype(np.uint8), compression_params)
del overlay_count
#skimage.io.imsave(out_file_mask_norm, mask_norm)
#skimage.io.imsave(out_file_mask_raw, mask_raw)
#skimage.io.imsave(out_file_count, overlay_count)
t3 = time.time()
logger1.info("Time to run stitch.py and create large masks: {} seconds".format(ttot))
logger1.info("Time to run stitch.py and create large masks (and save): {} seconds".format(t3-t0))
# print("Time to run stitch.py and create large masks:", ttot, "seconds")
print("Time to run stitch.py and create large masks (and save):", t3 - t0, "seconds")
return
def main():
'''See _arr_slicing_speed.ipynb for better tests'''
global logger1
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
##########
# Variables
t0 = time.time()
percentile = 85
dx, dy = 4, 4 # nearest neighbors patch size
min_z = 128 # min z value to consider a hit
N_plots = 20
figsize = (12, 12)
# best colors
node_color, edge_color = '#cc9900', '#ffbf00' # gold
default_node_size = 2
plot_width_key, plot_width_mult = 'inferred_speed_mph', 0.085 # 0.08 # variable width
#width_key, width_mult = 4, 1 # constant width
if config.num_classes == 8:
use_totband = True
else:
use_totband = False
save_shapefiles = True
use_weighted_mean = True
variable_edge_speed = False
run_08a_plot_graph_plus_im = False
verbose = False
##########
# input dirs
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
#path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
# get mask location, check if we are stitching together large images or not
out_dir_mask_norm = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_norm)
folds_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.merged_dir)
mask_prefix = ''
if os.path.exists(out_dir_mask_norm):
mask_dir = out_dir_mask_norm
else:
if config.num_folds > 1:
mask_dir = merge_dir
else:
mask_dir = folds_dir
mask_prefix = 'fold0_'
log_file = os.path.join(res_root_dir, 'skeleton_speed.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# output dirs
graph_speed_dir = os.path.join(res_root_dir, config.graph_dir + '_speed')
os.makedirs(graph_speed_dir, exist_ok=True)
logger1.info("graph_speed_dir: " + graph_speed_dir)
# speed conversion dataframes (see _speed_data_prep.ipynb)
speed_conversion_file = config.speed_conversion_file
# load conversion file
# get the conversion diction between pixel mask values and road speed (mph)
if config.num_classes > 1:
conv_df, conv_dict \
= load_speed_conversion_dict_binned(speed_conversion_file)
else:
conv_df, conv_dict \
= load_speed_conversion_dict_contin(speed_conversion_file)
logger1.info("speed conv_dict: " + str(conv_dict))
# Add travel time to entire dir
add_travel_time_dir(graph_dir,
mask_dir,
conv_dict,
graph_speed_dir,
min_z=min_z,
dx=dx,
dy=dy,
percentile=percentile,
use_totband=use_totband,
use_weighted_mean=use_weighted_mean,
variable_edge_speed=variable_edge_speed,
mask_prefix=mask_prefix,
save_shapefiles=save_shapefiles,
verbose=verbose)
t1 = time.time()
logger1.info(
"Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
# plot a few
if N_plots > 0:
logger1.info("\nPlot a few...")
# define output dir
graph_speed_plots_dir = os.path.join(res_root_dir,
config.graph_dir + '_speed_plots')
os.makedirs(graph_speed_plots_dir, exist_ok=True)
# plot graph on image (with width proportional to speed)
path_images = os.path.join(config.path_data_root,
config.test_data_refined_dir)
image_list = [z for z in os.listdir(path_images) if z.endswith('tif')]
if len(image_list) > N_plots:
image_names = np.random.choice(image_list, N_plots)
else:
image_names = sorted(image_list)
#logger1.info("image_names: " + image_names)
for i, image_name in enumerate(image_names):
if i > 1000:
break
image_path = os.path.join(path_images, image_name)
logger1.info("\n\nPlotting: " + image_name + " " + image_path)
pkl_path = os.path.join(graph_speed_dir,
image_name.split('.')[0] + '.gpickle')
logger1.info(" pkl_path: " + pkl_path)
if not os.path.exists(pkl_path):
logger1.info(" missing pkl: " + pkl_path)
continue
G = nx.read_gpickle(pkl_path)
#if not os.path.exists(image_path)
figname = os.path.join(graph_speed_plots_dir, image_name)
_ = plot_graph_on_im_yuge(G,
image_path,
figsize=figsize,
show_endnodes=True,
default_node_size=default_node_size,
width_key=plot_width_key,
width_mult=plot_width_mult,
node_color=node_color,
edge_color=edge_color,
title=image_name,
figname=figname,
verbose=True,
super_verbose=verbose)
t2 = time.time()
logger1.info(
"Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
logger1.info("Time to make plots: {x} seconds".format(x=t2 - t1))
logger1.info("Total time: {x} seconds".format(x=t2 - t0))
def main():
global logger1
spacenet_naming_convention = False # True
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
im_prefix = ''
# check if we are stitching together large images or not
out_dir_mask_norm = None
folds_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.folds_save_dir)
im_dir = folds_dir
os.makedirs(im_dir, exist_ok=True)
# outut files
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
outfile_csv = os.path.join(res_root_dir, config.wkt_submission)
# outfile_gpickle = os.path.join(res_root_dir, 'G_sknw.gpickle')
out_ske_dir = os.path.join(res_root_dir, config.skeleton_dir) # set to '' to not save
os.makedirs(out_ske_dir, exist_ok=True)
if len(config.skeleton_pkl_dir) > 0:
out_gdir = os.path.join(res_root_dir, config.skeleton_pkl_dir) # set to '' to not save
os.makedirs(out_gdir, exist_ok=True)
else:
out_gdir = ''
# # use stitched dir if it has been populated
# if os.path.exists(config.stitched_dir_norm):
# im_dir = config.stitched_dir_norm
# else:
# im_dir = config.merged_dir
# # merged_dir = os.path.join(config.results_dir, config.folder + config.out_suff, 'merged')
#
# # outut files
# res_root_dir = config.results_dir #os.path.dirname(im_dir)
# outfile_csv = os.path.join(res_root_dir, 'merged_wkt_list.csv')
# #outfile_gpickle = os.path.join(res_root_dir, 'G_sknw.gpickle')
# out_ske_dir = os.path.join(res_root_dir, 'skeleton') # set to '' to not save
# os.makedirs(out_ske_dir, exist_ok=True)
# out_gdir = os.path.join(res_root_dir, 'sknw_gpickle') # set to '' to not save
# os.makedirs(out_gdir, exist_ok=True)
print("im_dir:", im_dir)
print("out_ske_dir:", out_ske_dir)
print("out_gdir:", out_gdir)
thresh = config.skeleton_thresh
# # thresholds for each aoi
# threshes={'2': .3, '3': .3, '4': .3, '5': .2}
# thresh = threshes[config.aoi]
min_subgraph_length_pix = config.min_subgraph_length_pix
# min_subgraph_length_pix=200
debug = False
add_small = True
fix_borders = True
img_shape = () # (1300, 1300)
skel_replicate = 5
skel_clip = 2
img_mult = 255
hole_size = 300
cv2_kernel_close = 7
cv2_kernel_open = 7
# max_out_size=(16000, 8000) # works fine
# max_out_size=(8003, 16009) # works fine
max_out_size = (2000000, 2000000)
# ###############################################################################
# # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations
# # set up logging to file - see previous section for more details
# res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
# log_file = os.path.join(res_root_dir, 'skeleton.log')
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
# datefmt='%m-%d %H:%M',
# filename=log_file,
# filemode='w')
# # define a Handler which writes INFO messages or higher to the sys.stderr
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# # set a format which is simpler for console use
# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# # tell the handler to use this format
# console.setFormatter(formatter)
# # add the handler to the root logger
# logging.getLogger('').addHandler(console)
# logger1 = logging.getLogger('log')
# logger1.info("log file: {x}".format(x=log_file))
# ###############################################################################
log_file = os.path.join(res_root_dir, 'skeleton.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
print("Building wkts...")
t0 = time.time()
df = build_wkt_dir(im_dir, outfile_csv, out_ske_dir, out_gdir, thresh, # threshes={'2': .3, '3': .3, '4': .3, '5': .2},
debug=debug, add_small=add_small, fix_borders=fix_borders,
img_shape=img_shape,
skel_replicate=skel_replicate, skel_clip=skel_clip,
img_mult=img_mult, hole_size=hole_size,
min_subgraph_length_pix=min_subgraph_length_pix,
cv2_kernel_close=cv2_kernel_close, cv2_kernel_open=cv2_kernel_open,
max_out_size=max_out_size,
skeleton_band=config.skeleton_band,
num_classes=config.num_classes,
im_prefix=im_prefix,
spacenet_naming_convention=spacenet_naming_convention)
print("len df:", len(df))
print("outfile:", outfile_csv)
t1 = time.time()
logger1.info("Total time to run build_wkt_dir: {} seconds".format(t1 - t0))
def main():
'''See _arr_slicing_speed.ipynb for better tests'''
global logger1
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
##########
# Variables
t0 = time.time()
percentile = 85 # percentil filter (default = 85)
dx, dy = 6, 6 # nearest neighbors patch size (default = (4, 4))
min_z = 128 # min z value to consider a hit (default = 128)
N_plots = 0
n_threads = 12
# set speed bands, assume a total channel is appended to the speed channels
if config.skeleton_band > 0:
max_speed_band = config.skeleton_band - 1
else:
max_speed_band = config.num_channels - 1
#if config.num_classes == 8:
# use_totband = True
#else:
# use_totband = False
save_shapefiles = True
use_weighted_mean = True
variable_edge_speed = False
run_08a_plot_graph_plus_im = False
verbose = False
n_threads = 12
##########
# input dirs
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
#path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
# get mask location, check if we are stitching together large images or not
out_dir_mask_norm = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_norm)
folds_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.merged_dir)
mask_prefix = ''
if os.path.exists(out_dir_mask_norm):
mask_dir = out_dir_mask_norm
else:
if config.num_folds > 1:
mask_dir = merge_dir
else:
mask_dir = folds_dir
mask_prefix = 'fold0_'
#if os.path.exists(out_dir_mask_norm):
# mask_dir = out_dir_mask_norm
#else:
# mask_dir = merge_dir
log_file = os.path.join(res_root_dir, 'graph_speed.log')
console, logger1 = make_logger.make_logger(log_file,
logger_name='log',
write_to_console=bool(
config.log_to_console))
#console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# output dirs
graph_speed_dir = os.path.join(res_root_dir, config.graph_dir + '_speed')
os.makedirs(graph_speed_dir, exist_ok=True)
logger1.info("graph_speed_dir: " + graph_speed_dir)
# speed conversion dataframes (see _speed_data_prep.ipynb)
speed_conversion_file = config.speed_conversion_file
# get the conversion diction between pixel mask values and road speed (mph)
if config.num_classes > 1:
conv_df, conv_dict \
= load_speed_conversion_dict_binned(speed_conversion_file)
else:
conv_df, conv_dict \
= load_speed_conversion_dict_contin(speed_conversion_file)
logger1.info("speed conv_dict: " + str(conv_dict))
# Add travel time to entire dir
add_travel_time_dir(graph_dir,
mask_dir,
conv_dict,
graph_speed_dir,
min_z=min_z,
dx=dx,
dy=dy,
percentile=percentile,
max_speed_band=max_speed_band,
use_weighted_mean=use_weighted_mean,
variable_edge_speed=variable_edge_speed,
mask_prefix=mask_prefix,
save_shapefiles=save_shapefiles,
n_threads=n_threads,
verbose=verbose)
t1 = time.time()
logger1.info(
"Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
# plot a few
if N_plots > 0:
logger1.info("\nPlot a few...")
# plotting
figsize = (12, 12)
node_color, edge_color = 'OrangeRed', '#ff571a'
node_color, edge_color = '#00e699', '#1affb2' # aquamarine
node_color, edge_color = 'turquoise', '#65e6d9' # turqoise
node_color, edge_color = 'deepskyblue', '#33ccff'
node_color, edge_color = '#e68a00', '#ffa31a' # orange
node_color, edge_color = '#33ff99', 'SpringGreen'
node_color, edge_color = 'DarkViolet', 'BlueViolet' #'#b300ff'
node_color, edge_color = '#a446d2', 'BlueViolet'
#edge_color='#00a6ff',
#node_color = '#33cccc' #turquise
#edge_color = '#47d1d1'
#node_color, edge_color = '#ffd100', '#e09900', # orange gold
#node_color = '#29a329' # green
#edge_color = '#33cc33'
#node_color='#66ccff' # blue
#edge_color='#999999'
# best colors
node_color, edge_color = '#cc9900', '#ffbf00' # gold
#node_color, edge_color = 'l#4dff4d', '#00e600' # green
default_node_size = 2 #0.15 #4
plot_width_key, plot_width_mult = 'inferred_speed_mph', 0.085 # 0.08 # variable width
#width_key, width_mult = 4, 1 # constant width
# define output dir
graph_speed_plots_dir = os.path.join(res_root_dir,
config.graph_dir + '_speed_plots')
os.makedirs(graph_speed_plots_dir, exist_ok=True)
# plot graph on image (with width proportional to speed)
path_images = config.test_data_refined_dir
# path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
image_list = [z for z in os.listdir(path_images) if z.endswith('tif')]
if len(image_list) > N_plots:
image_names = np.random.choice(image_list, N_plots)
else:
image_names = sorted(image_list)
#logger1.info("image_names: " + image_names)
for i, image_name in enumerate(image_names):
if i > 10:
break
image_path = os.path.join(path_images, image_name)
logger1.info("\n\nPlotting: " + image_name + " " + image_path)
pkl_path = os.path.join(graph_speed_dir,
image_name.split('.')[0] + '.gpickle')
logger1.info(" pkl_path: " + pkl_path)
if not os.path.exists(pkl_path):
logger1.info(" missing pkl: " + pkl_path)
continue
G = nx.read_gpickle(pkl_path)
#if not os.path.exists(image_path)
figname = os.path.join(graph_speed_plots_dir, image_name)
figname = figname.replace('.tif', '.png')
_ = apls_plots.plot_graph_on_im_yuge(
G,
image_path,
figsize=figsize,
show_endnodes=True,
default_node_size=default_node_size,
width_key=plot_width_key,
width_mult=plot_width_mult,
node_color=node_color,
edge_color=edge_color,
title=image_name,
figname=figname,
verbose=True,
super_verbose=verbose)
t2 = time.time()
logger1.info(
"Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
logger1.info("Time to make plots: {x} seconds".format(x=t2 - t1))
logger1.info("Total time: {x} seconds".format(x=t2 - t0))
print("Total time: {x} seconds".format(x=t2 - t0))
def main():
global logger1
spacenet_naming_convention = False # True
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
im_prefix = ''
# check if we are stitching together large images or not
out_dir_mask_norm = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_norm)
folds_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.merged_dir)
if os.path.exists(out_dir_mask_norm):
im_dir = out_dir_mask_norm
else:
if config.num_folds > 1:
im_dir = merge_dir
else:
im_dir = folds_dir
im_prefix = 'fold0_'
os.makedirs(im_dir, exist_ok=True)
# outut files
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
outfile_csv = os.path.join(res_root_dir, config.wkt_submission)
#outfile_gpickle = os.path.join(res_root_dir, 'G_sknw.gpickle')
out_ske_dir = os.path.join(res_root_dir, config.skeleton_dir) # set to '' to not save
os.makedirs(out_ske_dir, exist_ok=True)
if len(config.skeleton_pkl_dir) > 0:
out_gdir = os.path.join(res_root_dir, config.skeleton_pkl_dir) # set to '' to not save
os.makedirs(out_gdir, exist_ok=True)
else:
out_gdir = ''
print ("im_dir:", im_dir)
print ("out_ske_dir:", out_ske_dir)
print ("out_gdir:", out_gdir)
thresh = config.skeleton_thresh
min_subgraph_length_pix = config.min_subgraph_length_pix
debug=False
add_small=True
fix_borders=True
img_shape=() #(1300, 1300)
skel_replicate=5
skel_clip=2
img_mult=255
hole_size=300
cv2_kernel_close=7
cv2_kernel_open=7
#max_out_size=(16000, 8000) # works fine
#max_out_size=(8003, 16009) # works fine
max_out_size=(2000000, 2000000)
log_file = os.path.join(res_root_dir, 'skeleton.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
print ("Building wkts...")
t0 = time.time()
df = build_wkt_dir(im_dir, outfile_csv, out_ske_dir, out_gdir, thresh, #threshes={'2': .3, '3': .3, '4': .3, '5': .2},
debug=debug, add_small=add_small, fix_borders=fix_borders,
img_shape=img_shape,
skel_replicate=skel_replicate, skel_clip=skel_clip,
img_mult=img_mult, hole_size=hole_size,
min_subgraph_length_pix=min_subgraph_length_pix,
cv2_kernel_close=cv2_kernel_close, cv2_kernel_open=cv2_kernel_open,
max_out_size=max_out_size,
skeleton_band=config.skeleton_band,
num_classes=config.num_classes,
im_prefix=im_prefix,
spacenet_naming_convention=spacenet_naming_convention)
print ("len df:", len(df))
print ("outfile:", outfile_csv)
t1 = time.time()
logger1.info("Total time to run build_wkt_dir: {} seconds".format(t1-t0))
def main():
global logger1
# min_subgraph_length_pix = 300
simplify_graph = True #True # False
verbose = True #sFalse
super_verbose = False
make_plots = False #True
save_shapefiles = False #False
pickle_protocol = 4 # 4 is most recent, python 2.7 can't read 4
node_iter = 10000 # start int for node naming
edge_iter = 10000 # start int for edge naming
manually_reproject_nodes = False #True
n_threads = 12
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
# outut files
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
path_images = os.path.join(config.test_data_refined_dir)
# path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
csv_file = os.path.join(res_root_dir, config.wkt_submission)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
log_file = os.path.join(res_root_dir, 'wkt_to_G.log')
os.makedirs(graph_dir, exist_ok=True)
min_subgraph_length_pix = config.min_subgraph_length_pix
min_spur_length_m = config.min_spur_length_m
console, logger1 = make_logger.make_logger(log_file, logger_name='log',
write_to_console=bool(config.log_to_console))
# read in wkt list
logger1.info("df_wkt at: {}".format(csv_file))
#print ("df_wkt at:", csv_file)
df_wkt = pd.read_csv(csv_file)
# iterate through image ids and create graphs
t0 = time.time()
image_ids = np.sort(np.unique(df_wkt['ImageId']))
nfiles = len(image_ids)
print("image_ids:", image_ids)
print("len image_ids:", len(image_ids))
n_threads = min(n_threads, nfiles)
params = []
for i,image_id in enumerate(image_ids):
# if image_id != 'SN5_roads_test_public_AOI_9_San_Juan_PS-RGB_chip98':
# continue
# print("\n")
out_file = os.path.join(graph_dir, image_id.split('.')[0] + '.gpickle')
if verbose:
logger1.info("\n{x} / {y}, {z}".format(x=i+1, y=len(image_ids), z=image_id))
#print ("\n")
#print (i, "/", len(image_ids), image_id)
# for geo referencing, im_file should be the raw image
if config.num_channels == 3:
im_file = os.path.join(path_images, 'RGB-PanSharpen_' + image_id + '.tif')
else:
im_file = os.path.join(path_images, 'MUL-PanSharpen_' + image_id + '.tif')
#im_file = os.path.join(path_images, image_id)
if not os.path.exists(im_file):
im_file = os.path.join(path_images, image_id + '.tif')
# filter
df_filt = df_wkt['WKT_Pix'][df_wkt['ImageId'] == image_id]
wkt_list = df_filt.values
#wkt_list = [z[1] for z in df_filt_vals]
# print a few values
if verbose:
logger1.info("\n{x} / {y}, num linestrings: {z}".format(x=i+1, y=len(image_ids), z=len(wkt_list)))
#print ("\n", i, "/", len(image_ids), "num linestrings:", len(wkt_list))
if verbose:
print ("image_file:", im_file)
print (" wkt_list[:2]", wkt_list[:2])
if (len(wkt_list) == 0) or (wkt_list[0] == 'LINESTRING EMPTY'):
G = nx.MultiDiGraph()
nx.write_gpickle(G, out_file, protocol=pickle_protocol)
continue
else:
params.append((wkt_list, im_file, min_subgraph_length_pix, \
node_iter, edge_iter, \
min_spur_length_m, simplify_graph, \
config.rdp_epsilon,
manually_reproject_nodes,
out_file, graph_dir, n_threads, verbose))
# exectute
if n_threads > 1:
pool = Pool(n_threads)
pool.map(wkt_to_G, params)
else:
wkt_to_G(params[0])
tf = time.time()
logger1.info("Time to run wkt_to_G.py: {} seconds".format(tf - t0))
print("Time to run wkt_to_G.py: {} seconds".format(tf - t0))
config.folds_save_dir)
print("save_dir:", save_dir)
os.makedirs(save_dir, exist_ok=True)
fn_mapping = {
'masks': lambda name: os.path.splitext(name)[0] + '.tif' #'.png'
}
image_suffix = '' #'img'
# set folds
print("paths:", paths)
print("fn_mapping:", fn_mapping)
print("image_suffix:", image_suffix)
###################
# set up logging
console, logger = make_logger.make_logger(log_file, logger_name='log')
logger.info("Testing: weight_dir: {x}".format(x=weight_dir))
# print ("Testing: weight_dir:", weight_dir)
# execute
t0 = time.time()
logging.info("Saving eval outputs to: {x}".format(x=save_dir))
#print ("Saving eval outputs to:", save_dir)
folds = eval_cresi(config,
paths,
fn_mapping,
image_suffix,
save_dir,
test=True,
weight_dir=weight_dir,
num_channels=config.num_channels,
def main():
# if using config instead of argparse
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
print("Running stitch.py...")
save_overlay_and_raw = False # switch to save the stitchin overlay and
# non-normalized image
# compression 0 to 9 (most compressed)
compression_params = [cv2.IMWRITE_PNG_COMPRESSION, 5]
folds_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.merged_dir)
if config.num_folds > 1:
im_dir = merge_dir
im_prefix = ''
else:
im_dir = folds_dir
im_prefix = 'fold0_'
# output dirs
out_dir_mask_raw = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_raw)
out_dir_count = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_count)
out_dir_mask_norm = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_norm)
# assume tile csv is in data dir, not root dir
path_tile_df_csv = os.path.join(config.path_data_root,
os.path.dirname(config.test_sliced_dir),
config.tile_df_csv)
# try tile_df_csv in results path
#path_tile_df_csv = os.path.join(config.path_results_root, config.test_results_dir, config.tile_df_csv)
#out_dir_mask_norm = config.stitched_dir_norm #os.path.join(config.stitched_dir ,'mask_norm')
#out_dir_mask_raw = config.stitched_dir_raw #os.path.join(config.stitched_dir, 'mask_raw')
#out_dir_count = config.stitched_dir_count #os.path.join(config.stitched_dir, 'mask_count')
#res_root_dir = os.path.dirname(config.merged_dir)
##out_dir_root = os.path.join(res_root_dir, 'stitched')
#out_dir_mask_norm = os.path.join(res_root_dir, 'stitched/mask_norm')
#out_dir_mask_raw = os.path.join(res_root_dir, 'stitched/mask_raw')
#out_dir_count = os.path.join(res_root_dir, 'stitched/mask_count')
# make dirs
os.makedirs(out_dir_mask_norm, exist_ok=True)
os.makedirs(out_dir_mask_raw, exist_ok=True)
os.makedirs(out_dir_count, exist_ok=True)
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
log_file = os.path.join(res_root_dir, 'stitch.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# ###############################################################################
# # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations
# # set up logging to file - see previous section for more details
# res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
# log_file = os.path.join(res_root_dir, 'stitch.log')
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
# datefmt='%m-%d %H:%M',
# filename=log_file,
# filemode='w')
# # define a Handler which writes INFO messages or higher to the sys.stderr
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# # set a format which is simpler for console use
# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# # tell the handler to use this format
# console.setFormatter(formatter)
# # add the handler to the root logger
# logging.getLogger('').addHandler(console)
# logger1 = logging.getLogger('log')
# logger1.info("log file: {x}".format(x=log_file))
# ###############################################################################
# read in df_pos
#df_file = os.path.join(out_dir_root, 'tile_df.csv')
df_pos_tot = pd.read_csv(path_tile_df_csv)
logger1.info("len df_pos_tot: {x}".format(x=len(df_pos_tot)))
#print ("len df_pos_tot:", len(df_pos_tot))
t0 = time.time()
ttot = 0
# save for each individual image
idxs = np.sort(np.unique(df_pos_tot['idx']))
logger1.info("image idxs: {x}".format(x=idxs))
#print ("image idxs:", idxs)
for idx in idxs:
logger1.info("\n")
logger1.info("idx: {x} / {y}".format(x=idx + 1, y=len(idxs)))
#print ("\nidx:", idx, "/", len(idxs))
# filter by idx
df_pos = df_pos_tot.loc[df_pos_tot['idx'] == idx]
logger1.info("len df_pos: {x}".format(x=len(df_pos)))
#print ("len df_pos:", len(df_pos))
# execute
t1 = time.time()
name, mask_norm, mask_raw, overlay_count = \
post_process_image(df_pos, im_dir,
im_prefix=im_prefix,
num_classes=config.num_classes,
super_verbose=False)
t2 = time.time()
ttot += t2 - t1
logger1.info("Time to run stitch for idx: {x} = {y} seconds".format(
x=idx, y=t2 - t1))
#print ("Time to run stitch for idx:", idx, "=", t2 - t1, "seconds")
logger1.info("mask_norm.shape: {x}".format(x=mask_norm.shape))
print("mask_norm.dtype:", mask_norm.dtype)
print("mask_raw.dtype:", mask_raw.dtype)
print("overlay_count.dtype:", overlay_count.dtype)
print("np.max(overlay_count):", np.max(overlay_count))
print("np.min(overlay_count):", np.min(overlay_count))
# write to files (cv2 can't handle reading enormous files, can write large ones)
print("Saving to files...")
# remove prefix, if required
if len(im_prefix) > 0:
out_file_root = name.split(im_prefix)[-1] + '.tif'
else:
out_file_root = name + '.tif'
logger1.info("out_file_root {x}:".format(x=out_file_root))
#print ("out_file_root:", out_file_root)
out_file_mask_norm = os.path.join(out_dir_mask_norm, out_file_root)
out_file_mask_raw = os.path.join(out_dir_mask_raw, out_file_root)
out_file_count = os.path.join(out_dir_count, out_file_root)
if config.num_classes == 1:
cv2.imwrite(out_file_mask_norm, mask_norm.astype(np.uint8),
compression_params)
del mask_norm
if save_overlay_and_raw:
cv2.imwrite(out_file_mask_raw, mask_raw.astype(np.uint8),
compression_params)
del mask_raw
else:
mask_norm = np.moveaxis(mask_norm, -1, 0).astype(np.uint8)
skimage.io.imsave(out_file_mask_norm, mask_norm, compress=1)
del mask_norm
if save_overlay_and_raw:
mask_raw = np.moveaxis(mask_raw, -1, 0).astype(np.uint8)
skimage.io.imsave(out_file_mask_raw, mask_raw, compress=1)
del mask_raw
if save_overlay_and_raw:
cv2.imwrite(out_file_count, overlay_count, compression_params)
#cv2.imwrite(out_file_count, overlay_count.astype(np.uint8), compression_params)
del overlay_count
#skimage.io.imsave(out_file_mask_norm, mask_norm)
#skimage.io.imsave(out_file_mask_raw, mask_raw)
#skimage.io.imsave(out_file_count, overlay_count)
t3 = time.time()
logger1.info(
"Time to run stitch.py and create large masks: {} seconds".format(
ttot))
logger1.info(
"Time to run stitch.py and create large masks (and save): {} seconds".
format(t3 - t0))
#print ("Time to run stitch.py and create large masks:", ttot, "seconds")
#print ("Time to run stitch.py and create large masks (and save):", t3 - t0, "seconds")
return
def main():
global logger1
# min_subgraph_length_pix = 300
min_spur_length_m = 0.001 # default = 5
local = False # True
verbose = True
super_verbose = False
make_plots = False # True
save_shapefiles = True # False
pickle_protocol = 4 # 4 is most recent, python 2.7 can't read 4
# local
if local:
pass
else:
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
# outut files
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
csv_file = os.path.join(res_root_dir, config.wkt_submission)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
log_file = os.path.join(res_root_dir, 'wkt_to_G.log')
os.makedirs(graph_dir, exist_ok=True)
min_subgraph_length_pix = config.min_subgraph_length_pix
min_spur_length_m = config.min_spur_length_m
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# read in wkt list
logger1.info("df_wkt at: {}".format(csv_file))
# print ("df_wkt at:", csv_file)
df_wkt = pd.read_csv(csv_file)
# columns=['ImageId', 'WKT_Pix'])
# iterate through image ids and create graphs
t0 = time.time()
image_ids = np.sort(np.unique(df_wkt['ImageId']))
print("image_ids:", image_ids)
print("len image_ids:", len(image_ids))
imfiles_args = [[image_id, config, graph_dir, path_images, df_wkt, pickle_protocol, min_spur_length_m,
min_subgraph_length_pix, super_verbose, verbose, save_shapefiles, make_plots] for image_id in
image_ids]
with multiprocessing.Pool(16) as pool:
list(pool.starmap(process_img, imfiles_args))
# for i,image_id in enumerate(image_ids):
#
# #if image_id != 'AOI_2_Vegas_img586':
# # continue
# out_file = os.path.join(graph_dir, image_id.split('.')[0] + '.gpickle')
#
# logger1.info("\n{x} / {y}, {z}".format(x=i+1, y=len(image_ids), z=image_id))
# #print ("\n")
# #print (i, "/", len(image_ids), image_id)
#
# # for geo referencing, im_file should be the raw image
# if config.num_channels == 3:
# im_file = os.path.join(path_images, 'RGB-PanSharpen_' + image_id + '.tif')
# else:
# im_file = os.path.join(path_images, 'MUL-PanSharpen_' + image_id + '.tif')
# #im_file = os.path.join(path_images, image_id)
# if not os.path.exists(im_file):
# im_file = os.path.join(path_images, image_id + '.tif')
#
# # filter
# df_filt = df_wkt['WKT_Pix'][df_wkt['ImageId'] == image_id]
# wkt_list = df_filt.values
# #wkt_list = [z[1] for z in df_filt_vals]
#
# # print a few values
# logger1.info("\n{x} / {y}, num linestrings: {z}".format(x=i+1, y=len(image_ids), z=len(wkt_list)))
# #print ("\n", i, "/", len(image_ids), "num linestrings:", len(wkt_list))
# if verbose:
# print ("image_file:", im_file)
# print (" wkt_list[:2]", wkt_list[:2])
#
# if (len(wkt_list) == 0) or (wkt_list[0] == 'LINESTRING EMPTY'):
# G = nx.MultiDiGraph()
# nx.write_gpickle(G, out_file, protocol=pickle_protocol)
# continue
#
# # create graph
# t1 = time.time()
# G = wkt_to_G(wkt_list, im_file=im_file,
# min_subgraph_length_pix=min_subgraph_length_pix,
# min_spur_length_m=min_spur_length_m,
# verbose=super_verbose)
# t2 = time.time()
# if verbose:
# logger1.info("Time to create graph: {} seconds".format(t2-t1))
# #print ("Time to create graph:", t2-t1, "seconds")
#
# if len(G.nodes()) == 0:
# nx.write_gpickle(G, out_file, protocol=pickle_protocol)
# continue
#
# # print a node
# node = list(G.nodes())[-1]
# print (node, "random node props:", G.nodes[node])
# # print an edge
# # edge_tmp = list(G.edges())[-1]
# #print (edge_tmp, "random edge props:", G.edges([edge_tmp[0], edge_tmp[1]])) #G.edge[edge_tmp[0]][edge_tmp[1]])
# # print (edge_tmp, "random edge props:", G.get_edge_data(edge_tmp[0], edge_tmp[1]))
#
# # save graph
# logger1.info("Saving graph to directory: {}".format(graph_dir))
# #print ("Saving graph to directory:", graph_dir)
# nx.write_gpickle(G, out_file, protocol=pickle_protocol)
#
# # save shapefile as well?
# if save_shapefiles:
# logger1.info("Saving shapefile to directory: {}".format(graph_dir))
# try:
# ox.save_graph_shapefile(G, filename=image_id.split('.')[0] , folder=graph_dir, encoding='utf-8')
# except:
# print("Cannot save shapefile...")
# #out_file2 = os.path.join(graph_dir, image_id.split('.')[0] + '.graphml')
# #ox.save_graphml(G, image_id.split('.')[0] + '.graphml', folder=graph_dir)
#
# # plot, if desired
# if make_plots:
# print ("Plotting graph...")
# outfile_plot = os.path.join(graph_dir, image_id)
# print ("outfile_plot:", outfile_plot)
# ox.plot_graph(G, fig_height=9, fig_width=9,
# #save=True, filename=outfile_plot, margin=0.01)
# )
# #plt.tight_layout()
# plt.savefig(outfile_plot, dpi=400)
#
# #if i > 30:
# # break
tf = time.time()
logger1.info("Time to run wkt_to_G.py: {} seconds".format(tf - t0))
def main():
global logger1
add_small = True
verbose = True
super_verbose = False
spacenet_naming_convention = False # True
debug = False
fix_borders = True
img_shape = () # (1300, 1300)
skel_replicate = 5
skel_clip = 2
img_mult = 255
hole_size = 300
cv2_kernel_close = 7
cv2_kernel_open = 7
kernel_blur = -1 # 25
min_background_frac = -1 # 0.2
max_out_size = (2000000, 2000000)
n_threads = 12
im_prefix = ''
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
min_spur_length_pix = int(np.rint(config.min_spur_length_m / config.GSD))
print("min_spur_length_pix:", min_spur_length_pix)
use_medial_axis = bool(config.use_medial_axis)
print("Use_medial_axis?", use_medial_axis)
pix_extent = config.eval_rows - (2 * config.padding)
# check if we are stitching together large images or not
out_dir_mask_norm = os.path.join(config.path_results_root,
config.test_results_dir,
config.stitched_dir_norm)
folds_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.folds_save_dir)
merge_dir = os.path.join(config.path_results_root, config.test_results_dir,
config.merged_dir)
if os.path.exists(out_dir_mask_norm):
im_dir = out_dir_mask_norm
else:
if config.num_folds > 1:
im_dir = merge_dir
else:
im_dir = folds_dir
im_prefix = 'fold0_'
os.makedirs(im_dir, exist_ok=True)
# outut files
res_root_dir = os.path.join(config.path_results_root,
config.test_results_dir)
outfile_csv = os.path.join(res_root_dir, config.wkt_submission)
#outfile_gpickle = os.path.join(res_root_dir, 'G_sknw.gpickle')
out_ske_dir = os.path.join(res_root_dir,
config.skeleton_dir) # set to '' to not save
os.makedirs(out_ske_dir, exist_ok=True)
if len(config.skeleton_pkl_dir) > 0:
out_gdir = os.path.join(
res_root_dir, config.skeleton_pkl_dir) # set to '' to not save
os.makedirs(out_gdir, exist_ok=True)
else:
out_gdir = ''
print("im_dir:", im_dir)
print("out_ske_dir:", out_ske_dir)
print("out_gdir:", out_gdir)
thresh = config.skeleton_thresh
# # thresholds for each aoi
# threshes={'2': .3, '3': .3, '4': .3, '5': .2}
# thresh = threshes[config.aoi]
min_subgraph_length_pix = config.min_subgraph_length_pix
#min_subgraph_length_pix=200
log_file = os.path.join(res_root_dir, 'skeleton.log')
console, logger1 = make_logger.make_logger(log_file,
logger_name='log',
write_to_console=bool(
config.log_to_console))
# print("Building wkts...")
t0 = time.time()
df = build_wkt_dir(im_dir,
outfile_csv,
out_ske_dir,
out_gdir,
thresh,
debug=debug,
add_small=add_small,
fix_borders=fix_borders,
img_shape=img_shape,
skel_replicate=skel_replicate,
skel_clip=skel_clip,
img_mult=img_mult,
hole_size=hole_size,
min_subgraph_length_pix=min_subgraph_length_pix,
min_spur_length_pix=min_spur_length_pix,
cv2_kernel_close=cv2_kernel_close,
cv2_kernel_open=cv2_kernel_open,
max_out_size=max_out_size,
skeleton_band=config.skeleton_band,
num_classes=config.num_classes,
im_prefix=im_prefix,
spacenet_naming_convention=spacenet_naming_convention,
use_medial_axis=use_medial_axis,
kernel_blur=kernel_blur,
min_background_frac=min_background_frac,
n_threads=n_threads,
verbose=verbose,
super_verbose=super_verbose)
print("len df:", len(df))
print("outfile:", outfile_csv)
t1 = time.time()
logger1.info("Total time to run build_wkt_dir: {} seconds".format(t1 - t0))
print("Total time to run build_wkt_dir:", t1 - t0, "seconds")
def main():
'''See _arr_slicing_speed.ipynb for better tests'''
global logger1
import argparse
import json
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
##########
# Variables
t0 = time.time()
percentile = 85
dx, dy = 4, 4 # nearest neighbors patch size
min_z = 128 # min z value to consider a hit
N_plots = 20
figsize = (12, 12)
node_color, edge_color = 'OrangeRed', '#ff571a'
node_color, edge_color = '#00e699', '#1affb2' # aquamarine
node_color, edge_color = 'turquoise', '#65e6d9' # turqoise
node_color, edge_color = 'deepskyblue', '#33ccff'
node_color, edge_color = '#e68a00', '#ffa31a' # orange
node_color, edge_color = '#33ff99', 'SpringGreen'
node_color, edge_color = 'DarkViolet', 'BlueViolet' # '#b300ff'
node_color, edge_color = '#a446d2', 'BlueViolet'
# node_color='#0086CC' # cosmiq logo color (blue)
# edge_color='#00a6ff',
# node_color = '#33cccc' #turquise
# edge_color = '#47d1d1'
# node_color, edge_color = '#ffd100', '#e09900', # orange gold
# node_color = '#29a329' # green
# edge_color = '#33cc33'
# node_color='#66ccff' # blue
# edge_color='#999999'
# best colors
node_color, edge_color = '#cc9900', '#ffbf00' # gold
# node_color, edge_color = 'l#4dff4d', '#00e600' # green
default_node_size = 2 # 0.15 #4
plot_width_key, plot_width_mult = 'inferred_speed_mph', 0.085 # 0.08 # variable width
# width_key, width_mult = 4, 1 # constant width
if config.num_classes >7:
use_totband = True
else:
use_totband = False
save_shapefiles = True
use_weighted_mean = True
variable_edge_speed = False
run_08a_plot_graph_plus_im = False
verbose = False
##########
# input dirs
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
# path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
# get mask location, check if we are stitching together large images or not
folds_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.folds_save_dir)
mask_prefix = ''
mask_dir = folds_dir
# if os.path.exists(out_dir_mask_norm):
# mask_dir = out_dir_mask_norm
# else:
# mask_dir = merge_dir
log_file = os.path.join(res_root_dir, 'skeleton_speed.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# ###############################################################################
# # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations
# # set up logging to file - see previous section for more details
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
# datefmt='%m-%d %H:%M',
# filename=log_file,
# filemode='w')
# # define a Handler which writes INFO messages or higher to the sys.stderr
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# # set a format which is simpler for console use
# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# # tell the handler to use this format
# console.setFormatter(formatter)
# # add the handler to the root logger
# logging.getLogger('').addHandler(console)
# logger1 = logging.getLogger('log')
# logger1.info("log file: {x}".format(x=log_file))
# ###############################################################################
# output dirs
graph_speed_dir = os.path.join(res_root_dir, config.graph_dir + '_speed')
os.makedirs(graph_speed_dir, exist_ok=True)
logger1.info("graph_speed_dir: " + graph_speed_dir)
speed_conversion_file_binned = 'speed_conversion_binned10.csv'
# load conversion file
# get the conversion diction between pixel mask values and road speed (mph)
conv_df, conv_dict \
= load_speed_conversion_dict_binned(speed_conversion_file_binned)
logger1.info("speed conv_dict: " + str(conv_dict))
# Add travel time to entire dir
add_travel_time_dir(graph_dir, mask_dir, conv_dict, graph_speed_dir,
min_z=min_z,
dx=dx, dy=dy,
percentile=percentile,
use_totband=use_totband,
use_weighted_mean=use_weighted_mean,
variable_edge_speed=variable_edge_speed,
mask_prefix=mask_prefix,
save_shapefiles=save_shapefiles,
verbose=verbose)
t1 = time.time()
logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
# plot a few
if N_plots > 0:
logger1.info("\nPlot a few...")
## import apls_tools (or just copy plot_graph_on_in() func he)
# local = False
## local
# if local:
# apls_dir = '/raid/cosmiq/apls/apls/src'
## dev box
# else:
# apls_dir = '/raid/local/src/apls/apls/src'
# sys.path.append(apls_dir)
# import apls_tools
# define output dir
graph_speed_plots_dir = os.path.join(res_root_dir, config.graph_dir + '_speed_plots')
os.makedirs(graph_speed_plots_dir, exist_ok=True)
# plot graph on image (with width proportional to speed)
path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
image_list = [z for z in os.listdir(path_images) if z.endswith('tif')]
if len(image_list) > N_plots:
image_names = np.random.choice(image_list, N_plots)
else:
image_names = sorted(image_list)
# logger1.info("image_names: " + image_names)
for i, image_name in enumerate(image_names):
if i > 1000:
break
image_path = os.path.join(path_images, image_name)
logger1.info("\n\nPlotting: " + image_name + " " + image_path)
pkl_path = os.path.join(graph_speed_dir, image_name.split('.')[0] + '.gpickle')
logger1.info(" pkl_path: " + pkl_path)
if not os.path.exists(pkl_path):
logger1.info(" missing pkl: " + pkl_path)
continue
G = nx.read_gpickle(pkl_path)
# if not os.path.exists(image_path)
figname = os.path.join(graph_speed_plots_dir, image_name)
_ = plot_graph_on_im_yuge(G, image_path, figsize=figsize,
show_endnodes=True,
default_node_size=default_node_size,
width_key=plot_width_key, width_mult=plot_width_mult,
node_color=node_color, edge_color=edge_color,
title=image_name, figname=figname,
verbose=True, super_verbose=verbose)
t2 = time.time()
logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
logger1.info("Time to make plots: {x} seconds".format(x=t2 - t1))
logger1.info("Total time: {x} seconds".format(x=t2 - t0))
