def random_training_data(raster_data_paths, extract_type, band_list, tile_size,
pixels_to_fatten_roads, label_data_files, tile_overlap):
"""Return lists of training images and matching labels."""
road_labels = []
naip_tiles = []
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
way_bitmap_npy = {}
for raster_data_path in raster_data_paths:
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = numpy.asarray(
way_bitmap_for_naip(waymap.extracter.ways, raster_data_path, raster_dataset, rows,
cols, pixels_to_fatten_roads))
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
new_tile = way_bitmap_npy[row:row + tile_size, col:col + tile_size]
road_labels.append((new_tile, (col, row), raster_data_path))
for tile in tile_naip(raster_data_path, raster_dataset, bands_data, band_list, tile_size,
tile_overlap):
naip_tiles.append(tile)
assert len(road_labels) == len(naip_tiles)
road_labels, naip_tiles = shuffle_in_unison(road_labels, naip_tiles)
return road_labels, naip_tiles, waymap
def create_tiled_training_data(raster_data_paths, extract_type, band_list, tile_size,
pixels_to_fatten_roads, label_data_files, tile_overlap, naip_state):
"""Return lists of training images and matching labels."""
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
for raster_data_path in raster_data_paths:
path_parts = raster_data_path.split('/')
filename = path_parts[len(path_parts) - 1]
labels_path = CACHE_PATH + filename + '-labels.npy'
images_path = CACHE_PATH + filename + '-images.npy'
if os.path.exists(labels_path) and os.path.exists(images_path):
print("TRAINING DATA for {} already tiled".format(filename))
continue
road_labels = []
naip_tiles = []
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = way_bitmap_for_naip(waymap.extracter.ways, raster_data_path,
raster_dataset, rows, cols, pixels_to_fatten_roads)
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
# tile the way bitmap
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
new_tile = way_bitmap_npy[row:row + tile_size, col:col + tile_size]
road_labels.append((new_tile, col, row, raster_data_path))
# tile the NAIP
for tile in tile_naip(raster_data_path, raster_dataset, bands_data, band_list, tile_size,
tile_overlap):
naip_tiles.append(tile)
assert len(naip_tiles) == len(road_labels)
# dump the tiled labels from the way bitmap to disk
with open(labels_path, 'w') as outfile:
numpy.save(outfile, numpy.asarray(road_labels))
# dump the tiled images from the NAIP to disk
with open(images_path, 'w') as outfile:
numpy.save(outfile, numpy.asarray(naip_tiles))
# dump the metadata to disk for configuring the analysis script later
training_info = {'bands': band_list, 'tile_size': tile_size, 'naip_state': naip_state}
with open(CACHE_PATH + METADATA_PATH, 'w') as outfile:
pickle.dump(training_info, outfile)
def create_tiled_training_data(raster_data_paths, extract_type, band_list,
tile_size, pixels_to_fatten_roads,
label_data_files, tile_overlap):
"""Return lists of training images and matching labels."""
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
for raster_data_path in raster_data_paths:
path_parts = raster_data_path.split('/')
filename = path_parts[len(path_parts) - 1]
labels_path = CACHE_PATH + filename + '-labels.npy'
images_path = CACHE_PATH + filename + '-images.npy'
if os.path.exists(labels_path) and os.path.exists(images_path):
print("TRAINING DATA for {} already tiled".format(filename))
continue
road_labels = []
naip_tiles = []
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = way_bitmap_for_naip(waymap.extracter.ways,
raster_data_path, raster_dataset,
rows, cols,
pixels_to_fatten_roads)
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
# tile the way bitmap
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
new_tile = way_bitmap_npy[row:row + tile_size,
col:col + tile_size]
road_labels.append((new_tile, col, row, raster_data_path))
# tile the NAIP
for tile in tile_naip(raster_data_path, raster_dataset, bands_data,
band_list, tile_size, tile_overlap):
naip_tiles.append(tile)
assert len(naip_tiles) == len(road_labels)
# dump the tiled labels from the way bitmap to disk
with open(labels_path, 'w') as outfile:
numpy.save(outfile, numpy.asarray(road_labels))
# dump the tiled images from the NAIP to disk
with open(images_path, 'w') as outfile:
numpy.save(outfile, numpy.asarray(naip_tiles))
def create_tiled_training_data(raster_data_paths, extract_type, band_list, tile_size,
pixels_to_fatten_roads, label_data_files, tile_overlap, naip_state):
"""Save tiles for training data to disk, file names are padded with 16 0s.
Images are named 0000000000000000.colors.
Labels are named 0000000000000000.lbl.
"""
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
tile_index = 0
for raster_data_path in raster_data_paths:
# TODO need new code to check cache
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = way_bitmap_for_naip(waymap.extracter.ways, raster_data_path,
raster_dataset, rows, cols, pixels_to_fatten_roads)
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
# tile the way bitmap
origin_tile_index = tile_index
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
file_suffix = '{0:016d}'.format(tile_index)
label_filepath = "{}{}.lbl".format(LABEL_CACHE_DIRECTORY, file_suffix)
new_tile = way_bitmap_npy[row:row + tile_size, col:col + tile_size]
with open(label_filepath, 'w') as outfile:
numpy.save(outfile, numpy.asarray((new_tile, col, row, raster_data_path)))
tile_index += 1
tile_index = origin_tile_index
# tile the NAIP
for tile in tile_naip(raster_data_path, raster_dataset, bands_data, band_list, tile_size,
tile_overlap):
file_suffix = '{0:016d}'.format(tile_index)
img_filepath = "{}{}.colors".format(IMAGE_CACHE_DIRECTORY, file_suffix)
with open(img_filepath, 'w') as outfile:
numpy.save(outfile, tile)
tile_index += 1
# dump the metadata to disk for configuring the analysis script later
training_info = {'bands': band_list, 'tile_size': tile_size, 'naip_state': naip_state}
with open(CACHE_PATH + METADATA_PATH, 'w') as outfile:
pickle.dump(training_info, outfile)
def random_training_data(raster_data_paths, extract_type, band_list, tile_size,
pixels_to_fatten_roads, label_data_files,
tile_overlap):
"""Return lists of training images and matching labels."""
road_labels = []
naip_tiles = []
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
way_bitmap_npy = {}
for raster_data_path in raster_data_paths:
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = numpy.asarray(
way_bitmap_for_naip(waymap.extracter.ways, raster_data_path,
raster_dataset, rows, cols,
pixels_to_fatten_roads))
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
new_tile = way_bitmap_npy[row:row + tile_size,
col:col + tile_size]
road_labels.append(
(new_tile, (col, row), raster_data_path))
for tile in tile_naip(raster_data_path, raster_dataset, bands_data,
band_list, tile_size, tile_overlap):
naip_tiles.append(tile)
assert len(road_labels) == len(naip_tiles)
road_labels, naip_tiles = shuffle_in_unison(road_labels, naip_tiles)
return road_labels, naip_tiles, waymap
def create_tiled_training_data(raster_data_paths, extract_type, band_list,
tile_size, pixels_to_fatten_roads,
label_data_files, tile_overlap, naip_state):
"""Save tiles for training data to disk, file names are padded with 16 0s.
Images are named 0000000000000000.colors.
Labels are named 0000000000000000.lbl.
"""
# tile images and labels
waymap = download_and_extract(label_data_files, extract_type)
tile_index = 0
for raster_data_path in raster_data_paths:
# TODO need new code to check cache
raster_dataset, bands_data = read_naip(raster_data_path, band_list)
rows = bands_data.shape[0]
cols = bands_data.shape[1]
way_bitmap_npy = way_bitmap_for_naip(waymap.extracter.ways,
raster_data_path, raster_dataset,
rows, cols,
pixels_to_fatten_roads)
left_x, right_x = NAIP_PIXEL_BUFFER, cols - NAIP_PIXEL_BUFFER
top_y, bottom_y = NAIP_PIXEL_BUFFER, rows - NAIP_PIXEL_BUFFER
# tile the way bitmap
origin_tile_index = tile_index
for col in range(left_x, right_x, tile_size / tile_overlap):
for row in range(top_y, bottom_y, tile_size / tile_overlap):
if row + tile_size < bottom_y and col + tile_size < right_x:
file_suffix = '{0:016d}'.format(tile_index)
label_filepath = "{}/{}.lbl".format(
LABEL_CACHE_DIRECTORY, file_suffix)
new_tile = way_bitmap_npy[row:row + tile_size,
col:col + tile_size]
with open(label_filepath, 'w') as outfile:
numpy.save(
outfile,
numpy.asarray(
(new_tile, col, row, raster_data_path)))
tile_index += 1
tile_index = origin_tile_index
# tile the NAIP
for tile in tile_naip(raster_data_path, raster_dataset, bands_data,
band_list, tile_size, tile_overlap):
file_suffix = '{0:016d}'.format(tile_index)
img_filepath = "{}/{}.colors".format(IMAGE_CACHE_DIRECTORY,
file_suffix)
with open(img_filepath, 'w') as outfile:
numpy.save(outfile, tile)
tile_index += 1
# dump the metadata to disk for configuring the analysis script later
training_info = {
'bands': band_list,
'tile_size': tile_size,
'naip_state': naip_state
}
with open(CACHE_PATH + METADATA_PATH, 'w') as outfile:
pickle.dump(training_info, outfile)