def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths, args.neural_net, args.bands,
args.tile_size, args.number_of_epochs)
if not args.omit_findings:
for path in raster_data_paths:
print path
labels, images = load_training_tiles(path)
if len(labels) == 0 or len(images) == 0:
print("WARNING, there is a borked naip image file")
continue
false_positives, false_negatives, fp_images, fn_images = list_findings(labels, images,
model)
path_parts = path.split('/')
filename = path_parts[len(path_parts) - 1]
print("FINDINGS: {} false pos and {} false neg, of {} tiles, from {}".format(
len(false_positives), len(false_negatives), len(images), filename))
render_results_for_analysis([path], false_positives, fp_images, args.bands,
args.tile_size)
if args.render_results:
predictions = predictions_for_tiles(test_images, model)
render_results_for_analysis(raster_data_paths, predictions, test_images, args.bands,
args.tile_size)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths, args.neural_net, args.bands,
args.tile_size)
if args.render_results:
predictions = predictions_for_tiles(test_images, model)
render_results_for_analysis(raster_data_paths, predictions, test_images, args.bands,
args.tile_size)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths,
args.neural_net, args.bands,
args.tile_size)
if args.render_results:
predictions = predictions_for_tiles(test_images, model)
render_results_for_analysis(raster_data_paths, predictions,
test_images, args.bands, args.tile_size)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths,
args.neural_net, args.bands,
args.tile_size,
args.number_of_epochs)
if not args.omit_findings:
findings = []
for path in raster_data_paths:
print path
labels, images = load_training_tiles(path)
if len(labels) == 0 or len(images) == 0:
print("WARNING, there is a borked naip image file")
continue
false_positives, false_negatives, fp_images, fn_images = list_findings(
labels, images, model)
path_parts = path.split('/')
filename = path_parts[len(path_parts) - 1]
print(
"FINDINGS: {} false pos and {} false neg, of {} tiles, from {}"
.format(len(false_positives), len(false_negatives),
len(images), filename))
# render JPEGs showing findings
render_results_for_analysis([path], false_positives, fp_images,
args.bands, args.tile_size)
# combine findings for all NAIP images analyzed
[
findings.append(f) for f in tag_with_locations(
fp_images, false_positives, args.tile_size)
]
# dump combined findings to disk as a pickle
with open(CACHE_PATH + 'findings.pickle', 'w') as outfile:
pickle.dump(findings, outfile)
# push pickle to S3
s3_client = boto3.client('s3')
s3_client.upload_file(CACHE_PATH + 'findings.pickle', 'deeposm',
'findings.pickle')
if args.render_results:
predictions = predictions_for_tiles(test_images, model)
render_results_for_analysis(raster_data_paths, predictions,
test_images, args.bands, args.tile_size)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
with open(CACHE_PATH + METADATA_PATH, 'r') as infile:
training_info = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths, args.neural_net,
training_info['bands'], training_info['tile_size'],
args.number_of_epochs)
with open(CACHE_PATH + 'model.pickle', 'w') as outfile:
pickle.dump(model, outfile)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
with open(CACHE_PATH + METADATA_PATH, 'r') as infile:
training_info = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths,
args.neural_net,
training_info['bands'],
training_info['tile_size'],
args.number_of_epochs)
with open(CACHE_PATH + 'model.pickle', 'w') as outfile:
pickle.dump(model, outfile)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
with open(CACHE_PATH + 'raster_data_paths.pickle', 'r') as infile:
raster_data_paths = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths, args.neural_net, args.bands,
args.tile_size, args.number_of_epochs)
if not args.omit_findings:
findings = []
for path in raster_data_paths:
print path
labels, images = load_training_tiles(path)
if len(labels) == 0 or len(images) == 0:
print("WARNING, there is a borked naip image file")
continue
false_positives, false_negatives, fp_images, fn_images = list_findings(labels, images,
model)
path_parts = path.split('/')
filename = path_parts[len(path_parts) - 1]
print("FINDINGS: {} false pos and {} false neg, of {} tiles, from {}".format(
len(false_positives), len(false_negatives), len(images), filename))
# render JPEGs showing findings
render_results_for_analysis([path], false_positives, fp_images, args.bands,
args.tile_size)
# combine findings for all NAIP images analyzed
[findings.append(f) for f in tag_with_locations(fp_images, false_positives,
args.tile_size)]
# dump combined findings to disk as a pickle
with open(CACHE_PATH + 'findings.pickle', 'w') as outfile:
pickle.dump(findings, outfile)
# push pickle to S3
s3_client = boto3.client('s3')
s3_client.upload_file(CACHE_PATH + 'findings.pickle', 'deeposm', 'findings.pickle')
if args.render_results:
predictions = predictions_for_tiles(test_images, model)
render_results_for_analysis(raster_data_paths, predictions, test_images, args.bands,
args.tile_size)
def main():
"""Do each state one by one."""
# randomize_naips = False
naip_year = 2013
naip_states = {'de': ['http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf'],
'me': ['http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf']}
number_of_naips = 25
extract_type = 'highway'
bands = [1, 1, 1, 1]
tile_size = 64
pixels_to_fatten_roads = 3
tile_overlap = 1
neural_net = 'one_layer_relu_conv'
number_of_epochs = 25
for state, filenames in naip_states:
naiper = NAIPDownloader(number_of_naips, state, naip_year)
raster_data_paths = naiper.download_naips()
cache_paths(raster_data_paths)
create_tiled_training_data(raster_data_paths, extract_type, bands, tile_size,
pixels_to_fatten_roads, filenames,
tile_overlap, state)
with open(CACHE_PATH + METADATA_PATH, 'r') as infile:
training_info = pickle.load(infile)
test_images, model = train_on_cached_data(raster_data_paths, neural_net,
training_info['bands'], training_info['tile_size'],
number_of_epochs)
with open(CACHE_PATH + MODEL_METADATA_PATH, 'r') as infile:
model_info = pickle.load(infile)
model = load_model(model_info['neural_net_type'], model_info['tile_size'],
len(model_info['bands']))
post_findings_to_s3(raster_data_paths, model, training_info)
def main():
"""Analyze each state and publish results to deeposm.org."""
naip_year = 2013
naip_states = {'de': ['http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf'],
'ia': ['http://download.geofabrik.de/north-america/us/iowa-latest.osm.pbf'],
'me': ['http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf']
}
number_of_naips = 10
extract_type = 'highway'
bands = [1, 1, 1, 1]
tile_size = 64
pixels_to_fatten_roads = 3
tile_overlap = 1
neural_net = 'two_layer_relu_conv'
number_of_epochs = 10
randomize_naips = False
for state in naip_states:
filenames = naip_states[state]
raster_data_paths = download_and_serialize(number_of_naips,
randomize_naips,
state,
naip_year,
extract_type,
bands,
tile_size,
pixels_to_fatten_roads,
filenames,
tile_overlap)
model = train_on_cached_data(neural_net, number_of_epochs)
with open(CACHE_PATH + METADATA_PATH, 'r') as infile:
training_info = pickle.load(infile)
post_findings_to_s3(raster_data_paths, model, training_info, training_info['bands'], False)
requests.get('http://www.deeposm.org/refresh_findings/')
def main():
"""Analyze each state and publish results to deeposm.org."""
naip_year = 2013
naip_states = {
'de': [
'http://download.geofabrik.de/north-america/us/delaware-latest.osm.pbf'
],
'ia':
['http://download.geofabrik.de/north-america/us/iowa-latest.osm.pbf'],
'me':
['http://download.geofabrik.de/north-america/us/maine-latest.osm.pbf']
}
number_of_naips = 175
extract_type = 'highway'
bands = [1, 1, 1, 1]
tile_size = 64
pixels_to_fatten_roads = 3
tile_overlap = 1
naip_extent = None # WMIV 3/31/17
neural_net = 'two_layer_relu_conv'
number_of_epochs = 10
randomize_naips = False
for state in naip_states:
filenames = naip_states[state]
raster_data_paths = download_and_serialize(
number_of_naips, randomize_naips, state, naip_year, naip_extent,
extract_type, bands, tile_size, pixels_to_fatten_roads, filenames,
tile_overlap)
model = train_on_cached_data(neural_net, number_of_epochs)
with open(METADATA_FILE, 'r') as infile:
training_info = pickle.load(infile)
post_findings_to_s3(raster_data_paths, model, training_info,
training_info['bands'], False)
requests.get('http://www.deeposm.org/refresh_findings/')
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
train_on_cached_data(args.neural_net, args.number_of_epochs)
def main():
"""Use local data to train the neural net, probably made by bin/create_training_data.py."""
parser = create_parser()
args = parser.parse_args()
train_on_cached_data(args.neural_net, args.number_of_epochs)