def create_multi_scale_dataset(batch_size):
train_data_dict = {}
for scale in [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]:
grid_size = scale // 32
anchors = config.yolo_anchors / scale
# Training dataset setting
AUTOTUNE = tf.data.experimental.AUTOTUNE # 自動調整模式
combined_split = tfds.Split.TRAIN + tfds.Split.VALIDATION
train_data = tfds.load("voc2007", split=combined_split, data_dir=dataset_path) # 取得訓練數據
train_data = train_data.shuffle(1000) # 打散資料集
train_data = train_data.map(lambda dataset: parse_aug_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE)
train_data = train_data.batch(batch_size)
train_data = train_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size),
num_parallel_calls=AUTOTUNE)
train_data = train_data.prefetch(buffer_size=AUTOTUNE)
train_data_dict[scale] = train_data
# Validation dataset setting
grid_size = 416 // 32
anchors = config.yolo_anchors / 416
val_data = tfds.load("voc2007", split=tfds.Split.TEST, data_dir=dataset_path)
val_data = val_data.map(lambda dataset: parse_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE)
val_data = val_data.batch(batch_size)
val_data = val_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size),
num_parallel_calls=AUTOTUNE)
val_data = val_data.prefetch(buffer_size=AUTOTUNE)
return train_data_dict, val_data
def predict(image_paths):
X = np.array(
list(
map(lambda path: val_preprocess_mobilenet(parse_fn(path)),
image_paths)))
scores = model.predict(X, batch_size=32, verbose=1)
return scores
def multi_scale_training_model(model, callbacks, num_classes=80, step=1):
if step == 1:
batch_size = 30
epoch_step = 1
else:
batch_size = 8
epoch_step = 10
start_epoch = 0
# for lr in [1e-3, 1e-3, 1e-4]:
# for scale in [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]:
memory_used = []
for lr in [1e-3]:
for scale in [320, 352, 384]:
print('scale: {}, learning rate: {}'.format(scale, lr))
# if scale == 416 and lr == 1e-4:
# print('Last round')
# epoch_step = 50
anchors = config.yolo_anchors / scale
grid_size = scale // 32
# Training dataset setting
AUTOTUNE = tf.data.experimental.AUTOTUNE # 自動調整模式
combined_split = tfds.Split.TRAIN + tfds.Split.VALIDATION
train_data = tfds.load("voc2007", split=combined_split, data_dir='/home/share/dataset/tensorflow-datasets') # 取得訓練數據
train_data = train_data.shuffle(1000) # 打散資料集
train_data = train_data.map(lambda dataset: parse_aug_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE)
train_data = train_data.batch(batch_size)
train_data = train_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size),
num_parallel_calls=AUTOTUNE)
train_data = train_data.prefetch(buffer_size=AUTOTUNE)
# Validation dataset setting
val_data = tfds.load("voc2007", split=tfds.Split.TEST, data_dir='/home/share/dataset/tensorflow-datasets')
val_data = val_data.map(lambda dataset: parse_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE)
val_data = val_data.batch(batch_size)
val_data = val_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size),
num_parallel_calls=AUTOTUNE)
val_data = val_data.prefetch(buffer_size=AUTOTUNE)
# Training
optimizer = tf.keras.optimizers.Adam(lr=lr)
model.compile(optimizer=optimizer,
loss=[YoloLoss(anchors[mask], num_classes=num_classes) for mask in anchor_masks],
run_eagerly=False)
model.fit(train_data,
epochs=start_epoch + epoch_step,
callbacks=callbacks,
# validation_data=val_data,
initial_epoch=start_epoch)
start_epoch += epoch_step
memory_used.append(psutil.virtual_memory().used / 2 ** 30)
gc.collect()
plt.plot(memory_used)
plt.title('Evolution of memory')
plt.xlabel('iteration')
plt.ylabel('memory used (GB)')
def find_neighbors(self, elev_source_files):
neighbors = {
fn: {
'left': '',
'right': '',
'top': '',
'bottom': '',
'top-left': '',
'top-right': '',
'bottom-right': '',
'bottom-left': ''
}
for fn in elev_source_files
}
coords = np.array([parse_fn(fn) for fn in elev_source_files])
# find the left neighbors (and right)
top = 2
bot = 0
left = 1
right = 3
# Sort the coordinates to find neighbors faster
coords1, I = sortrows(coords.copy(), index_out=True, recurse=True)
f_right = lambda c1, c2: c2[bot] == c1[bot] and c2[top] == c1[top] \
and c2[right] > c1[right] and c2[left] <= c1[right]
neighbors = find_neighbors(neighbors, coords1, I, elev_source_files,
f_right, ['right', 'left'])
# Right takes care of left on a grid (should always be true)
coords1, I = sortrows(coords.copy(), i=1, index_out=True, recurse=True)
f_top = lambda c1, c2: c2[left] == c1[left] and c2[right] == c1[right] \
and c2[top] > c1[top] and c2[bot] <= c1[top]
neighbors = find_neighbors(neighbors, coords1, I, elev_source_files,
f_top, ['top', 'bottom'])
# Hard part is done. now for convenience, let's find the rest of the
# neighbors
for key in neighbors.keys():
for tb in ['top', 'bottom']:
for lr in ['left', 'right']:
top_neig = neighbors[key][tb]
if top_neig != '':
neighbors[key]['-'.join([tb, lr])] = \
neighbors[top_neig][lr]
if neighbors[key]['-'.join([tb, lr])] == '' and \
neighbors[key][lr] != '': # try other option
neighbors[key]['-'.join([tb, lr])] = \
neighbors[neighbors[key][lr]][tb]
return neighbors
def training_model(model, callbacks, num_classes=80, step=1):
if step == 1:
batch_size = config.step1_batch_size
learning_rate = config.step1_learning_rate
start_epochs = config.step1_start_epochs
end_epochs = config.step1_end_epochs
else:
batch_size = config.step2_batch_size
learning_rate = config.step2_learning_rate
start_epochs = config.step2_start_epochs
end_epochs = config.step2_end_epochs
anchors = config.yolo_anchors / 416
# Training dataset setting
AUTOTUNE = tf.data.experimental.AUTOTUNE
combined_split = "train+validation"
train_data = tfds.load("voc", split=combined_split)
train_data = train_data.shuffle(1000)
train_data = train_data.map(lambda dataset: parse_aug_fn(dataset),
num_parallel_calls=AUTOTUNE)
train_data = train_data.batch(batch_size)
train_data = train_data.map(
lambda x, y: transform_targets(x, y, anchors, anchor_masks),
num_parallel_calls=AUTOTUNE)
train_data = train_data.prefetch(buffer_size=AUTOTUNE)
# Validation dataset setting
val_data = tfds.load("voc", split=tfds.Split.TEST)
val_data = val_data.map(lambda dataset: parse_fn(dataset),
num_parallel_calls=AUTOTUNE)
val_data = val_data.batch(batch_size)
val_data = val_data.map(
lambda x, y: transform_targets(x, y, anchors, anchor_masks),
num_parallel_calls=AUTOTUNE)
val_data = val_data.prefetch(buffer_size=AUTOTUNE)
# Training
optimizer = tf.keras.optimizers.Adam(lr=learning_rate)
model.compile(optimizer=optimizer,
loss=[
YoloLoss(anchors[mask], num_classes=num_classes)
for mask in anchor_masks
],
run_eagerly=False)
model.fit(train_data,
epochs=end_epochs,
callbacks=callbacks,
validation_data=val_data,
initial_epoch=start_epochs)
def __init__(self, fn, slice_, save_path, overwrite=False):
self.fn = fn
self.coords = parse_fn(fn)
self.slice = slice_
self.save_path = save_path
self.post_fn = '_' + str(slice_).replace(' ', '').replace(',', '-')
# When initialized we also have to create and initialize the files
# that go with this edge
self.fn_coords = self.get_fn('coords')
self.fn_data = self.get_fn('data')
self.fn_done = self.get_fn('done')
self.fn_todo = self.get_fn('todo')
# Open the elevation file and strip out the coordinates, then save init
# data
elev_file = GdalReader(file_name=fn)
elev, = elev_file.raster_layers
gc = elev.grid_coordinates
del elev_file # close file
del elev # make sure it's closed
points = np.meshgrid(gc.x_axis, gc.y_axis)
coordinates = np.column_stack([pts[slice_].ravel() for pts in points])
# flip xy coordinates for regular grid interpolator
coordinates = coordinates[:, ::-1]
init_data = np.zeros(coordinates.shape[0], float)
init_bool = np.zeros(coordinates.shape[0], bool)
if not os.path.exists(self.fn_coords) and not overwrite:
self.save_data(coordinates, 'coords')
if not os.path.exists(self.fn_data) and not overwrite:
self.save_data(init_data, 'data')
if not os.path.exists(self.fn_done) and not overwrite:
self.save_data(init_bool, 'done')
if not os.path.exists(self.fn_todo) and not overwrite:
self.save_data(~init_bool, 'todo')
self.update_metrics()
def process_command(self, command, save_name='custom', index=None):
"""
Processes the hillshading
Parameters
-----------
index : int/slice (optional)
Default: None - process all tiles in source directory. Otherwise,
will only process the index/indices of the files as listed in
self.elev_source_files
"""
if index is not None:
elev_source_files = [self.elev_source_files[index]]
else:
elev_source_files = self.elev_source_files
save_root = os.path.join(self.save_path, save_name)
if not os.path.exists(save_root):
os.makedirs(save_root)
for i, esfile in enumerate(elev_source_files):
try:
status = 'Success' # optimism
# Check if file is locked
lckfn = _get_lockfile_name(esfile)
coords = parse_fn(esfile)
fn = get_fn_from_coords(coords, save_name)
fn = os.path.join(save_root, fn)
if os.path.exists(lckfn): # another process is working on it
print fn, 'is locked'
status = 'locked'
elif os.path.exists(fn):
print fn, 'already exists'
status = 'cached'
else: # lock this tile
print fn, '... calculating ', save_name
fid = file(lckfn, 'w')
fid.close()
# Calculate the custom process for this tile
status = command(esfile, fn)
os.remove(lckfn)
if index is None:
self.custom_status[i] = status
else:
self.custom_status[index] = status
except:
lckfn = _get_lockfile_name(esfile)
try:
os.remove(lckfn)
except:
pass
traceback.print_exc()
print traceback.format_exc()
if index is None:
self.custom_status[i] = "Error " + traceback.format_exc()
else:
self.custom_status[
index] = "Error " + traceback.format_exc()
def calculate_twi(self,
esfile,
save_path,
use_cache=True,
do_edges=False,
skip_uca_twi=False):
"""
Calculates twi for supplied elevation file
Parameters
-----------
esfile : str
Path to elevation file to be processed
save_path: str
Root path to location where TWI will be saved. TWI will be saved in
a subdirectory 'twi'.
use_cache : bool (optional)
Default True. If a temporary file exists (from a previous run),
the cached file will be used. Otherwise, if False, existing files
will be recomputed
do_edges : bool (optional)
See :py:func:`process_twi` for details on this argument.
skip_uca_twi : bool (optional)
Skips the calculation of the UCA and TWI (only calculates the
magnitude and direction)
"""
if os.path.exists(os.path.join(save_path, 'tile_edge.pkl')) and \
self.tile_edge is None:
with open(os.path.join(save_path, 'tile_edge.pkl'), 'r') as fid:
self.tile_edge = cPickle.load(fid)
elif self.tile_edge is None:
self.tile_edge = TileEdgeFile(self.elev_source_files, save_path)
with open(os.path.join(save_path, 'tile_edge.pkl'), 'wb') as fid:
cPickle.dump(self.tile_edge, fid)
status = 'Success' # optimism
# Check if file is locked
lckfn = _get_lockfile_name(esfile)
coords = parse_fn(esfile)
fn = get_fn_from_coords(coords, 'twi')
print '*' * 79
if skip_uca_twi:
print '*' * 10, fn, 'Slope Calculation starting...:', '*' * 10
else:
print '*' * 10, fn, 'TWI Calculation starting...:', '*' * 10
print '*' * 79
if os.path.exists(lckfn): # another process is working on it
print fn, 'is locked'
return fn, "Locked"
else: # lock this tile
fid = file(lckfn, 'w')
fid.close()
dem_proc = DEMProcessor(esfile)
# check if the slope already exists for the file. If yes, we should
# move on to the next tile without doing anything else
if skip_uca_twi \
and os.path.exists(dem_proc.get_full_fn('mag', save_path)
+ '.npz') \
and os.path.exists(dem_proc.get_full_fn('ang', save_path)
+ '.npz'):
print dem_proc.get_full_fn('mag',
save_path) + '.npz', 'already exists'
print dem_proc.get_full_fn('ang',
save_path) + '.npz', 'already exists'
# remove lock file
os.remove(lckfn)
return fn, 'Cached: Slope'
# check if the twi already exists for the file. If not in the edge
# resolution round, we should move on to the next tile
if os.path.exists(dem_proc.get_full_fn('twi', save_path)) \
and (do_edges is False):
print dem_proc.get_full_fn('twi', save_path), 'already exists'
# remove lock file
os.remove(lckfn)
return fn, 'Cached'
# only calculate the slopes and direction if they do not exist in cache
fn_ang = dem_proc.get_full_fn('ang', save_path)
fn_mag = dem_proc.get_full_fn('mag', save_path)
if os.path.exists(fn_ang + '.npz') and os.path.exists(fn_mag + '.npz')\
and not self.overwrite_cache:
dem_proc.load_direction(fn_ang)
dem_proc.load_slope(fn_mag)
dem_proc.find_flats()
else:
if os.path.exists(fn_ang + '.npz') and os.path_exists(fn_mag + '.npz')\
and self.overwrite_cache:
os.remove(fn_ang)
os.remove(fn_mag)
dem_proc.calc_slopes_directions()
dem_proc.save_slope(save_path, raw=True)
dem_proc.save_direction(save_path, raw=True)
if self._DEBUG:
dem_proc.save_slope(save_path, as_int=False)
dem_proc.save_direction(save_path, as_int=False)
if skip_uca_twi:
# remove lock file
os.remove(lckfn)
return fn, status + ":mag-dir-only"
fn_uca = dem_proc.get_full_fn('uca', save_path)
fn_uca_ec = dem_proc.get_full_fn('uca_edge_corrected', save_path)
fn_twi = dem_proc.get_full_fn('twi', save_path)
# check if edge structure exists for this tile and initialize
edge_init_data, edge_init_done, edge_init_todo = \
self.tile_edge.get_edge_init_data(esfile, save_path)
# Check if uca data exists (if yes, we are in the
# edge-resolution round)
uca_init = None
if os.path.exists(fn_uca + '.npz'):
if os.path.exists(fn_uca_ec + '.npz'):
dem_proc.load_uca(fn_uca_ec)
else:
dem_proc.load_uca(fn_uca)
uca_init = dem_proc.uca
if do_edges or uca_init is None:
dem_proc.calc_uca(uca_init=uca_init,
edge_init_data=[
edge_init_data, edge_init_done,
edge_init_todo
])
if uca_init is None:
dem_proc.save_uca(save_path, raw=True)
if self._DEBUG:
# Also save a geotiff for debugging
dem_proc.save_uca(save_path, as_int=False)
else:
if os.path.exists(fn_uca_ec):
os.remove(fn_uca_ec)
dem_proc.save_array(dem_proc.uca,
None,
'uca_edge_corrected',
save_path,
raw=True)
if self._DEBUG:
dem_proc.save_array(dem_proc.uca,
None,
'uca_edge_corrected',
save_path,
as_int=False)
# Saving Edge Data, and updating edges
self.tile_edge.update_edges(esfile, dem_proc)
dem_proc.calc_twi()
if os.path.exists(fn_twi):
os.remove(fn_twi)
dem_proc.save_twi(save_path, raw=False)
# clean up for in case
gc.collect()
# remove lock file
os.remove(lckfn)
# Save last-used dem_proc for debugging purposes
if self._DEBUG:
self.dem_proc = dem_proc
return fn, status
def visualize_neighbors(self, neighbors=None):
if neighbors is None:
neighbors = self.neighbors
import matplotlib.pyplot as plt
coords = np.array([parse_fn(key) for key in neighbors.keys()])
n_coords = [
np.array(
[parse_fn(neighbors[key][side]) for key in neighbors.keys()])
for side in [
'left', 'right', 'top', 'bottom', 'top-right', 'top-left',
'bottom-right', 'bottom-left'
]
]
top = 2
bot = 0
left = 1
right = 3
x = (coords[:, left] + coords[:, right]) / 2.0
y = (coords[:, top] + coords[:, bot]) / 2.0
n_x = [(n_coord[:, left] + n_coord[:, right]) / 2.0 - x
for n_coord in n_coords]
n_y = [(n_coord[:, top] + n_coord[:, bot]) / 2.0 - y
for n_coord in n_coords]
self.fill_percent_done()
colors = np.array([self.percent_done[key] for key in neighbors.keys()])
plt.scatter(x, y, c=colors, s=400, cmap='CMRmap_r')
plt.clim(0, 100)
plt.colorbar()
for coord in coords:
plt.plot([
coord[left], coord[right], coord[right], coord[left],
coord[left]
], [coord[top], coord[top], coord[bot], coord[bot], coord[top]])
for nx, ny in zip(n_x, n_y):
plt.quiver(x,
y,
nx,
ny,
angles='xy',
scale_units='xy',
scale=1,
width=0.005)
plt.xlim(coords[:, left].min(), coords[:, right].max())
plt.ylim(coords[:, bot].min(), coords[:, top].max())
count = 0
for key, edge in self.edges.iteritems():
for side in ['left', 'right', 'top', 'bottom']:
ed = edge[side]
coordinates = ed.get_coordinates()
todo = ed.get('todo')
done = ed.get('done')
data = (ed.get('data') > 0)
y, x = coordinates.T
if side in ['left', 'top']:
plt.plot(x[todo & done & data],
y[todo & done & data],
'bo',
mec='b',
mfc='none',
mew=1,
label='could do (left/top)')
plt.plot(x[todo & ~done],
y[todo & ~done],
'xr',
label='not done, could not do (left/top)')
plt.plot(x[~todo & done],
y[~todo & done],
'<g',
mec='g',
label='done (left/top)')
else:
plt.plot(x[todo & done & data],
y[todo & done & data],
'bs',
mec='b',
mfc='none',
mew=1,
label='could do (right/bot)')
plt.plot(x[todo & ~done],
y[todo & ~done],
'+r',
label='not done, could not do (right/bot)')
plt.plot(x[~todo & done],
y[~todo & done],
'>g',
mec='g',
label='done (right/bot)')
if count == 0:
plt.plot(x[0],
y[0],
'bs',
mec='b',
mfc='none',
mew=1,
label='could do (right/bot)')
plt.plot(x[0],
y[0],
'+r',
label='not done, could not do (right/bot)')
plt.plot(x[0],
y[0],
'>g',
mec='g',
label='done (right/bot)')
plt.legend(loc=0)
count += 1
# clean up
del coordinates
del todo
del done
del data
