def create_multi_scale_dataset(batch_size): train_data_dict = {} for scale in [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]: grid_size = scale // 32 anchors = config.yolo_anchors / scale # Training dataset setting AUTOTUNE = tf.data.experimental.AUTOTUNE # 自動調整模式 combined_split = tfds.Split.TRAIN + tfds.Split.VALIDATION train_data = tfds.load("voc2007", split=combined_split, data_dir=dataset_path) # 取得訓練數據 train_data = train_data.shuffle(1000) # 打散資料集 train_data = train_data.map(lambda dataset: parse_aug_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE) train_data = train_data.batch(batch_size) train_data = train_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size), num_parallel_calls=AUTOTUNE) train_data = train_data.prefetch(buffer_size=AUTOTUNE) train_data_dict[scale] = train_data # Validation dataset setting grid_size = 416 // 32 anchors = config.yolo_anchors / 416 val_data = tfds.load("voc2007", split=tfds.Split.TEST, data_dir=dataset_path) val_data = val_data.map(lambda dataset: parse_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE) val_data = val_data.batch(batch_size) val_data = val_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size), num_parallel_calls=AUTOTUNE) val_data = val_data.prefetch(buffer_size=AUTOTUNE) return train_data_dict, val_data
def predict(image_paths): X = np.array( list( map(lambda path: val_preprocess_mobilenet(parse_fn(path)), image_paths))) scores = model.predict(X, batch_size=32, verbose=1) return scores
def multi_scale_training_model(model, callbacks, num_classes=80, step=1): if step == 1: batch_size = 30 epoch_step = 1 else: batch_size = 8 epoch_step = 10 start_epoch = 0 # for lr in [1e-3, 1e-3, 1e-4]: # for scale in [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]: memory_used = [] for lr in [1e-3]: for scale in [320, 352, 384]: print('scale: {}, learning rate: {}'.format(scale, lr)) # if scale == 416 and lr == 1e-4: # print('Last round') # epoch_step = 50 anchors = config.yolo_anchors / scale grid_size = scale // 32 # Training dataset setting AUTOTUNE = tf.data.experimental.AUTOTUNE # 自動調整模式 combined_split = tfds.Split.TRAIN + tfds.Split.VALIDATION train_data = tfds.load("voc2007", split=combined_split, data_dir='/home/share/dataset/tensorflow-datasets') # 取得訓練數據 train_data = train_data.shuffle(1000) # 打散資料集 train_data = train_data.map(lambda dataset: parse_aug_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE) train_data = train_data.batch(batch_size) train_data = train_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size), num_parallel_calls=AUTOTUNE) train_data = train_data.prefetch(buffer_size=AUTOTUNE) # Validation dataset setting val_data = tfds.load("voc2007", split=tfds.Split.TEST, data_dir='/home/share/dataset/tensorflow-datasets') val_data = val_data.map(lambda dataset: parse_fn(dataset, (scale, scale)), num_parallel_calls=AUTOTUNE) val_data = val_data.batch(batch_size) val_data = val_data.map(lambda x, y: transform_targets(x, y, anchors, anchor_masks, grid_size), num_parallel_calls=AUTOTUNE) val_data = val_data.prefetch(buffer_size=AUTOTUNE) # Training optimizer = tf.keras.optimizers.Adam(lr=lr) model.compile(optimizer=optimizer, loss=[YoloLoss(anchors[mask], num_classes=num_classes) for mask in anchor_masks], run_eagerly=False) model.fit(train_data, epochs=start_epoch + epoch_step, callbacks=callbacks, # validation_data=val_data, initial_epoch=start_epoch) start_epoch += epoch_step memory_used.append(psutil.virtual_memory().used / 2 ** 30) gc.collect() plt.plot(memory_used) plt.title('Evolution of memory') plt.xlabel('iteration') plt.ylabel('memory used (GB)')
def find_neighbors(self, elev_source_files): neighbors = { fn: { 'left': '', 'right': '', 'top': '', 'bottom': '', 'top-left': '', 'top-right': '', 'bottom-right': '', 'bottom-left': '' } for fn in elev_source_files } coords = np.array([parse_fn(fn) for fn in elev_source_files]) # find the left neighbors (and right) top = 2 bot = 0 left = 1 right = 3 # Sort the coordinates to find neighbors faster coords1, I = sortrows(coords.copy(), index_out=True, recurse=True) f_right = lambda c1, c2: c2[bot] == c1[bot] and c2[top] == c1[top] \ and c2[right] > c1[right] and c2[left] <= c1[right] neighbors = find_neighbors(neighbors, coords1, I, elev_source_files, f_right, ['right', 'left']) # Right takes care of left on a grid (should always be true) coords1, I = sortrows(coords.copy(), i=1, index_out=True, recurse=True) f_top = lambda c1, c2: c2[left] == c1[left] and c2[right] == c1[right] \ and c2[top] > c1[top] and c2[bot] <= c1[top] neighbors = find_neighbors(neighbors, coords1, I, elev_source_files, f_top, ['top', 'bottom']) # Hard part is done. now for convenience, let's find the rest of the # neighbors for key in neighbors.keys(): for tb in ['top', 'bottom']: for lr in ['left', 'right']: top_neig = neighbors[key][tb] if top_neig != '': neighbors[key]['-'.join([tb, lr])] = \ neighbors[top_neig][lr] if neighbors[key]['-'.join([tb, lr])] == '' and \ neighbors[key][lr] != '': # try other option neighbors[key]['-'.join([tb, lr])] = \ neighbors[neighbors[key][lr]][tb] return neighbors
def training_model(model, callbacks, num_classes=80, step=1): if step == 1: batch_size = config.step1_batch_size learning_rate = config.step1_learning_rate start_epochs = config.step1_start_epochs end_epochs = config.step1_end_epochs else: batch_size = config.step2_batch_size learning_rate = config.step2_learning_rate start_epochs = config.step2_start_epochs end_epochs = config.step2_end_epochs anchors = config.yolo_anchors / 416 # Training dataset setting AUTOTUNE = tf.data.experimental.AUTOTUNE combined_split = "train+validation" train_data = tfds.load("voc", split=combined_split) train_data = train_data.shuffle(1000) train_data = train_data.map(lambda dataset: parse_aug_fn(dataset), num_parallel_calls=AUTOTUNE) train_data = train_data.batch(batch_size) train_data = train_data.map( lambda x, y: transform_targets(x, y, anchors, anchor_masks), num_parallel_calls=AUTOTUNE) train_data = train_data.prefetch(buffer_size=AUTOTUNE) # Validation dataset setting val_data = tfds.load("voc", split=tfds.Split.TEST) val_data = val_data.map(lambda dataset: parse_fn(dataset), num_parallel_calls=AUTOTUNE) val_data = val_data.batch(batch_size) val_data = val_data.map( lambda x, y: transform_targets(x, y, anchors, anchor_masks), num_parallel_calls=AUTOTUNE) val_data = val_data.prefetch(buffer_size=AUTOTUNE) # Training optimizer = tf.keras.optimizers.Adam(lr=learning_rate) model.compile(optimizer=optimizer, loss=[ YoloLoss(anchors[mask], num_classes=num_classes) for mask in anchor_masks ], run_eagerly=False) model.fit(train_data, epochs=end_epochs, callbacks=callbacks, validation_data=val_data, initial_epoch=start_epochs)
def __init__(self, fn, slice_, save_path, overwrite=False): self.fn = fn self.coords = parse_fn(fn) self.slice = slice_ self.save_path = save_path self.post_fn = '_' + str(slice_).replace(' ', '').replace(',', '-') # When initialized we also have to create and initialize the files # that go with this edge self.fn_coords = self.get_fn('coords') self.fn_data = self.get_fn('data') self.fn_done = self.get_fn('done') self.fn_todo = self.get_fn('todo') # Open the elevation file and strip out the coordinates, then save init # data elev_file = GdalReader(file_name=fn) elev, = elev_file.raster_layers gc = elev.grid_coordinates del elev_file # close file del elev # make sure it's closed points = np.meshgrid(gc.x_axis, gc.y_axis) coordinates = np.column_stack([pts[slice_].ravel() for pts in points]) # flip xy coordinates for regular grid interpolator coordinates = coordinates[:, ::-1] init_data = np.zeros(coordinates.shape[0], float) init_bool = np.zeros(coordinates.shape[0], bool) if not os.path.exists(self.fn_coords) and not overwrite: self.save_data(coordinates, 'coords') if not os.path.exists(self.fn_data) and not overwrite: self.save_data(init_data, 'data') if not os.path.exists(self.fn_done) and not overwrite: self.save_data(init_bool, 'done') if not os.path.exists(self.fn_todo) and not overwrite: self.save_data(~init_bool, 'todo') self.update_metrics()
def process_command(self, command, save_name='custom', index=None): """ Processes the hillshading Parameters ----------- index : int/slice (optional) Default: None - process all tiles in source directory. Otherwise, will only process the index/indices of the files as listed in self.elev_source_files """ if index is not None: elev_source_files = [self.elev_source_files[index]] else: elev_source_files = self.elev_source_files save_root = os.path.join(self.save_path, save_name) if not os.path.exists(save_root): os.makedirs(save_root) for i, esfile in enumerate(elev_source_files): try: status = 'Success' # optimism # Check if file is locked lckfn = _get_lockfile_name(esfile) coords = parse_fn(esfile) fn = get_fn_from_coords(coords, save_name) fn = os.path.join(save_root, fn) if os.path.exists(lckfn): # another process is working on it print fn, 'is locked' status = 'locked' elif os.path.exists(fn): print fn, 'already exists' status = 'cached' else: # lock this tile print fn, '... calculating ', save_name fid = file(lckfn, 'w') fid.close() # Calculate the custom process for this tile status = command(esfile, fn) os.remove(lckfn) if index is None: self.custom_status[i] = status else: self.custom_status[index] = status except: lckfn = _get_lockfile_name(esfile) try: os.remove(lckfn) except: pass traceback.print_exc() print traceback.format_exc() if index is None: self.custom_status[i] = "Error " + traceback.format_exc() else: self.custom_status[ index] = "Error " + traceback.format_exc()
def calculate_twi(self, esfile, save_path, use_cache=True, do_edges=False, skip_uca_twi=False): """ Calculates twi for supplied elevation file Parameters ----------- esfile : str Path to elevation file to be processed save_path: str Root path to location where TWI will be saved. TWI will be saved in a subdirectory 'twi'. use_cache : bool (optional) Default True. If a temporary file exists (from a previous run), the cached file will be used. Otherwise, if False, existing files will be recomputed do_edges : bool (optional) See :py:func:`process_twi` for details on this argument. skip_uca_twi : bool (optional) Skips the calculation of the UCA and TWI (only calculates the magnitude and direction) """ if os.path.exists(os.path.join(save_path, 'tile_edge.pkl')) and \ self.tile_edge is None: with open(os.path.join(save_path, 'tile_edge.pkl'), 'r') as fid: self.tile_edge = cPickle.load(fid) elif self.tile_edge is None: self.tile_edge = TileEdgeFile(self.elev_source_files, save_path) with open(os.path.join(save_path, 'tile_edge.pkl'), 'wb') as fid: cPickle.dump(self.tile_edge, fid) status = 'Success' # optimism # Check if file is locked lckfn = _get_lockfile_name(esfile) coords = parse_fn(esfile) fn = get_fn_from_coords(coords, 'twi') print '*' * 79 if skip_uca_twi: print '*' * 10, fn, 'Slope Calculation starting...:', '*' * 10 else: print '*' * 10, fn, 'TWI Calculation starting...:', '*' * 10 print '*' * 79 if os.path.exists(lckfn): # another process is working on it print fn, 'is locked' return fn, "Locked" else: # lock this tile fid = file(lckfn, 'w') fid.close() dem_proc = DEMProcessor(esfile) # check if the slope already exists for the file. If yes, we should # move on to the next tile without doing anything else if skip_uca_twi \ and os.path.exists(dem_proc.get_full_fn('mag', save_path) + '.npz') \ and os.path.exists(dem_proc.get_full_fn('ang', save_path) + '.npz'): print dem_proc.get_full_fn('mag', save_path) + '.npz', 'already exists' print dem_proc.get_full_fn('ang', save_path) + '.npz', 'already exists' # remove lock file os.remove(lckfn) return fn, 'Cached: Slope' # check if the twi already exists for the file. If not in the edge # resolution round, we should move on to the next tile if os.path.exists(dem_proc.get_full_fn('twi', save_path)) \ and (do_edges is False): print dem_proc.get_full_fn('twi', save_path), 'already exists' # remove lock file os.remove(lckfn) return fn, 'Cached' # only calculate the slopes and direction if they do not exist in cache fn_ang = dem_proc.get_full_fn('ang', save_path) fn_mag = dem_proc.get_full_fn('mag', save_path) if os.path.exists(fn_ang + '.npz') and os.path.exists(fn_mag + '.npz')\ and not self.overwrite_cache: dem_proc.load_direction(fn_ang) dem_proc.load_slope(fn_mag) dem_proc.find_flats() else: if os.path.exists(fn_ang + '.npz') and os.path_exists(fn_mag + '.npz')\ and self.overwrite_cache: os.remove(fn_ang) os.remove(fn_mag) dem_proc.calc_slopes_directions() dem_proc.save_slope(save_path, raw=True) dem_proc.save_direction(save_path, raw=True) if self._DEBUG: dem_proc.save_slope(save_path, as_int=False) dem_proc.save_direction(save_path, as_int=False) if skip_uca_twi: # remove lock file os.remove(lckfn) return fn, status + ":mag-dir-only" fn_uca = dem_proc.get_full_fn('uca', save_path) fn_uca_ec = dem_proc.get_full_fn('uca_edge_corrected', save_path) fn_twi = dem_proc.get_full_fn('twi', save_path) # check if edge structure exists for this tile and initialize edge_init_data, edge_init_done, edge_init_todo = \ self.tile_edge.get_edge_init_data(esfile, save_path) # Check if uca data exists (if yes, we are in the # edge-resolution round) uca_init = None if os.path.exists(fn_uca + '.npz'): if os.path.exists(fn_uca_ec + '.npz'): dem_proc.load_uca(fn_uca_ec) else: dem_proc.load_uca(fn_uca) uca_init = dem_proc.uca if do_edges or uca_init is None: dem_proc.calc_uca(uca_init=uca_init, edge_init_data=[ edge_init_data, edge_init_done, edge_init_todo ]) if uca_init is None: dem_proc.save_uca(save_path, raw=True) if self._DEBUG: # Also save a geotiff for debugging dem_proc.save_uca(save_path, as_int=False) else: if os.path.exists(fn_uca_ec): os.remove(fn_uca_ec) dem_proc.save_array(dem_proc.uca, None, 'uca_edge_corrected', save_path, raw=True) if self._DEBUG: dem_proc.save_array(dem_proc.uca, None, 'uca_edge_corrected', save_path, as_int=False) # Saving Edge Data, and updating edges self.tile_edge.update_edges(esfile, dem_proc) dem_proc.calc_twi() if os.path.exists(fn_twi): os.remove(fn_twi) dem_proc.save_twi(save_path, raw=False) # clean up for in case gc.collect() # remove lock file os.remove(lckfn) # Save last-used dem_proc for debugging purposes if self._DEBUG: self.dem_proc = dem_proc return fn, status
def visualize_neighbors(self, neighbors=None): if neighbors is None: neighbors = self.neighbors import matplotlib.pyplot as plt coords = np.array([parse_fn(key) for key in neighbors.keys()]) n_coords = [ np.array( [parse_fn(neighbors[key][side]) for key in neighbors.keys()]) for side in [ 'left', 'right', 'top', 'bottom', 'top-right', 'top-left', 'bottom-right', 'bottom-left' ] ] top = 2 bot = 0 left = 1 right = 3 x = (coords[:, left] + coords[:, right]) / 2.0 y = (coords[:, top] + coords[:, bot]) / 2.0 n_x = [(n_coord[:, left] + n_coord[:, right]) / 2.0 - x for n_coord in n_coords] n_y = [(n_coord[:, top] + n_coord[:, bot]) / 2.0 - y for n_coord in n_coords] self.fill_percent_done() colors = np.array([self.percent_done[key] for key in neighbors.keys()]) plt.scatter(x, y, c=colors, s=400, cmap='CMRmap_r') plt.clim(0, 100) plt.colorbar() for coord in coords: plt.plot([ coord[left], coord[right], coord[right], coord[left], coord[left] ], [coord[top], coord[top], coord[bot], coord[bot], coord[top]]) for nx, ny in zip(n_x, n_y): plt.quiver(x, y, nx, ny, angles='xy', scale_units='xy', scale=1, width=0.005) plt.xlim(coords[:, left].min(), coords[:, right].max()) plt.ylim(coords[:, bot].min(), coords[:, top].max()) count = 0 for key, edge in self.edges.iteritems(): for side in ['left', 'right', 'top', 'bottom']: ed = edge[side] coordinates = ed.get_coordinates() todo = ed.get('todo') done = ed.get('done') data = (ed.get('data') > 0) y, x = coordinates.T if side in ['left', 'top']: plt.plot(x[todo & done & data], y[todo & done & data], 'bo', mec='b', mfc='none', mew=1, label='could do (left/top)') plt.plot(x[todo & ~done], y[todo & ~done], 'xr', label='not done, could not do (left/top)') plt.plot(x[~todo & done], y[~todo & done], '<g', mec='g', label='done (left/top)') else: plt.plot(x[todo & done & data], y[todo & done & data], 'bs', mec='b', mfc='none', mew=1, label='could do (right/bot)') plt.plot(x[todo & ~done], y[todo & ~done], '+r', label='not done, could not do (right/bot)') plt.plot(x[~todo & done], y[~todo & done], '>g', mec='g', label='done (right/bot)') if count == 0: plt.plot(x[0], y[0], 'bs', mec='b', mfc='none', mew=1, label='could do (right/bot)') plt.plot(x[0], y[0], '+r', label='not done, could not do (right/bot)') plt.plot(x[0], y[0], '>g', mec='g', label='done (right/bot)') plt.legend(loc=0) count += 1 # clean up del coordinates del todo del done del data