def __init__(self):
# Training set path
self.train_annotation_path = args.yolo_train_file
# Validation set path
self.val_annotation_path = args.yolo_val_file
# Detecter setting
self.classes_path = args.classes_file
self.anchors_path = args.anchors_file
self.class_names = self.get_classes(self.classes_path)
self.num_classes = len(self.class_names)
self.anchors = self.get_anchors(self.anchors_path)
self.input_shape = (416, 416) # multiple of 32, hw
self.shape = (416, 416, 3)
# training batch size
self.step1_batch_size = 32
self.step2_batch_size = 8 # note that more GPU memory is required after unfreezing the body
self.is_tiny_version = len(self.anchors) == 6 # default setting
if self.is_tiny_version:
self.yolo_model, self.yolo_body = self.create_model_tiny(yolo_weights_path='model_data/tiny_yolo_weights.h5')
else:
self.yolo_model, self.yolo_body = self.create_model(yolo_weights_path='model_data/yolo_weights.h5')
# PIL setting
self.image_size = (1280, 720, 3)
self.colors = np.array(cm.hsv(np.linspace(0, 1, self.num_classes)).tolist()) * 255
# mAP setting
self.min_overlap = 0.5
self.gt_counter_per_class = defaultdict(int) # dictionary with counter per class
# Temp file path
self.tmp_gt_files_path = "tmp_gt_files"
self.tmp_pred_files_path = "tmp_pred_files"
if not os.path.exists(self.tmp_gt_files_path):
os.mkdir(self.tmp_gt_files_path)
self.train_data, self.val_data, self.val_images = self.read_txt_file()
shutil.copytree(self.tmp_gt_files_path, self.tmp_gt_files_path + '_org')
if not os.path.exists(self.tmp_pred_files_path):
os.mkdir(self.tmp_pred_files_path)
# Highlight images
np.random.seed(10101)
images_choose = [self.val_images[i] for i in np.random.randint(0, len(self.val_images), 50)]
self.eval_save_images_id = [os.path.split(img_path)[-1].split('.')[0] for img_path in images_choose]
# Evaluate setting parameter
self.score = 0.3
self.iou = 0.45
self.input_image_shape = K.placeholder(shape=(2, ))
self.boxes, self.scores, self.classes, self.eval_inputs = yolo_eval_v2(self.yolo_body.output_shape, self.anchors,
len(self.class_names), self.input_image_shape,
score_threshold=self.score, iou_threshold=self.iou)
self.sess = K.get_session()
# Create tensorboard logger
self.callback = TensorBoard(LOGS_PATH)
self.callback.set_model(self.yolo_model)
def plot_convergence(result_list,
true_minimum=None,
yscale=None,
title="Convergence plot"):
ax = plt.gca()
ax.set_title(title)
ax.set_xlabel("Number of calls $n$")
ax.set_ylabel(r"$\min f(x)$ after $n$ calls")
ax.grid()
if yscale is not None:
ax.set_yscale(yscale)
colors = cm.hsv(np.linspace(0.25, 1.0, len(result_list)))
for results, color in zip(result_list, colors):
name, results = results
n_calls = len(results[0].x_iters)
iterations = range(1, n_calls + 1)
mins = [[np.min(r.func_vals[:i]) for i in iterations] for r in results]
ax.plot(iterations, np.mean(mins, axis=0), c=color, label=name)
#ax.errorbar(iterations, np.mean(mins, axis=0),
# yerr=np.std(mins, axis=0), c=color, label=name)
if true_minimum:
ax.axhline(true_minimum,
linestyle="--",
color="r",
lw=1,
label="True minimum")
ax.legend(loc="best")
return ax
def __init__(
self,
data,
anchors,
config,
tensorboard=None,
verbose=1
):
""" Evaluate a given dataset using a given model at the end of every epoch during training.
# Arguments
generator : The generator that represents the dataset to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
tensorboard : Instance of keras.callbacks.TensorBoard used to log the mAP value.
weighted_average : Compute the mAP using the weighted average of precisions among classes.
verbose : Set the verbosity level, by default this is set to 1.
"""
self.val_data = data
self.tensorboard = tensorboard
self.verbose = verbose
self.vis_id=[i for i in np.random.randint(0, len(data), 200)]
self.batch_size = max(config['batch_size']//2,1)
self.colors = np.array(cm.hsv(np.linspace(0, 1, 10)).tolist()) * 255
self.input_shape = (config['input_size'], config['input_size']) # multiple of 32, hw
self.config=config
self.word_embed=spacy.load(config['word_embed'])
self.word_len = config['word_len']
self.anchors=anchors
self.use_nls=config['use_nls']
# mAP setting
self.det_acc_thresh = config['det_acc_thresh']
self.seg_min_overlap=config['segment_thresh']
if self.tensorboard is not None:
self.log_images=config['log_images']
else:
self.log_images=0
self.input_image_shape = K.placeholder(shape=(2,))
self.sess = K.get_session()
self.eval_save_images_id = [i for i in np.random.randint(0, len(self.val_data), 200)]
super(Evaluate, self).__init__()
def plot_layer_2d(mat):
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
fig = plt.figure()
ax = fig.add_subplot(111)
x, y = np.meshgrid(range(mat.shape[1]), range(mat.shape[0]))
xs = x.reshape(-1)
ys = y.reshape(-1)
cols = mat.reshape(-1) - np.min(mat)
colors = cm.hsv(cols / max(cols))
colormap = cm.ScalarMappable(cmap=cm.hsv)
colormap.set_array(cols)
ax.scatter(xs, ys, c=colors)
cb = fig.colorbar(colormap)
plt.title("subtracted:{}".format(np.min(mat)))
plt.show()
def plot_layer_3d(mat):
import matplotlib.pyplot as plt
from matplotlib.pyplot import cm
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
x, y, z = np.meshgrid(range(mat.shape[1]), range(mat.shape[0]),
range(mat.shape[2]))
xs = x.reshape(-1)
ys = y.reshape(-1)
zs = z.reshape(-1)
cols = mat.reshape(-1) - np.min(mat)
colors = cm.hsv(cols / max(cols))
colormap = cm.ScalarMappable(cmap=cm.hsv)
colormap.set_array(cols)
ax.scatter(xs, ys, zs, c=colors)
cb = fig.colorbar(colormap)
plt.title("subtracted:{}".format(np.min(mat)))
plt.show()
#prediction df for kaggle submission
subm2 = pd.read_csv('sample_submission.csv')
subm2.drop('SalePrice', axis=1, inplace=True)
test_df = test_df1.loc[:, test_df1.columns != 'SalePrice'].astype(float)
ridge_pred = pd.Series(ridge.predict(test_df))
subm2['SalePrice'] = ridge_pred.apply(lambda price: exp(price))
subm2.to_csv('subm2.csv', index=False)
#...........feature importance
feature_importance = list(zip(X.columns, rf.feature_importances_))
dtype = [('feature', 'S10'), ('importance', 'float')]
feature_importance = np.array(feature_importance, dtype=dtype)
feature_sort = np.sort(feature_importance, order='importance')[::-1]
name, score = zip(*list(feature_sort))
pd.DataFrame({'name': name, 'score': score})[:25].plot.bar(x='name', y='score')
plt.title('Feature Importance from Random Forest')
#....feature importance
from matplotlib.pyplot import cm
feature_importance = list(zip(X.columns, gbr.feature_importances_))
dtype = [('feature', 'S10'), ('importance', 'float')]
feature_importance = np.array(feature_importance, dtype=dtype)
feature_sort = np.sort(feature_importance, order='importance')[::-1]
name, score = zip(*list(feature_sort))
colors = cm.hsv(y / float(max(y)))
pd.DataFrame({
'name': name,
'score': score
})[:15].plot.bar(x='name', y='score', color=colors)
plt.title('Feature Importance from Gradient Boosting Regressor')
def get_highlighted_transitions_pos_with_colors(self,
highlighted_transitions,
filter_by_nb_keys=None):
obs_labels = self._get_dense_obs_labels()
keys_idxs = tuple(idx for idx, label in enumerate(obs_labels)
if 'key_' in label)
transitions_targets_unique = set()
for source_state, target_states in highlighted_transitions.items():
source_obs = self.state2obs(source_state)
nb_keys_taken = np.sum(1 - np.take(source_obs, keys_idxs))
if (filter_by_nb_keys is not None
and nb_keys_taken != filter_by_nb_keys):
continue
for target_state in target_states.values():
transitions_targets_unique.add(target_state)
nb_unique_target_states = len(transitions_targets_unique)
colors = {
transition_target: (255 * np.array(to_rgb(color))).astype(int)
for transition_target, color in zip(
transitions_targets_unique,
cm.hsv(np.linspace(0, 1, 1 + nb_unique_target_states)))
}
invalid_transition_sources = {room_loc: [] for room_loc in self.rooms}
invalid_transition_targets = {room_loc: [] for room_loc in self.rooms}
room_loc_idxs = obs_labels.index('.loc_y'), obs_labels.index('.loc_x')
agent_pos_idxs = (obs_labels.index('agent_y'),
obs_labels.index('agent_x'))
for state, highlighted_transitions in highlighted_transitions.items():
obs = self.state2obs(state)
room_loc = tuple(np.take(obs, room_loc_idxs))
agent_pos = tuple(np.take(obs, agent_pos_idxs))
keys_taken = np.sum(1 - np.take(obs, keys_idxs)).astype(int)
if (filter_by_nb_keys is not None
and keys_taken != filter_by_nb_keys):
continue
for action, target_state in highlighted_transitions.items():
target_obs = self.state2obs(target_state)
next_room_loc = tuple(np.take(target_obs, room_loc_idxs))
next_agent_pos = tuple(np.take(target_obs, agent_pos_idxs))
color = colors[target_state]
invalid_transition_sources[room_loc].append(
(agent_pos, action, color))
if (next_agent_pos[0] < 0
or next_agent_pos[0] >= self.room.size[0]
or next_agent_pos[1] < 0
or next_agent_pos[1] >= self.room.size[1]
or next_room_loc not in self.rooms):
continue
invalid_transition_targets[next_room_loc].append(
(next_agent_pos, color))
return invalid_transition_sources, invalid_transition_targets
plot_surface(pd.DataFrame([T.loc[indx]]),
data_pts=True,
metric_name="DIP Rate",
zlim=(-0.35, 0.35),
elev=24,
azim=10,
alpha=.9,
fname=None,
path=fil.split('2018')[0])
plt.xlim((-11, 0))
plt.ylim((-11, 0))
plt.savefig('mx_dose_{:.2f}_surface.pdf'.format(mx_dose))
plt.savefig('L2-norm parameter fits.pdf')
cols = cm.hsv(np.linspace(0, 1, len(T['beta_true'].unique())))
plt.figure()
ax = []
fit_alg = ['pso_mcmc']
for e, fa in enumerate(fit_alg):
sub_T = T[T['MCMC_converge'] == 1 & (T['fit_algorithm'] == fa)
& (T['log_alpha1_true'] == 0.) & (T['R2'] > .9)]
ax.append(plt.subplot(1, 2, e + 1))
x = sub_T.groupby(['beta_true', 'max_conc_d1'
])[['beta_std']].mean().index.levels[0].values
for e1, xx in enumerate(x):
ax[-1].plot(sub_T.groupby(['beta_true', 'max_conc_d1'
])[['beta_std']].mean().loc[xx],
c=cols[e1],
label=r'$\beta=$' + str(xx))
