def load_datum(datum, cam):
refer = "\n".join([" ".join(refexp) for refexp in datum['refexp']])
# pprint(datum)
n_steps = len(datum['act_deg_list'])
image_path = datum['img_src']
waldo_position_lng = datum['gt_longitude']
waldo_position_lat = datum['gt_latitude']
pred_lng = datum['pred_xlng_deg']
pred_lat = datum['pred_ylat_deg']
gt_x, gt_y = get_coordinates(waldo_position_lng, waldo_position_lat)
pred_x, pred_y = get_coordinates(pred_lng, pred_lat)
print("image_path", image_path)
print(refer)
print("gt:", waldo_position_lat, waldo_position_lng)
print("gt xy:", gt_x, gt_y)
print("pred:", pred_lat, pred_lng)
print("pred xy:", pred_x, pred_y)
print('n_steps:', n_steps)
panels = {}
panels['original'] = cv2.imread(image_path, cv2.IMREAD_COLOR)
panels['grid'] = cv2.imread(grid_path, cv2.IMREAD_COLOR)
panels['mask'] = np.zeros((full_h, full_w, 3), dtype='uint8')
#add_square(panels['original'], gt_x, gt_y, color=color_green)
#add_square(panels['original'], pred_x, pred_y, color=color_red)
#add_square(panels['grid'], gt_x, gt_y, color=color_green)
#add_square(panels['grid'], pred_x, pred_y, color=color_red)
start_loc = datum['act_deg_list'][0][0]
start_x, start_y = get_coordinates(start_loc[0],
start_loc[1],
full_w=full_w,
full_h=full_h)
start_fov, _ = get_nearest(nodes, start_x, start_y)
gt_path = [start_fov]
for kk, act_list in enumerate(datum['act_deg_list']):
act = act_list[-1]
lng, lat = act
x, y = get_coordinates(lng, lat, full_w=full_w, full_h=full_h)
min_n, _ = get_nearest(nodes, x, y)
if gt_path[-1] != min_n:
gt_path.append(min_n)
if len(gt_path) >= 2:
start = gt_path[-2]
end = gt_path[-1]
intermediate_path = cam.paths[start][end]
panels['grid'] = visualize_path(intermediate_path, cam.nodes,
cam.edges, panels['grid'])
print('intermediate_path:', intermediate_path)
print('gt_path:', gt_path)
return image_path, gt_path, panels, waldo_position_lng, waldo_position_lat, pred_lng, pred_lat
def __init__(self, config):
self.config = config
self.device = config.device
self.max_itr = config.max_itr
self.batch_size = config.batch_size
self.img_size = config.img_size
self.dim_z = config.dim_z
self.dim_c = config.dim_c
self.scale = config.scale
self.n_gen = config.n_gen
self.start_itr = 1
dataloader = DataLoader(
config.data_root, config.dataset_name, config.img_size, config.batch_size, config.with_label
)
train_loader, test_loader = dataloader.get_loader(only_train=True)
self.dataloader = train_loader
self.dataloader = endless_dataloader(self.dataloader)
self.generator = Generator(config).to(config.device)
self.discriminator = Discriminator(config).to(config.device)
self.optim_g = torch.optim.Adam(self.generator.parameters(), lr=config.lr_g, betas=(config.beta1, config.beta2))
self.optim_d = torch.optim.Adam(self.discriminator.parameters(), lr=config.lr_d, betas=(config.beta1, config.beta2))
self.criterion = GANLoss()
if not self.config.checkpoint_path == '':
self._load_models(self.config.checkpoint_path)
self.x, self.y, self.r = get_coordinates(self.img_size, self.img_size, self.scale, self.batch_size)
self.x, self.y, self.r = self.x.to(self.device), self.y.to(self.device), self.r.to(self.device)
self.writer = SummaryWriter(log_dir=config.log_dir)
def configure(self):
coordinates = get_coordinates('main', self.root)
self.root.geometry('%dx%d+%d+%d' %
(coordinates['w'], coordinates['h'],
coordinates['x'], coordinates['y']))
self.root.resizable(True, True)
self.root.configure(background='#48B484')
def plot_vars(f_step, projection, load_all=False):
# The one employed for the figure name when exported
variable_name = 'gph_t_850'
# Build the name of the output image
run_string, _ = get_run()
if load_all:
f_steps = list(range(0, 79)) + list(range(81, 121, 3))
else:
f_steps = [f_step]
filenames = ['/tmp/' + projection + '_' + variable_name +
'_%s_%03d.png' % (run_string, f_step) for f_step in f_steps]
test_filenames = [os.path.exists(f) for f in filenames]
if all(test_filenames): # means the files already exist
return filenames
# otherwise do the plots
dset = get_dset(vars_3d=['t@850', 'fi@500'], f_times=f_steps).squeeze()
# Add a fictictious 1-D time dimension just to avoid problems
if 'step' not in dset.dims.keys():
dset = dset.expand_dims('step')
#
dset = subset_arrays(dset, projection)
time = pd.to_datetime(dset.valid_time.values)
cum_hour = dset.step.values.astype(int)
temp_850 = dset['t'] - 273.15
z_500 = dset['z']
gph_500 = mpcalc.geopotential_to_height(z_500)
gph_500 = xr.DataArray(gph_500.magnitude, coords=z_500.coords,
attrs={'standard_name': 'geopotential height',
'units': gph_500.units})
levels_temp = np.arange(-30., 30., 1.)
levels_gph = np.arange(4700., 6000., 70.)
lon, lat = get_coordinates(temp_850)
lon2d, lat2d = np.meshgrid(lon, lat)
cmap = get_colormap('temp')
args = dict(filenames=filenames, projection=projection, levels_temp=levels_temp,
cmap=cmap, lon2d=lon2d, lat2d=lat2d, lon=lon, lat=lat, temp_850=temp_850.values,
gph_500=gph_500.values, levels_gph=levels_gph, time=time, run_string=run_string)
if load_all:
single_plot_param = partial(single_plot, **args)
iterator = range(0, len(f_steps))
pool = Pool(cpu_count())
results = pool.map(single_plot_param, iterator)
pool.close()
pool.join()
else:
results = single_plot(0, **args)
return results
def main():
#开始测试
for file in dirs:
image_path = os.path.join(text_floder, os.path.basename(file))
#print(os.path.basename(file))
#print(image_path)
# 加载灰度图像和灰度图片
rgb_image = load_image(image_path, grayscale=False, color_mode = "rgb")
gray_image = load_image(image_path, grayscale=True,color_mode = "grayscale")
# 去掉维度为1的维度（只留下宽高，去掉灰度维度）
gray_image = np.squeeze(gray_image)
gray_image = gray_image.astype("uint8")
faces = detect_faces(face_detection, gray_image)
#print("-----")
#print(len(faces))
for face_coordinates in faces:
#获取人脸在图像中的矩形坐标的对角两点
x1, x2, y1, y2 = get_coordinates(face_coordinates)
#print(x1, x2, y1, y2 )
# 截取人脸图像像素数组
gray_face = gray_image[y1:y2, x1:x2]
try:
# 将人脸reshape模型需要的尺寸
gray_face = cv2.resize(gray_face,(emotion_target_size))
except:
continue
gray_face = preprocessing_input(gray_face)
gray_face = np.expand_dims(gray_face, 0)
gray_face = np.expand_dims(gray_face, -1)
# print(gray_face.shape)
# 预测
emotion_label_arg = np.argmax(emotion_classifier.predict(gray_face))
emotion_text = emotion_labels[emotion_label_arg]
color = (255,0,0)
# 画边框
draw_bounding_box(face_coordinates, rgb_image, color)
# 画表情说明
draw_text(face_coordinates, rgb_image, emotion_text, color, 0, face_coordinates[3]+30, 1, 2)
# 将图像转换为BGR模式显示
bgr_image = cv2.cvtColor(rgb_image, cv2.COLOR_RGB2BGR)
cv2.imwrite("./pic_test/"+"predict"+os.path.basename(file), bgr_image)
cv2.waitKey(1)
cv2.destroyAllWindows()
print("已识别%d张图片" % int(len(dirs)))
def __init__(self, config):
self.config = config
self.device = config.device
self.max_itr = config.max_itr
self.batch_size = config.batch_size
self.img_size = config.img_size
self.dim_z = config.dim_z
self.dim_x = config.dim_x
self.dim_y = config.dim_y
self.dim_c = config.dim_c
self.scale = config.scale
self.generator = Generator(config).to(config.device)
self.generator.eval()
self._load_models(config.checkpoint_path)
self.x, self.y, self.r = get_coordinates(self.dim_x, self.dim_y,
self.scale, 1)
self.x, self.y, self.r = self.x.to(self.device), self.y.to(
self.device), self.r.to(self.device)
def main():
config = get_config()
print(config)
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
config.device = device
print(f'device: {config.device}')
model = CPPN(config).to(device)
x, y, r = get_coordinates(config.dim_x, config.dim_y, config.scale)
x, y, r = x.to(config.device), y.to(config.device), r.to(config.device)
z = torch.randn(1, config.dim_z).to(device)
scale = torch.ones((config.dim_x * config.dim_y, 1)).to(config.device)
z_scaled = torch.matmul(scale, z)
result = model(z_scaled, x, y, r)
result = result.view(-1, config.dim_x, config.dim_y, config.dim_c).cpu()
result = result.permute((0, 3, 1, 2))
torchvision.utils.save_image(torchvision.utils.make_grid(result),
'sample.jpg')
def generate_grid(full_w=4552, full_h=2276, degree=5, size=10):
left_w = int(full_w * (degree / 360) + 1)
dx = full_w * (degree / 360)
dy = full_h * (degree / 180)
DISTANCE = (dx**2 + dy**2)**0.5 + 10
node_dict = dict()
objects = []
nodes = []
positions = {}
kk = 0
for lat in range(75, -75, -degree):
for lng in range(-180, 180, degree):
x, y = get_coordinates(lng, lat, full_w=full_w, full_h=full_h)
objects.append((lng, lat, 1, x, y, []))
nodes.append([x, y])
positions[kk] = [x, y]
n = {
'idx': kk,
'lng': lng,
'lat': lat,
'obj_label': '', # grid fov label
'obj_id': '1', # grid fov
'x': x,
'y': y,
'boxes': [],
'neighbors': [],
'dir2neighbor': {},
'neighbor2dir': {}
}
node_dict[kk] = n
kk += 1
canvas = np.zeros((full_h, full_w, 3), dtype='uint8')
n_nodes = len(nodes)
order2nid = {i: i for i in range(n_nodes)}
idx = n_nodes
new_nodes = nodes
for ii, n in enumerate(nodes):
if n[0] < left_w:
order2nid[idx] = ii
new_nodes.append((n[0] + full_w, n[1]))
idx += 1
edges = {}
G = nx.Graph()
for ii, s1 in enumerate(new_nodes):
for jj, s2 in enumerate(new_nodes):
if ii == jj:
continue
d = ((s1[0] - s2[0])**2 + (s1[1] - s2[1])**2)**0.5
if d <= DISTANCE:
n0 = order2nid[ii]
n1 = order2nid[jj]
node_dict[n0]['neighbors'] += [n1]
node_dict[n1]['neighbors'] += [n0]
if s1[0] < s2[0] and s1[1] < s2[1]:
direction = 'dr'
inverse_dir = 'ul'
elif s1[0] < s2[0] and s1[1] == s2[1]:
direction = 'r'
inverse_dir = 'l'
elif s1[0] < s2[0] and s1[1] > s2[1]:
direction = 'ur'
inverse_dir = 'dl'
elif s1[0] == s2[0] and s1[1] < s2[1]:
direction = 'd'
inverse_dir = 'r'
elif s1[0] == s2[0] and s1[1] > s2[1]:
direction = 'u'
inverse_dir = 'd'
elif s1[0] > s2[0] and s1[1] < s2[1]:
direction = 'dl'
inverse_dir = 'ur'
elif s1[0] > s2[0] and s1[1] == s2[1]:
direction = 'l'
inverse_dir = 'r'
elif s1[0] > s2[0] and s1[1] > s2[1]:
direction = 'ul'
inverse_dir = 'dr'
node_dict[n0]['dir2neighbor'][direction] = n1
node_dict[n1]['dir2neighbor'][inverse_dir] = n0
node_dict[n0]['neighbor2dir'][n1] = direction
node_dict[n1]['neighbor2dir'][n0] = inverse_dir
edges[n0, n1] = ((s1[0], s1[1]), (s2[0], s2[1]))
cv2.line(canvas, (s1[0], s1[1]), (s2[0], s2[1]), color_gray, 3,
8)
G.add_edge(n0, n1, weight=d)
for kk, o in enumerate(objects):
ox, oy = o[3], o[4]
add_square(canvas, ox, oy, size=10, color=color_gray)
nx.set_node_attributes(G, values=positions, name='position')
paths = dict(nx.all_pairs_dijkstra_path(G))
distances = dict(nx.all_pairs_dijkstra_path_length(G))
return node_dict, canvas, edges, G, paths, distances
def fit(self, A):
"""
Fit model parameters U, V.
:param X:
Data matrix of shape (m, n)
Unknown values are assumed to take the value of zero (0).
"""
start_time = time.time()
# permute matrix A, columns minimum increasing
columns_perm = np.argsort(np.min(A, axis = 0))
A = A[:, columns_perm]
m = A.shape[0] # number of rows
n = A.shape[1] # number of columns
# initialization of U matrix
U_initial = self.initialize_U(A, m)
V = min_plus(ma.transpose(np.negative(U_initial)), A)
U = min_plus(A, ma.transpose(np.negative(V)))
D = np.subtract(A, max_plus(U, V))
# initialization of f values needed for convergence test
f_old = 1
f_new = self.b_norm(D)
f = f_new
i_list, j_list, k_list = range(m), range(n), range(self.rank)
U_new, V_new = U, V
comb = get_coordinates(A)
iterations = 0
while abs(f_old - f_new) > self.epsilon:
f = f_new
for x in comb:
i = x[0]
j = x[1]
temp = False
temp_1 = False
for k in k_list:
iterations += 1
V_new = copy.deepcopy(V)
V_new[k, j] = A[i, j] - U[i, k]
U_new = min_plus(A, np.transpose(np.negative(V_new)))
V_new = min_plus(np.transpose(np.negative(U_new)), A)
f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
if f_new < f:
temp_1 = True
break
if temp_1:
break # only executed if the inner loop DID break
for k in k_list:
iterations += 1
U_new = copy.deepcopy(U)
U_new[i, k] = A[i, j] - V[k, j]
V_new = min_plus(np.transpose(np.negative(U_new)), A)
U_new = min_plus(A, np.transpose(np.negative(V_new)))
f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
if f_new < f:
temp = True
break
if temp:
break # only executed if the inner loop DID break
if f_new < f:
U = U_new
V = V_new
f_old = f
f = f_new
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
else:
print("no solution found!")
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
def configure(self):
coordinates = get_coordinates('main', self.root)
self.root.geometry('%dx%d+%d+%d' % (coordinates['w'], coordinates['h'],
coordinates['x'], coordinates['y']))
self.root.resizable(True, True)
self.root.configure(background='#48B484')
def plot_var(f_step, projection):
# NOTE!
# If we are inside this function it means that the picture does not exist
# The one employed for the figure name when exported
variable_name = 'gph_t_850'
# Build the name of the output image
run_string, _ = get_run()
filename = '/tmp/' + projection + '_' + \
variable_name + '_%s_%03d.png' % (run_string, f_step)
"""In the main function we basically read the files and prepare the variables to be plotted.
This is not included in utils.py as it can change from case to case."""
dset = get_dset(vars_3d=['t@850', 'fi@500'], f_times=f_step).squeeze()
dset = subset_arrays(dset, projection)
time = pd.to_datetime(dset.valid_time.values)
cum_hour = dset.step.values.astype(int)
temp_850 = dset['t'] - 273.15
z_500 = dset['z']
gph_500 = mpcalc.geopotential_to_height(z_500)
gph_500 = xr.DataArray(gph_500.magnitude, coords=z_500.coords,
attrs={'standard_name': 'geopotential height',
'units': gph_500.units})
levels_temp = np.arange(-30., 30., 1.)
levels_gph = np.arange(4700., 6000., 70.)
cmap = get_colormap('temp')
fig = plt.figure(figsize=(figsize_x, figsize_y))
ax = plt.gca()
lon, lat = get_coordinates(temp_850)
lon2d, lat2d = np.meshgrid(lon, lat)
ax = get_projection_cartopy(plt, projection, compute_projection=True)
if projection == 'euratl':
norm = BoundaryNorm(levels_temp, ncolors=cmap.N)
cs = ax.pcolormesh(lon2d, lat2d, temp_850, cmap=cmap, norm=norm)
else:
cs = ax.contourf(lon2d, lat2d, temp_850, extend='both',
cmap=cmap, levels=levels_temp)
c = ax.contour(lon2d, lat2d, gph_500, levels=levels_gph,
colors='white', linewidths=1.)
labels = ax.clabel(c, c.levels, inline=True, fmt='%4.0f', fontsize=6)
maxlabels = plot_maxmin_points(ax, lon, lat, gph_500,
'max', 80, symbol='H', color='royalblue', random=True)
minlabels = plot_maxmin_points(ax, lon, lat, gph_500,
'min', 80, symbol='L', color='coral', random=True)
an_fc = annotation_forecast(ax, time)
an_var = annotation(
ax, 'Geopotential height @500hPa [m] and temperature @850hPa [C]', loc='lower left', fontsize=6)
an_run = annotation_run(ax, time)
plt.colorbar(cs, orientation='horizontal',
label='Temperature', pad=0.03, fraction=0.04)
plt.savefig(filename, **options_savefig)
plt.clf()
return filename
def dump_td_datasets(splits,
output_file,
task='continuous_grounding',
task_root='',
graph_root='',
full_w=3000,
full_h=1500,
obj_dict_file='../data/vg_object_dictionaries.all.json',
degree=30,
cache_root='../data/cached_td_data30degrees/'): # FIX
'''Prepares and dumps dataset to an npy file.
'''
if task_root != '' and not os.path.exists(task_root):
try:
os.makedirs(task_root)
except:
print('Cannot create folder {}'.format(task_root))
quit(1)
if cache_root:
meta_file = os.path.join(cache_root, 'meta.npy')
meta = np.load(meta_file, allow_pickle=True)[()]
cached_nodes = meta['nodes']
cached_paths = meta['paths']
add_cached_path = True
else:
raise NotImplementedError()
vg2idx = json.load(open(obj_dict_file, 'r'))['vg2idx']
data_list = []
all_sentences = []
stats = defaultdict(int)
for split in splits:
data = []
td_data_file = '../data/td_data/{}.json'.format(split)
lines = [(ii, line) for ii, line in enumerate(open(td_data_file))]
pbar = tqdm(lines)
count_err = 0
for ii, line in pbar:
datum = {}
instance = {}
td_instance = json.loads(line)
instance['img_idx'] = td_instance['main_pano']
instance['img_cat'] = 'street'
instance['img_loc'] = 'outdoor'
instance['img_src'] = '../data/td_data/images/' + \
td_instance['main_pano'] + '.jpg'
instance['annotationid'] = td_instance['route_id']
center = json.loads(td_instance['main_static_center'])
gt_x, gt_y = int(center['x'] * full_w), int(center['y'] * full_h)
xlng_deg, ylat_deg = coordinate2degrees(gt_x,
gt_y,
full_w=full_w,
full_h=full_h)
instance['xlng_deg'], instance['ylat_deg'] = xlng_deg, ylat_deg
instance['refexp'] = [td_instance['td_location_text'].split(' ')]
n_rows = int(360 / degree)
start_fov = np.random.randint(n_rows)
start_node = cached_nodes[start_fov]
assert start_node['idx'] == start_fov
instance['actions'] = [{
'act_deg_list': [[[start_node['lng'], start_node['lat']],
[xlng_deg, ylat_deg]]],
'actionid':
instance['annotationid']
}]
all_moves = [[(start_node['lng'], start_node['lat']),
(xlng_deg, ylat_deg)]]
xlongitude, ylatitude = instance['xlng_deg'], instance['ylat_deg']
datum['annotationid'] = instance['annotationid']
datum['gt_lng'] = xlongitude
datum['gt_lat'] = ylatitude
datum['gt_x'] = gt_x
datum['gt_y'] = gt_y
img_cat = instance['img_cat']
img_loc = instance['img_loc']
img_src = instance['img_src']
datum['img_src'] = img_src
datum['img_category'] = img_cat
stats[img_loc] += 1
stats[img_cat] += 1
sent_queue = []
sentences = instance['refexp']
for refexp in sentences:
sent_queue += [refexp]
datum['gt_moves'] = all_moves
datum['refexps'] = sentences
all_sentences += sent_queue
start_loc = instance['actions'][0]['act_deg_list'][0][0]
start_x, start_y = get_coordinates(start_loc[0],
start_loc[1],
full_w=full_w,
full_h=full_h)
start_fov, _ = get_nearest(cached_nodes, start_x, start_y)
gt_path = [start_fov]
path = []
intermediate_paths = []
if add_cached_path:
for kk, act_list in enumerate(
instance['actions'][0]['act_deg_list']):
act = act_list[-1]
lng, lat = act
x, y = get_coordinates(lng,
lat,
full_w=full_w,
full_h=full_h)
min_n, _ = get_nearest(cached_nodes, x, y)
gt_path.append(min_n)
path = [gt_path[0]]
for kk in range(len(gt_path) - 1):
start = gt_path[kk]
end = gt_path[kk + 1]
intermediate_path = cached_paths[start][end]
path += intermediate_path[1:]
intermediate_paths.append(intermediate_path)
assert len(gt_path) <= len(
path), 'len(gt_path) <= len(path) {} > {}'.format(
len(gt_path), len(path))
assert len(datum['refexps']) == len(
intermediate_paths
), 'len(refepxs) != len(intermediate_paths) {} != {}'.format(
len(datum['refexps']), len(intermediate_paths))
datum['gt_path'] = gt_path
datum['path'] = path
datum['intermediate_paths'] = intermediate_paths
datum['actionid'] = instance['actions'][0]['actionid']
datum['actions'] = instance['actions']
if task == 'continuous_grounding':
data.append(datum)
elif task == 'cached_fov_pretraining':
pano = instance['img_idx']
node_path = os.path.join(graph_root, '{}.npy'.format(pano))
nodes = np.load(node_path, allow_pickle=True)[()]
for n in cached_nodes:
node = cached_nodes[n]
mdatum = {}
fov_id = node['idx']
mdatum['fov_id'] = fov_id
mdatum['move_max'] = len(sentences)
mdatum['img_src'] = datum['img_src']
# mdatum['actionid'] = move_id
mdatum['annotationid'] = instance['annotationid']
ylat, xlng = node['lat'], node['lng']
mx, my = node['x'], node['y']
mdatum['xlongitude'] = xlng
mdatum['ylatitude'] = ylat
mdatum['x'] = mx
mdatum['y'] = my
mdatum['refexps'] = sentences
fov_file = os.path.join(
cache_root, 'fovs', task_root,
'pano_{}.{}.jpg'.format(pano, fov_id))
mdatum['fov_file'] = fov_file
regions, obj_list = get_objects(mx,
my,
nodes,
vg2idx,
full_w=full_w,
full_h=full_h,
include_vectors=False)
mdatum['regions'] = regions
mdatum['obj_list'] = obj_list
directions = [
len(obj_list['navigation'][d])
for d in obj_list['navigation'].keys()
]
if sum(directions) > 0:
data.append(mdatum)
data_list.append(data)
print('{} instances have errors'.format(count_err))
pbar.close()
n_instances = sum([len(l) for l in data_list])
print('Dumping {} instances to {}'.format(n_instances, output_file))
np.save(open(output_file, 'wb'), {
'data_list': data_list,
'sentences': all_sentences
})