def insert_batch(data, label, last_batch=False):
global h5_batch_data, h5_batch_label
global buffer_size, h5_index
data_size = data.shape[0]
# If there is enough space, just insert
if buffer_size + data_size <= h5_batch_data.shape[0]:
h5_batch_data[buffer_size:buffer_size + data_size, ...] = data
h5_batch_label[buffer_size:buffer_size + data_size] = label
buffer_size += data_size
else: # not enough space
capacity = h5_batch_data.shape[0] - buffer_size
assert (capacity >= 0)
if capacity > 0:
h5_batch_data[buffer_size:buffer_size + capacity,
...] = data[0:capacity, ...]
h5_batch_label[buffer_size:buffer_size + capacity,
...] = label[0:capacity, ...]
# Save batch data and label to h5 file, reset buffer_size
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data, h5_batch_label,
data_dtype, label_dtype)
print('Stored {0} with size {1}'.format(h5_filename,
h5_batch_data.shape[0]))
fout_all_files.write(h5_filename + '\n')
# -------------store the data to show -----------------
if SHOW_DATA:
fout = open(output_filename_prefix + '_' + str(h5_index) + '.txt',
'w')
for batch in range(h5_batch_data.shape[0]):
for p in range(h5_batch_data.shape[1]):
fout.write('v %f %f %f \n' % (h5_batch_data[batch, p, 0],
h5_batch_data[batch, p, 1],
h5_batch_data[batch, p, 2]))
fout.close()
# --------------------------------------------------
h5_index += 1
buffer_size = 0
# recursive call
insert_batch(data[capacity:, ...], label[capacity:, ...], last_batch)
if last_batch and buffer_size > 0:
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data[0:buffer_size, ...],
h5_batch_label[0:buffer_size,
...], data_dtype, label_dtype)
print('Stored {0} with size {1}'.format(h5_filename, buffer_size))
fout_all_files.write(h5_filename + '\n')
# -------------store the data to show -----------------
if SHOW_DATA:
fout = open(output_filename_prefix + '_' + str(h5_index) + '.txt',
'w')
for batch in range(h5_batch_data.shape[0]):
for p in range(h5_batch_data.shape[1]):
fout.write('v %f %f %f \n' % (h5_batch_data[batch, p, 0],
h5_batch_data[batch, p, 1],
h5_batch_data[batch, p, 2]))
fout.close()
# --------------------------------------------------
h5_index += 1
buffer_size = 0
return
def insert_batch(data, label, last_batch=False):
global h5_batch_data, h5_batch_label
global buffer_size, h5_index
data_size = data.shape[0]
# If there is enough space, just insert
if buffer_size + data_size <= h5_batch_data.shape[0]:
h5_batch_data[buffer_size:buffer_size + data_size, ...] = data
h5_batch_label[buffer_size:buffer_size + data_size] = label
buffer_size += data_size
else: # not enough space
capacity = h5_batch_data.shape[0] - buffer_size
assert (capacity >= 0)
if capacity > 0:
h5_batch_data[buffer_size:buffer_size + capacity,
...] = data[0:capacity, ...]
h5_batch_label[buffer_size:buffer_size + capacity,
...] = label[0:capacity, ...]
# Save batch data and label to h5 file, reset buffer_size
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data, h5_batch_label,
data_dtype, label_dtype)
#print('Stored {0} with size {1}'.format(h5_filename, h5_batch_data.shape[0]))
h5_index += 1
buffer_size = 0
# recursive call
insert_batch(data[capacity:, ...], label[capacity:, ...], last_batch)
if last_batch and buffer_size > 0:
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data[0:buffer_size, ...],
h5_batch_label[0:buffer_size,
...], data_dtype, label_dtype)
#print('Stored {0} with size {1}'.format(h5_filename, buffer_size))
h5_index += 1
buffer_size = 0
return
def insert_batch(data, label, last_batch=False):
global h5_batch_data, h5_batch_label
global buffer_size, h5_index
data_size = data.shape[0]
# If there is enough space, just insert
if buffer_size + data_size <= h5_batch_data.shape[0]:
h5_batch_data[buffer_size:buffer_size+data_size, ...] = data
h5_batch_label[buffer_size:buffer_size+data_size] = label
buffer_size += data_size
else: # not enough space
capacity = h5_batch_data.shape[0] - buffer_size
assert(capacity>=0)
if capacity > 0:
h5_batch_data[buffer_size:buffer_size+capacity, ...] = data[0:capacity, ...]
h5_batch_label[buffer_size:buffer_size+capacity, ...] = label[0:capacity, ...]
# Save batch data and label to h5 file, reset buffer_size
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data, h5_batch_label, data_dtype, label_dtype)
print('Stored {0} with size {1}'.format(h5_filename, h5_batch_data.shape[0]))
h5_index += 1
buffer_size = 0
# recursive call
insert_batch(data[capacity:, ...], label[capacity:, ...], last_batch)
if last_batch and buffer_size > 0:
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
data_prep_util.save_h5(h5_filename, h5_batch_data[0:buffer_size, ...], h5_batch_label[0:buffer_size, ...], data_dtype, label_dtype)
print('Stored {0} with size {1}'.format(h5_filename, buffer_size))
h5_index += 1
buffer_size = 0
return
def save_to_h5_files(data, labels, split):
for i in range(math.ceil(len(data) / FILE_SIZE)):
file_name = split + '_aligned_modelnet_' + str(i) + '.h5'
data_util.save_h5('data/modelnet40_aligned/' + file_name,
data,
labels,
data_dtype='float32')
doc_file = open('data/modelnet40_aligned/' + split + '_files.txt', 'a')
doc_file.write('data/modelnet40_aligned/' + file_name)
doc_file.close()
def insert_batch(data, label, output_dir, is_last_batch=False):
global h5_batch_data, h5_batch_label
global buffer_size, h5_index
data_size = data.shape[0]
# print(f'data.shape = {data.shape}')
all_files = os.path.join(output_dir, 'all_files.txt')
print(f'all_files = {all_files}')
# If there is enough space, just insert
if buffer_size + data_size <= h5_batch_data.shape[0]:
h5_batch_data[buffer_size:buffer_size + data_size, ...] = data
h5_batch_label[buffer_size:buffer_size + data_size] = label
buffer_size += data_size
else: # not enough space
capacity = h5_batch_data.shape[0] - buffer_size
assert (capacity >= 0)
if capacity > 0:
h5_batch_data[buffer_size:buffer_size + capacity,
...] = data[0:capacity, ...]
h5_batch_label[buffer_size:buffer_size + capacity,
...] = label[0:capacity, ...]
# Save batch data and label to h5 file, reset buffer_size
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
h5_filename = os.path.join(output_dir, f'ply_data_{h5_index!s}.h5')
print(f'h5_filename = {h5_filename}')
data_prep_util.save_h5(h5_filename, h5_batch_data, h5_batch_label,
data_dtype, label_dtype)
write_to_all_files(all_files, h5_filename)
h5_index += 1
buffer_size = 0
# recursive call
insert_batch(data[capacity:, ...], label[capacity:, ...], output_dir,
is_last_batch)
if is_last_batch and buffer_size > 0:
h5_filename = output_filename_prefix + '_' + str(h5_index) + '.h5'
h5_filename = os.path.join(output_dir, f'ply_data_{h5_index!s}.h5')
print(f'is last batch')
print(f'h5_filename = {h5_filename}')
data_prep_util.save_h5(h5_filename, h5_batch_data[0:buffer_size, ...],
h5_batch_label[0:buffer_size,
...], data_dtype, label_dtype)
write_to_all_files(all_files, h5_filename)
h5_index += 1
buffer_size = 0
return
def eval_one_epoch(sess, ops):
is_training = False
total_seen = 0
for fn in range(len(INFER_FILES)):
log_string('---- file number ' + str(fn+1) + ' out of ' + str(len(INFER_FILES)) + ' files ----')
current_data, current_label = provider.loadDataFile(INFER_FILES[fn])
current_data = current_data[:, 0:NUM_IN_POINTS, :]
out_data_generated = np.zeros(
(current_data.shape[0], NUM_GENERATED_POINTS, current_data.shape[2]))
out_data_sampled_fw_plus_simple_continued_fps = np.zeros(
(current_data.shape[0], NUM_OUT_POINTS, current_data.shape[2]))
current_label_orig = current_label
current_label = np.squeeze(current_label)
print(current_data.shape)
file_size = current_data.shape[0]
num_batches = file_size // BATCH_SIZE
print(file_size)
for batch_idx in range(num_batches):
print str(batch_idx) + '/' + str(num_batches-1)
start_idx = batch_idx * BATCH_SIZE
end_idx = (batch_idx + 1) * BATCH_SIZE
feed_dict = {ops['pointclouds_pl']: current_data[start_idx:end_idx], ops['labels_pl']: current_label[start_idx:end_idx], ops['is_training_pl']: is_training}
generated_points, idx = sess.run([ops['generated_points'], ops['idx']], feed_dict=feed_dict)
out_data_generated[start_idx:end_idx, :, :] = generated_points
outcloud_fw_plus_simple_continued_fps = SAMPLER_MODEL.nn_matching(current_data[start_idx:end_idx], idx, NUM_OUT_POINTS)
out_data_sampled_fw_plus_simple_continued_fps[start_idx:end_idx, :, :] = outcloud_fw_plus_simple_continued_fps[:, 0:NUM_OUT_POINTS, :]
total_seen += BATCH_SIZE
file_name = os.path.split(INFER_FILES[fn])
if not os.path.exists(OUT_DATA_PATH + '/generated/'): os.makedirs(OUT_DATA_PATH + '/generated/')
data_prep_util.save_h5(OUT_DATA_PATH + '/generated/' + file_name[1], out_data_generated, current_label_orig, data_dtype, label_dtype)
if not os.path.exists(OUT_DATA_PATH + '/sampled/'): os.makedirs(OUT_DATA_PATH + '/sampled/')
data_prep_util.save_h5(OUT_DATA_PATH + '/sampled/' + file_name[1], out_data_sampled_fw_plus_simple_continued_fps, current_label_orig, data_dtype, label_dtype)
def eval_one_epoch(sess, ops):
is_training = False
total_seen = 0
for fn in range(len(INFER_FILES)):
log_string("---- file number " + str(fn + 1) + " out of " +
str(len(INFER_FILES)) + " files ----")
current_data, current_label = provider.loadDataFile(INFER_FILES[fn])
current_data = current_data[:, 0:NUM_IN_POINTS, :]
out_data_simplified = np.zeros(
(current_data.shape[0], NUM_SIMPLIFIED_POINTS,
current_data.shape[2]))
out_data_soft_projected = np.zeros(
(current_data.shape[0], NUM_SIMPLIFIED_POINTS,
current_data.shape[2]))
out_data_hard_projected = np.zeros(
(current_data.shape[0], NUM_SIMPLIFIED_POINTS,
current_data.shape[2]))
out_data_sampled = np.zeros(
(current_data.shape[0], NUM_OUT_POINTS, current_data.shape[2]))
current_label_orig = current_label
current_label = np.squeeze(current_label)
print(current_data.shape)
file_size = current_data.shape[0]
num_batches = file_size // BATCH_SIZE
print(file_size)
for batch_idx in range(num_batches):
print(str(batch_idx) + "/" + str(num_batches - 1))
start_idx = batch_idx * BATCH_SIZE
end_idx = (batch_idx + 1) * BATCH_SIZE
feed_dict = {
ops["pointclouds_pl"]: current_data[start_idx:end_idx],
ops["labels_pl"]: current_label[start_idx:end_idx],
ops["is_training_pl"]: is_training,
}
(
simplified_points,
soft_projected_points,
hard_projected_points,
nn_indices,
) = sess.run(
[
ops["simplified_points"],
ops["soft_projected_points"],
ops["hard_projected_points"],
ops["idx"],
],
feed_dict=feed_dict,
)
out_data_simplified[start_idx:end_idx, :, :] = simplified_points
out_data_soft_projected[
start_idx:end_idx, :, :] = soft_projected_points
out_data_hard_projected[
start_idx:end_idx, :, :] = hard_projected_points
outcloud_sampled = SAMPLER_MODEL.nn_matching(
current_data[start_idx:end_idx], nn_indices, NUM_OUT_POINTS)
out_data_sampled[
start_idx:end_idx, :, :] = outcloud_sampled[:, 0:
NUM_OUT_POINTS, :]
total_seen += BATCH_SIZE
file_name = os.path.split(INFER_FILES[fn])
if not os.path.exists(OUT_DATA_PATH + "/simplified/"):
os.makedirs(OUT_DATA_PATH + "/simplified/")
data_prep_util.save_h5(
OUT_DATA_PATH + "/simplified/" + file_name[1],
out_data_simplified,
current_label_orig,
data_dtype,
label_dtype,
)
if not os.path.exists(OUT_DATA_PATH + "/soft_projected/"):
os.makedirs(OUT_DATA_PATH + "/soft_projected/")
data_prep_util.save_h5(
OUT_DATA_PATH + "/soft_projected/" + file_name[1],
out_data_soft_projected,
current_label_orig,
data_dtype,
label_dtype,
)
if not os.path.exists(OUT_DATA_PATH + "/hard_projected/"):
os.makedirs(OUT_DATA_PATH + "/hard_projected/")
data_prep_util.save_h5(
OUT_DATA_PATH + "/hard_projected/" + file_name[1],
out_data_hard_projected,
current_label_orig,
data_dtype,
label_dtype,
)
if not os.path.exists(OUT_DATA_PATH + "/sampled/"):
os.makedirs(OUT_DATA_PATH + "/sampled/")
data_prep_util.save_h5(
OUT_DATA_PATH + "/sampled/" + file_name[1],
out_data_sampled,
current_label_orig,
data_dtype,
label_dtype,
)
def eval_one_epoch(sess, ops):
is_training = False
total_correct = 0
total_seen = 0
loss_sum = 0
num_unique_idx = 0
total_seen_class = [0 for _ in range(NUM_CLASSES)]
total_correct_class = [0 for _ in range(NUM_CLASSES)]
fout = open(os.path.join(DUMP_DIR, 'pred_label.txt'), 'w')
for fn in range(len(TEST_FILES)):
log_string('---- file number ' + str(fn + 1) + ' out of ' + str(len(TEST_FILES)) + ' files ----')
current_data, current_label = provider.loadDataFile(TEST_FILES[fn])
current_data = current_data[:, 0:NUM_IN_POINTS, :]
current_label_orig = current_label
current_label = np.squeeze(current_label)
print(current_data.shape)
file_size = current_data.shape[0]
num_batches = file_size // BATCH_SIZE
print(file_size)
out_data_generated = np.zeros((current_data.shape[0], NUM_OUT_POINTS, current_data.shape[2]))
out_data_sampled = np.zeros((current_data.shape[0], NUM_OUT_POINTS, current_data.shape[2]))
for batch_idx in range(num_batches):
print str(batch_idx) + '/' + str(num_batches - 1)
start_idx = batch_idx * BATCH_SIZE
end_idx = (batch_idx + 1) * BATCH_SIZE
cur_batch_size = end_idx - start_idx
# Aggregating BEG
batch_loss_sum = 0 # sum of losses for the batch
batch_pred_sum = np.zeros((cur_batch_size, NUM_CLASSES)) # score for classes
batch_pred_classes = np.zeros((cur_batch_size, NUM_CLASSES)) # 0/1 for classes
rotated_data = current_data[start_idx:end_idx, :, :]
feed_dict = {ops['pointclouds_pl']: rotated_data, ops['labels_pl']: current_label[start_idx:end_idx], ops['is_training_pl']: is_training}
generated_points, idx = sess.run([ops['generated_points'], ops['idx']], feed_dict=feed_dict)
if MATCH_OUTPUT:
outcloud = SAMPLER_MODEL.nn_matching(rotated_data, idx, NUM_OUT_POINTS)
else:
outcloud = generated_points
for ii in range(0, BATCH_SIZE):
num_unique_idx += np.size(np.unique(idx[ii]))
feed_dict = {ops['pointclouds_pl']: rotated_data, ops['outcloud']: outcloud, ops['labels_pl']: current_label[start_idx:end_idx], ops['is_training_pl']: is_training}
loss_val, pred_val = sess.run([ops['loss'], ops['pred']], feed_dict=feed_dict)
out_data_generated[start_idx:end_idx, :, :] = generated_points
out_data_sampled[start_idx:end_idx, :, :] = outcloud
batch_pred_sum += pred_val
batch_pred_val = np.argmax(pred_val, 1)
for el_idx in range(cur_batch_size):
batch_pred_classes[el_idx, batch_pred_val[el_idx]] += 1
batch_loss_sum += loss_val * cur_batch_size
pred_val = np.argmax(batch_pred_sum, 1)
# Aggregating END
correct = np.sum(pred_val == current_label[start_idx:end_idx])
total_correct += correct
total_seen += cur_batch_size
loss_sum += batch_loss_sum
for i in range(start_idx, end_idx):
l = current_label[i]
total_seen_class[l] += 1
total_correct_class[l] += (pred_val[i - start_idx] == l)
fout.write('%d, %d\n' % (pred_val[i - start_idx], l))
if SAVE_POINTS:
if not os.path.exists(OUT_DATA_PATH + '/generated/'):
os.makedirs(OUT_DATA_PATH + '/generated/')
if not os.path.exists(OUT_DATA_PATH + '/sampled/'):
os.makedirs(OUT_DATA_PATH + '/sampled/')
file_name = os.path.split(TEST_FILES[fn])
data_prep_util.save_h5(OUT_DATA_PATH + '/generated/' + file_name[1], out_data_generated, current_label_orig, data_dtype, label_dtype)
data_prep_util.save_h5(OUT_DATA_PATH + '/sampled/' + file_name[1], out_data_sampled, current_label_orig, data_dtype, label_dtype)
log_string('eval mean loss: %f' % (loss_sum / float(total_seen)))
log_string('eval accuracy: %f' % (total_correct / float(total_seen)))
log_string('eval avg class acc: %f' % (np.mean(np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float))))
log_string('total_seen: %f' % (total_seen))
class_accuracies = np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float)
for i, name in enumerate(SHAPE_NAMES):
log_string('%10s:\t%0.3f' % (name, class_accuracies[i]))
