def _check_sst_quality(self, dataset, product_type):
mask_specs = product_type.get_mask_consistency_check_specs()
if len(mask_specs) == 0:
return
sst_variable_names = product_type.get_sst_variable_names()
if len(sst_variable_names) == 0:
return
quality_variable_name = mask_specs[0][2]
quality_data = dataset.variables[quality_variable_name][:]
valid_retrieval_quality = ma.masked_less(quality_data, 2)
self.report["sst_valid_retrieval"] = float(valid_retrieval_quality.count())
failed_retrieval_quality = ma.masked_not_equal(quality_data, 1)
sst_variable = dataset.variables[sst_variable_names[0]]
fill_value = sst_variable.getncattr('_FillValue')
sst_quality_one_data = ma.array(sst_variable[:], mask=failed_retrieval_quality.mask)
invalid_retrieval = ma.masked_equal(sst_quality_one_data, fill_value)
self.report["sst_invalid_retrieval"] = float(invalid_retrieval.count())
failed_retrieval = ma.masked_not_equal(sst_quality_one_data, fill_value)
self.report["sst_failed_retrieval"] = float(failed_retrieval.count())
not_ocean = ma.masked_not_equal(quality_data, 0)
self.report["not_ocean"] = float(not_ocean.count())
def logisticRegression(x, t, K, alpha=0.01, epsilon=0.001, steps=500):
"""
Return w: vector with dimension M(K)
"""
#initialize w, phi, _t
phi = basisNone(x)
M = phi.shape[1]
for _ in range(K - 1):
phi = scipy.linalg.block_diag(phi, basisNone(x))
w = np.random.random(phi.shape[1])
_t = np.empty(0)
for k in range(K):
_t = np.append(_t, ma.masked_not_equal(ma.masked_not_equal(t, k + 1).filled(0), 0).filled(1))
error = 0
de = 0
#gradient descent on error function
for step in range(steps):
previous = error
_y = y(w.reshape((K, M)), basisNone(x))
w = w_new(w, _y, _t, phi, K, alpha)
error = np.dot(-_t, np.array(map(math.log, abs(_y))))
de = error - previous
if abs(de) < epsilon:
print "Finished after step %d with delta-error = %f error = %f" % (step, de, error)
break
return w
def logisticRegression(x, t, K, alpha=0.01, epsilon=0.001, steps=500):
"""
Return w: vector with dimension M(K)
"""
#initialize w, phi, _t
phi = basisNone(x)
M = phi.shape[1]
for _ in range(K - 1):
phi = scipy.linalg.block_diag(phi, basisNone(x))
w = np.random.random(phi.shape[1])
_t = np.empty(0)
for k in range(K):
_t = np.append(
_t,
ma.masked_not_equal(ma.masked_not_equal(t, k + 1).filled(0),
0).filled(1))
error = 0
de = 0
#gradient descent on error function
for step in range(steps):
previous = error
_y = y(w.reshape((K, M)), basisNone(x))
w = w_new(w, _y, _t, phi, K, alpha)
error = np.dot(-_t, np.array(map(math.log, abs(_y))))
de = error - previous
if abs(de) < epsilon:
print "Finished after step %d with delta-error = %f error = %f" % (
step, de, error)
break
return w
def _check_sst_quality(self, dataset, product_type):
mask_specs = product_type.get_mask_consistency_check_specs()
if len(mask_specs) == 0:
return
sst_variable_names = product_type.get_sst_variable_names()
if len(sst_variable_names) == 0:
return
quality_variable_name = mask_specs[0][2]
quality_data = dataset.variables[quality_variable_name][:]
valid_retrieval_quality = ma.masked_less(quality_data, 2)
self.report["sst_valid_retrieval"] = float(
valid_retrieval_quality.count())
failed_retrieval_quality = ma.masked_not_equal(quality_data, 1)
sst_variable = dataset.variables[sst_variable_names[0]]
fill_value = sst_variable.getncattr('_FillValue')
sst_quality_one_data = ma.array(sst_variable[:],
mask=failed_retrieval_quality.mask)
invalid_retrieval = ma.masked_equal(sst_quality_one_data, fill_value)
self.report["sst_invalid_retrieval"] = float(invalid_retrieval.count())
failed_retrieval = ma.masked_not_equal(sst_quality_one_data,
fill_value)
self.report["sst_failed_retrieval"] = float(failed_retrieval.count())
not_ocean = ma.masked_not_equal(quality_data, 0)
self.report["not_ocean"] = float(not_ocean.count())
def __call__(self, meta_data, img, depth, points):
meta_data_occ = copy.deepcopy(meta_data)
label = meta_data_occ['mask']
for k in range(5):
subpath = random.choice(self.syn)
front = np.array(
Image.open('{0}/{1}-color.png'.format(self.dataset_root,
subpath)).convert("RGB"))
f_label = np.array(
Image.open('{0}/{1}-label.png'.format(self.dataset_root,
subpath)))
front_label = np.unique(f_label).tolist()[1:]
if len(front_label) < self.front_num:
continue
front_label = random.sample(front_label, self.front_num)
for f_i in front_label:
mk = ma.getmaskarray(ma.masked_not_equal(f_label, f_i))
if f_i == front_label[0]:
mask_front = mk
else:
mask_front = mask_front * mk
t_label = label * mask_front
if len(t_label.nonzero()[0]) > 1000:
label = t_label
break
mask_front = np.expand_dims(mask_front, 2)
img_occ = img * mask_front + front * ~mask_front
meta_data_occ['mask'] = label
return meta_data_occ, img_occ, depth, points
def parse_segments(seg, msk_modes):
"""Parse the label segments.
Each channel corresponds to a different region of the tumor, decouple and stack these
mode_to_key_value = {"necrotic": 1, "edema": 2, "GD": 4}
Args:
seg: The segmentation labels
msk_modes: Label mode to parse for the model
Returns:
The processed mask labels
"""
msks_parsed = []
for slice in range(seg.shape[-1]):
# which mask values indicicate which label mode
mode_to_key_value = {"necrotic": 1, "edema": 2, "GD": 4}
curr = seg[:, :, slice]
this_msk_parts = []
for mode in msk_modes:
this_msk_parts.append(
ma.masked_not_equal(
curr, mode_to_key_value[mode]).filled(fill_value=0))
msks_parsed.append(np.dstack(this_msk_parts))
# Replace all tumorous areas with 1 (previously marked as 1, 2 or 4)
mask = np.asarray(msks_parsed)
mask[mask > 0] = 1
return mask
def update_aseg_fs(subject):
"""
Merge Freesurfer brain segmentation with the lesion segmentation
performed by samseg in Freesurfer's space.
Parameters
----------
subject : TYPE str
Subject id.
Returns
-------
None.
"""
# Load lesion mask
lesion_image = nib.load(SUBJECTS_DIR + "/sub-{0}_MPRAGE.nii/mri/lesions_fs.mgz".format(subject))
lesion_mx = lesion_image.get_fdata()
lesion_mask = ma.masked_not_equal(lesion_mx,1)
# Load Freesurfer segmentation mask (aseg)
aseg_image = nib.load(SUBJECTS_DIR + "/sub-{0}_MPRAGE.nii/mri/aseg.mgz".format(subject))
aseg_mx = aseg_image.get_fdata()
# Set all voxels of aseg marked as lesion in the lesion mask to 99 (Freesurfer lesion id)
aseg_mask = ma.masked_array(aseg_mx, np.logical_not(lesion_mask.mask), fill_value=99).filled()
# Save resulting matrix to nifti file
nifti_out = nib.Nifti1Image(aseg_mask,affine=aseg_image.affine)
nib.save(nifti_out, SUBJECTS_DIR +"/sub-{0}_MPRAGE.nii/mri/aseg_lesions.nii.gz".format(subject))
def func(V, Y): count_each_feature_value = Counter(V) feature_value = np.unique(V) num_value_feature = len(feature_value) subtotal = 0 count = 0 for ele in feature_value:# each value in this feature weight = count_each_feature_value[feature_value[count]] / float(sampleNum) after_mask = ma.masked_not_equal(V,ele) ma.set_fill_value(after_mask,0) after_mask = after_mask.filled() mat_each_value_feature = np.dot(after_mask, Y) # tmp = mat_each_value_feature tmp_sum = np.sum(mat_each_value_feature) mat_each_value_feature = np.dot(mat_each_value_feature, 1.0 / tmp_sum) current = 0 #after_compress = ma.masked_equal(mat_each_value_feature, 0) #after_compress = after_compress.compressed() #t5 = time.clock() #current = np.dot(mat_each_value_feature, np.log2(after_compress)) #t6 = time.clock() #mat_each_value_feature = mat_each_value_feature[-mat_each_value_feature.mask] #ma.set_fill_value(mat_each_value_feature,1) #mat_each_value_feature = mat_each_value_feature.filled() #current = np.dot(mat_each_value_feature, np.log2(mat_each_value_feature)) #current = 0 for k in mat_each_value_feature: if k == 0: continue; current += k * math.log(k,2) subtotal += current * weight count += 1 subtotal = - subtotal return subtotal
def openData(self, ccd, ap, save=False, mask=True):
target = self.target.replace(' ', '_')
filters = self.filters[::-1]
ccd = str(ccd)
ap = str(ap)
if ccd not in self.apnames.keys():
raise ValueError('{} not a valid CCD'.format(ccd))
if ap not in self.apnames[ccd]:
raise ValueError('{} not a valid aperture'.format(ap))
data = self.logf.tseries(ccd, ap)
obstime = Time(data.t,
format='mjd',
scale='utc',
location=self.tel_location)
data.t = obstime.tdb.value
exp = self.logf[ccd]['Exptim'] / 86400
weights = np.ones(len(data.t))
m = data.get_mask()
zero_flux_mask = ma.getmask(ma.masked_not_equal(data.y, 0))
data_mask = np.logical_and(~m, zero_flux_mask)
if mask:
out = np.column_stack([
data.t[data_mask], exp[data_mask], data.y[data_mask],
data.ye[data_mask], weights[data_mask], weights[data_mask]
])
else:
out = np.column_stack(
[data.t, exp, data.y, data.ye, weights, weights])
return out, data_mask
def func(V, Y):
count_each_feature_value = Counter(V)
feature_value = np.unique(V)
num_value_feature = len(feature_value)
subtotal = 0
count = 0
for ele in feature_value: # each value in this feature
weight = count_each_feature_value[feature_value[count]] / float(
sampleNum)
after_mask = ma.masked_not_equal(V, ele)
ma.set_fill_value(after_mask, 0)
after_mask = after_mask.filled()
mat_each_value_feature = np.dot(after_mask, Y) #
tmp = mat_each_value_feature
tmp_sum = np.sum(mat_each_value_feature)
mat_each_value_feature = np.dot(mat_each_value_feature, 1.0 / tmp_sum)
current = 0
#after_compress = ma.masked_equal(mat_each_value_feature, 0)
#after_compress = after_compress.compressed()
#t5 = time.clock()
#current = np.dot(mat_each_value_feature, np.log2(after_compress))
#t6 = time.clock()
#mat_each_value_feature = mat_each_value_feature[-mat_each_value_feature.mask]
#ma.set_fill_value(mat_each_value_feature,1)
#mat_each_value_feature = mat_each_value_feature.filled()
#current = np.dot(mat_each_value_feature, np.log2(mat_each_value_feature))
#current = 0
for k in mat_each_value_feature:
if k == 0:
continue
current += k * math.log(k, 2)
subtotal += current * weight
count += 1
subtotal = -subtotal
return subtotal
def update_aseg_norm(subject):
"""
Merge normalized Freesurfer brain segmentation with the lesion segmentation
performed by samseg.
Parameters
----------
subject : TYPE str
Subject id.
Returns
-------
None.
"""
# Load lesion mask
lesion_image = nib.load(MAIN_DIR+'/derivatives/segmentations/sub-{0}/ses-{1}/sub-{0}_lesions_binary.nii.gz'.format(subject,SESSION))
lesion_mx = lesion_image.get_fdata()
lesion_mask = ma.masked_not_equal(lesion_mx,1)
# Load Freesurfer segmentation mask (aseg)
aseg_image = nib.load(MAIN_DIR + "/derivatives/segmentations/sub-{0}/ses-{1}/sub-{0}_aseg_normalized.nii.gz".format(subject,SESSION))
aseg_mx = aseg_image.get_fdata()
# Set all voxels of aseg marked as lesion in the lesion mask to 99 (Freesurfer lesion id)
aseg_mask = ma.masked_array(aseg_mx, np.logical_not(lesion_mask.mask), fill_value=99).filled()
# Save resulting matrix to nifti file
nifti_out = nib.Nifti1Image(aseg_mask,affine=aseg_image.affine)
nib.save(nifti_out, MAIN_DIR + "/derivatives/segmentations/sub-{0}/ses-{1}/sub-{0}_aseg_lesions.nii.gz".format(subject,SESSION))
def windows(self,
w1=np.array([0, 0]),
w2=np.array([0, 0]),
r_w=0,
w3=np.array([0, 0])):
self.window_1 = w1 #array of start and ending time for window of use #1
self.window_2 = w2 #array of start and ending time for window of use #2
self.window_3 = w3 #array of start and ending time for window of use #3
self.random_var_w = r_w #percentage of variability in the start and ending times of the windows
self.daily_use = np.zeros(1440) #create an empty daily use profile
self.daily_use[w1[0]:(w1[1])] = np.full(
np.diff(w1), 0.001
) #fills the daily use profile with infinitesimal values that are just used to identify the functioning windows
self.daily_use[w2[0]:(w2[1])] = np.full(
np.diff(w2), 0.001) #same as above for window2
self.daily_use[w3[0]:(w3[1])] = np.full(
np.diff(w3), 0.001) #same as above for window3
self.daily_use_masked = np.zeros_like(
ma.masked_not_equal(self.daily_use, 0.001)
) #apply a python mask to the daily_use array to make only functioning windows 'visibile'
self.random_var_1 = int(
r_w * np.diff(w1)
) #calculate the random variability of window1, i.e. the maximum range of time they can be enlarged or shortened
self.random_var_2 = int(r_w * np.diff(w2)) #same as above
self.random_var_3 = int(r_w * np.diff(w3)) #same as above
self.user.App_list.append(
self
) #automatically appends the appliance to the user's appliance list
def belowhorizon(z):
"""Return masked z values that are below the horizon.
Below the horizon means either than z is negative or
the z has a nonzero imaginary part.
"""
imagz_ma = ma.getmaskarray(ma.masked_not_equal(z.imag, 0.))
negz_ma = ma.getmaskarray(ma.masked_less(z, .0))
belowhrz = ma.mask_or(imagz_ma, negz_ma)
return belowhrz
def parse_segments(seg):
# Each channel corresponds to a different region of the tumor, decouple and stack these
msks_parsed = []
for slice in range(seg.shape[-1]):
curr = seg[:, :, slice]
GD = ma.masked_not_equal(curr, 4).filled(fill_value=0)
edema = ma.masked_not_equal(curr, 2).filled(fill_value=0)
necrotic = ma.masked_not_equal(curr, 1).filled(fill_value=0)
none = ma.masked_not_equal(curr, 0).filled(fill_value=0)
msks_parsed.append(np.dstack((none, necrotic, edema, GD)))
# Replace all tumorous areas with 1 (previously marked as 1, 2 or 4)
mask = np.asarray(msks_parsed)
mask[mask > 0] = 1
return mask
def zone( self, position ):
"""
Return a masked array representing the zone for the input position
@param position - i,j coordinates from which to derive zone
@return - NumPy masked array representing the geometry of the zone for pixel at input position
"""
array = self.gField.ReadAsArray()
val = array[position[0], position[1]]
maskArray = ma.masked_not_equal( array, val ) #All values not equal to zone value of input are masked
return maskArray
def get_indexes(v, val):
"""
Returns the indexes of the v array which have the value 'val':
Cases:
if v = column of a matrix:
returns the rows which have the value 'val'
if v = row of a matrix:
returns the columns which have the value 'val'
"""
max_mask = ma.getmask(ma.masked_not_equal(v, val))
return list(ma.array(np.arange(len(v)), mask=max_mask).compressed())
def __getitem__(self, index):
self.counter += 1
if not self.use_real_img:
return self.gen_synthetic()
if index > self.num_real_images - 1:
return self.gen_synthetic()
# for DA, enforces that each batch has at least one source sample
if self.one_syn_per_batch and self.counter % self.batch_size == 0:
return self.gen_synthetic()
else:
prefix = self.train_paths[index]
# get raw image
raw = cv2.imread(prefix + "-color.png")
img = cv2.resize(raw, (self.input_height, self.input_width))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# load class info
meta = loadmat(prefix + '-meta.mat')
class_ids = meta['cls_indexes']
# get segmentation gt, note 0 is for background
label_img = cv2.imread(prefix + "-label.png")[:, :, 0]
label_img = cv2.resize(label_img, (self.target_h, self.target_w),
interpolation=cv2.INTER_NEAREST)
# generate kp gt map of (nH, nW, nV)
kp_gt_map_x = np.zeros((self.target_h, self.target_w, self.n_kp))
kp_gt_map_y = np.zeros((self.target_h, self.target_w, self.n_kp))
in_pkl = prefix + '-bb8_2d.pkl'
with open(in_pkl, 'rb') as f:
bb8_2d = pickle.load(f)
for i, cid in enumerate(class_ids):
class_mask = np.where(label_img == cid[0])
kp_gt_map_x[class_mask] = bb8_2d[:, :, 0][i]
kp_gt_map_y[class_mask] = bb8_2d[:, :, 1][i]
mask_front = ma.getmaskarray(ma.masked_not_equal(label_img,
0)).astype(int)
# debug: save 100 real imgages
#cv2.imwrite("./real_images/img-" + str(random.randint(0, 100)) + ".jpg",cv2.cvtColor(img, cv2.COLOR_RGB2BGR))
#TODO: get mask weighted by class
return (torch.from_numpy(img.transpose(2, 0,
1)).float().div(255.0),
torch.from_numpy(label_img).long(),
torch.from_numpy(kp_gt_map_x).float(),
torch.from_numpy(kp_gt_map_y).float(),
torch.from_numpy(mask_front).float(),
torch.ones(self.target_w, self.target_h).long())
def zone(self, position):
"""
Return a masked array representing the zone for the input position
@param position - i,j coordinates from which to derive zone
@return - NumPy masked array representing the geometry of the zone for pixel at input position
"""
array = self.gField.ReadAsArray()
val = array[position[0], position[1]]
maskArray = ma.masked_not_equal(
array,
val) #All values not equal to zone value of input are masked
return maskArray
def _check_mask_consistency(self, dataset, product_type):
"""
:type dataset: Dataset
:type product_type: ProductType
"""
consistency_check_specs = product_type.get_mask_consistency_check_specs(
)
if len(consistency_check_specs) == 0:
return
quality_variable_name = consistency_check_specs[0][2]
quality_data = dataset.variables[quality_variable_name][:]
quality_masks = {}
for level in range(0, 6):
level_mask = ma.masked_not_equal(quality_data, level).mask
quality_masks.update({level: level_mask})
for spec in consistency_check_specs:
reference_variable_name = spec[0]
objective_variable_name = spec[1]
quality_variable_name = spec[2]
if quality_variable_name in dataset.variables:
quality_levels = spec[3]
for l in quality_levels:
level_mask = quality_masks[l]
a = SstProductVerifier.__get_data_of_quality(
dataset, reference_variable_name, level_mask).mask
b = SstProductVerifier.__get_data_of_quality(
dataset, objective_variable_name, level_mask).mask
# false negatives: element is not masked in a, but masked in b
check_name = objective_variable_name + '.' + 'mask_false_negative_check_' + str(
l)
self.__check_false_negatives(a, b, check_name)
# false positives: element is masked in a, but not masked in b
check_name = objective_variable_name + '.' + 'mask_false_positive_check_' + str(
l)
self.__check_false_positives(a, b, check_name)
else:
a = SstProductVerifier.__get_data(dataset,
reference_variable_name).mask
b = SstProductVerifier.__get_data(dataset,
objective_variable_name).mask
# false negatives: element is not masked in a, but masked in b
check_name = objective_variable_name + '.mask_false_negative_check'
self.__check_false_negatives(a, b, check_name)
# false positives: element is masked in a, but not masked in b
check_name = objective_variable_name + '.' + 'mask_false_positive_check'
self.__check_false_positives(a, b, check_name)
def get_mask(self, dep, img, masked):
img_dep = dep.detach().numpy()
if masked:
img_mask_1 = ma.masked_not_equal(img_dep, 0)
img_mask_2 = ma.masked_less(img_dep, (70 / self.max_value))
img_mask = ~img_mask_1.mask + ~img_mask_2.mask
img_mask = np.array(img_mask, dtype=np.uint8)
return 1 - img_mask
else:
mask = np.all(np.swapaxes(img.detach().numpy(), 0, 2) != [0, 0, 0], axis=-1)
if mask.size != 1:
return np.expand_dims(np.swapaxes(mask, 0, 1), axis=0)
else:
return np.ones(dep.shape, dtype=np.uint8)
def __getitem__(self, index):
# get a single training sample
if not self.use_real_img:
# use synthetic images
return self.gen_synthetic()
if index > len(self.train_paths) - 1:
# generate synthetic images if index out of original data range
return self.gen_synthetic()
else:
# use real images from YCB videos
prefix = self.train_paths[index]
# get raw image
raw = imread(prefix + "-color.png")
img = cv2.resize(raw, (self.input_height, self.input_width))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# load class info
meta = loadmat(prefix + '-meta.mat')
class_ids = meta['cls_indexes']
# get segmentation gt, note 0 is for background
label_img = imread(prefix + "-label.png")
label_img = cv2.resize(label_img, (self.target_h, self.target_w),
interpolation=cv2.INTER_NEAREST)
# generate kp gt map of (nH, nW, nV)
kp_gt_map_x = np.zeros((self.target_h, self.target_w, self.n_kp))
kp_gt_map_y = np.zeros((self.target_h, self.target_w, self.n_kp))
in_pkl = prefix + '-bb8_2d.pkl'
with open(in_pkl, 'rb') as f:
bb8_2d = pickle.load(f)
for i, cid in enumerate(class_ids):
class_mask = np.where(label_img == cid[0])
kp_gt_map_x[class_mask] = bb8_2d[:, :, 0][i]
kp_gt_map_y[class_mask] = bb8_2d[:, :, 1][i]
# get image mask front (used to compute loss)
mask_front = ma.getmaskarray(ma.masked_not_equal(label_img,
0)).astype(int)
# return training data
# input : normalized RGB image
# output : segmentation mask, x ground truth map, y ground truth map & mask front
return (torch.from_numpy(img.transpose(2, 0,
1)).float().div(255.0),
torch.from_numpy(label_img).long(),
torch.from_numpy(kp_gt_map_x).float(),
torch.from_numpy(kp_gt_map_y).float(),
torch.from_numpy(mask_front).float())
def getMetaData(self, index, mask=False, bbox=False, camera_matrix=False):
returned_dict = {}
object_label = self.list_obj[index]
returned_dict['object_label'] = object_label
# Transform matrix should be in 4x4 format
rank = self.list_rank[index]
if object_label == 2:
for i in range(0, len(self.meta[object_label][rank])):
if self.meta[object_label][rank][i]['obj_id'] == 2:
meta = self.meta[object_label][rank][i]
break
else:
meta = self.meta[object_label][rank][0]
target_r = np.resize(np.array(meta['cam_R_m2c']), (3, 3))
target_t = np.array(meta['cam_t_m2c'])
transform_mat = np.identity(4)
transform_mat[:3, :3] = target_r
transform_mat[:3, 3] = target_t / 1000.0
returned_dict['transform_mat'] = transform_mat
if mask:
path = self.list_depth[index]
depth = self.getDepthImage(index)
label = np.array(Image.open(self.list_label[index]))
if (len(label.shape) > 2):
label = label[:, :, 0]
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255)))
mask = mask_label * mask_depth
returned_dict['mask'] = mask
if bbox: # needs to return x,y,w,h
bbox = get_bbox(meta['obj_bb'])
returned_dict['bbox'] = bbox
if camera_matrix:
returned_dict['camera_scale'] = 1.0
returned_dict['camera_cx'] = 325.26110
returned_dict['camera_cy'] = 242.04899
returned_dict['camera_fx'] = 572.41140
returned_dict['camera_fy'] = 573.57043
return returned_dict
def __get_masked_data_of_quality(variable, quality_data, level):
"""
:type variable: Variable
:type quality_data: Object
:type level: int
:rtype : ma.MaskedArray
"""
mask = ma.masked_not_equal(quality_data, level).mask
data = ma.array(variable[:], mask=mask)
try:
fill_value = variable.getncattr('_FillValue')
return ma.array(data, mask=ma.masked_equal(data, fill_value).mask)
except AttributeError:
return data
def __get_masked_data_of_quality(variable, quality_data, level):
"""
:type variable: Variable
:type quality_data: Object
:type level: int
:rtype : ma.MaskedArray
"""
mask = ma.masked_not_equal(quality_data, level).mask
data = ma.array(variable[:], mask=mask)
try:
fill_value = variable.getncattr('_FillValue')
return ma.array(data, mask=ma.masked_equal(data, fill_value).mask)
except AttributeError:
return data
def _check_mask_consistency(self, dataset, product_type):
"""
:type dataset: Dataset
:type product_type: ProductType
"""
consistency_check_specs = product_type.get_mask_consistency_check_specs()
if len(consistency_check_specs) == 0:
return
quality_variable_name = consistency_check_specs[0][2]
quality_data = dataset.variables[quality_variable_name][:]
quality_masks = {}
for level in range(0, 6):
level_mask = ma.masked_not_equal(quality_data, level).mask
quality_masks.update({level: level_mask})
for spec in consistency_check_specs:
reference_variable_name = spec[0]
objective_variable_name = spec[1]
quality_variable_name = spec[2]
if quality_variable_name in dataset.variables:
quality_levels = spec[3]
for l in quality_levels:
level_mask = quality_masks[l]
a = SstProductVerifier.__get_data_of_quality(dataset, reference_variable_name, level_mask).mask
b = SstProductVerifier.__get_data_of_quality(dataset, objective_variable_name, level_mask).mask
# false negatives: element is not masked in a, but masked in b
check_name = objective_variable_name + '.' + 'mask_false_negative_check_' + str(l)
self.__check_false_negatives(a, b, check_name)
# false positives: element is masked in a, but not masked in b
check_name = objective_variable_name + '.' + 'mask_false_positive_check_' + str(l)
self.__check_false_positives(a, b, check_name)
else:
a = SstProductVerifier.__get_data(dataset, reference_variable_name).mask
b = SstProductVerifier.__get_data(dataset, objective_variable_name).mask
# false negatives: element is not masked in a, but masked in b
check_name = objective_variable_name + '.mask_false_negative_check'
self.__check_false_negatives(a, b, check_name)
# false positives: element is masked in a, but not masked in b
check_name = objective_variable_name + '.' + 'mask_false_positive_check'
self.__check_false_positives(a, b, check_name)
def filter_auxfiles(data, isets, auxf, auxpar, auxconds):
if auxf:
data = ma.asarray(data)
for f in range(len(auxf)):
catlg = cl.ClumpCatalog.from_file([auxf[f]])
size = np.shape(catlg.clumps)[0]
npars = len(auxpar[f])
tmparr = np.zeros((npars, size))
for i in range(size):
for j in range(npars):
tmparr[j, i] = catlg.clumps[i].record[auxpar[f][j]]
tmparr = ma.asarray(tmparr)
fcond = auxconds[f]
col = 0
while fcond:
cond = fcond.pop(0)
cut = fcond.pop(0)
if cond == ">":
mask_aux = ma.masked_greater(tmparr[col, :], cut)
elif cond == "=":
mask_aux = ma.masked_equal(tmparr[col, :], cut)
elif cond == "<":
mask_aux = ma.masked_less(tmparr[col, :], cut)
elif cond == "!=":
mask_aux = ma.masked_not_equal(tmparr[col, :], cut)
col += 1
tmparr.mask = mask_aux.mask
if np.shape(tmparr.mask):
data.mask = tmparr.mask
else:
data = data.data
return data
def filter_clumps(varmap, ifiles, defs, conds, inclumps):
"""Filter data in clumps"""
mapinclumps = np.empty(ifiles, dtype=maps.Map)
print("inclumps")
if defs:
print("defs")
for i in range(ifiles):
mapinclumps[i] = varmap[i].masked_where(ma.getmask(inclumps))
print(" >>> number of valid pixels in clumps:", i, "=>",
mapinclumps[i].count())
while len(conds) > 0:
cond = conds.pop(0)
val = conds.pop(0)
if cond == ">":
mskcl = ma.masked_greater(inclumps, val)
elif cond == "=":
mskcl = ma.masked_not_equal(inclumps, val)
elif cond == "<":
mskcl = ma.masked_less(inclumps, val)
else:
print(" ++ wrong condition in filter clumps", cond)
sys.exit(1)
for i in range(ifiles):
mapinclumps[i] = mapinclumps[i].masked_where(ma.getmask(mskcl))
print(" >>> number of valid pixels after clump filter:", i,
"=>", mapinclumps[i].count())
else:
for i in range(ifiles):
mapinclumps[i] = varmap[i].copy()
return mapinclumps
def count_matching_lesion_voxels(lesion_mx, segmentation_mx, lesion,
segmentation):
"""
Counts the number of voxels where
lesion_mx[index] = lesion and
segmentation_mx[index] = segmentation
for the same index
Parameters
----------
lesion_mx : 3D numpy.ndarray
Matrix of labeled lesions.
segmentation_mx : 3D numpy.ndarray
Matrix of labeled brain segmentation.
lesion : TYPE <int>
Label for the lesion of interest.
segmentation : TYPE <int>
Label for the brain structure of interest.
Returns
-------
TYPE <int>
Count of matching voxels.
"""
les_mx = lesion_mx.copy()
# Set all voxels whose value differ from the lesion id to -1
les_mx = ma.masked_not_equal(les_mx, lesion)
les_mx.fill_value = -1
les_mx = les_mx.filled()
# Set all voxels whose value equal to the lesion id to 1
les_mx[les_mx == lesion] = 1
return np.sum(les_mx == segmentation_mx)
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(eq(masked_inside(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 1, 3).mask,
[1, 1, 1, 1, 0]))
assert_(eq(masked_outside(array(list(range(5)),
mask=[0, 1, 0, 0, 0]), 1, 3).mask,
[1, 1, 0, 0, 1]))
assert_(eq(masked_equal(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 0]))
assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1],
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 1]))
assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))
def printCurve(take_idx, criterion):
conf_tp_or_fn = [[] for i in range(5)]
conf_fp = [[] for i in range(5)]
prec = [[] for i in range(5)]
recall = [[] for i in range(5)]
if take_idx is 0:
file_1_name = 'occ/under60.txt'
file_2_name = 'occ/under60_frames.txt'
elif take_idx is 1:
file_1_name = 'occ/from60to80.txt'
file_2_name = 'occ/f60t80_frames.txt'
else:
file_1_name = 'occ/up80.txt'
file_2_name = 'occ/up80_frames.txt'
xmap = np.array([[j for i in range(640)] for j in range(480)])
ymap = np.array([[i for i in range(640)] for j in range(480)])
with open(file_1_name, 'r') as f1:
with open(file_2_name, 'r') as f2:
while 1:
input_line_test = f2.readline()
# print(input_line_test)
if not input_line_test:
break
if input_line_test[-1:] == '\n':
input_line_test = input_line_test[:-1]
_, test_scene_id, test_frame_id = input_line_test.split('/')
input_line_test = '/'.join(
['data_v1', test_scene_id, test_frame_id])
# import pdb;pdb.set_trace()
input_line_test_2 = f1.readline()
test_obj_id = int(float(input_line_test_2.split()[2])) + 1
test_idx = int(float(input_line_test_2.split()[1]))
# import pdb;pdb.set_trace()
img = Image.open('{0}/{1}-color.png'.format(
opt.dataset_root, input_line_test))
depth = np.array(
Image.open('{0}/{1}-depth.png'.format(
opt.dataset_root, input_line_test)))
label = np.array(
Image.open('{0}/{1}-label.png'.format(
opt.dataset_root, input_line_test)))
meta = scio.loadmat('{0}/{1}-meta.mat'.format(
opt.dataset_root, input_line_test))
cam_cx = 312.9869
cam_cy = 241.3109
cam_fx = 1066.778
cam_fy = 1067.487
mask_back = ma.getmaskarray(ma.masked_equal(label, 0))
print('scene index: ', test_scene_id)
print('object index: ', test_obj_id)
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
mask_label = ma.getmaskarray(
ma.masked_equal(label, test_obj_id))
mask = mask_label * mask_depth
if not (len(mask.nonzero()[0]) > 50
and len(opt.symmetry[test_obj_id]['mirror']) > 0):
continue
rmin, rmax, cmin, cmax = get_bbox(mask_label)
img_temp = np.transpose(np.array(img)[:, :, :3],
(2, 0, 1))[:, rmin:rmax, cmin:cmax]
img_masked = img_temp
target_r = meta['poses'][:, :, test_idx][:, 0:3]
target_t = np.array(meta['poses'][:, :,
test_idx][:, 3:4].flatten())
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
if len(choose) > opt.num_points:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:opt.num_points] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
choose = np.pad(choose, (0, opt.num_points - len(choose)),
'wrap')
depth_masked = depth[
rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(np.float32)
xmap_masked = xmap[
rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(np.float32)
ymap_masked = ymap[
rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(np.float32)
choose = np.array([choose])
cam_scale = meta['factor_depth'][0][0]
pt2 = depth_masked / cam_scale
pt0 = (ymap_masked - cam_cx) * pt2 / cam_fx
pt1 = (xmap_masked - cam_cy) * pt2 / cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
target_sym = []
for sym in opt.symmetry[test_obj_id]['mirror']:
target_sym.append(np.dot(sym, target_r.T))
target_sym = np.array(target_sym)
target_cen = np.add(opt.symmetry[test_obj_id]['center'],
target_t)
# print('ground truth norm: ', target_sym)
# print('ground truth center: ', target_cen)
points_ten, choose_ten, img_ten, target_sym_ten, target_cen_ten, idx_ten = \
torch.from_numpy(cloud.astype(np.float32)).unsqueeze(0), \
torch.LongTensor(choose.astype(np.int32)).unsqueeze(0), \
opt.norm(torch.from_numpy(img_masked.astype(np.float32))).unsqueeze(0), \
torch.from_numpy(target_sym.astype(np.float32)).unsqueeze(0), \
torch.from_numpy(target_cen.astype(np.float32)).unsqueeze(0), \
torch.LongTensor([test_obj_id-1]).unsqueeze(0)
points_ten, choose_ten, img_ten, target_sym_ten, target_cen_ten, idx_ten = Variable(points_ten), \
Variable(choose_ten), \
Variable(img_ten), \
Variable(target_sym_ten), \
Variable(target_cen_ten), \
Variable(idx_ten)
pred_norm, pred_on_plane, emb = opt.estimator(
img_ten, points_ten, choose_ten, idx_ten)
bs, num_p, _ = pred_on_plane.size()
pred_norm = pred_norm / (torch.norm(pred_norm, dim=2).view(
bs, num_p, 1))
pred_norm = pred_norm.detach().numpy()
pred_on_plane = pred_on_plane.detach().numpy()
points = points_ten.detach().numpy()
clustering_points_idx = np.where(pred_on_plane > max(
0.5,
pred_on_plane.max() * PRED_ON_PLANE_FACTOR +
pred_on_plane.mean() * (1 - PRED_ON_PLANE_FACTOR)))[1]
clustering_norm = pred_norm[0, clustering_points_idx, :]
clustering_points = points[0, clustering_points_idx, :]
num_points = len(clustering_points_idx)
# print(pred_on_plane.max())
# import pdb;pdb.set_trace()
close_thresh = 2e-3
broad_thresh = 3e-3
sym_conf = np.zeros((5, target_sym.shape[0]))
count_pred = 0
# import pdb; pdb.set_trace()
while True:
count_pred += 1
if num_points <= 20 or count_pred > 3:
break
best_fit_num = 0
count_try = 0
for j in range(10):
pick_idx = np.random.randint(0, num_points - 1)
pick_point = clustering_points[pick_idx]
# proposal_norm = np.array(Plane(Point3D(pick_points[0]),Point3D(pick_points[1]),Point3D(pick_points[2])).normal_vector).astype(np.float32)
proposal_norm = clustering_norm[pick_idx]
proposal_norm = proposal_norm[:, np.newaxis]
# import pdb;pdb.set_trace()
proposal_point = pick_point
clustering_diff = clustering_points - proposal_point
clustering_dist = np.abs(
np.matmul(clustering_diff, proposal_norm))
broad_inliers = np.where(
clustering_dist < broad_thresh)[0]
broad_inlier_num = len(broad_inliers)
close_inliers = np.where(
clustering_dist < close_thresh)[0]
close_inlier_num = len(close_inliers)
norm_dist = np.abs(clustering_norm -
np.transpose(proposal_norm)).sum(1)
close_norm_idx = np.where(norm_dist < 0.6)[0]
close_norm_num = len(close_norm_idx)
if close_inlier_num >= best_fit_num and broad_inlier_num >= num_points / (
4 - count_pred
) * 0.9 and close_norm_num >= num_points / (
4 - count_pred) * 0.9:
best_fit_num = close_inlier_num
best_fit_norm = proposal_norm
best_fit_cen = clustering_points[
close_inliers].mean(0)
best_fit_idx = clustering_points_idx[close_inliers]
best_norm_dist = norm_dist
best_close_norm_idx = np.where(
best_norm_dist < 0.6)[0]
if best_fit_num == 0 or num_points <= 20:
break
clustering_points_same_sym = clustering_points[
best_close_norm_idx]
clustering_diff_same_sym = clustering_points_same_sym - best_fit_cen
clustering_dist_same_sym = np.abs(
np.matmul(clustering_diff_same_sym, best_fit_norm))
close_inliers = np.where(
clustering_dist_same_sym < close_thresh)[0]
close_inlier_num = len(close_inliers)
best_fit_num = close_inlier_num
broad_inliers = np.where(
clustering_dist_same_sym < broad_thresh)[0]
broad_inlier_num = len(broad_inliers)
def f(x):
dist = 0
# import pdb;pdb.set_trace()
for point in clustering_points_same_sym[broad_inliers]:
dist += np.abs(
(point * x[0:3]).sum() + x[3]) / np.sqrt(
np.sum(np.square(x[0:3]), axis=0))
return dist
start_point = np.zeros(4)
start_point[0:3] = np.copy(best_fit_norm[:, 0])
start_point[3] = (-best_fit_cen *
best_fit_norm[:, 0]).sum()
min_point = fmin(f, start_point, maxiter=50)
# import pdb;pdb.set_trace()
min_point = min_point / np.sqrt(
np.sum(np.square(min_point[0:3]), axis=0))
x_val = -(min_point[3] + best_fit_cen[1] * min_point[1] +
best_fit_cen[2] * min_point[2]) / min_point[0]
y_val = -(min_point[3] + best_fit_cen[0] * min_point[0] +
best_fit_cen[2] * min_point[2]) / min_point[1]
z_val = -(min_point[3] + best_fit_cen[0] * min_point[0] +
best_fit_cen[1] * min_point[1]) / min_point[2]
if np.abs(x_val) < 1:
new_pred_loc = np.array(
[x_val, best_fit_cen[1], best_fit_cen[2]])
elif np.abs(z_val) < 1:
new_pred_loc = np.array(
[best_fit_cen[0], best_fit_cen[1], z_val])
else:
new_pred_loc = np.array(
[best_fit_cen[0], y_val, best_fit_cen[2]])
new_proposal_norm = min_point[0:3]
clustering_diff = clustering_points_same_sym - new_pred_loc
clustering_dist = np.abs(
np.matmul(clustering_diff, new_proposal_norm))
close_inliers = np.where(clustering_dist < close_thresh)[0]
new_close_inlier_num = len(close_inliers)
broad_inliers = np.where(clustering_dist < broad_thresh)[0]
new_broad_inlier_num = len(broad_inliers)
# import pdb;pdb.set_trace()
if new_close_inlier_num >= close_inlier_num:
best_fit_num = new_close_inlier_num
best_fit_norm = new_proposal_norm[:, np.newaxis]
best_fit_cen = new_pred_loc
if best_fit_num == 0:
break
else:
print('predicted norm:{}, predicted point:{}'.format(
best_fit_norm, best_fit_cen))
max_idx = np.argmax(
np.abs(np.matmul(target_sym, best_fit_norm)))
sym_product = np.abs(
np.matmul(target_sym, best_fit_norm)[max_idx][0])
sym_dist = np.abs((target_sym[max_idx] *
(best_fit_cen - target_cen)).sum())
norm_dist = np.abs(clustering_norm -
np.transpose(best_fit_norm)).sum(1)
scrub_close_norm_idx = np.where(norm_dist < 1.3)[0]
# import pdb;pdb.set_trace()
predicted_confidence = best_fit_num / len(
best_close_norm_idx) - np.abs(
clustering_norm[best_close_norm_idx] -
np.transpose(best_fit_norm)).mean() * 3 * 1.5
predicted_confidence = max(0, predicted_confidence)
for dist_idx in range(5):
if sym_product > PRODUCT_THRESHOLD and sym_dist < (
dist_idx + 1) * 0.01:
# import pdb;pdb.set_trace()
sym_conf[dist_idx, max_idx] = max(
sym_conf[dist_idx, max_idx],
predicted_confidence)
else:
conf_fp[dist_idx].append(predicted_confidence)
clustering_points_idx = np.setdiff1d(
clustering_points_idx,
clustering_points_idx[scrub_close_norm_idx])
clustering_norm = pred_norm[0,
clustering_points_idx, :]
clustering_points = points[0, clustering_points_idx, :]
num_points = len(clustering_points_idx)
# import pdb;pdb.set_trace()
# import pdb;pdb.set_trace()
for dist_idx in range(5):
for i in range(target_sym.shape[0]):
conf_tp_or_fn[dist_idx].append(sym_conf[dist_idx, i])
# import pdb;pdb.set_trace()
# import pdb;pdb.set_trace()
print(conf_tp_or_fn)
print(conf_fp)
# import pdb;pdb.set_trace()
for dist_idx in range(5):
for t in range(1, 1001):
conf_thresh = t / 1000
true_positives = len(
np.where(np.array(conf_tp_or_fn[dist_idx]) >= conf_thresh)[0])
false_negatives = len(
np.where(np.array(conf_tp_or_fn[dist_idx]) < conf_thresh)[0])
false_positives = len(
np.where(np.array(conf_fp[dist_idx]) >= conf_thresh)[0])
if false_positives + true_positives > 0 and true_positives + false_negatives > 0:
prec[dist_idx].append(true_positives /
(false_positives + true_positives))
recall[dist_idx].append(true_positives /
(true_positives + false_negatives))
return prec, recall
def __getitem__(self, index):
img = Image.open('{0}/{1}-color.png'.format(self.root,
self.list[index]))
depth = np.array(
Image.open('{0}/{1}-depth.png'.format(self.root,
self.list[index])))
label = np.array(
Image.open('{0}/{1}-label.png'.format(self.root,
self.list[index])))
meta = scio.loadmat('{0}/{1}-meta.mat'.format(self.root,
self.list[index]))
if self.list[index][:8] != 'data_syn' and int(
self.list[index][5:9]) >= 60:
cam_cx = self.cam_cx_2
cam_cy = self.cam_cy_2
cam_fx = self.cam_fx_2
cam_fy = self.cam_fy_2
else:
cam_cx = self.cam_cx_1
cam_cy = self.cam_cy_1
cam_fx = self.cam_fx_1
cam_fy = self.cam_fy_1
mask_back = ma.getmaskarray(ma.masked_equal(label, 0))
add_front = False
if self.add_noise:
for k in range(5):
seed = random.choice(self.syn)
front = np.array(
self.trancolor(
Image.open('{0}/{1}-color.png'.format(
self.root, seed)).convert("RGB")))
front = np.transpose(front, (2, 0, 1))
f_label = np.array(
Image.open('{0}/{1}-label.png'.format(self.root, seed)))
front_label = np.unique(f_label).tolist()[1:]
if len(front_label) < self.front_num:
continue
front_label = random.sample(front_label, self.front_num)
for f_i in front_label:
mk = ma.getmaskarray(ma.masked_not_equal(f_label, f_i))
if f_i == front_label[0]:
mask_front = mk
else:
mask_front = mask_front * mk
t_label = label * mask_front
if len(t_label.nonzero()[0]) > 1000:
label = t_label
add_front = True
break
obj = meta['cls_indexes'].flatten().astype(np.int32)
while 1:
idx = np.random.randint(0, len(obj))
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
mask_label = ma.getmaskarray(ma.masked_equal(label, obj[idx]))
mask = mask_label * mask_depth
if len(mask.nonzero()[0]) > self.minimum_num_pt:
break
if self.add_noise:
img = self.trancolor(img)
rmin, rmax, cmin, cmax = get_bbox(mask_label)
img = np.transpose(np.array(img)[:, :, :3], (2, 0, 1))[:, rmin:rmax,
cmin:cmax]
if self.list[index][:8] == 'data_syn':
seed = random.choice(self.real)
back = np.array(
self.trancolor(
Image.open('{0}/{1}-color.png'.format(
self.root, seed)).convert("RGB")))
back = np.transpose(back, (2, 0, 1))[:, rmin:rmax, cmin:cmax]
img_masked = back * mask_back[rmin:rmax, cmin:cmax] + img
else:
img_masked = img
if self.add_noise and add_front:
img_masked = img_masked * mask_front[
rmin:rmax, cmin:cmax] + front[:, rmin:rmax, cmin:cmax] * ~(
mask_front[rmin:rmax, cmin:cmax])
if self.list[index][:8] == 'data_syn':
img_masked = img_masked + np.random.normal(
loc=0.0, scale=7.0, size=img_masked.shape)
# p_img = np.transpose(img_masked, (1, 2, 0))
# scipy.misc.imsave('temp/{0}_input.png'.format(index), p_img)
# scipy.misc.imsave('temp/{0}_label.png'.format(index), mask[rmin:rmax, cmin:cmax].astype(np.int32))
target_r = meta['poses'][:, :, idx][:, 0:3]
target_t = np.array([meta['poses'][:, :, idx][:, 3:4].flatten()])
add_t = np.array([
random.uniform(-self.noise_trans, self.noise_trans)
for i in range(3)
])
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
if len(choose) > self.num_pt:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:self.num_pt] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
choose = np.pad(choose, (0, self.num_pt - len(choose)), 'wrap')
depth_masked = depth[rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(
np.float32)
xmap_masked = self.xmap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
ymap_masked = self.ymap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
choose = np.array([choose])
cam_scale = meta['factor_depth'][0][0]
pt2 = depth_masked / cam_scale
pt0 = (ymap_masked - cam_cx) * pt2 / cam_fx
pt1 = (xmap_masked - cam_cy) * pt2 / cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
if self.add_noise:
cloud = np.add(cloud, add_t)
# fw = open('temp/{0}_cld.xyz'.format(index), 'w')
# for it in cloud:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
# fw.close()
dellist = [j for j in range(0, len(self.cld[obj[idx]]))]
if self.refine:
dellist = random.sample(
dellist,
len(self.cld[obj[idx]]) - self.num_pt_mesh_large)
else:
dellist = random.sample(
dellist,
len(self.cld[obj[idx]]) - self.num_pt_mesh_small)
model_points = np.delete(self.cld[obj[idx]], dellist, axis=0)
# fw = open('temp/{0}_model_points.xyz'.format(index), 'w')
# for it in model_points:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
# fw.close()
target = np.dot(model_points, target_r.T)
if self.add_noise:
target = np.add(target, target_t + add_t)
else:
target = np.add(target, target_t)
# fw = open('temp/{0}_tar.xyz'.format(index), 'w')
# for it in target:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
# fw.close()
return torch.from_numpy(cloud.astype(np.float32)), \
torch.LongTensor(choose.astype(np.int32)), \
self.norm(torch.from_numpy(img_masked.astype(np.float32))), \
torch.from_numpy(target.astype(np.float32)), \
torch.from_numpy(model_points.astype(np.float32)), \
torch.LongTensor([int(obj[idx]) - 1])
def __getitem__(self, index):
img = Image.open(self.list_rgb[index])
ori_img = np.array(img)
depth = np.array(Image.open(self.list_depth[index]))
label = np.array(Image.open(self.list_label[index]))
obj = self.list_obj[index]
rank = self.list_rank[index]
if obj == 2:
for i in range(0, len(self.meta[obj][rank])):
if self.meta[obj][rank][i]['obj_id'] == 2:
meta = self.meta[obj][rank][i]
break
else:
meta = self.meta[obj][rank][0]
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
if self.mode == 'eval':
mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255)))
else:
mask_label = ma.getmaskarray(ma.masked_equal(label, np.array([255, 255, 255])))[:, :, 0]
mask = mask_label * mask_depth
if self.add_noise:
img = self.trancolor(img)
img = np.array(img)[:, :, :3]
img = np.transpose(img, (2, 0, 1))
img_masked = img
rmin, rmax, cmin, cmax = get_bbox(meta['obj_bb'])
img_masked = img_masked[:, rmin:rmax, cmin:cmax]
#p_img = np.transpose(img_masked, (1, 2, 0))
#scipy.misc.imsave('evaluation_result/{0}_input.png'.format(index), p_img)
target_r = np.resize(np.array(meta['cam_R_m2c']), (3, 3))
target_t = np.array(meta['cam_t_m2c'])
add_t = np.array([random.uniform(-self.noise_trans, self.noise_trans) for i in range(3)])
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
if len(choose) == 0:
cc = torch.LongTensor([0])
return(cc, cc, cc, cc, cc, cc)
if len(choose) > self.num:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:self.num] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
choose = np.pad(choose, (0, self.num - len(choose)), 'wrap')
depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
xmap_masked = self.xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
ymap_masked = self.ymap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
choose = np.array([choose])
cam_scale = 1.0
pt2 = depth_masked / cam_scale
pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx
pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
cloud = np.add(cloud, -1.0 * target_t) / 1000.0
cloud = np.add(cloud, target_t / 1000.0)
if self.add_noise:
cloud = np.add(cloud, add_t)
#fw = open('evaluation_result/{0}_cld.xyz'.format(index), 'w')
#for it in cloud:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
#fw.close()
model_points = self.pt[obj] / 1000.0
dellist = [j for j in range(0, len(model_points))]
dellist = random.sample(dellist, len(model_points) - self.num_pt_mesh_small)
model_points = np.delete(model_points, dellist, axis=0)
#fw = open('evaluation_result/{0}_model_points.xyz'.format(index), 'w')
#for it in model_points:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
#fw.close()
target = np.dot(model_points, target_r.T)
if self.add_noise:
target = np.add(target, target_t / 1000.0 + add_t)
out_t = target_t / 1000.0 + add_t
else:
target = np.add(target, target_t / 1000.0)
out_t = target_t / 1000.0
#fw = open('evaluation_result/{0}_tar.xyz'.format(index), 'w')
#for it in target:
# fw.write('{0} {1} {2}\n'.format(it[0], it[1], it[2]))
#fw.close()
return torch.from_numpy(cloud.astype(np.float32)), \
torch.LongTensor(choose.astype(np.int32)), \
self.norm(torch.from_numpy(img_masked.astype(np.float32))), \
torch.from_numpy(target.astype(np.float32)), \
torch.from_numpy(model_points.astype(np.float32)), \
torch.LongTensor([self.objlist.index(obj)])
my_result_wo_refine = []
my_result = []
for idx in range(len(detected_classIDs)):
itemid = detected_classIDs[idx]
maskid = idx + 1
try:
mask = ma.getmaskarray(ma.masked_equal(masks, maskid))
rmin, rmax, cmin, cmax = get_bbox(mask)
print('itemid: {0}\n'.format(itemid))
print('rmin {0}, rmax {1}, cmin {2}, cmax {3}'.format(rmin, rmax, cmin, cmax))
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
mask_label = ma.getmaskarray(ma.masked_equal(masks, maskid))
mask = mask_label * mask_depth
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
if len(choose) > num_points:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:num_points] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
choose = np.pad(choose, (0, num_points - len(choose)), 'wrap')
depth_masked = depth[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
xmap_masked = xmap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
ymap_masked = ymap[rmin:rmax, cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
def callback(self, rgb, depth):
if DEBUG:
print('received depth image of type: ' + depth.encoding)
print('received rgb image of type: ' + rgb.encoding)
#https://answers.ros.org/question/64318/how-do-i-convert-an-ros-image-into-a-numpy-array/
depth = np.frombuffer(depth.data,
dtype=np.uint16).reshape(depth.height,
depth.width, -1)
rgb = np.frombuffer(rgb.data,
dtype=np.uint8).reshape(rgb.height, rgb.width, -1)
rgb_original = rgb
#cv2.imshow('depth', depth)
#time1 = time.time()
rgb = np.transpose(rgb, (2, 0, 1))
rgb = norm(torch.from_numpy(rgb.astype(np.float32)))
rgb = Variable(rgb).cuda()
semantic = self.model(rgb.unsqueeze(0))
_, pred = torch.max(semantic, dim=1)
pred = pred * 255
pred = np.transpose(pred, (1, 2, 0)) # (CxHxW)->(HxWxC)
#print(pred.shape)
#ret, threshold = cv2.threshold(pred.cpu().numpy(), 1, 255, cv2.THRESH_BINARY) #pred is already binary, therefore, this line is unnecessary
contours, hierarchy = cv2.findContours(np.uint8(pred),
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnt = max(contours, key=cv2.contourArea)
x, y, w, h = cv2.boundingRect(cnt)
rmin, rmax, cmin, cmax = get_bbox([x, y, w, h])
#cv2.rectangle(rgb_original,(cmin,rmin), (cmax,rmax) , (0,255,0),2)
#cv2.imwrite('depth.png', depth) #save depth image
mask_depth = ma.getmasksarray(ma.masked_not_equal(depth, 0))
mask_label = ma.getmaskarray(ma.masked_equal(pred, np.array(255)))
mask = mask_depth * mask_label
#print(rgb.shape) #torch.Size([3, 480, 640])
#print(rgb_original.shape) #(480, 640, 3)
img = np.transpose(rgb_original, (2, 0, 1))
img_masked = img[:, rmin:rmax, cmin:cmax]
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
#print("length of choose is :{0}".format(len(choose)))
if len(choose) == 0:
cc = torch.LongTensor([0])
return (cc, cc, cc, cc, cc, cc)
if len(choose) > num_points:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:
num_points] = 1 # if number of object pixels are bigger than 500, we select just 500
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()] # now len(choose) = 500
else:
choose = np.pad(choose, (0, num_points - len(choose)), 'wrap')
depth_masked = depth[rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(
np.float32)
xmap_masked = self.xmap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
ymap_masked = self.ymap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
choose = np.array([choose])
pt2 = depth_masked
#print(pt2)
pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx
pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
cloud = cloud / 1000
points = torch.from_numpy(cloud.astype(np.float32))
choose = torch.LongTensor(choose.astype(np.int32))
img = norm(torch.from_numpy(img_masked.astype(np.float32)))
idx = torch.LongTensor([self.object_index])
img = Variable(img).cuda().unsqueeze(0)
points = Variable(points).cuda().unsqueeze(0)
choose = Variable(choose).cuda().unsqueeze(0)
idx = Variable(idx).cuda().unsqueeze(0)
pred_r, pred_t, pred_c, emb = self.estimator(img, points, choose, idx)
pred_r = pred_r / torch.norm(pred_r, dim=2).view(1, num_points, 1)
pred_c = pred_c.view(bs, num_points)
how_max, which_max = torch.max(pred_c, 1)
pred_t = pred_t.view(bs * num_points, 1, 3)
my_r = pred_r[0][which_max[0]].view(-1).cpu().data.numpy()
my_t = (points.view(bs * num_points, 1, 3) +
pred_t)[which_max[0]].view(-1).cpu().data.numpy()
my_pred = np.append(my_r, my_t)
for ite in range(0, iteration):
T = Variable(torch.from_numpy(
my_t.astype(np.float32))).cuda().view(1, 3).repeat(
num_points, 1).contiguous().view(1, num_points, 3)
my_mat = quaternion_matrix(my_r)
R = Variable(torch.from_numpy(my_mat[:3, :3].astype(
np.float32))).cuda().view(1, 3, 3)
my_mat[0:3, 3] = my_t
new_points = torch.bmm((points - T), R).contiguous()
pred_r, pred_t = self.refiner(new_points, emb, idx)
pred_r = pred_r.view(1, 1, -1)
pred_r = pred_r / (torch.norm(pred_r, dim=2).view(1, 1, 1))
my_r_2 = pred_r.view(-1).cpu().data.numpy()
my_t_2 = pred_t.view(-1).cpu().data.numpy()
my_mat_2 = quaternion_matrix(my_r_2)
my_mat_2[0:3, 3] = my_t_2
my_mat_final = np.dot(
my_mat,
my_mat_2) # refine pose means two matrix multiplication
my_r_final = copy.deepcopy(my_mat_final)
my_r_final[0:3, 3] = 0
my_r_final = quaternion_from_matrix(my_r_final, True)
my_t_final = np.array(
[my_mat_final[0][3], my_mat_final[1][3], my_mat_final[2][3]])
my_pred = np.append(my_r_final, my_t_final)
my_r = my_r_final
my_t = my_t_final
my_r = quaternion_matrix(my_r)[:3, :3]
#print(my_t.shape)
my_t = np.array(my_t)
#print(my_t.shape)
#print(my_r.shape)
target = np.dot(self.scaled, my_r.T)
target = np.add(target, my_t)
p0 = (int((target[0][0] / target[0][2]) * self.cam_fx + self.cam_cx),
int((target[0][1] / target[0][2]) * self.cam_fy + self.cam_cy))
p1 = (int((target[1][0] / target[1][2]) * self.cam_fx + self.cam_cx),
int((target[1][1] / target[1][2]) * self.cam_fy + self.cam_cy))
p2 = (int((target[2][0] / target[2][2]) * self.cam_fx + self.cam_cx),
int((target[2][1] / target[2][2]) * self.cam_fy + self.cam_cy))
p3 = (int((target[3][0] / target[3][2]) * self.cam_fx + self.cam_cx),
int((target[3][1] / target[3][2]) * self.cam_fy + self.cam_cy))
p4 = (int((target[4][0] / target[4][2]) * self.cam_fx + self.cam_cx),
int((target[4][1] / target[4][2]) * self.cam_fy + self.cam_cy))
p5 = (int((target[5][0] / target[5][2]) * self.cam_fx + self.cam_cx),
int((target[5][1] / target[5][2]) * self.cam_fy + self.cam_cy))
p6 = (int((target[6][0] / target[6][2]) * self.cam_fx + self.cam_cx),
int((target[6][1] / target[6][2]) * self.cam_fy + self.cam_cy))
p7 = (int((target[7][0] / target[7][2]) * self.cam_fx + self.cam_cx),
int((target[7][1] / target[7][2]) * self.cam_fy + self.cam_cy))
cv2.line(rgb_original, p0, p1, (255, 255, 255), 2)
cv2.line(rgb_original, p0, p3, (255, 255, 255), 2)
cv2.line(rgb_original, p0, p4, (255, 255, 255), 2)
cv2.line(rgb_original, p1, p2, (255, 255, 255), 2)
cv2.line(rgb_original, p1, p5, (255, 255, 255), 2)
cv2.line(rgb_original, p2, p3, (255, 255, 255), 2)
cv2.line(rgb_original, p2, p6, (255, 255, 255), 2)
cv2.line(rgb_original, p3, p7, (255, 255, 255), 2)
cv2.line(rgb_original, p4, p5, (255, 255, 255), 2)
cv2.line(rgb_original, p4, p7, (255, 255, 255), 2)
cv2.line(rgb_original, p5, p6, (255, 255, 255), 2)
cv2.line(rgb_original, p6, p7, (255, 255, 255), 2)
#print('estimated rotation is :{0}'.format(my_r))
#print('estimated translation is :{0}'.format(my_t))
#time2 = time.time()
#print('inference time is :{0}'.format(time2-time1))
cv2.imshow('rgb',
cv2.cvtColor(rgb_original,
cv2.COLOR_BGR2RGB)) # OpenCV uses BGR model
cv2.waitKey(
1
) # pass any integr except 0, as 0 will freeze the display windows
import numpy as np import numpy.ma as ma a = np.arange(4) a ma.masked_not_equal(a, 2)
def cf_jackknife(self, ignore_regions=[], estimator='landy-szalay',
random_oversample=None, save_steps_file=None,
name='jackknife', clobber=False):
#This takes a divided mask and performs the correlation
#function calculation on the field with each sub-region
#removed in turn.
if (name in self._cfs.keys()) and not clobber:
raise ValueError("CorrelationFunction.cf_jackknife says: "
"There's already a CF by that name. Please "
"choose another or overwrite by calling with "
"clobber=True")
#Check to make sure we have everything we need
self.__check_cf_setup(need_subregions=True, check_trees=False,
random_oversample=random_oversample)
#Make a new CorrelationFunction instance and set the basic info
#First make a dictionary of the arguments to pass because it's ugly
info={'name' : name,
'cf_type' : 'jackknife',
'ngals' : self._n_objects,
'theta_bin_object' : copy.deepcopy(self._theta_bins),
'estimator' : estimator
}
self._cfs[name] = cfclass.CorrelationFunction(**info)
centers, edges = self._cfs[name].get_thetas(unit='degrees')
#pull out the unique subregion numbers and figure out which to use
regions=np.asarray(list(set(self._subregion_number)))
use_regions=[r for r in regions if (r not in ignore_regions) and (r != -1)]
use_regions=np.array(use_regions)
n_jacks=len(use_regions)
#Figure out where the randoms are
random_subregions=self._image_mask.return_subregions(self._ra_random,
self._dec_random)
#Now loop through the regions that you should be using
#and calculate the correlation function leaving out each
jackknife_jacks = {}
#Make a mask that takes out all the galaxies that aren't in use_regions
valid_subregion = ma.masked_not_equal(self._subregion_number, -1).mask
random_valid_subregion=ma.masked_not_equal(random_subregions, -1).mask
for bad_reg in ignore_regions:
this_mask = ma.masked_not_equal(self._subregion_number, bad_reg).mask
valid_subregion = valid_subregion & this_mask
this_mask = ma.masked_not_equal(random_subregions, bad_reg).mask
random_valid_subregion = random_valid_subregion & this_mask
temp = np.zeros((n_jacks, len(self._cf_thetas)))
for i, r in enumerate(use_regions):
#Make the mask for the data
not_region_r = ma.masked_not_equal(self._subregion_number, r).mask
this_jackknife = valid_subregion & not_region_r & self._use
#Make the mask for the randoms
random_not_region_r = ma.masked_not_equal(random_subregions, r).mask
random_this_jackknife = random_not_region_r & random_valid_subregion
#Do the calculation for this jackknife and store it
print "calculating jackknife", i
jackknife_jacks[r] = corr.two_point_angular(self._ra[this_jackknife],
self._dec[this_jackknife],
edges, method=estimator,
ra_R = self._ra_random[random_this_jackknife],
dec_R = self._dec_random[random_this_jackknife])
temp[i]=jackknife_jacks[r]
if (save_steps_file is not None):
jackknife_cf=np.nanmean(temp[0:i+1], axis=0)
jackknife_cf_err=np.nanstd(temp[0:i+1], axis=0)
self._cfs[name].set_cf(jackknife_cf, jackknife_cf_err,
iterations=bootstrap_boots)
self.save_cfs(save_steps_file, cf_keys=[name])
#Now that we have all of the jackknifes (jackknives?), calculate the mean
# and variance.
jackknife_cf=np.nanmean(temp, axis=0)
jackknife_cf_err=np.nanstd(temp, axis=0)
self._cfs[name].set_cf(jackknife_cf, jackknife_cf_err,
iterations=bootstrap_boots)
def main():
cfg = setup_config()
pipeline = rs.pipeline()
realsense_cfg = setup_realsense()
pipeline.start(realsense_cfg) # Start streaming
visualizer = predictor.VisualizationDemo(cfg)
ref_frame_axies = []
ref_frame_label = []
min_distance = 0.9
label_cnt = 0
frameth = 0
my_t_pool = {}
my_r_pool = {}
while True:
frameth += 1
cur_frame_axies = []
cur_frame_label = []
my_t_per_frame = []
my_r_per_frame = []
align = rs.align(rs.stream.color)
frames = pipeline.wait_for_frames()
aligned_frames = align.process(frames)
rgb = aligned_frames.get_color_frame()
rgb = np.asanyarray(rgb.get_data())
frame = rgb.copy()
# Do instance segmentation
start = time.time()
segmentation, vis = visualizer.run_on_image(frame)
#print("Time = " + str(time.time()-start))
cv2.imshow('Mask', vis)
cv2.waitKey(1)
# Get segmentation mask
ori_label = segmentation['instances'].pred_masks.cpu().numpy()
label = np.sum(ori_label, axis=0).astype(np.uint8)
label = np.where(label != 0, 255, label)
label = Image.fromarray(label).convert("L")
label = np.asarray(label.convert('RGB')).astype(np.uint8)
bboxes = segmentation['instances'].pred_boxes.tensor.cpu().numpy()
xyxy_bboxes = bboxes
bboxes = bbox_convert(bboxes)
if len(bboxes) > 0:
#depth_frames = frames.get_depth_frame()
depth_frames = aligned_frames.get_depth_frame()
video_profile = depth_frames.profile.as_video_stream_profile()
intr = video_profile.get_intrinsics()
depth = np.asanyarray(depth_frames.get_data())
#centers = segmentation['instances'].pred_boxes.get_centers()
if len(my_t_pool) > 0:
last_key = list(my_t_pool.keys())[-1]
for i in range(0, len(bboxes)):
bbox_xyxy = np.array(list(xyxy_bboxes[i]))
bbox = list(bboxes[i])
print("Bounding Box:" + str(bbox))
#center = bboxes[i].get_centers()
#center = centers[i].cpu().numpy()
num_idx = float('nan')
max_value = 0
label_of_object = ori_label[i].astype(np.uint8)
label_of_object = np.where(label_of_object != 0, 255,
label_of_object)
label_of_object = Image.fromarray(label_of_object).convert("L")
label_of_object = np.asarray(
label_of_object.convert('RGB')).astype(np.uint8)
if len(ref_frame_label) > 0:
iou_list = []
b = bbox_xyxy
a = np.array(ref_frame_axies)
for k in range(len(ref_frame_axies)):
iou = iou_score(a[k], b)
iou_list.append(iou)
iou_list = np.array(iou_list)
max_value = iou_list.max()
if (max_value > min_distance):
min_idx = np.where(iou_list == max_value)[0][0]
num_idx = ref_frame_label[min_idx]
if (math.isnan(num_idx)):
num_idx = label_cnt
label_cnt += 1
cur_frame_label.append(num_idx)
cur_frame_axies.append(bbox_xyxy)
print(max_value)
if (frameth == 1) or (max_value < 0.9) or (
i > len(my_t_pool[last_key]) - 1) or (frameth % 20
== 0):
pos_text = (bbox[0], bbox[1])
class_id = segmentation['instances'].pred_classes[i].cpu(
).data.numpy()
print("Class: " + str(class_id))
#idx = class_id
if class_id == 0:
idx = 0
if class_id == 2:
idx = 1
model_points = model_points_list[idx]
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0))
#mask_label = ma.getmaskarray(ma.masked_equal(label, np.array(255)))
mask_label = ma.getmaskarray(
ma.masked_equal(label_of_object,
np.array([255, 255, 255])))[:, :, 0]
mask = mask_label * mask_depth
rmin, rmax, cmin, cmax = posenet_deploy.get_bbox(bbox)
# choose
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
if len(choose) == 0:
choose = torch.LongTensor([0])
if len(choose) > num_points:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:num_points] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
choose = np.pad(choose, (0, num_points - len(choose)),
'wrap')
depth_masked = depth[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(
np.float32)
xmap_masked = xmap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(
np.float32)
ymap_masked = ymap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(
np.float32)
choose = np.array([choose])
# point cloud
pt2 = depth_masked / cam_scale
pt0 = (ymap_masked - cam_cx) * pt2 / cam_fx
pt1 = (xmap_masked - cam_cy) * pt2 / cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
cloud = cloud / 1000.0
# print(cloud.shape)
# cropped img
#img_masked = rgb[:, :, :3]
img_masked = rgb[:, :, ::-1] # bgr to rgb
img_masked = np.transpose(img_masked, (2, 0, 1))
img_masked = img_masked[:, rmin:rmax, cmin:cmax]
my_mask = np.transpose(label_of_object, (2, 0, 1))
my_mask = my_mask[:, rmin:rmax, cmin:
cmax] ## Added by me to crop the mask
mask_img = np.transpose(my_mask, (1, 2, 0))
img_rgb = np.transpose(img_masked, (1, 2, 0))
croped_img_mask = cv2.bitwise_and(img_rgb, mask_img)
crop_image_to_check = croped_img_mask.copy()
cv2.imshow("mask_crop", croped_img_mask)
croped_img_mask = np.transpose(croped_img_mask, (2, 0, 1))
# Variables
cloud = torch.from_numpy(cloud.astype(
np.float32)).unsqueeze(0)
choose = torch.LongTensor(choose.astype(
np.int32)).unsqueeze(0)
#img_masked = torch.from_numpy(img_masked.astype(np.float32)).unsqueeze(0)
img_masked = torch.from_numpy(
croped_img_mask.astype(np.float32)).unsqueeze(0)
index = torch.LongTensor([idx]).unsqueeze(
0) # Specify which object
cloud = Variable(cloud).cuda()
choose = Variable(choose).cuda()
img_masked = Variable(img_masked).cuda()
index = Variable(index).cuda()
# Deploy
with torch.no_grad():
pred_r, pred_t, pred_c, emb = estimator(
img_masked, cloud, choose, index)
pred_r = pred_r / torch.norm(pred_r, dim=2).view(
1, num_points, 1)
pred_c = pred_c.view(bs, num_points)
how_max, which_max = torch.max(pred_c, 1)
pred_t = pred_t.view(bs * num_points, 1, 3)
points = cloud.view(bs * num_points, 1, 3)
my_r = pred_r[0][which_max[0]].view(-1).cpu().data.numpy()
my_t = (points.view(bs * num_points, 1, 3) +
pred_t)[which_max[0]].view(-1).cpu().data.numpy()
my_pred = np.append(my_r, my_t)
# Refinement
for ite in range(0, iteration):
T = Variable(torch.from_numpy(my_t.astype(
np.float32))).cuda().view(1, 3).repeat(
num_points,
1).contiguous().view(1, num_points, 3)
my_mat = quaternion_matrix(my_r)
R = Variable(
torch.from_numpy(my_mat[:3, :3].astype(
np.float32))).cuda().view(1, 3, 3)
my_mat[0:3, 3] = my_t
new_cloud = torch.bmm((cloud - T), R).contiguous()
pred_r, pred_t = refiner(new_cloud, emb, index)
pred_r = pred_r.view(1, 1, -1)
pred_r = pred_r / (torch.norm(pred_r, dim=2).view(
1, 1, 1))
my_r_2 = pred_r.view(-1).cpu().data.numpy()
my_t_2 = pred_t.view(-1).cpu().data.numpy()
my_mat_2 = quaternion_matrix(my_r_2)
my_mat_2[0:3, 3] = my_t_2
my_mat_final = np.dot(my_mat, my_mat_2)
my_r_final = copy.deepcopy(my_mat_final)
my_r_final[0:3, 3] = 0
my_r_final = quaternion_from_matrix(my_r_final, True)
my_t_final = np.array([
my_mat_final[0][3], my_mat_final[1][3],
my_mat_final[2][3]
])
my_pred = np.append(my_r_final, my_t_final)
my_r = my_r_final
my_t = my_t_final
my_r_matrix = quaternion_matrix(my_r)[:3, :3]
#print("Time = " + str(time.time()-start))
my_t_per_frame.append(my_t)
my_r_per_frame.append(my_r_matrix)
#rotation = Rot.from_matrix(my_r_matrix)
#angle = rotation.as_euler('xyz', degrees=True)
my_t = np.around(my_t, 5)
#print("translation vector = " + str(my_t))
#print("rotation angles = " + str(my_r))
frame = posenet_deploy.get_3d_bbox(frame, model_points,
my_r_matrix, my_t)
frame = posenet_deploy.draw_axes(frame, my_r_matrix, my_t)
if check_inverted(crop_image_to_check):
cv2.putText(frame,
str(num_idx) + "_inverted", pos_text,
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2, cv2.LINE_AA)
else:
cv2.putText(frame, str(num_idx), pos_text,
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2, cv2.LINE_AA)
#cv2.putText(frame, str(num_idx), pos_text, cv2.FONT_HERSHEY_SIMPLEX,
# 0.5, (0,255,0), 2, cv2.LINE_AA)
posenet_deploy.putText(frame, i, num_idx, class_id, my_t)
#cv2.imshow('Result', rgb)
#cv2.waitKey(1)
else:
rmin, rmax, cmin, cmax = posenet_deploy.get_bbox(bbox)
img_masked = rgb[:, :, ::-1] # bgr to rgb
img_masked = np.transpose(img_masked, (2, 0, 1))
img_masked = img_masked[:, rmin:rmax, cmin:cmax]
my_mask = np.transpose(label_of_object, (2, 0, 1))
my_mask = my_mask[:, rmin:rmax, cmin:
cmax] ## Added by me to crop the mask
mask_img = np.transpose(my_mask, (1, 2, 0))
img_rgb = np.transpose(img_masked, (1, 2, 0))
croped_img_mask = cv2.bitwise_and(img_rgb, mask_img)
crop_image_to_check = croped_img_mask.copy()
pos_text = (bbox[0], bbox[1])
last_key = list(my_t_pool.keys())[-1]
print("POOL: " + str(my_t_pool[last_key]))
class_id = segmentation['instances'].pred_classes[i].cpu(
).data.numpy()
my_t = my_t_pool[last_key][min_idx]
my_r_matrix = my_r_pool[last_key][min_idx]
my_t_per_frame.append(my_t)
my_r_per_frame.append(my_r_matrix)
frame = posenet_deploy.get_3d_bbox(frame, model_points,
my_r_matrix, my_t)
frame = posenet_deploy.draw_axes(frame, my_r_matrix, my_t)
if check_inverted(crop_image_to_check):
cv2.putText(frame,
str(num_idx) + "_inverted", pos_text,
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2, cv2.LINE_AA)
else:
cv2.putText(frame, str(num_idx), pos_text,
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0),
2, cv2.LINE_AA)
#cv2.putText(frame, str(num_idx), pos_text, cv2.FONT_HERSHEY_SIMPLEX,
# 0.5, (0,255,0), 2, cv2.LINE_AA)
posenet_deploy.putText(frame, i, num_idx, class_id, my_t)
if len(my_t_per_frame) > 0:
my_t_pool[frameth] = my_t_per_frame
my_r_pool[frameth] = my_r_per_frame
ref_frame_label = cur_frame_label
ref_frame_axies = cur_frame_axies
end = time.time() - start
cv2.putText(frame,
"Time processing: " + str(round(end, 3)) + " seconds",
(100, 700), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255),
2, cv2.LINE_AA)
cv2.imshow('Result', frame)
cv2.waitKey(1)
else:
# Show images
#video_writer.write(rgb)
cv2.imshow('Result', rgb)
cv2.waitKey(1)
pipeline.stop()
def DenseFusion(self, img, depth, posecnn_res):
my_result_wo_refine = []
itemid = 1 # this is simplified for single label decttion, if multi-label used, check DFYW3.py for more
depth = np.array(depth)
# img = img
seg_res = posecnn_res
x1, y1, x2, y2 = seg_res["box"]
banana_bbox_draw = self.posecnn.get_box_rcwh(seg_res["box"])
rmin, rmax, cmin, cmax = int(y1), int(y2), int(x1), int(x2)
try:
depth = depth[:, :,
1] # because depth has 3 dimensions RGB but they are the all the same with each other
except:
# depth=depth
pass
depth = np.nan_to_num(depth) #DIY
mask_depth = ma.getmaskarray(ma.masked_not_equal(depth, 0)) # ok
mask_depth_nonzeros = mask_depth[:].nonzero()
label_banana = np.squeeze(seg_res["mask"])
label_banana = ma.getmaskarray(ma.masked_greater(label_banana, 0.5))
label_banana_nonzeros = label_banana.flatten().nonzero()
mask_label = ma.getmaskarray(ma.masked_equal(
label_banana, itemid)) # label from banana label
mask_label_nonzeros = mask_label[:].nonzero()
mask = mask_label * mask_depth
mask_nonzeros = mask[:].flatten().nonzero()
mask_target = mask[rmin:rmax, cmin:cmax]
choose = mask[rmin:rmax, cmin:cmax].flatten().nonzero()[0]
res_len_choose = len(choose)
if len(choose) > self.num_points:
c_mask = np.zeros(len(choose), dtype=int)
c_mask[:self.num_points] = 1
np.random.shuffle(c_mask)
choose = choose[c_mask.nonzero()]
else:
# print("(?)len of choose is 0, check error")
print("Info, DenseFusion: len(choose)=", len(choose))
# return "ERROR, img broken (?)"
# choose = np.pad(choose, (0, self.num_points - len(choose)), 'wrap')
return None
depth_masked = depth[rmin:rmax,
cmin:cmax].flatten()[choose][:,
np.newaxis].astype(
np.float32)
xmap_masked = self.xmap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
ymap_masked = self.ymap[
rmin:rmax,
cmin:cmax].flatten()[choose][:, np.newaxis].astype(np.float32)
choose = np.array([choose])
pt2 = depth_masked / self.cam_scale
pt0 = (ymap_masked - self.cam_cx) * pt2 / self.cam_fx
pt1 = (xmap_masked - self.cam_cy) * pt2 / self.cam_fy
cloud = np.concatenate((pt0, pt1, pt2), axis=1)
img_masked = np.array(img)[:, :, :3]
img_masked = np.transpose(img_masked, (2, 0, 1))
img_masked = img_masked[:, rmin:rmax, cmin:cmax]
cloud = torch.from_numpy(cloud.astype(np.float32))
choose = torch.LongTensor(choose.astype(np.int32))
img_masked = self.norm(torch.from_numpy(img_masked.astype(np.float32)))
index = torch.LongTensor([itemid - 1])
cloud = Variable(cloud).cuda()
choose = Variable(choose).cuda()
img_masked = Variable(img_masked).cuda()
index = Variable(index).cuda()
cloud = cloud.view(1, self.num_points, 3)
img_masked = img_masked.view(1, 3,
img_masked.size()[1],
img_masked.size()[2])
pred_r, pred_t, pred_c, emb = self.estimator(img_masked, cloud, choose,
index)
pred_r = pred_r / torch.norm(pred_r, dim=2).view(1, self.num_points, 1)
pred_c = pred_c.view(self.bs, self.num_points)
how_max, which_max = torch.max(pred_c, 1)
pred_t = pred_t.view(self.bs * self.num_points, 1, 3)
points = cloud.view(self.bs * self.num_points, 1, 3)
my_r = pred_r[0][which_max[0]].view(-1).cpu().data.numpy()
my_t = (points + pred_t)[which_max[0]].view(-1).cpu().data.numpy()
my_pred = np.append(my_r, my_t)
my_result_wo_refine.append(my_pred.tolist())
my_result = []
for ite in range(0, self.iteration):
T = Variable(torch.from_numpy(
my_t.astype(np.float32))).cuda().view(1, 3).repeat(
self.num_points,
1).contiguous().view(1, self.num_points, 3)
my_mat = quaternion_matrix(my_r)
R = Variable(torch.from_numpy(my_mat[:3, :3].astype(
np.float32))).cuda().view(1, 3, 3)
my_mat[0:3, 3] = my_t
new_cloud = torch.bmm((cloud - T), R).contiguous()
pred_r, pred_t = self.refiner(new_cloud, emb, index)
pred_r = pred_r.view(1, 1, -1)
pred_r = pred_r / (torch.norm(pred_r, dim=2).view(1, 1, 1))
my_r_2 = pred_r.view(-1).cpu().data.numpy()
my_t_2 = pred_t.view(-1).cpu().data.numpy()
my_mat_2 = quaternion_matrix(my_r_2)
my_mat_2[0:3, 3] = my_t_2
my_mat_final = np.dot(my_mat, my_mat_2)
my_r_final = copy.deepcopy(my_mat_final)
my_r_final[0:3, 3] = 0
my_r_final = quaternion_from_matrix(my_r_final, True)
my_t_final = np.array(
[my_mat_final[0][3], my_mat_final[1][3], my_mat_final[2][3]])
my_pred = np.append(my_r_final, my_t_final)
my_result.append(my_pred.tolist())
my_result_np = np.array(my_result)
my_result_mean = np.mean(my_result, axis=0)
my_r = my_result_mean[:4]
my_t = my_result_mean[4:]
my_r_quaternion = my_r
return my_r_quaternion, my_t
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(eq(masked_inside(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 1, 3).mask,
[1, 1, 1, 1, 0]))
assert_(eq(masked_outside(array(list(range(5)),
mask=[0, 1, 0, 0, 0]), 1, 3).mask,
[1, 1, 0, 0, 1]))
assert_(eq(masked_equal(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 0]))
assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1],
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 1]))
assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))