def _nearest_tri_edge(pt_tris, to_pt, pqs, dist, tri_geom): """Get nearest location from a point to the edge of a set of triangles""" # We might do something intelligent here. However, for now # it is ok to do it in the hard way aa = tri_geom['a'][pt_tris] bb = tri_geom['b'][pt_tris] cc = tri_geom['c'][pt_tris] pp = pqs[0] qq = pqs[1] # Find the nearest point from a triangle: # Side 1 -> 2 p0 = np.minimum(np.maximum(pp + 0.5 * (qq * cc) / aa, 0.0), 1.0) q0 = np.zeros_like(p0) # Side 2 -> 3 t1 = (0.5 * ((2.0 * aa - cc) * (1.0 - pp) + (2.0 * bb - cc) * qq) / (aa + bb - cc)) t1 = np.minimum(np.maximum(t1, 0.0), 1.0) p1 = 1.0 - t1 q1 = t1 # Side 1 -> 3 q2 = np.minimum(np.maximum(qq + 0.5 * (pp * cc) / bb, 0.0), 1.0) p2 = np.zeros_like(q2) # figure out which one had the lowest distance dist0 = _get_tri_dist(pp, qq, p0, q0, aa, bb, cc, dist) dist1 = _get_tri_dist(pp, qq, p1, q1, aa, bb, cc, dist) dist2 = _get_tri_dist(pp, qq, p2, q2, aa, bb, cc, dist) pp = np.r_[p0, p1, p2] qq = np.r_[q0, q1, q2] dists = np.r_[dist0, dist1, dist2] ii = np.argmin(np.abs(dists)) p, q, pt, dist = pp[ii], qq[ii], pt_tris[ii % len(pt_tris)], dists[ii] return p, q, pt, dist
def scale(self, factor_x, factor_y=None): """ Expand or contract the bounding box or contour around its center by a given factor :param factor_x: The multiplicative scale parameter in the x direction :type factor_x: float :param factor_y: The multiplicative scale parameter in the y direction :type factor_y: float .. note:: if factor_y parameter is omitted, then the factor_x is used in both directions .. note:: The scaling is done with respect to the contour's centroid as computed by the get_centroid methods. :Example: :: shape = (100, 100, 3) image = np.zeros(shape, dtype=np.uint8) d = bounding_region(shape, contour=np.array([[[10, 20]], [[25, 15]], [[80, 65]], [[60, 70]], [[20, 75]], [[5, 50]]])) d.draw_contour(image, color=(0, 255, 0)) # Scale to half the size d.scale(0.5) d.draw_contour(image, color=(255, 255, 0)) d.draw_box(image) cv2.imshow("Two contours", image) cv2.waitKey(0) """ if self._empty: return if factor_y is None: factor_y = factor_x if self.image_shape is None: raise Exception("Image shape is nescessary to compute the relative coordinates") if self.box_is_primary: shift_x = self.box[2] * (1.-factor_x) * 0.5 shift_y = self.box[3] * (1.-factor_y) * 0.5 self.box = np.array([np.maximum(self.box[0]+shift_x, 0), np.maximum(self.box[1]+shift_y, 0), np.minimum(self.box[2]*factor_x, self.image_shape[1]), np.minimum(self.box[3]*factor_y, self.image_shape[0])]).astype(np.int32) self._contour_from_box() self._update_internals() else: (cx, cy) = self.get_centroid_pixels() new_contour = np.zeros_like(self.contour, dtype=np.int32) for i in xrange(self.contour.shape[0]): new_contour[i][0][0] = np.clip(int(cx + (self.contour[i][0][0]-cx)*factor_x), a_min=0, a_max=self.image_shape[1]) new_contour[i][0][1] = np.clip(int(cy + (self.contour[i][0][1]-cy)*factor_y), a_min=0, a_max=self.image_shape[0]) self.contour = new_contour self._box_from_contour() self._update_internals()
def autoRange(self,zmin=None,zmax=None,coordList=None): """ determine X,Y,Z limits """ if coordList==None: coordList = self.coordList # autorange in X,Y minX = 1.e50 maxX = -1.e50 minY = 1.e50 maxY = -1.e50 minZ = 1.e50 maxZ = -1.e50 for iCoord in coordList: #if self.interpPresent: # X,Y = self.interpGrids[iCoord] # Z = self.interpValues[iCoord] #else: X,Y,Z = self.getXYZpoints(coordList=coordList) minX = numpy.minimum(minX,X.min()) maxX = numpy.maximum(maxX,X.max()) minY = numpy.minimum(minY,Y.min()) maxY = numpy.maximum(maxY,Y.max()) minZ = numpy.minimum(minZ,Z.min()) maxZ = numpy.maximum(maxZ,Z.max()) if zmin!=None: minZ = zmin if zmax!=None: maxZ = zmax return minX,maxX,minY,maxY,minZ,maxZ
def __PlotWeightedAtlas(self, sortedfunctiontuples, weightdict): # initialize the data median = self.__Curves[sortedfunctiontuples[0][0]] minimum = np.Inf + np.zeros(len(median)) maximum = -np.Inf + np.zeros(len(median)) outliers = [] cumulativeWeight = weightdict[sortedfunctiontuples[0][0]] i = 0 # determine the functions in inter quartile range (IQR) while cumulativeWeight < .5: maximum = np.maximum( maximum, self.__Curves[sortedfunctiontuples[i][0]]) minimum = np.minimum( minimum, self.__Curves[sortedfunctiontuples[i][0]]) cumulativeWeight += weightdict[sortedfunctiontuples[i][0]] i += 1 upperfence = maximum[:] lowerfence = minimum[:] fences = AtlasMath.BandDepth.GenerateFences(minimum, maximum, median) # Determine the functions in the 99.3% confidence interval while cumulativeWeight < .993: upperfence = np.maximum( upperfence, self.__Curves[sortedfunctiontuples[i][0]]) lowerfence = np.minimum( lowerfence, self.__Curves[sortedfunctiontuples[i][0]]) cumulativeWeight += weightdict[sortedfunctiontuples[i][0]] i += 1 fences = [ np.minimum(upperfence, fences[0]), np.maximum(lowerfence, fences[1])] # the left over functions are outliers for pair in sortedfunctiontuples[i:]: outliers.append(self.__Curves[pair[0]]) self.__PlotLines(median, maximum, minimum, outliers, fences) return
def util_analysis(arrps): cpu_u, cpu_u_orig, dead_miss_orig, dead_miss = [], [], [], [] for ix, arrp in enumerate(arrps): original_deadline_misses = len(arrp[(arrp['abs_deadline'] - arrp['abs_start'] < arrp['original_duration'])])*100.0/len(arrp) deadline_misses = len(arrp[arrp['miss']==1])*100.0/len(arrp) sum_my_offload_task_dur = sum(arrps[(ix+1)%3]['dur_offload']) exp_dur = arrp['abs_end'][-1] - arrp['abs_start'][0] original_cpu_util = sum(np.minimum(arrp['original_duration'], arrp['abs_deadline'] - arrp['abs_start']))*100.0/exp_dur cpu_util = (sum_my_offload_task_dur + sum(np.minimum(arrp['duration'], arrp['abs_deadline'] - arrp['abs_start'])))*100.0/exp_dur print 'CPU %d, Util %2.2f --> %2.2f, Deadline miss %2.5f --> %2.5f'%(ix, original_cpu_util, cpu_util, original_deadline_misses, deadline_misses) cpu_u_orig.append(original_cpu_util) cpu_u.append(cpu_util) dead_miss_orig.append(original_deadline_misses) dead_miss.append(deadline_misses) result = [cpu_u, cpu_u_orig, dead_miss_orig, dead_miss] return result
def SetZoomAxes(self): x = self.zoompoint[0] y = self.zoompoint[1] W = params.params.movie.get_width() H = params.params.movie.get_height() h = H/self.zoomfactor w = W/self.zoomfactor x1 = x-w/2 x2 = x+w/2 y1 = y-h/2 y2 = y+h/2 if x1 < 0: x2 -= x1 x1 = 0 elif x2 > W-1: x1 -= (x2 - W + 1) x2 = W-1 if y1 < 0: y2 -= y1 y1 = 0 elif y2 > H-1: y1 -= (y2 - H + 1) y2 = H-1 x1 = num.maximum(int(x1),0) x2 = num.minimum(int(x2),W-1) y1 = num.maximum(int(y1),0) y2 = num.minimum(int(y2),H-1) self.zoomaxes = [x1,x2,y1,y2] self.ShowImage()
def process_chunk(self, t0, t1, intensity, weights, pp_intensity, pp_weights): # Loop over intensity/weights in chunks of size v1_chunk for ichunk in xrange(0, self.nt_chunk, self.v1_chunk): for frequency in xrange(self.nfreq): # Calculate the v1 for each frequency self.v1_tmp[frequency] = self._v1(intensity[frequency, ichunk:ichunk+self.v1_chunk], weights[frequency, ichunk:ichunk+self.v1_chunk]) # Once v1s have been calculated for each frequency, update the weights and running variance non_zero_v1 = self.v1_tmp != 0 zero_v1 = np.logical_not(non_zero_v1) # For nonzero (successful) v1s, increase the weights (if possible) and update the running variance self.running_weights[non_zero_v1] = np.minimum(2.0, self.running_weights[non_zero_v1] + self.w_clamp) self.v1_tmp[non_zero_v1] = np.minimum((1-self.var_weight) * self.running_var[non_zero_v1] + self.var_weight * self.v1_tmp[non_zero_v1], self.running_var[non_zero_v1] + self.var_clamp_add + self.running_var[non_zero_v1] * self.var_clamp_mult) self.v1_tmp[non_zero_v1] = np.maximum(self.v1_tmp[non_zero_v1], self.running_var[non_zero_v1] - self.var_clamp_add - self.running_var[non_zero_v1] * self.var_clamp_mult) self.running_var[non_zero_v1] = self.v1_tmp[non_zero_v1] # For unsuccessful v1s, decrease the weights (if possible) and do not modify the running variance self.running_weights[zero_v1] = np.maximum(0, self.running_weights[zero_v1] - self.w_clamp) # Mask fill! intensity_valid = (weights[:, ichunk:ichunk+self.v1_chunk] > self.w_cutoff) rand_intensity = np.random.standard_normal(size=intensity[:, ichunk:ichunk+self.v1_chunk].shape) for (ifreq,v) in enumerate(self.running_var): if v > 0.0: rand_intensity[ifreq, :] *= v**0.5 intensity[:, ichunk:ichunk+self.v1_chunk] = np.where(intensity_valid, intensity[:, ichunk:ichunk+self.v1_chunk], rand_intensity) weights[:, ichunk:ichunk+self.v1_chunk] = np.repeat(self.running_weights, self.v1_chunk).reshape(self.nfreq, self.v1_chunk)
def append_flipped_images(self): num_images = self.num_images print 'number images %d' % (num_images) #widths = self._get_widths() print 'loading widths from file' #widths = np.save('/home/zhongjie/CodesDown/jointmulticut/joint-multicut/py-faster-rcnn/widths2.npy',widths) widths = np.load('/home/zhongjie/CodesDown/jointmulticut/joint-multicut/py-faster-rcnn/widths.npy') print 'done loading widths' for i in xrange(num_images): #import ipdb; ipdb.set_trace() boxes = self.roidb[i]['boxes'].copy() oldx1 = boxes[:, 0].copy() oldx2 = boxes[:, 2].copy() boxes[:, 0] = widths[i] - np.minimum(oldx2, widths[i]) # for imagenet, no need to subtract 1, ...- 1 boxes[:, 2] = widths[i] - np.minimum(oldx1, widths[i]) # same as above, ...- 1 #import ipdb; ipdb.set_trace() #print 'flip no. %07d / %d' % (i,num_images) assert (boxes[:, 2] >= boxes[:, 0]).all() entry = {'boxes' : boxes, 'gt_overlaps' : self.roidb[i]['gt_overlaps'], 'gt_classes' : self.roidb[i]['gt_classes'], 'flipped' : True} self.roidb.append(entry) #import ipdb; ipdb.set_trace() self._image_index = self._image_index * 2
def compute_overlap(a, b): """ Parameters ---------- a: (N, 4) ndarray of float b: (K, 4) ndarray of float Returns ------- overlaps: (N, K) ndarray of overlap between boxes and query_boxes """ area = (b[:, 2] - b[:, 0] + 1) * (b[:, 3] - b[:, 1] + 1) iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0]) + 1 ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1]) + 1 iw = np.maximum(iw, 0) ih = np.maximum(ih, 0) ua = np.expand_dims((a[:, 2] - a[:, 0] + 1) * (a[:, 3] - a[:, 1] + 1), axis=1) + area - iw * ih ua = np.maximum(ua, np.finfo(float).eps) intersection = iw * ih return intersection / ua
def ensure_within_bounds(box, size): box['xmin'] = np.minimum(np.maximum(box['xmin'], 0), size['width']-1) box['xmax'] = np.minimum(np.maximum(box['xmax'], 0), size['width']-1) box['ymin'] = np.minimum(np.maximum(box['ymin'], 0), size['height']-1) box['ymax'] = np.minimum(np.maximum(box['ymax'], 0), size['height']-1) return box
def reflective_transformation(y, lb, ub): """Compute reflective transformation and its gradient.""" if in_bounds(y, lb, ub): return y, np.ones_like(y) lb_finite = np.isfinite(lb) ub_finite = np.isfinite(ub) x = y.copy() g_negative = np.zeros_like(y, dtype=bool) mask = lb_finite & ~ub_finite x[mask] = np.maximum(y[mask], 2 * lb[mask] - y[mask]) g_negative[mask] = y[mask] < lb[mask] mask = ~lb_finite & ub_finite x[mask] = np.minimum(y[mask], 2 * ub[mask] - y[mask]) g_negative[mask] = y[mask] > ub[mask] mask = lb_finite & ub_finite d = ub - lb t = np.remainder(y[mask] - lb[mask], 2 * d[mask]) x[mask] = lb[mask] + np.minimum(t, 2 * d[mask] - t) g_negative[mask] = t > d[mask] g = np.ones_like(y) g[g_negative] = -1 return x, g
def rp_gumbel_original(p_zero, loc, scale, flvol, max_return_period=1e9): """ Transforms a unique, or array of flood volumes into the belonging return periods, according to gumbel parameters (belonging to non-zero part of the distribution) and a zero probability Inputs: p_zero: probability that flood volume is zero loc: Gumbel location parameter (of non-zero part of distribution) scale: Gumbel scale parameter (of non-zero part of distribution) flvol: Flood volume that will be transformed to return period max_return_period: maximum return period considered. This maximum is needed to prevent that floating point precision becomes a problem (default: 1e9) This function is copied from: https://repos.deltares.nl/repos/Hydrology/trunk/GLOFRIS/src/rp_bias_corr.py """ np.seterr(divide='ignore') np.seterr(invalid='ignore') max_p = 1-1./max_return_period max_p_residual = np.minimum(np.maximum((max_p-np.float64(p_zero))/(1-np.float64(p_zero)), 0), 1) max_reduced_variate = -np.log(-np.log(np.float64(max_p_residual))) # compute the gumbel reduced variate belonging to the Gumbel distribution (excluding any zero-values) # make sure that the reduced variate does not exceed the one, resembling the 1,000,000 year return period reduced_variate = np.minimum((flvol-loc)/scale, max_reduced_variate) # reduced_variate = (flvol-loc)/scale # transform the reduced variate into a probability (residual after removing the zero volume probability) p_residual = np.minimum(np.maximum(np.exp(-np.exp(-np.float64(reduced_variate))), 0), 1) # tranform from non-zero only distribution to zero-included distribution p = np.minimum(np.maximum(p_residual*(1-p_zero) + p_zero, p_zero), max_p) # Never larger than max_p # transform into a return period return_period = 1./(1-p) test_p = p == 1 return return_period, test_p
def clip_upper(arr,upper_bound): """ In-place, one-sided version of numpy.clip(). i.e. numpy.clip(arr,a_max=upper_bound,out=arr) if it existed. """ minimum(arr,upper_bound,arr)
def nms(boxes, threshold, method): if boxes.size == 0: return np.empty((0, 3)) x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] s = boxes[:, 4] area = (x2 - x1 + 1) * (y2 - y1 + 1) I = np.argsort(s) pick = np.zeros_like(s, dtype=np.int16) counter = 0 while I.size > 0: i = I[-1] pick[counter] = i counter += 1 idx = I[0:-1] xx1 = np.maximum(x1[i], x1[idx]) yy1 = np.maximum(y1[i], y1[idx]) xx2 = np.minimum(x2[i], x2[idx]) yy2 = np.minimum(y2[i], y2[idx]) w = np.maximum(0.0, xx2 - xx1 + 1) h = np.maximum(0.0, yy2 - yy1 + 1) inter = w * h if method is 'Min': o = inter / np.minimum(area[i], area[idx]) else: o = inter / (area[i] + area[idx] - inter) I = I[np.where(o <= threshold)] pick = pick[0:counter] return pick
def nms2d(boxes, overlap=0.3): """Compute the nms given a set of scored boxes, as numpy array with 5 columns <x1> <y1> <x2> <y2> <score> return the indices of the tubelets to keep """ if boxes.size == 0: return np.array([],dtype=np.int32) x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] scores = boxes[:, 4] areas = (x2-x1+1) * (y2-y1+1) I = np.argsort(scores) indices = np.zeros(scores.shape, dtype=np.int32) counter = 0 while I.size > 0: i = I[-1] indices[counter] = i counter += 1 xx1 = np.maximum(x1[i],x1[I[:-1]]) yy1 = np.maximum(y1[i],y1[I[:-1]]) xx2 = np.minimum(x2[i],x2[I[:-1]]) yy2 = np.minimum(y2[i],y2[I[:-1]]) inter = np.maximum(0.0, xx2 - xx1 + 1) * np.maximum(0.0, yy2 - yy1 + 1) iou = inter / (areas[i] + areas[I[:-1]] - inter) I = I[np.where(iou <= overlap)[0]] return indices[:counter]
def opponent_histogram_key_vector(sR, sG, sB): """ New method plus -- vector ops with inlined scalar values (exec time <0.072s) """ return ( ( numpy.minimum( numpy.floor( ((((sR - sG) / ROOT_2) + TWOFIFTYFIVE_OVER_ROOT2).astype("float32") / FIVETEN_OVER_ROOT2) * 4.0 ), 3.0, ) ).astype("int") + ( numpy.minimum( numpy.floor( ((((sR + sG - 2 * sB) / ROOT_6).astype("float32") + FIVETEN_OVER_ROOT6) / TENTWENTY_OVER_ROOT6) * 4.0 ), 3.0, ) ).astype("int") * 4 + ( numpy.minimum( 3.0, numpy.floor((((sR + sG + sB) / ROOT_3).astype("float32") / THREE_TIMES_TWOFIFTYFIVE_OVER_ROOT3) * 4.0), ) ).astype("int") * FOURBYFOUR )
def test_elementwise_min_grad(self, n, m, d, gc, dc): go = np.random.rand(n, m, d).astype(np.float32) X = np.random.rand(n, m, d).astype(np.float32) Y = np.random.rand(n, m, d).astype(np.float32) Z = np.random.rand(n, m, d).astype(np.float32) mx = np.minimum(np.minimum(X, Y), Z) inputs = [mx, go, X, Y, Z] def min_grad_op(mx, go, X, Y, Z): def mx_grad(a): return go * (mx == a) return [mx_grad(a) for a in [X, Y, Z]] op = core.CreateOperator( "MinGradient", ["mx", "go", "X", "Y", "Z"], ["gX", "gY", "gZ"] ) self.assertReferenceChecks( device_option=gc, op=op, inputs=inputs, reference=min_grad_op, ) self.assertDeviceChecks(dc, op, inputs, [0, 1, 2])
def nms(dets, thresh): x1 = dets[:, 0] y1 = dets[:, 1] x2 = dets[:, 2] y2 = dets[:, 3] scores = dets[:, 4] areas = (x2 - x1 + 1) * (y2 - y1 + 1) order = scores.argsort()[::-1] keep = [] while order.size > 0: i = order[0] keep.append(i) xx1 = np.maximum(x1[i], x1[order[1:]]) yy1 = np.maximum(y1[i], y1[order[1:]]) xx2 = np.minimum(x2[i], x2[order[1:]]) yy2 = np.minimum(y2[i], y2[order[1:]]) w = np.maximum(0.0, xx2 - xx1 + 1) h = np.maximum(0.0, yy2 - yy1 + 1) inter = w * h ovr = inter / ((areas[i] + areas[order[1:]] - inter)+0.0000001) inds = np.where(ovr <= thresh)[0] order = order[inds + 1] return keep
def opponent_histogram_key_vector_ORIG(sR, sG, sB): """ New method -- everything results in vector ops (exec time <0.076s) """ # TAKE NOTE: `numpy.minimum(a)` behaves like `numpy.vectorize(__builtins__['min'])(a)` # ... which is DIFFERENT from `numpy.min(a)`, which acts like `min(list(a.flatten()))` # ... (the latter returns a scalar whereas the former returns an array shaped like `a`) return ( ( numpy.minimum( numpy.floor(((((sR - sG) / ROOT_2) + 255.0 / ROOT_2).astype("float32") / (510.0 / ROOT_2)) * 4.0), 3.0 ) ).astype("int") + ( numpy.minimum( numpy.floor( ((((sR + sG - 2 * sB) / ROOT_6).astype("float32") + 510.0 / ROOT_6) / (1020.0 / ROOT_6)) * 4.0 ), 3.0, ) ).astype("int") * 4 + ( numpy.minimum( 3.0, numpy.floor((((sR + sG + sB) / ROOT_3).astype("float32") / (3.0 * 255.0 / ROOT_3)) * 4.0) ) ).astype("int") * 4 * 4 )
def nms(dets, thresh): # ------------------------- # Pure Python NMS baseline. # Written by Ross Girshick # ------------------------- x1 = dets[:, 0] - dets[:, 2] / 2. y1 = dets[:, 1] - dets[:, 3] / 2. x2 = dets[:, 0] + dets[:, 2] / 2. y2 = dets[:, 1] + dets[:, 3] / 2. scores = dets[:, 4]# 预测得分 areas = (x2 - x1 + 1) * (y2 - y1 + 1) order = scores.argsort()[::-1]#排序 keep = [] while order.size > 0: i = order[0] keep.append(i) xx1 = np.maximum(x1[i], x1[order[1:]]) yy1 = np.maximum(y1[i], y1[order[1:]]) xx2 = np.minimum(x2[i], x2[order[1:]]) yy2 = np.minimum(y2[i], y2[order[1:]]) w = np.maximum(0.0, xx2 - xx1 + 1) h = np.maximum(0.0, yy2 - yy1 + 1) inter = w * h ovr = inter / (areas[i] + areas[order[1:]] - inter) inds = np.where(ovr <= thresh)[0] order = order[inds + 1] return dets[np.require(keep), :]
def conv_backward_naive(dout, cache): """ A naive implementation of the backward pass for a convolutional layer. Inputs: - dout: Upstream derivatives. - cache: A tuple of (x, w, b, conv_param) as in conv_forward_naive Returns a tuple of: - dx: Gradient with respect to x - dw: Gradient with respect to w - db: Gradient with respect to b """ x, w, b, conv_param = cache p = conv_param['pad'] s = conv_param['stride'] dx = np.zeros_like(x) N, C, H, W = x.shape F, C, HH, WW = w.shape lmin = 0 lmax = int((H + 2*p - HH)/s) kmin = 0 kmax = int((W + 2*p - WW)/s) for n in range(N): for c in range(C): for h in range(H): for g in range(W): delta = 0.0 l0 = int(np.maximum(lmin, np.ceil((h+p-HH+1)/s))) l1 = int(np.minimum(lmax, np.floor((h+p)/s))) k0 = int(np.maximum(kmin,np.ceil((g+p-WW+1)/s))) k1 = int(np.minimum(kmax, np.floor((g+p)/s))) for l in range(l0,l1+1): for k in range(k0, k1+1): for f in range(F): delta += w[f,c,h + p - l * s, g + p - k*s] * dout[n,f,l,k] dx[n,c,h,g] = delta dw = np.zeros_like(w) for f in range(F): for c in range(C): for i in range(HH): for j in range(WW): delta = 0.0 l0 = int(np.maximum(lmin, np.ceil((p-i)/s))) l1 = int(np.minimum(lmax, np.floor((p-i+H-1)/s))) k0 = int(np.maximum(kmin, np.ceil((p-j)/s))) k1 = int(np.minimum(kmax, np.floor((p-j+W-1)/s))) for l in range(l0, l1 + 1): for k in range(k0, k1 + 1): for n in range(N): delta += x[n,c,-p+l*s+i, -p+k*s+j] * dout[n,f,l,k] dw[f,c,i,j] = delta db = np.sum(dout, axis=(0,2,3)) ########## return dx, dw, db
def image_cart_to_polar(image, center, min_radius, max_radius, phase_width, zoom_factor=1): '''Converts an image from cartesian to polar coordinates around center''' # Upsample image if zoom_factor != 1: image = zoom(image, (zoom_factor, zoom_factor), order=4) center = (center[0]*zoom_factor + zoom_factor/2, center[1]*zoom_factor + zoom_factor/2) min_radius = min_radius * zoom_factor max_radius = max_radius * zoom_factor # pad if necessary max_x, max_y = image.shape[0], image.shape[1] pad_dist_x = np.max([(center[0] + max_radius) - max_x, -(center[0] - max_radius)]) pad_dist_y = np.max([(center[1] + max_radius) - max_y, -(center[1] - max_radius)]) pad_dist = int(np.max([0, pad_dist_x, pad_dist_y])) if pad_dist != 0: image = np.pad(image, pad_dist, 'constant') # coordinate conversion theta, r = np.meshgrid(np.linspace(0, 2*np.pi, phase_width), np.arange(min_radius, max_radius)) x, y = coord_polar_to_cart(r, theta, center) x, y = np.round(x), np.round(y) x, y = x.astype(int), y.astype(int) x = np.maximum(x, 0) y = np.maximum(y, 0) x = np.minimum(x, max_x-1) y = np.minimum(y, max_y-1) polar = image[x, y] polar.reshape((max_radius - min_radius, phase_width)) return polar
def connect_extrema(im_pos, target, markers, visualize=False): ''' im_pos : XYZ positions of each point in image formation (n x m x 3) ''' height, width,_ = im_pos.shape centroid = np.array(target) im_pos = np.ascontiguousarray(im_pos.astype(np.int16)) cost_map = np.ascontiguousarray(np.zeros([height, width], dtype=np.uint16)) extrema = dgn.geodesic_map_MPI(cost_map, im_pos, np.array(centroid, dtype=np.int16), 1, 1) cost_map = extrema[-1] trails = [] for m in markers: trail = dgn.geodesic_trail(cost_map.copy()+(32000*(im_pos[:,:,2]==0)).astype(np.uint16), np.array(m, dtype=np.int16)) trails += [trail.copy()] if visualize: cost_map = deepcopy(cost_map) circ = circle(markers[0][0],markers[0][1], 5) circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) cost_map[circ[0], circ[1]] = 0 for i,t in enumerate(trails[1:]): # embed() cost_map[t[:,0], t[:,1]] = 0 circ = circle(markers[i+1][0],markers[i+1][1], 5) circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) cost_map[circ[0], circ[1]] = 0 return trails, cost_map else: return trails
def onmouse(event, x, y, flags, param): global selection, drag_start, tracking_state, show_backproj, down_x, down_y, selcFrame global mouseX, mouseY, trackBoxShow x, y = np.int16([x, y]) #[sic] BUG mouseX = x mouseY = y if event == cv2.EVENT_LBUTTONDOWN: down_x = x down_y = y drag_start = (x, y) tracking_state = 0 trackBoxShow = True if event == cv2.EVENT_LBUTTONUP: trackBoxShow = False if drag_start: if flags & cv2.EVENT_FLAG_LBUTTON: h, w = selcFrame.shape[:2] xo, yo = drag_start x0, y0 = np.maximum(0, np.minimum([xo, yo], [x, y])) x1, y1 = np.minimum([w, h], np.maximum([xo, yo], [x, y])) selection = None if x1-x0 > 0 and y1-y0 > 0: selection = (x0, y0, x1, y1) else: drag_start = None if selection is not None: tracking_state = 1
def __getitem__(self, index): if self._representation == 'mv': representation_idx = 1 elif self._representation == 'residual': representation_idx = 2 else: representation_idx = 0 if self._is_train: video_path, label, num_frames = random.choice(self._video_list) else: video_path, label, num_frames = self._video_list[index] frames = [] for seg in range(self._num_segments): if self._is_train: gop_index, gop_pos = self._get_train_frame_index(num_frames, seg) else: gop_index, gop_pos = self._get_test_frame_index(num_frames, seg) img = load(video_path, gop_index, gop_pos, representation_idx, self._accumulate) if img is None: print('Error: loading video %s failed.' % video_path) img = np.zeros((256, 256, 2)) if self._representation == 'mv' else np.zeros((256, 256, 3)) else: if self._representation == 'mv': img = clip_and_scale(img, 20) img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) elif self._representation == 'residual': img += 128 img = (np.minimum(np.maximum(img, 0), 255)).astype(np.uint8) if self._representation == 'iframe': img = color_aug(img) # BGR to RGB. (PyTorch uses RGB according to doc.) img = img[..., ::-1] frames.append(img) frames = self._transform(frames) frames = np.array(frames) frames = np.transpose(frames, (0, 3, 1, 2)) input = torch.from_numpy(frames).float() / 255.0 if self._representation == 'iframe': input = (input - self._input_mean) / self._input_std elif self._representation == 'residual': input = (input - 0.5) / self._input_std elif self._representation == 'mv': input = (input - 0.5) return input, label
def minimum_pension(self, trim_wages_reg, trim_wages_all, pension_reg, pension_all): ''' MICO du régime général : allocation différentielle RQ : ASPA et minimum vieillesse sont gérés par OF Il est attribué quels que soient les revenus dont dispose le retraité en plus de ses pensions : loyers, revenus du capital, activité professionnelle... + mécanisme de répartition si cotisations à plusieurs régimes TODO: coder toutes les évolutions et rebondissements 2004/2008''' P = reduce(getattr, self.param_name.split('.'), self.P) # pension_RG, pension, trim_RG, trim_cot, trim trimesters = trim_wages_reg['trimesters'] trim_regime = trimesters['regime'].sum() + sum(trim_wages_reg['maj'].values()) coeff = minimum(1, divide(trim_regime, P.prorat.n_trim)) if P.mico.dispositif == 0: # Avant le 1er janvier 1983, comparé à l'AVTS min_pension = self.P.common.avts return maximum(min_pension - pension_reg,0)*coeff elif P.mico.dispositif == 1: # TODO: Voir comment gérer la limite de cumul relativement complexe (Doc n°5 du COR) mico = P.mico.entier return maximum(mico - pension_reg,0)*coeff elif P.mico.dispositif == 2: # A partir du 1er janvier 2004 les périodes cotisées interviennent (+ dispositif transitoire de 2004) nb_trim = P.prorat.n_trim trim_regime = trimesters['regime'].sum() #+ sum(trim_wages_regime['maj'].values()) trim_cot_regime = sum(trimesters[key].sum() for key in trimesters.keys() if 'cot' in key) mico_entier = P.mico.entier*minimum(divide(trim_regime, nb_trim), 1) maj = (P.mico.entier_maj - P.mico.entier)*divide(trim_cot_regime, nb_trim) mico = mico_entier + maj*(trim_cot_regime >= P.mico.trim_min) return (mico - pension_reg)*(mico > pension_reg)*(pension_reg>0)
def calculate_coeff_proratisation(self, info_ind, trim_wage_regime, trim_wage_all): ''' Calcul du coefficient de proratisation ''' def _assurance_corrigee(trim_regime, agem): ''' Deux types de corrections : - correction de 1948-1982 - Détermination de la durée d'assurance corrigée introduite par la réforme Boulin (majoration quand départ à la retraite après 65 ans) à partir de 1983''' P = reduce(getattr, self.param_name.split('.'), self.P) if P.prorat.dispositif == 1: correction = (P.prorat.n_trim - trim_regime)/2 return trim_regime + correction elif P.prorat.dispositif == 2: age_taux_plein = P.decote.age_null trim_majo = divide(agem - age_taux_plein, 3)*(agem > age_taux_plein) elig_majo = (trim_regime < P.prorat.n_trim) correction = trim_regime*P.tx_maj*trim_majo*elig_majo return trim_regime + correction else: return trim_regime P = reduce(getattr, self.param_name.split('.'), self.P) trim_regime = trim_wage_regime['trimesters']['regime'].sum(1) trim_regime_maj = sum(trim_wage_regime['maj'].values()) agem = info_ind['agem'] trim_regime = trim_regime_maj + trim_regime # _assurance_corrigee(trim_regime, agem) #disposition pour montée en charge de la loi Boulin (ne s'applique qu'entre 72 et 74) : if P.prorat.application_plaf == 1: trim_regime = minimum(trim_regime, P.prorat.plaf) CP = minimum(1, divide(trim_regime, P.prorat.n_trim)) return CP
def test_get_output_for(self, ParametricRectifierLayer, init_alpha): input_shape = (3, 3, 28, 28) # random input tensor input = np.random.randn(*input_shape).astype(theano.config.floatX) # default: alphas shared only along 2nd axis layer = ParametricRectifierLayer(input_shape, alpha=init_alpha) alpha_v = layer.alpha.get_value() expected = np.maximum(input, 0) + np.minimum(input, 0) * \ alpha_v[None, :, None, None] assert np.allclose(layer.get_output_for(input).eval(), expected) # scalar alpha layer = ParametricRectifierLayer(input_shape, alpha=init_alpha, shared_axes='all') alpha_v = layer.alpha.get_value() expected = np.maximum(input, 0) + np.minimum(input, 0) * alpha_v assert np.allclose(layer.get_output_for(input).eval(), expected) # alphas shared over the 1st axis layer = ParametricRectifierLayer(input_shape, alpha=init_alpha, shared_axes=0) alpha_v = layer.alpha.get_value() expected = np.maximum(input, 0) + np.minimum(input, 0) * \ alpha_v[None, :, :, :] assert np.allclose(layer.get_output_for(input).eval(), expected) # alphas shared over the 1st and 4th axes layer = ParametricRectifierLayer(input_shape, shared_axes=(0, 3), alpha=init_alpha) alpha_v = layer.alpha.get_value() expected = np.maximum(input, 0) + np.minimum(input, 0) * \ alpha_v[None, :, :, None] assert np.allclose(layer.get_output_for(input).eval(), expected)
def step(self): """ Perform single automaton step. """ gsum = sum(self.grid) # Compute burn touched cells maximum(1, self.grid, self.burn_map) self.burn_map -= 1 # Correlate cells for next set of fires correlate(self.burn_map, self.spread, mode='constant', cval=0, output=self.next_burn_map) # And cutoff at 1 and multiply by grid to remove # barren cells. self.next_burn_map *= self.grid minimum(1, self.next_burn_map, self.next_burn_map) # Finally ignite next set of trees and top at barren self.grid += self.next_burn_map self.grid += self.burn_map minimum(3, self.grid, self.grid) if p.sleep: __import__('time').sleep(p.sleep) # No more fire? return gsum < sum(self.grid)
def update_qm_region(self, atoms, potential_energies=None, ): """Update the QM region while the simulation is running Parameters ---------- atoms : ase.Atoms whole structure potential_energies : array Potential energy per atom Returns ------- list of lists of ints list of individual clusters as lists of atoms """ # Make sure the right atoms object is in # ------ Increase the energy by a common factor - makes it more readable in some cases if (self.energy_increase is not None): potential_energies *= self.energy_increase # ------ Cap maximum energy according to the flag if (self.energy_cap is not None): np.minimum(potential_energies, self.energy_cap, potential_energies) # ------ Get the energized atoms list flagged_atoms_dict = {} flagged_atoms_dict["potential_energies"] = self.get_energized_list(atoms, potential_energies, "avg_potential_energies", self.qm_flag_potential_energies) energized_set = set() for key in flagged_atoms_dict: energized_set = set(flagged_atoms_dict[key]) | energized_set energized_list = list(energized_set) self.old_energized_list = list(energized_list) if (len(energized_list) != 0): self.mediator.neighbour_list.update(atoms) # TODO if energized list include the whole system just pass it along for array_i, atom_i in enumerate(energized_list): energized_list[array_i] = self.create_cluster_around_atom(atoms, atom_i, hydrogenate=False) self.qm_atoms_list = energized_list if (len(self.qm_atoms_list) > 0): self.join_clusters() self.expand_cluster(self.mediator.special_atoms_list) self.join_clusters() if self.only_heavy is False: for index in range(len(self.qm_atoms_list)): self.qm_atoms_list[index] = self.hydrogenate_cluster(atoms, self.qm_atoms_list[index]) self.qm_atoms_list = list(map(list, self.qm_atoms_list)) return self.qm_atoms_list
def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes): '''Preprocess true boxes to training input format Parameters ---------- true_boxes: array, shape=(m, T, 5) Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape. input_shape: array-like, hw, multiples of 32 anchors: array, shape=(N, 2), wh num_classes: integer Returns ------- y_true: list of array, shape like yolo_outputs, xywh are reletive value ''' assert (true_boxes[..., 4] < num_classes).all(), 'class id must be less than num_classes' num_layers = len(anchors) // 3 # default setting anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2] ] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]] true_boxes = np.array(true_boxes, dtype='float32') input_shape = np.array(input_shape, dtype='int32') boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2 boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] true_boxes[..., 0:2] = boxes_xy / input_shape[::-1] true_boxes[..., 2:4] = boxes_wh / input_shape[::-1] m = true_boxes.shape[0] grid_shapes = [ input_shape // { 0: 32, 1: 16, 2: 8 }[l] for l in range(num_layers) ] y_true = [ np.zeros((m, grid_shapes[l][0], grid_shapes[l][1], len( anchor_mask[l]), 5 + num_classes), dtype='float32') for l in range(num_layers) ] # Expand dim to apply broadcasting. anchors = np.expand_dims(anchors, 0) anchor_maxes = anchors / 2. anchor_mins = -anchor_maxes valid_mask = boxes_wh[..., 0] > 0 for b in range(m): # Discard zero rows. wh = boxes_wh[b, valid_mask[b]] if len(wh) == 0: continue # Expand dim to apply broadcasting. wh = np.expand_dims(wh, -2) box_maxes = wh / 2. box_mins = -box_maxes intersect_mins = np.maximum(box_mins, anchor_mins) intersect_maxes = np.minimum(box_maxes, anchor_maxes) intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.) intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] box_area = wh[..., 0] * wh[..., 1] anchor_area = anchors[..., 0] * anchors[..., 1] iou = intersect_area / (box_area + anchor_area - intersect_area) # Find best anchor for each true box best_anchor = np.argmax(iou, axis=-1) for t, n in enumerate(best_anchor): for l in range(num_layers): if n in anchor_mask[l]: i = np.floor(true_boxes[b, t, 0] * grid_shapes[l][1]).astype('int32') j = np.floor(true_boxes[b, t, 1] * grid_shapes[l][0]).astype('int32') k = anchor_mask[l].index(n) c = true_boxes[b, t, 4].astype('int32') y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4] y_true[l][b, j, i, k, 4] = 1 y_true[l][b, j, i, k, 5 + c] = 1 return y_true
def loo(trace, model=None, pointwise=False): """Calculates leave-one-out (LOO) cross-validation for out of sample predictive model fit, following Vehtari et al. (2015). Cross-validation is computed using Pareto-smoothed importance sampling (PSIS). Parameters ---------- trace : result of MCMC run model : PyMC Model Optional model. Default None, taken from context. pointwise: bool if True the pointwise predictive accuracy will be returned. Default False Returns ------- namedtuple with the following elements: loo: approximated Leave-one-out cross-validation loo_se: standard error of loo p_loo: effective number of parameters loo_i: and array of the pointwise predictive accuracy, only if pointwise True """ model = modelcontext(model) log_py = _log_post_trace(trace, model) if log_py.size == 0: raise ValueError('The model does not contain observed values.') # Importance ratios r = np.exp(-log_py) r_sorted = np.sort(r, axis=0) # Extract largest 20% of importance ratios and fit generalized Pareto to each # (returns tuple with shape, location, scale) q80 = int(len(log_py) * 0.8) pareto_fit = np.apply_along_axis(lambda x: pareto.fit(x, floc=0), 0, r_sorted[q80:]) if np.any(pareto_fit[0] > 0.7): warnings.warn("""Estimated shape parameter of Pareto distribution is greater than 0.7 for one or more samples. You should consider using a more robust model, this is because importance sampling is less likely to work well if the marginal posterior and LOO posterior are very different. This is more likely to happen with a non-robust model and highly influential observations.""") elif np.any(pareto_fit[0] > 0.5): warnings.warn("""Estimated shape parameter of Pareto distribution is greater than 0.5 for one or more samples. This may indicate that the variance of the Pareto smoothed importance sampling estimate is very large.""") # Calculate expected values of the order statistics of the fitted Pareto S = len(r_sorted) M = S - q80 z = (np.arange(M) + 0.5) / M expvals = map(lambda x: pareto.ppf(z, x[0], scale=x[2]), pareto_fit.T) # Replace importance ratios with order statistics of fitted Pareto r_sorted[q80:] = np.vstack(expvals).T # Unsort ratios (within columns) before using them as weights r_new = np.array([ r[np.argsort(i)] for r, i in zip(r_sorted.T, np.argsort(r.T, axis=1)) ]).T # Truncate weights to guarantee finite variance w = np.minimum(r_new, r_new.mean(axis=0) * S**0.75) loo_lppd_i = -2. * logsumexp(log_py, axis=0, b=w / np.sum(w, axis=0)) loo_lppd_se = np.sqrt(len(loo_lppd_i) * np.var(loo_lppd_i)) loo_lppd = np.sum(loo_lppd_i) lppd = np.sum(logsumexp(log_py, axis=0, b=1. / log_py.shape[0])) p_loo = lppd + (0.5 * loo_lppd) if pointwise: LOO_r = namedtuple('LOO_r', 'LOO, LOO_se, p_LOO, LOO_i') return LOO_r(loo_lppd, loo_lppd_se, p_loo, loo_lppd_i) else: LOO_r = namedtuple('LOO_r', 'LOO, LOO_se, p_LOO') return LOO_r(loo_lppd, loo_lppd_se, p_loo)
def float_2_cv(x): """ Converts given numeric to code value. """ return np.maximum(CV_min, np.minimum(CV_max, np.round(x)))
def inner_point(X, y, max_iter=1000): m, n = X.shape X = np.column_stack((X, np.ones((m, 1)))) y = y.astype(np.float64) data_num = len(y) C = 1.0 kernel = np.dot(X, np.transpose(X)) # + np.diag(np.ones(data_num, np.float64)) * .5/C p = np.matrix(np.multiply(kernel, np.outer(y, y))) q = np.matrix(-np.ones([data_num, 1], np.float64)) bounds = (0, C) low, up = bounds x = np.zeros([m, 1]) for k in range(max_iter): # heavy on matrix operations for i in range(m): # remove optimal alpha g = p*x+q if low < x[i] and x[i] < up: if g[i] == 0: continue if x[i] == low: if np.minimum(0, g[i]) == 0: continue if x[i] == up: if np.maximum(0, g[i]) == 0: continue tmpx = x.copy() tmpx[i, 0] = 0 temp = (p[i, :] * tmpx) + q[i] # if temp > 0 and x[i] == 0: # continue temp = temp.item() if p[i, i] > 0: xi = -(temp / p[i, i]).item() xi = np.maximum(low, xi) xi = np.minimum(up, xi) elif p[i, i] < 0: xi = -1 #print('error') else: if temp > 0: xi = low else: xi = up x[i, 0] = xi # for u in range(m): # i = -1-u # tmpx = x.copy() # tmpx[i, 0] = 0 # temp = (p[i, :] * tmpx) + q[i] # # if temp > 0 and x[i] == 0: # # continue # temp = temp.item() # if p[i, i] > 0: # xi = -(temp / p[i, i]).item() # xi = np.maximum(low, xi) # xi = np.minimum(up, xi) # elif p[i, i] < 0: # print('error') # else: # if temp > 0: # xi = low # else: # xi = up # x[i, 0] = xi dual = -(0.5 * x.T * (p * x) + q.T * x) dual = dual.item() y1 = np.reshape(y, (-1, 1)) lambda1 = np.multiply(x, y1) w = np.dot(X.T, lambda1) w = np.matrix(w).reshape(-1, 1) tmp = np.maximum(1 - np.multiply(y1, X * w), 0) primal = 0.5 * np.linalg.norm(w)**2 + 1 * np.sum(tmp) primal = primal.item() # stop criteria #if k % 10 == 0: #print(np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal))) # print(np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal))) #----bug---- #if np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)) < 1e-12: if np.abs(dual - primal) / (1 + np.abs(dual) + np.abs(primal)) < 4.061554383439379: #print('success') break return w
def grf_solution( time, rad, storage, conductivity, dim=2, lat_ext=1.0, rate=-1e-4, h_bound=0.0, struc_grid=True, ): """ The general radial flow (GRF) model for a pumping test. Parameters ---------- time : :class:`numpy.ndarray` Array with all time-points where the function should be evaluated. rad : :class:`numpy.ndarray` Array with all radii where the function should be evaluated. storage : :class:`float` Storage coefficient of the aquifer. conductivity : :class:`float` Conductivity of the aquifer. dim : :class:`float`, optional Fractional dimension of the aquifer. Default: ``2.0`` lat_ext : :class:`float`, optional Lateral extend of the aquifer. Default: ``1.0`` rate : :class:`float`, optional Pumpingrate at the well. Default: -1e-4 h_bound : :class:`float`, optional Reference head at the outer boundary at infinity. Default: ``0.0`` struc_grid : :class:`bool`, optional If this is set to "False", the "rad" and "time" array will be merged and interpreted as single, r-t points. In this case they need to have the same shapes. Otherwise a structured r-t grid is created. Default: ``True`` Returns ------- head : :class:`numpy.ndarray` Array with all heads at the given radii and time-points. Raises ------ ValueError If ``rad`` is not positiv. ValueError If ``time`` is negative. ValueError If shape of ``rad`` and ``time`` differ in case of ``struc_grid`` is ``True``. ValueError If ``conductivity`` is not positiv. ValueError If ``storage`` is not positiv. """ Input = Shaper(time, rad, struc_grid) if not conductivity > 0.0: raise ValueError("The Conductivity needs to be positive.") if not storage > 0.0: raise ValueError("The Storage needs to be positive.") time_mat = Input.time_mat rad_mat = Input.rad_mat u = storage * rad_mat**2 / (4 * conductivity * time_mat) nu = 1.0 - dim / 2.0 res = np.zeros((Input.time_no, Input.rad_no)) res[Input.time_gz, :] = inc_gamma(-nu, u) res[Input.time_gz, :] *= ( rate / (4.0 * np.pi**(1 - nu) * conductivity * lat_ext**(3.0 - dim)) * rad_mat**(2 * nu)) res = Input.reshape(res) if rate > 0: res = np.maximum(res, 0) else: res = np.minimum(res, 0) # add the reference head res += h_bound return res
def well_solution( time, rad, storage, transmissivity, rate=-1e-4, h_bound=0.0, struc_grid=True, ): """ The classical Theis solution. The classical Theis solution for transient flow under a pumping condition in a confined and homogeneous aquifer. This solution was presented in ''Theis 1935''[R9]_. Parameters ---------- time : :class:`numpy.ndarray` Array with all time-points where the function should be evaluated. rad : :class:`numpy.ndarray` Array with all radii where the function should be evaluated. storage : :class:`float` Storage of the aquifer. transmissivity : :class:`float` Transmissivity of the aquifer. rate : :class:`float`, optional Pumpingrate at the well. Default: -1e-4 h_bound : :class:`float`, optional Reference head at the outer boundary at infinity. Default: ``0.0`` struc_grid : :class:`bool`, optional If this is set to "False", the "rad" and "time" array will be merged and interpreted as single, r-t points. In this case they need to have the same shapes. Otherwise a structured r-t grid is created. Default: ``True`` Returns ------- head : :class:`numpy.ndarray` Array with all heads at the given radii and time-points. Raises ------ ValueError If ``rad`` is not positiv. ValueError If ``time`` is negative. ValueError If shape of ``rad`` and ``time`` differ in case of ``struc_grid`` is ``True``. ValueError If ``transmissivity`` is not positiv. ValueError If ``storage`` is not positiv. References ---------- .. [R9] Theis, C., ''The relation between the lowering of the piezometric surface and the rate and duration of discharge of a well using groundwater storage'', Trans. Am. Geophys. Union, 16, 519-524, 1935 Notes ----- The parameters ``rad``, ``T`` and ``S`` will be checked for positivity. If you want to use cartesian coordiantes, just use the formula ``r = sqrt(x**2 + y**2)`` Examples -------- >>> well_solution([10,100], [1,2,3], 0.001, 0.001, -0.001) array([[-0.24959541, -0.14506368, -0.08971485], [-0.43105106, -0.32132823, -0.25778313]]) """ Input = Shaper(time, rad, struc_grid) if not transmissivity > 0.0: raise ValueError("The Transmissivity needs to be positive.") if not storage > 0.0: raise ValueError("The Storage needs to be positive.") time_mat = Input.time_mat rad_mat = Input.rad_mat res = np.zeros((Input.time_no, Input.rad_no)) res[Input.time_gz, :] = (rate / (4.0 * np.pi * transmissivity) * exp1(rad_mat**2 * storage / (4 * transmissivity * time_mat))) res = Input.reshape(res) if rate > 0: res = np.maximum(res, 0) else: res = np.minimum(res, 0) # add the reference head res += h_bound return res
def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes): '''Preprocess true boxes to training input format Parameters ---------- true_boxes: array, shape=(m, T, 5) Absolute x_min, y_min, x_max, y_max, class_id relative to input_shape. input_shape: array-like, hw, multiples of 32 anchors: array, shape=(N, 2), wh num_classes: integer Returns ------- y_true: list of array, shape like yolo_outputs, xywh are reletive value ''' # 检测类别序号是否小于类别数,避免异常数据 assert (true_boxes[..., 4] < num_classes).all(), 'class id must be less than num_classes' # 每一层anchor box的数量num_layers; # 预设anchor box的掩码anchor_mask,第1层678,第2层345,第3层012,倒序排列 num_layers = len(anchors) // 3 # default setting anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2] ] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]] """ 计算true_boxes: true_boxes:真值框,左上和右下2个坐标值和1个类别,如[184, 299, 191, 310, 0.0] 结构是(16, 20, 5),16是批次数,20是框的最大数,5是框的5个值; boxes_xy:xy是box的中心点,结构是(16, 20, 2); boxes_wh:wh是box的宽和高,结构也是(16, 20, 2); input_shape:输入尺寸416x416; true_boxes:第0和1位设置为xy,除以416,归一化,第2和3位设置为wh,除以416,归一化,如[0.449, 0.730, 0.016, 0.026, 0.0]。 """ true_boxes = np.array(true_boxes, dtype='float32') input_shape = np.array(input_shape, dtype='int32') boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2 boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] true_boxes[..., 0:2] = boxes_xy / input_shape[::-1] true_boxes[..., 2:4] = boxes_wh / input_shape[::-1] """ 设置y_true的初始值: m是批次16; grid_shape是input_shape等比例降低,即[[13,13], [26,26], [52,52]]; y_true是全0矩阵(np.zeros)列表,即[(16,13,13,3,5+num_classes), (16,26,26,3,6), (16,52,52,3,6)] """ m = true_boxes.shape[0] grid_shapes = [ input_shape // { 0: 32, 1: 16, 2: 8 }[l] for l in range(num_layers) ] y_true = [ np.zeros((m, grid_shapes[l][0], grid_shapes[l][1], len( anchor_mask[l]), 5 + num_classes), dtype='float32') for l in range(num_layers) ] # Expand dim to apply broadcasting. """ 设置anchors的值: 将anchors增加1维expand_dims,由(9,2)转为(1,9,2); anchor_maxes,是anchors值除以2; anchor_mins,是负的anchor_maxes; valid_mask,将boxes_wh中宽w大于0的位,设为True,即含有box,结构是(16,20); """ anchors = np.expand_dims(anchors, 0) anchor_maxes = anchors / 2. anchor_mins = -anchor_maxes valid_mask = boxes_wh[..., 0] > 0 for b in range(m): # Discard zero rows. """ 循环m处理批次中的每个图像和标注框: 只选择存在标注框的wh,例如:wh的shape是(7,2); np.expand_dims(wh, -2)是wh倒数第2个添加1位,即(7,2)->(7,1,2); box_maxes和box_mins,与anchor_maxes和anchor_mins的操作类似。 """ wh = boxes_wh[b, valid_mask[b]] if len(wh) == 0: continue # Expand dim to apply broadcasting. wh = np.expand_dims(wh, -2) box_maxes = wh / 2. box_mins = -box_maxes """ 计算标注框box与anchor box的iou值,计算方式很巧妙: box_mins的shape是(7,1,2),anchor_mins的shape是(1,9,2), intersect_mins的shape是(7,9,2),即两两组合的值; intersect_area的shape是(7,9);box_area的shape是(7,1);anchor_area的shape是(1,9); iou的shape是(7,9); IoU数据,即anchor box与检测框box,两两匹配的iou值。 """ intersect_mins = np.maximum(box_mins, anchor_mins) intersect_maxes = np.minimum(box_maxes, anchor_maxes) intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.) intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1] box_area = wh[..., 0] * wh[..., 1] anchor_area = anchors[..., 0] * anchors[..., 1] iou = intersect_area / (box_area + anchor_area - intersect_area) # Find best anchor for each true box # 选择IoU最大的anchor索引 best_anchor = np.argmax(iou, axis=-1) """ 设置y_true的值: t是box的序号;n是最优anchor的序号;l是层号; 如果最优anchor在层l中,则设置其中的值,否则默认为0; true_boxes是(16, 20, 5),即批次、box数、框值; true_boxes[b, t, 0],其中b是批次序号、t是box序号,第0位是x,第1位是y; grid_shapes是3个检测图的尺寸,将归一化的值,与框长宽相乘,恢复为具体值; k是在anchor box中的序号; c是类别,true_boxes的第4位; 将xy和wh放入y_true中,将y_true的第4位框的置信度设为1,第5~n位的类别设为1; """ for t, n in enumerate(best_anchor): for l in range(num_layers): if n in anchor_mask[l]: i = np.floor(true_boxes[b, t, 0] * grid_shapes[l][1]).astype('int32') j = np.floor(true_boxes[b, t, 1] * grid_shapes[l][0]).astype('int32') k = anchor_mask[l].index(n) c = true_boxes[b, t, 4].astype('int32') y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4] y_true[l][b, j, i, k, 4] = 1 y_true[l][b, j, i, k, 5 + c] = 1 # y_true的第0和1位是中心点xy,范围是(0~1), # 第2和3位是宽高wh,范围是(0~1), # 第4位是置信度1或0, # 第5~n位是类别为1其余为0 return y_true
b = float(pt.y) - centroid_y tmp = np.array([a, b]) x.append(tmp) x = np.array(x) #print(h,w) #print(img.shape) X_ind = bfm.kpt_ind # index of keypoints in 3DMM. fixed. # fit fitted_sp, fitted_ep, fitted_s, fitted_angles, fitted_t = bfm.fit( x, X_ind, max_iter=3) tp = bfm.get_tex_para('random') colors = bfm.generate_colors(tp) colors = np.minimum(np.maximum(colors, 0), 1) fitted_vertices = bfm.generate_vertices(fitted_sp, fitted_ep) transformed_vertices = bfm.transform(fitted_vertices, fitted_s, fitted_angles, fitted_t) image_vertices = mesh.transform.to_image(transformed_vertices, h, w) #Invert y and z axis to make rendering image normal z = image_vertices[:, 2:] z = 0 - z image_vertices[:, 2:] = z y = image_vertices[:, 1:2] y = w - y image_vertices[:, 1:2] = y fitted_image = mesh.render.render_colors(image_vertices, bfm.triangles, colors, h, w)
def histogram(x, y, **kwargs): # Histogram kernel implemented as a callable. assert kwargs == {} # no kernel_params that we didn't ask for return np.minimum(x, y).sum()
def plot (arg1, arg2=None, xrange=None, yrange=None, ps=0, thick=1, xtitle=None, ytitle=None, color='black', noerase=False, overplot=False,position=None, ylog=False, xlog=False, xr=None, yr=None, title=None, label=None, nodata=False, linestyle=None, markersize=None, xaxis_formatter=None, yaxis_formatter=None, autoscalex=False, autoscaley=False, markerfacecolor=None, markeredgecolor=None, axis=None, transpose=False, **kw): """ Plot your data in an IDL-way Example: plot(x,y,xrange=[0,39],yrange=[-1,10],ps=4,xtitle="X",\ color='black',position=[0.1,0.1,0.9,0.9], xlog=True) """ if arg2 is None: y=listToArrFlat(arg1) x=numpy.arange(len(y)) else: x=listToArrFlat(arg1) y=listToArrFlat(arg2) if not noerase: plt.gcf().clf() if position is not None and axis is None: mypos=position[:] mypos[2]=position[2]-position[0] mypos[3]=position[3]-position[1] plt.axes(mypos) if axis is None: axis = plt.gca() if xlog: axis.set_xscale('log',subx=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) if ylog: axis.set_yscale('log',suby=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) if xaxis_formatter is not None: axis.xaxis.set_major_formatter(xaxis_formatter) if yaxis_formatter is not None: axis.yaxis.set_major_formatter(yaxis_formatter) marker, linestyle = get_marker(ps, linestyle) if xr is not None: xrange=xr if yr is not None: yrange=yr if xrange is None and yrange is None: ind = numpy.isfinite(x) if not ind.any(): xrange=[0,1] else: xrange=[numpy.min(x[ind]),numpy.max(x[ind])] ind = numpy.isfinite(y) if not ind.any(): yrange=[0,1] else: yrange=[numpy.min(y[ind]),numpy.max(y[ind])] del ind elif xrange is None and yrange is not None: ind=(y<numpy.maximum(yrange[1],yrange[0])) & (y>numpy.minimum(yrange[0],yrange[1])) & numpy.isfinite(x) if ind.any(): xrange=[numpy.min(x[ind]),numpy.max(x[ind])] else: xrange=[numpy.min(x),numpy.max(x)] del ind elif xrange is not None and yrange is None: ind=(x<numpy.maximum(xrange[1],xrange[0])) & (x>numpy.minimum(xrange[0],xrange[1])) & numpy.isfinite(y) if ind.any(): yrange=[numpy.min(y[ind]),numpy.max(y[ind])] else: yrange=[numpy.min(y),numpy.max(y)] del ind if len(yrange)!=2 or len(xrange)!=2: raise ValueError("Wrong xrange or yrange") if not overplot: axis.minorticks_on() if xtitle is not None: axis.set_xlabel(xtitle) if ytitle is not None: axis.set_ylabel(ytitle) axis.set_autoscalex_on(autoscalex) axis.set_autoscaley_on(autoscaley) if transpose: xrange,yrange=yrange,xrange if not overplot: axis.axis(numpy.concatenate((xrange,yrange))) if title is not None: plt.title(title) if not nodata: if transpose: x,y=y,x if markersize is None: axis.plot(x, y, marker=marker, linestyle=linestyle, linewidth=thick, color=color, label=label, markerfacecolor=markerfacecolor, markeredgecolor=markeredgecolor,**kw) else: axis.plot(x, y, marker=marker, linestyle=linestyle, linewidth=thick, color=color, label=label, markersize=markersize, markerfacecolor=markerfacecolor, markeredgecolor=markeredgecolor,**kw)
def _compute_clipped_weights(self, advantages: np.ndarray) -> np.ndarray: weights = np.exp(advantages / self._beta) return np.minimum(weights, self._max_weight)
def vg_eval( detpath, gt_roidb, image_index, classindex, ovthresh=0.5, use_07_metric=False, eval_attributes=False): """rec, prec, ap, sorted_scores, npos = voc_eval( detpath, gt_roidb, image_index, classindex, [ovthresh], [use_07_metric]) Top level function that does the Visual Genome evaluation. detpath: Path to detections gt_roidb: List of ground truth structs. image_index: List of image ids. classindex: Category index [ovthresh]: Overlap threshold (default = 0.5) [use_07_metric]: Whether to use VOC07's 11 point AP computation (default False) """ # extract gt objects for this class class_recs = {} npos = 0 for item,imagename in zip(gt_roidb,image_index): if eval_attributes: bbox = item['boxes'][np.where(np.any(item['gt_attributes'].toarray() == classindex, axis=1))[0], :] else: bbox = item['boxes'][np.where(item['gt_classes'] == classindex)[0], :] difficult = np.zeros((bbox.shape[0],)).astype(np.bool) det = [False] * bbox.shape[0] npos = npos + sum(~difficult) class_recs[str(imagename)] = {'bbox': bbox, 'difficult': difficult, 'det': det} if npos == 0: # No ground truth examples return 0,0,0,0,npos # read dets with open(detpath, 'r') as f: lines = f.readlines() if len(lines) == 0: # No detection examples return 0,0,0,0,npos splitlines = [x.strip().split(' ') for x in lines] image_ids = [x[0] for x in splitlines] confidence = np.array([float(x[1]) for x in splitlines]) BB = np.array([[float(z) for z in x[2:]] for x in splitlines]) # sort by confidence sorted_ind = np.argsort(-confidence) sorted_scores = -np.sort(-confidence) BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] # go down dets and mark TPs and FPs nd = len(image_ids) tp = np.zeros(nd) fp = np.zeros(nd) for d in range(nd): R = class_recs[image_ids[d]] bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) if BBGT.size > 0: # compute overlaps # intersection ixmin = np.maximum(BBGT[:, 0], bb[0]) iymin = np.maximum(BBGT[:, 1], bb[1]) ixmax = np.minimum(BBGT[:, 2], bb[2]) iymax = np.minimum(BBGT[:, 3], bb[3]) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + (BBGT[:, 2] - BBGT[:, 0] + 1.) * (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) overlaps = inters / uni ovmax = np.max(overlaps) jmax = np.argmax(overlaps) if ovmax > ovthresh: if not R['difficult'][jmax]: if not R['det'][jmax]: tp[d] = 1. R['det'][jmax] = 1 else: fp[d] = 1. else: fp[d] = 1. # compute precision recall fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(npos) # avoid divide by zero in case the first detection matches a difficult # ground truth prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, use_07_metric) return rec, prec, ap, sorted_scores, npos
def one_hot(ints, width): oh = np.zeros((ints.shape[0], width)) hot_ixs = np.minimum(ints, width - 1) oh[np.arange(ints.shape[0]), hot_ixs] = 1 return oh
def main(): start_line = 0 # 471526 end_line = 50000 os.environ['CUDA_VISIBLE_DEVICES'] = '0' model_dir = '/net/per920a/export/das14a/satoh-lab/wangz/ins2018/src/facenet-master/model/' key_frames_path = '/net/per610a/export/das11g/caizhizhu/ins/ins2013/frames_png/' output_feature_dir = '/net/per920a/export/das14a/satoh-lab/wangz/ins2018/data/face_gallery/' shot_list_file = 'clips.txt' frame_list_file = 'frames.txt' detect_multiple_faces = True margin = 44 image_size = 182 minsize = 20 # minimum size of face threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold factor = 0.709 # scale factor print('Creating networks and loading parameters') g1 = tf.Graph() # detect and align g2 = tf.Graph() # feature with g1.as_default(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess1 = tf.Session(graph=g1) with sess1.as_default(): pnet, rnet, onet = align.detect_face.create_mtcnn(sess1, None) with g2.as_default(): sess2 = tf.Session(graph=g2) with sess2.as_default(): facenet.load_model(model_dir) images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0") embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0") phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0") embedding_size = embeddings.get_shape()[1] output_feature_dir = os.path.expanduser(output_feature_dir) if not os.path.exists(output_feature_dir): os.makedirs(output_feature_dir) shot_id_list_file = open(os.path.join(key_frames_path, shot_list_file), 'r') for shot_id in shot_id_list_file.readlines()[start_line:end_line]: shot_id = shot_id.strip('\n') shot_id = shot_id.strip() shot_face_feature = None shot_face_num = 0 frame_id_list_file = open(os.path.join(key_frames_path, shot_id, frame_list_file), 'r') for frame_id in frame_id_list_file.readlines(): frame_id = frame_id.strip('\n') frame_id = frame_id.strip() img = misc.imread(os.path.join(key_frames_path, shot_id, frame_id)) if img.ndim == 2: img = facenet.to_rgb(img) img = img[:, :, 0:3] bounding_boxes, _ = align.detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor) nrof_faces = bounding_boxes.shape[0] if nrof_faces > 0: det = bounding_boxes[:, 0:4] det_arr = [] img_size = np.asarray(img.shape)[0:2] if nrof_faces > 1: if detect_multiple_faces: for i in range(nrof_faces): det_arr.append(np.squeeze(det[i])) else: bounding_box_size = (det[:, 2] - det[:, 0]) * (det[:, 3] - det[:, 1]) img_center = img_size / 2 offsets = np.vstack([(det[:, 0] + det[:, 2]) / 2 - img_center[1], (det[:, 1] + det[:, 3]) / 2 - img_center[0]]) offset_dist_squared = np.sum(np.power(offsets, 2.0), 0) index = np.argmax( bounding_box_size - offset_dist_squared * 2.0) # some extra weight on the centering det_arr.append(det[index, :]) else: det_arr.append(np.squeeze(det)) # for each detected face for i, det in enumerate(det_arr): det = np.squeeze(det) bb = np.zeros(4, dtype=np.int32) bb[0] = np.maximum(det[0] - margin / 2, 0) bb[1] = np.maximum(det[1] - margin / 2, 0) bb[2] = np.minimum(det[2] + margin / 2, img_size[1]) bb[3] = np.minimum(det[3] + margin / 2, img_size[0]) cropped = img[bb[1]:bb[3], bb[0]:bb[2], :] scaled = misc.imresize(cropped, (image_size, image_size), interp='bilinear') scaled = misc.imresize(scaled, (160, 160), interp='bilinear') # output_filename = os.path.join(output_feature_dir, shot_id + '.' + frame_id + '.png') # filename_base, file_extension = os.path.splitext(output_filename) # if detect_multiple_faces: # output_filename_n = "{}_{}{}".format(filename_base, i, file_extension) # else: # output_filename_n = "{}{}".format(filename_base, file_extension) # misc.imsave(output_filename_n, scaled) scaled_reshape = [] pre_img = facenet.prewhiten(scaled) scaled_reshape.append(pre_img.reshape(-1, 160, 160, 3)) emb_temp = np.zeros((1, embedding_size)) emb_temp[0, :] = sess2.run(embeddings, feed_dict={images_placeholder: scaled_reshape[0], phase_train_placeholder: False})[0] if shot_face_num == 0: shot_face_feature = emb_temp[0, :] else: shot_face_feature = np.concatenate((shot_face_feature, emb_temp[0, :]), axis=0) shot_face_num = shot_face_num + 1 frame_id_list_file.close() if shot_face_num > 0 : shot_face_feature = shot_face_feature.reshape(shot_face_num, embedding_size) print(shot_id) np.save(os.path.join(output_feature_dir, shot_id + '.npy'), shot_face_feature) shot_id_list_file.close()
def voc_eval(detpath, annopath, imagesetfile, classname, cachedir, ovthresh=0.5, use_07_metric=False): """rec, prec, ap = voc_eval(detpath, annopath, imagesetfile, classname, [ovthresh], [use_07_metric]) Top level function that does the PASCAL VOC evaluation. detpath: Path to detections detpath.format(classname) should produce the detection results file. annopath: Path to annotations annopath.format(imagename) should be the xml annotations file. imagesetfile: Text file containing the list of images, one image per line. classname: Category name (duh) cachedir: Directory for caching the annotations [ovthresh]: Overlap threshold (default = 0.5) [use_07_metric]: Whether to use VOC07's 11 point AP computation (default False) """ # assumes detections are in detpath.format(classname) # assumes annotations are in annopath.format(imagename) # assumes imagesetfile is a text file with each line an image name # cachedir caches the annotations in a pickle file # first load gt if not os.path.isdir(cachedir): os.mkdir(cachedir) imageset = os.path.splitext(os.path.basename(imagesetfile))[0] cachefile = os.path.join(cachedir, imageset + '_annots.pkl') # read list of images with open(imagesetfile, 'r') as f: lines = f.readlines() imagenames = [x.strip() for x in lines] if not os.path.isfile(cachefile): # load annots recs = {} for i, imagename in enumerate(imagenames): recs[imagename] = parse_rec(annopath.format(imagename)) if i % 100 == 0: logger.info( 'Reading annotation for {:d}/{:d}'.format( i + 1, len(imagenames))) # save logger.info('Saving cached annotations to {:s}'.format(cachefile)) save_object(recs, cachefile) else: recs = load_object(cachefile) # extract gt objects for this class class_recs = {} npos = 0 for imagename in imagenames: R = [obj for obj in recs[imagename] if obj['name'] == classname] bbox = np.array([x['bbox'] for x in R]) difficult = np.array([x['difficult'] for x in R]).astype(np.bool) det = [False] * len(R) npos = npos + sum(~difficult) class_recs[imagename] = {'bbox': bbox, 'difficult': difficult, 'det': det} # read dets detfile = detpath.format(classname) with open(detfile, 'r') as f: lines = f.readlines() splitlines = [x.strip().split(' ') for x in lines] image_ids = [x[0] for x in splitlines] confidence = np.array([float(x[1]) for x in splitlines]) BB = np.array([[float(z) for z in x[2:]] for x in splitlines]) # sort by confidence sorted_ind = np.argsort(-confidence) BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] # go down dets and mark TPs and FPs nd = len(image_ids) tp = np.zeros(nd) fp = np.zeros(nd) for d in range(nd): R = class_recs[image_ids[d]] bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) if BBGT.size > 0: # compute overlaps # intersection ixmin = np.maximum(BBGT[:, 0], bb[0]) iymin = np.maximum(BBGT[:, 1], bb[1]) ixmax = np.minimum(BBGT[:, 2], bb[2]) iymax = np.minimum(BBGT[:, 3], bb[3]) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + (BBGT[:, 2] - BBGT[:, 0] + 1.) * (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) overlaps = inters / uni ovmax = np.max(overlaps) jmax = np.argmax(overlaps) if ovmax > ovthresh: if not R['difficult'][jmax]: if not R['det'][jmax]: tp[d] = 1. R['det'][jmax] = 1 else: fp[d] = 1. else: fp[d] = 1. # compute precision recall fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(npos) # avoid divide by zero in case the first detection matches a difficult # ground truth prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, use_07_metric) return rec, prec, ap
def train_loop(hps, X_train, y_train): num_batches = int(math.ceil(hps.data_size / float(hps.batch_size))) global_steps = 0 model = net.Net(hps) model.build_graph() truth = tf.argmax(model.labels, axis=1) predictions = tf.argmax(model.predictions, axis=1) precision = tf.reduce_mean(tf.to_float(tf.equal(predictions, truth))) # 'Saver' op to save and restore all the variables # saver = tf.train.Saver() # init = tf.global_variables_initializer() # with tf.Session() as sess: # sess.run(init) # # for _ in hps.max_number_of_steps: # if global_steps % num_batches: # start = hps.batch_size * global_steps # end = np.minimum(hps.data_size, hps.batch_size * (global_steps + 1)) # # X = X_train[start:end] # y = y_train[start:end] # fd = {model.X: X, # model.y: y} # # _, global_steps, loss, acc = sess.run([model.train_op, # model.cost, # precision], # feed_dict=fd) summary_hook = tf.train.SummarySaverHook( save_steps=100, output_dir=FLAGS.train_dir, summary_op=tf.summary.merge([model.summaries, tf.summary.scalar('Precision', precision)])) logging_hook = tf.train.LoggingTensorHook( tensors={'step': model.global_step, 'loss': model.cost, 'precision': precision}, every_n_iter=100) class _LearningRateSetterHook(tf.train.SessionRunHook): """Sets learning_rate based on global step.""" def begin(self): self._lrn_rate = 0.01 def before_run(self, run_context): return tf.train.SessionRunArgs( model.global_step, # Asks for global step value. feed_dict={model.lrn_rate: self._lrn_rate}) # Sets learning rate def after_run(self, run_context, run_values): train_step = run_values.results if train_step < 2000: self._lrn_rate = 0.01 elif train_step < 6000: self._lrn_rate = 0.008 elif train_step < 20000: self._lrn_rate = 0.001 else: self._lrn_rate = 0.0002 with tf.train.MonitoredTrainingSession( checkpoint_dir=FLAGS.log_root, hooks=[logging_hook, _LearningRateSetterHook()], chief_only_hooks=[summary_hook], save_checkpoint_secs=60, # Since we provide a SummarySaverHook, we need to disable default # SummarySaverHook. To do that we set save_summaries_steps to 0. save_summaries_steps=0, config=tf.ConfigProto(allow_soft_placement=True)) as mon_sess: while not mon_sess.should_stop(): curr_batch_start = global_steps % num_batches global_steps += 1 if curr_batch_start == 0: shuffle_indices = np.random.permutation(np.arange(X_train.shape[0])) X_train = X_train[shuffle_indices] y_train = y_train[shuffle_indices] start = hps.batch_size * curr_batch_start end = np.minimum(hps.data_size, hps.batch_size * (curr_batch_start + 1)) xx = X_train[start:end] yy = y_train[start:end] fd = {model.X: xx, model.y: yy} mon_sess.run(model.train_op, feed_dict=fd)
def voc_eval_corloc(detpath, annopath, imagesetfile, classname, cachedir, ovthresh=0.5, use_07_metric=False): # assumes detections are in detpath.format(classname) # assumes annotations are in annopath.format(imagename) # assumes imagesetfile is a text file with each line an image name # cachedir caches the annotations in a pickle file # first load gt if not os.path.isdir(cachedir): os.mkdir(cachedir) imageset = os.path.splitext(os.path.basename(imagesetfile))[0] cachefile = os.path.join(cachedir, imageset + '_annots.pkl') # read list of images with open(imagesetfile, 'r') as f: lines = f.readlines() imagenames = [x.strip() for x in lines] if not os.path.isfile(cachefile): # load annots recs = {} for i, imagename in enumerate(imagenames): recs[imagename] = parse_rec(annopath.format(imagename)) if i % 100 == 0: logger.info( 'Reading annotation for {:d}/{:d}'.format( i + 1, len(imagenames))) # save logger.info('Saving cached annotations to {:s}'.format(cachefile)) save_object(recs, cachefile) else: # load recs = load_object(cachefile) # extract gt objects for this class class_recs = {} npos = 0 npos_im = 0 for imagename in imagenames: R = [obj for obj in recs[imagename] if obj['name'] == classname] bbox = np.array([x['bbox'] for x in R]) difficult = np.array([x['difficult'] for x in R]).astype(np.bool) det = [False] * len(R) npos = npos + sum(~difficult) class_recs[imagename] = {'bbox': bbox, 'difficult': difficult, 'det': det} if len(R) > 0: npos_im += min(1, sum(~difficult)) # read dets detfile = detpath.format(classname) with open(detfile, 'r') as f: lines = f.readlines() if len(lines) == 0: return 0.0 splitlines = [x.strip().split(' ') for x in lines] image_ids = [x[0] for x in splitlines] confidence = np.array([float(x[1]) for x in splitlines]) BB = np.array([[float(z) for z in x[2:]] for x in splitlines]) # sort by confidence sorted_ind = np.argsort(-confidence) sorted_scores = np.sort(-confidence) BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] nd = len(image_ids) T = [] F = [] too_min = 0 for d in range(nd): if image_ids[d] in T or image_ids[d] in F: continue R = class_recs[image_ids[d]] all_difficult = True for difficult in R['difficult']: if not difficult: all_difficult = False if all_difficult: continue bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) if BBGT.size > 0: # compute overlaps # intersection ixmin = np.maximum(BBGT[:, 0], bb[0]) iymin = np.maximum(BBGT[:, 1], bb[1]) ixmax = np.minimum(BBGT[:, 2], bb[2]) iymax = np.minimum(BBGT[:, 3], bb[3]) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + (BBGT[:, 2] - BBGT[:, 0] + 1.) * (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) overlaps = inters / uni ovmax = np.max(overlaps) jmax = np.argmax(overlaps) if ovmax > ovthresh: T.append(image_ids[d]) else: F.append(image_ids[d]) uni = (bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) overlaps = inters / uni ovmax = np.max(overlaps) if ovmax > ovthresh: too_min += 1 if len(F) == 0: too_min_rate = 0.0 else: too_min_rate = 1.0 * too_min / len(F) print('npos_im: ', npos_im, '#F: ', len(F), ' too_min: ', too_min, ' rate: ', too_min_rate) return 1.0 * len(T) / npos_im, too_min_rate
def train_model(learning_rate, steps, batch_size, input_feature="total_rooms"): """Trains a linear regression model of one feature. Args: learning_rate: A `float`, the learning rate. steps: A non-zero `int`, the total number of training steps. A training step consists of a forward and backward pass using a single batch. batch_size: A non-zero `int`, the batch size. input_feature: A `string` specifying a column from `california_housing_dataframe` to use as input feature. """ periods = 10 steps_per_period = steps / periods my_feature = input_feature my_feature_data = california_housing_dataframe[[my_feature]] my_label = "median_house_value" targets = california_housing_dataframe[my_label] # Create feature columns feature_columns = [tf.feature_column.numeric_column(my_feature)] # Create input functions training_input_fn = lambda: my_input_fn( my_feature_data, targets, batch_size=batch_size) prediction_input_fn = lambda: my_input_fn( my_feature_data, targets, num_epochs=1, shuffle=False) # Create a linear regressor object. my_optimizer = tf.train.GradientDescentOptimizer( learning_rate=learning_rate) my_optimizer = tf.contrib.estimator.clip_gradients_by_norm( my_optimizer, 5.0) linear_regressor = tf.estimator.LinearRegressor( feature_columns=feature_columns, optimizer=my_optimizer) # Set up to plot the state of our model's line each period. plt.figure(figsize=(15, 6)) plt.subplot(1, 2, 1) plt.title("Learned Line by Period") plt.ylabel(my_label) plt.xlabel(my_feature) sample = california_housing_dataframe.sample(n=300) plt.scatter(sample[my_feature], sample[my_label]) colors = [cm.coolwarm(x) for x in np.linspace(-1, 1, periods)] # Train the model, but do so inside a loop so that we can periodically assess # loss metrics. print("Training model...") print("RMSE (on training data):") root_mean_squared_errors = [] for period in range(0, periods): # Train the model, starting from the prior state. linear_regressor.train(input_fn=training_input_fn, steps=steps_per_period) # Take a break and compute predictions. predictions = linear_regressor.predict(input_fn=prediction_input_fn) predictions = np.array( [item['predictions'][0] for item in predictions]) # Compute loss. root_mean_squared_error = math.sqrt( metrics.mean_squared_error(predictions, targets)) # Occasionally print the current loss. print(" period %02d : %0.2f" % (period, root_mean_squared_error)) # Add the loss metrics from this period to our list. root_mean_squared_errors.append(root_mean_squared_error) # Finally, track the weights and biases over time. # Apply some math to ensure that the data and line are plotted neatly. y_extents = np.array([0, sample[my_label].max()]) weight = linear_regressor.get_variable_value( 'linear/linear_model/%s/weights' % input_feature)[0] bias = linear_regressor.get_variable_value( 'linear/linear_model/bias_weights') x_extents = (y_extents - bias) / weight x_extents = np.maximum(np.minimum(x_extents, sample[my_feature].max()), sample[my_feature].min()) y_extents = weight * x_extents + bias plt.plot(x_extents, y_extents, color=colors[period]) print("Model training finished.") # Output a graph of loss metrics over periods. plt.subplot(1, 2, 2) plt.ylabel('RMSE') plt.xlabel('Periods') plt.title("Root Mean Squared Error vs. Periods") plt.tight_layout() plt.plot(root_mean_squared_errors) # Output a table with calibration data. calibration_data = pd.DataFrame() calibration_data["predictions"] = pd.Series(predictions) calibration_data["targets"] = pd.Series(targets) display.display(calibration_data.describe()) print("Final RMSE (on training data): %0.2f" % root_mean_squared_error)
def TPmatrix(choice): # if '%f_%f_%f.npz'%(choice[0],choice[1],choice[2]) in os.listdir(directory2): # continue start = time.time() #print('choice',choice,'started in period', period) rows_ind = [] cols_ind = [] data_val = [] stdata.loc[:, 'quality'] = choice[1] stdata.loc[:, 'numedits_totalothers_accepted'] = EAE.predict( stdata[['quality', 'AnswerNum', 'Seniority_days']]) meanup_main = stdata['lambda_up'] + choice[0] * EUV.predict(stdata[[ 'quality', 'AnswerNum', 'Seniority_days', 'numedits_totalothers_accepted' ]]) meandown_main = stdata['lambda_down'] + choice[0] * EDV.predict(stdata[[ 'quality', 'AnswerNum', 'Seniority_days', 'numedits_totalothers_accepted' ]]) fvup_main = np.arange( sst.poisson.ppf(0.001, mu=meanup_main.min()), sst.poisson.ppf(0.999, mu=meanup_main.max()) + 1) # plus one to include right boundary (as it was done in simultion) fvdown_main = np.arange( sst.poisson.ppf(0.001, mu=meandown_main.min()), sst.poisson.ppf(0.999, mu=meandown_main.max()) + 1) # plus one to include right boundary (as it was done in simultion) accedits_main = np.arange( 0, choice[2] + 1) # number of suggested edits potentially approved extra_main_new_up = ssp.csc_matrix(uppoints * fvup_main) colsup = np.repeat([i for i in range(len(fvup_main))], len(fvdown_main) * len(accedits_main)) extra_main_new_up = extra_main_new_up[:, colsup] extra_main_new_down = ssp.csc_matrix(downpoints * fvdown_main) colsdown = np.tile( np.repeat([i for i in range(len(fvdown_main))], len(accedits_main)), (len(fvup_main))) extra_main_new_down = extra_main_new_down[:, colsdown] extra_main_new_edits = ssp.csc_matrix(approvalpoints * accedits_main) colsedits = np.tile([i for i in range(len(accedits_main))], (int(extra_main_new_up.shape[1] / len(accedits_main)))) extra_main_new_edits = extra_main_new_edits[:, colsedits] extra_main_new = extra_main_new_up - extra_main_new_down + extra_main_new_edits rows = np.repeat(0, len(stdata), axis=0) extra_main_new = extra_main_new[rows, :] # probabilities of new points arriving / of values at a) tpup_main_new = ssp.csc_matrix( sst.poisson.pmf(fvup_main, mu=meanup_main[:, np.newaxis])) tpup_main_new = tpup_main_new[:, colsup] # use column indeces created before tpdown_main_new = ssp.csc_matrix( sst.poisson.pmf(fvdown_main, mu=meandown_main[:, np.newaxis])) tpdown_main_new = tpdown_main_new[:, colsdown] tpedits_main = ssp.csc_matrix( sst.binom.pmf( accedits_main, choice[2], prob_acceptance * stdata['editIsSuggested'][:, np.newaxis])) tpedits_main = tpedits_main[:, colsedits] tp_main_new = tpup_main_new.multiply(tpdown_main_new).multiply( tpedits_main) # reduce dimensionality fut_vals_main = extra_main_new[0, :].toarray()[0, :] unique_fut_vals_main = np.unique(fut_vals_main) tp_reduced_main = ssp.lil_matrix( (extra_main_new.shape[0], unique_fut_vals_main.shape[0])) cols_tokeep = [] # columns to keep in extra_main count = 0 # to keep order of columns in new tp_main for val in list(unique_fut_vals_main): colsval = np.where(fut_vals_main == val)[0] cols_tokeep.append( colsval[0] ) # collects one column for each value to retain from extra_main tp_reduced_main[:, count] = tp_main_new[:, colsval].sum(axis=1) count += 1 tp_main_new = tp_reduced_main.tocsr() extra_main_new = extra_main_new[:, cols_tokeep] fut_dta = pd.DataFrame() fut_dta['lambda_up'] = np.minimum( np.around(meanup_main * np.exp(-1 / tau_up), 2), max(possible_lambdaup)) fut_dta['lambda_down'] = np.minimum( np.around(meandown_main * np.exp(-1 / tau_down), 2), max(possible_lambdadown)) fut_dta['AnswerNum'] = np.minimum(stdata['AnswerNum'] + choice[0], max(possible_answernum)) fut_dta['avail'] = stdata['avail'] # no beliefs on evolution for now fut_dta['Seniority_days'] = np.minimum(stdata['Seniority_days'] + 1, max(possible_seniority)) for col in range(extra_main_new.shape[1]): fut_dta.loc[:, 'rep_cum'] = np.maximum( np.minimum( stdata['rep_cum'][:, np.newaxis] + extra_main_new[:, col].toarray(), max(possible_points)), 0)[:, 0] prob = tp_main_new[:, col].toarray() fut_dta = pd.merge(fut_dta, statesreset, on=state_names, how='left') rows_ind.extend(fut_dta.index.tolist()) cols_ind.extend(fut_dta['index'].values.tolist()) data_val.extend(prob.flatten().tolist()) fut_dta.drop('index', axis=1, inplace=True) print(time.time() - start) matrix = ssp.csr_matrix((data_val, (rows_ind, cols_ind)), shape=(len(stdata), len(stdata))) ssp.save_npz( directory2 + '%f_%f_%f.npz' % (choice[0], choice[1], choice[2]), matrix) print('choice {} completed'.format(choice))
def voc_eval_visualization( detpath, annopath, imagesetfile, classname, cachedir, image_path, output_dir): # assumes detections are in detpath.format(classname) # assumes annotations are in annopath.format(imagename) # assumes imagesetfile is a text file with each line an image name # cachedir caches the annotations in a pickle file # read list of images with open(imagesetfile, 'r') as f: lines = f.readlines() imagenames = [x.strip() for x in lines] # load annots recs = {} for i, imagename in enumerate(imagenames): recs[imagename] = parse_rec(annopath.format(imagename)) if i % 100 == 0: logger.info( 'Reading annotation for {:d}/{:d}'.format( i + 1, len(imagenames))) # extract gt objects for this class class_recs = {} npos = 0 for imagename in imagenames: R = [obj for obj in recs[imagename] if obj['name'] == classname] bbox = np.array([x['bbox'] for x in R]) difficult = np.array([x['difficult'] for x in R]).astype(np.bool) det = [False] * len(R) npos = npos + sum(~difficult) class_recs[imagename] = {'bbox': bbox, 'difficult': difficult, 'det': det} # read dets detfile = detpath.format(classname) with open(detfile, 'r') as f: lines = f.readlines() splitlines = [x.strip().split(' ') for x in lines] image_ids = [x[0] for x in splitlines] confidence = np.array([float(x[1]) for x in splitlines]) BB = np.array([[float(z) for z in x[2:]] for x in splitlines]) # sort by confidence sorted_ind = np.argsort(-confidence) sorted_scores = np.sort(-confidence) BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] nd = len(image_ids) T = [] F = [] for d in range(nd): if d % 1000 == 0: logger.info('{:s}: {:d} / {:d}'.format(classname, d + 1, nd)) if image_ids[d] in T or image_ids[d] in F: continue R = class_recs[image_ids[d]] all_difficult = True for difficult in R['difficult']: if not difficult: all_difficult = False if all_difficult: continue bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) if BBGT.size > 0: # compute overlaps # intersection ixmin = np.maximum(BBGT[:, 0], bb[0]) iymin = np.maximum(BBGT[:, 1], bb[1]) ixmax = np.minimum(BBGT[:, 2], bb[2]) iymax = np.minimum(BBGT[:, 3], bb[3]) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + (BBGT[:, 2] - BBGT[:, 0] + 1.) * (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) overlaps = inters / uni ovmax = np.max(overlaps) jmax = np.argmax(overlaps) if ovmax > 0.5: T.append(image_ids[d]) else: F.append(image_ids[d]) img = cv2.imread(image_path.format(image_ids[d])) for box in BBGT: x1 = int(box[0]) y1 = int(box[1]) x2 = int(box[2]) y2 = int(box[3]) img = cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 255), 4) x1 = int(bb[0]) y1 = int(bb[1]) x2 = int(bb[2]) y2 = int(bb[3]) if ovmax>0.5: img = cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 4) else: img = cv2.rectangle(img, (x1, y1), (x2, y2), (0, 0, 255), 4) cv2.imwrite(os.path.join(output_dir, image_ids[d] + '.png'), img)
def extract_storm_objects(label_grid, data, x_grid, y_grid, times, dx=1, dt=1, obj_buffer=0): """ After storms are labeled, this method extracts the storm objects from the grid and places them into STObjects. The STObjects contain intensity, location, and shape information about each storm at each timestep. Args: label_grid: 2D or 3D array output by label_storm_objects. data: 2D or 3D array used as input to label_storm_objects. x_grid: 2D array of x-coordinate data, preferably on a uniform spatial grid with units of length. y_grid: 2D array of y-coordinate data. times: List or array of time values, preferably as integers dx: grid spacing in same units as x_grid and y_grid. dt: period elapsed between times obj_buffer: number of extra pixels beyond bounding box of object to store in each STObject Returns: storm_objects: list of lists containing STObjects identified at each time. """ storm_objects = [] if len(label_grid.shape) == 3: ij_grid = np.indices(label_grid.shape[1:]) for t, time in enumerate(times): storm_objects.append([]) object_slices = list( find_objects(label_grid[t], label_grid[t].max())) if len(object_slices) > 0: for o, obj_slice in enumerate(object_slices): if obj_buffer > 0: obj_slice_buff = [ slice( np.maximum(0, osl.start - obj_buffer), np.minimum(osl.stop + obj_buffer, label_grid.shape[l + 1])) for l, osl in enumerate(obj_slice) ] else: obj_slice_buff = obj_slice storm_objects[-1].append( STObject(data[t][obj_slice_buff], np.where( label_grid[t][obj_slice_buff] == o + 1, 1, 0), x_grid[obj_slice_buff], y_grid[obj_slice_buff], ij_grid[0][obj_slice_buff], ij_grid[1][obj_slice_buff], time, time, dx=dx, step=dt)) if t > 0: dims = storm_objects[-1][-1].timesteps[0].shape storm_objects[-1][-1].estimate_motion( time, data[t - 1], dims[1], dims[0]) else: ij_grid = np.indices(label_grid.shape) storm_objects.append([]) object_slices = list(find_objects(label_grid, label_grid.max())) if len(object_slices) > 0: for o, obj_slice in enumerate(object_slices): if obj_buffer > 0: obj_slice_buff = [ slice( np.maximum(0, osl.start - obj_buffer), np.minimum(osl.stop + obj_buffer, label_grid.shape[l + 1])) for l, osl in enumerate(obj_slice) ] else: obj_slice_buff = obj_slice storm_objects[-1].append( STObject(data[obj_slice_buff], np.where(label_grid[obj_slice_buff] == o + 1, 1, 0), x_grid[obj_slice_buff], y_grid[obj_slice_buff], ij_grid[0][obj_slice_buff], ij_grid[1][obj_slice_buff], times, times, dx=dx, step=dt)) return storm_objects
def predict_kitti_to_anno(net, example, class_names, center_limit_range=None, lidar_input=True, global_set=None): batch_image_shape = example['image_shape'] batch_imgidx = example['image_idx'] predictions_dicts = net(example) # t = time.time() annos = [] for i, preds_dict in enumerate(predictions_dicts): image_shape = batch_image_shape[i] img_idx = preds_dict["image_idx"] if preds_dict["bbox"] is not None: box_2d_preds = preds_dict["bbox"].detach().cpu().numpy() box_preds = preds_dict["box3d_camera"].detach().cpu().numpy() scores = preds_dict["scores"].detach().cpu().numpy() box_preds_lidar = preds_dict["box3d_lidar"].detach().cpu().numpy() # write pred to file label_preds = preds_dict["label_preds"].detach().cpu().numpy() # label_preds = np.zeros([box_2d_preds.shape[0]], dtype=np.int32) anno = kitti.get_start_result_anno() num_example = 0 for box, box_lidar, bbox, score, label in zip( box_preds, box_preds_lidar, box_2d_preds, scores, label_preds): if not lidar_input: if bbox[0] > image_shape[1] or bbox[1] > image_shape[0]: continue if bbox[2] < 0 or bbox[3] < 0: continue # print(img_shape) if center_limit_range is not None: limit_range = np.array(center_limit_range) if (np.any(box_lidar[:3] < limit_range[:3]) or np.any(box_lidar[:3] > limit_range[3:])): continue bbox[2:] = np.minimum(bbox[2:], image_shape[::-1]) bbox[:2] = np.maximum(bbox[:2], [0, 0]) anno["name"].append(class_names[int(label)]) anno["truncated"].append(0.0) anno["occluded"].append(0) anno["alpha"].append(-np.arctan2(-box_lidar[1], box_lidar[0]) + box[6]) anno["bbox"].append(bbox) anno["dimensions"].append(box[3:6]) anno["location"].append(box[:3]) anno["rotation_y"].append(box[6]) if global_set is not None: for i in range(100000): if score in global_set: score -= 1 / 100000 else: global_set.add(score) break anno["score"].append(score) num_example += 1 if num_example != 0: anno = {n: np.stack(v) for n, v in anno.items()} annos.append(anno) else: annos.append(kitti.empty_result_anno()) else: annos.append(kitti.empty_result_anno()) num_example = annos[-1]["name"].shape[0] annos[-1]["image_idx"] = np.array([img_idx] * num_example, dtype=np.int64) return annos
def soft_nms_pytorch(dets, box_scores, sigma=0.5, thresh=0.001, cuda=0): """ Build a pytorch implement of Soft NMS algorithm. # Augments dets: boxes coordinate tensor (format:[y1, x1, y2, x2]) box_scores: box score tensors sigma: variance of Gaussian function thresh: score thresh cuda: CUDA flag # Return the index of the selected boxes """ # Indexes concatenate boxes with the last column N = dets.shape[0] if cuda: indexes = torch.arange(0, N, dtype=torch.float).cuda().view(N, 1) else: indexes = torch.arange(0, N, dtype=torch.float).view(N, 1) dets = torch.cat((dets, indexes), dim=1) # The order of boxes coordinate is [y1,x1,y2,x2] y1 = dets[:, 0] x1 = dets[:, 1] y2 = dets[:, 2] x2 = dets[:, 3] scores = box_scores areas = (x2 - x1 + 1) * (y2 - y1 + 1) for i in range(N): # intermediate parameters for later parameters exchange tscore = scores[i].clone() pos = i + 1 if i != N - 1: maxscore, maxpos = torch.max(scores[pos:], dim=0) if tscore < maxscore: dets[i], dets[maxpos.item() + i + 1] = dets[maxpos.item() + i + 1].clone(), dets[i].clone() scores[i], scores[maxpos.item() + i + 1] = scores[maxpos.item() + i + 1].clone(), scores[i].clone() areas[i], areas[maxpos + i + 1] = areas[maxpos + i + 1].clone(), areas[i].clone() # IoU calculate yy1 = np.maximum(dets[i, 0].to("cpu").numpy(), dets[pos:, 0].to("cpu").numpy()) xx1 = np.maximum(dets[i, 1].to("cpu").numpy(), dets[pos:, 1].to("cpu").numpy()) yy2 = np.minimum(dets[i, 2].to("cpu").numpy(), dets[pos:, 2].to("cpu").numpy()) xx2 = np.minimum(dets[i, 3].to("cpu").numpy(), dets[pos:, 3].to("cpu").numpy()) w = np.maximum(0.0, xx2 - xx1 + 1) h = np.maximum(0.0, yy2 - yy1 + 1) inter = torch.tensor(w * h).cuda() if cuda else torch.tensor(w * h) ovr = torch.div(inter, (areas[i] + areas[pos:] - inter)) # Gaussian decay weight = torch.exp(-(ovr * ovr) / sigma) scores[pos:] = weight * scores[pos:] # select the boxes and keep the corresponding indexes keep = dets[:, 4][scores > thresh].int() return keep
meandown = states['lambda_down'] + states['quantity'] * EDV.predict( states[[ 'quality', 'AnswerNum', 'Seniority_days', 'numedits_totalothers_accepted' ]]) upvotes = np.random.poisson(meanup) downvotes = np.random.poisson(meandown) accedits = np.random.binomial(states['numedits'], prob_acceptance * (1 - states['isEditor'])) points = uppoints * upvotes - downpoints * downvotes + approvalpoints * accedits # fill in simulated dataframe simdata.loc[simdata['periods'] == period + 1, 'rep_cum'] = np.maximum( np.minimum(states['rep_cum'].values + points, max(possible_points)), 0) simdata.loc[simdata['periods'] == period + 1, 'lambda_up'] = np.minimum( np.around(meanup.values * np.exp(-1 / tau_up), 2), max(possible_lambdaup)) simdata.loc[simdata['periods'] == period + 1, 'lambda_down'] = np.minimum( np.around(meandown.values * np.exp(-1 / tau_down), 2), max(possible_lambdadown)) simdata.loc[simdata['periods'] == period + 1, 'Seniority_days'] = np.minimum( states['Seniority_days'].values + 1, max(possible_seniority)) simdata.loc[simdata['periods'] == period + 1, 'AnswerNum'] = np.minimum( states['AnswerNum'].values + states['quantity'].values, max(possible_answernum)) simdata.to_csv(directory + 'simdata_type{}.csv'.format(typenum), index=False)
def _voc_computePrecisionRecallAp(class_recs, confidence, image_ids, BB, ovthresh=0.5, use_07_metric=False): ''' Computes precision, recall. and average precision ''' if len(BB) == 0: return 0.0, 0.0, 0.0 # sort by confidence sorted_ind = np.argsort(-confidence) BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] # go down dets and mark TPs and FPs nd = len(image_ids) tp = np.zeros(nd) fp = np.zeros(nd) for d in range(nd): R = class_recs[image_ids[d]] bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) if BBGT.size > 0: # compute overlaps ixmin = np.maximum(BBGT[:, 0], bb[0]) iymin = np.maximum(BBGT[:, 1], bb[1]) ixmax = np.minimum(BBGT[:, 2], bb[2]) iymax = np.minimum(BBGT[:, 3], bb[3]) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) + (BBGT[:, 2] - BBGT[:, 0] + 1.) * (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters) overlaps = inters / uni ovmax = np.max(overlaps) jmax = np.argmax(overlaps) if ovmax > ovthresh: if not R['difficult'][jmax]: if not R['det'][jmax]: tp[d] = 1. R['det'][jmax] = 1 else: fp[d] = 1. else: fp[d] = 1. # compute precision recall npos = sum([len(cr['bbox']) for cr in class_recs]) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(npos) # avoid divide by zero in case the first detection matches a difficult ground truth prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap = computeAveragePrecision(rec, prec, use_07_metric) return rec, prec, ap
def clip(im): return np.maximum(0.,np.minimum(1.,im))
def compute_tDCF(bonafide_score_cm, spoof_score_cm, Pfa_asv, Pmiss_asv, Pmiss_spoof_asv, cost_model, print_cost): """ Compute Tandem Detection Cost Function (t-DCF) [1] for a fixed ASV system. In brief, t-DCF returns a detection cost of a cascaded system of this form, Speech waveform -> [CM] -> [ASV] -> decision where CM stands for countermeasure and ASV for automatic speaker verification. The CM is therefore used as a 'gate' to decided whether or not the input speech sample should be passed onwards to the ASV system. Generally, both CM and ASV can do detection errors. Not all those errors are necessarily equally cost, and not all types of users are necessarily equally likely. The tandem t-DCF gives a principled with to compare different spoofing countermeasures under a detection cost function framework that takes that information into account. INPUTS: bonafide_score_cm A vector of POSITIVE CLASS (bona fide or human) detection scores obtained by executing a spoofing countermeasure (CM) on some positive evaluation trials. trial represents a bona fide case. spoof_score_cm A vector of NEGATIVE CLASS (spoofing attack) detection scores obtained by executing a spoofing CM on some negative evaluation trials. Pfa_asv False alarm (false acceptance) rate of the ASV system that is evaluated in tandem with the CM. Assumed to be in fractions, not percentages. Pmiss_asv Miss (false rejection) rate of the ASV system that is evaluated in tandem with the spoofing CM. Assumed to be in fractions, not percentages. Pmiss_spoof_asv Miss rate of spoof samples of the ASV system that is evaluated in tandem with the spoofing CM. That is, the fraction of spoof samples that were rejected by the ASV system. cost_model A struct that contains the parameters of t-DCF, with the following fields. Ptar Prior probability of target speaker. Pnon Prior probability of nontarget speaker (zero-effort impostor) Psoof Prior probability of spoofing attack. Cmiss_asv Cost of ASV falsely rejecting target. Cfa_asv Cost of ASV falsely accepting nontarget. Cmiss_cm Cost of CM falsely rejecting target. Cfa_cm Cost of CM falsely accepting spoof. print_cost Print a summary of the cost parameters and the implied t-DCF cost function? OUTPUTS: tDCF_norm Normalized t-DCF curve across the different CM system operating points; see [2] for more details. Normalized t-DCF > 1 indicates a useless countermeasure (as the tandem system would do better without it). min(tDCF_norm) will be the minimum t-DCF used in ASVspoof 2019 [2]. CM_thresholds Vector of same size as tDCF_norm corresponding to the CM threshold (operating point). NOTE: o In relative terms, higher detection scores values are assumed to indicate stronger support for the bona fide hypothesis. o You should provide real-valued soft scores, NOT hard decisions. The recommendation is that the scores are log-likelihood ratios (LLRs) from a bonafide-vs-spoof hypothesis based on some statistical model. This, however, is NOT required. The scores can have arbitrary range and scaling. o Pfa_asv, Pmiss_asv, Pmiss_spoof_asv are in fractions, not percentages. References: [1] T. Kinnunen, K.-A. Lee, H. Delgado, N. Evans, M. Todisco, M. Sahidullah, J. Yamagishi, D.A. Reynolds: "t-DCF: a Detection Cost Function for the Tandem Assessment of Spoofing Countermeasures and Automatic Speaker Verification", Proc. Odyssey 2018: the Speaker and Language Recognition Workshop, pp. 312--319, Les Sables d'Olonne, France, June 2018 (https://www.isca-speech.org/archive/Odyssey_2018/pdfs/68.pdf) [2] ASVspoof 2019 challenge evaluation plan TODO: <add link> """ # Sanity check of cost parameters if cost_model['Cfa_asv'] < 0 or cost_model['Cmiss_asv'] < 0 or \ cost_model['Cfa_cm'] < 0 or cost_model['Cmiss_cm'] < 0: print('WARNING: Usually the cost values should be positive!') if cost_model['Ptar'] < 0 or cost_model['Pnon'] < 0 or cost_model['Pspoof'] < 0 or \ np.abs(cost_model['Ptar'] + cost_model['Pnon'] + cost_model['Pspoof'] - 1) > 1e-10: sys.exit('ERROR: Your prior probabilities should be positive and sum up to one.') # Unless we evaluate worst-case model, we need to have some spoof tests against asv if Pmiss_spoof_asv is None: sys.exit('ERROR: you should provide miss rate of spoof tests against your ASV system.') # Sanity check of scores combined_scores = np.concatenate((bonafide_score_cm, spoof_score_cm)) if np.isnan(combined_scores).any() or np.isinf(combined_scores).any(): sys.exit('ERROR: Your scores contain nan or inf.') # Sanity check that inputs are scores and not decisions n_uniq = np.unique(combined_scores).size if n_uniq < 3: sys.exit('ERROR: You should provide soft CM scores - not binary decisions') # Obtain miss and false alarm rates of CM Pmiss_cm, Pfa_cm, CM_thresholds = compute_det_curve(bonafide_score_cm, spoof_score_cm) # Constants - see ASVspoof 2019 evaluation plan C1 = cost_model['Ptar'] * (cost_model['Cmiss_cm'] - cost_model['Cmiss_asv'] * Pmiss_asv) - \ cost_model['Pnon'] * cost_model['Cfa_asv'] * Pfa_asv C2 = cost_model['Cfa_cm'] * cost_model['Pspoof'] * (1 - Pmiss_spoof_asv) # Sanity check of the weights if C1 < 0 or C2 < 0: sys.exit( 'You should never see this error but I cannot evalute tDCF with negative weights - please check whether your ASV error rates are correctly computed?') # Obtain t-DCF curve for all thresholds tDCF = C1 * Pmiss_cm + C2 * Pfa_cm # Normalized t-DCF tDCF_norm = tDCF / np.minimum(C1, C2) # Everything should be fine if reaching here. if print_cost: print( 't-DCF evaluation from [Nbona={}, Nspoof={}] trials\n'.format(bonafide_score_cm.size, spoof_score_cm.size)) print('t-DCF MODEL') print(' Ptar = {:8.5f} (Prior probability of target user)'.format(cost_model['Ptar'])) print(' Pnon = {:8.5f} (Prior probability of nontarget user)'.format(cost_model['Pnon'])) print(' Pspoof = {:8.5f} (Prior probability of spoofing attack)'.format(cost_model['Pspoof'])) print(' Cfa_asv = {:8.5f} (Cost of ASV falsely accepting a nontarget)'.format(cost_model['Cfa_asv'])) print( ' Cmiss_asv = {:8.5f} (Cost of ASV falsely rejecting target speaker)'.format(cost_model['Cmiss_asv'])) print( ' Cfa_cm = {:8.5f} (Cost of CM falsely passing a spoof to ASV system)'.format(cost_model['Cfa_cm'])) print(' Cmiss_cm = {:8.5f} (Cost of CM falsely blocking target utterance which never reaches ASV)'.format( cost_model['Cmiss_cm'])) print('\n Implied normalized t-DCF function (depends on t-DCF parameters and ASV errors), s=CM threshold)') if C2 == np.minimum(C1, C2): print(' tDCF_norm(s) = {:8.5f} x Pmiss_cm(s) + Pfa_cm(s)\n'.format(C1 / C2)) else: print(' tDCF_norm(s) = Pmiss_cm(s) + {:8.5f} x Pfa_cm(s)\n'.format(C2 / C1)) return tDCF_norm, CM_thresholds
def update_all(self, boxes, start_frame): colored_boxes = [] frame = start_frame for detections in boxes: if len(detections) > 0: iou_matrix = np.zeros((len(detections), len(self.tracks) + len(detections)), dtype=np.float32) for d, det in enumerate(detections): for t, track in enumerate(self.tracks): iou_matrix[d, t] = -iou(det, track.get_prediction(), self.inc_size, self.soft_iou_k) if iou_matrix[d, t] < 0: if not track.boxes[-1].phantom: iou_matrix[d, t] -= 1.0 if det[4] < self.track_creation_score: iou_matrix[d, len(self.tracks) + d] = +0.001 else: iou_matrix[d, len(self.tracks) + d] = -0.001 matched_indices = linear_assignment(iou_matrix) old_length = len(self.tracks) for row in matched_indices: b = detections[row[0]] if row[1] >= old_length: id = self.next_id self.next_id += 1 self.tracks.append(Track(id, Box(id, b[0], b[1], b[2], b[3], frame, False, b[4], b[5]), self.vx, self.vy, self.ax, self.ay)) elif iou_matrix[row[0], row[1]] < 0: track = self.tracks[row[1]] box = Box(track.track_id, b[0], b[1], b[2], b[3], frame, False, b[4], b[5]) track.update_real(box, self.non_decrease, self.real_detection) if not track.counted and track.real_boxes >= self.detections_count: self.objects += 1 track.counted = True boxes = [] active_tracks = [] for track in self.tracks: if track.last_frame < frame: track.update_phantom() phantom_threshold = np.minimum(self.max_phantom_omit, np.maximum(self.min_phantom_omit, self.phantom_coef * track.get_max_phantoms())) box = track.boxes[-1] if track.phantom_boxes > phantom_threshold \ or box.left > self.max_x \ or box.right < self.min_x \ or box.top < self.min_y \ or box.bottom > self.max_y: self.deleted_tracks.append(track) else: active_tracks.append(track) else: active_tracks.append(track) box = track.boxes[-1] colored_box = [int(box.left), int(box.bottom), int(box.right), int(box.top), int(track.track_id), int(box.phantom)] boxes.append(colored_box) colored_boxes.append(boxes) self.tracks = active_tracks frame += 1 return colored_boxes
def multi_ROC_w_StratKFold(X, y, K, classifiers, classifier_names): cv = StratifiedKFold(n_splits=K) # initialize several dictionaries that will be populated by classifier # e.g. the tprs dictionary will have the true positive rates for each K fold across the fpr_range (X-axis) tprs = {} aucs = {} mean_tprs = {} mean_auc = {} std_auc = {} fpr_range = np.linspace(0, 1, 100) # X-axis range # create the subplot num_axes = len(classifiers) fig, axs = plt.subplots(1, num_axes, figsize=(10 * num_axes, 10)) for i, classifier in enumerate(classifiers): interp_tprs = [] roc_aucs = [] for j, (train, test) in enumerate(cv.split(X, y)): classifier.fit(X.iloc[train], y.iloc[train]) viz = plot_roc_curve(classifier, X.iloc[test], y.iloc[test], name='ROC fold {}'.format(j), alpha=0.3, lw=1, ax=axs[i]) # interpolate the true and false pos values across the X range interp_tpr = np.interp(fpr_range, viz.fpr, viz.tpr) interp_tpr[0] = 0.0 interp_tprs.append(interp_tpr) # store the AUC for each K fold roc_aucs.append(viz.roc_auc) # store the TPRs and AUCs by classifier for the all K folds in their dictionaries tprs[classifier] = interp_tprs aucs[classifier] = roc_aucs # plot dotted line from (0,0) to (1,1) axs[i].plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Chance', alpha=.8) # calculate mean TPRs and mean and std AUCs by classifier and plot mean_tprs[classifier] = np.mean(tprs[classifier], axis=0) mean_tprs[classifier][-1] = 1.0 mean_auc[classifier] = auc(fpr_range, mean_tprs[classifier]) std_auc[classifier] = np.std(aucs[classifier]) axs[i].plot(fpr_range, mean_tprs[classifier], color='b', label=r'Mean ROC (AUC = %0.3f $\pm$ %0.3f)' % (mean_auc[classifier], std_auc[classifier]), lw=2, alpha=.8) # shade the area 1 std below and above std_tpr = np.std(tprs[classifier], axis=0) tprs_upper = np.minimum(mean_tprs[classifier] + std_tpr, 1) tprs_lower = np.maximum(mean_tprs[classifier] - std_tpr, 0) axs[i].fill_between(fpr_range, tprs_lower, tprs_upper, color='grey', alpha=.2, label=r'$\pm$ 1 std. dev.') # a bit of formatting axs[i].set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05], title="ROC Curve: {}".format(classifier_names[i])) axs[i].legend(loc="lower right") plt.show()
def test_CNN(model,X_train,y_train,X_valid,y_valid,w_id,batch_size,num_epochs,preprocessed=False): num_samples = X_train.shape[0] num_batches = int(np.ceil(num_samples / float(batch_size))) l1 = preprocessing.LabelEncoder() t1 = l1.fit_transform(y_train) l2 = preprocessing.LabelEncoder() t2 = l2.fit_transform(y_valid) num_test_samples = X_valid.shape[0] num_test_batches = int(np.ceil(num_test_samples / float(batch_size))) # setting up lists for handling loss/accuracy train_loss, val_loss = [], [] train_cost, val_cost = [], [] for epoch in range(num_epochs): # Forward -> Backprob -> Update params ## Train correct = 0 model.train() for i in range(num_batches): if i % 10 == 0: print("\n {}, still training...".format(i), end='') idx = range(i * batch_size, np.minimum((i + 1) * batch_size, num_samples)) index = idx[-1]-idx[0]+1 if preprocessed==False: batch_image = np.zeros((index,224,224)) for j in range(index): image_resized = resize(X_train[idx[j]], (224, 224), anti_aliasing=True) batch_image[j,:,:] = image_resized X_batch_tr = Variable(torch.from_numpy(batch_image)) y_batch_tr = Variable(torch.from_numpy(t1[idx]).long()) optimizer.zero_grad() output = model(X_batch_tr.unsqueeze(1).float()) else: X_batch_tr = X_train[idx,:,:,:] y_batch_tr = Variable(torch.from_numpy(t1[idx]).long()) optimizer.zero_grad() output = model(X_batch_tr.float()) batch_loss = criterion(output, y_batch_tr) train_loss.append(batch_loss.data.numpy()) batch_loss.backward() optimizer.step() preds = np.argmax(output.data.numpy(), axis=-1) correct += np.sum(y_batch_tr.data.numpy() == preds) train_acc = correct / float(num_samples) train_cost.append(np.mean(train_loss)) correct2 = 0 model.eval() wrong_guesses = [] wrong_predictions = [] all_predictions = [] for i in range(num_test_batches): if i % 10 == 0: print("\n {}, now validation...".format(i), end='') idx = range(i * batch_size, np.minimum((i + 1) * batch_size, num_test_samples)) index = idx[-1] - idx[0] + 1 if preprocessed==False: batch_image = np.zeros((index,224,224)) for j in range(index): image_resized = resize(X_valid[idx[j]], (224, 224), anti_aliasing=True) batch_image[j,:,:] = image_resized X_batch_v = Variable(torch.from_numpy(batch_image)) y_batch_v = Variable(torch.from_numpy(t2[idx]).long()) output = model(X_batch_v.unsqueeze(1).float()) else: X_batch_v = X_valid[idx,:,:,:] y_batch_v = Variable(torch.from_numpy(t2[idx]).long()) output = model(X_batch_v.float()) batch_loss = criterion(output, y_batch_v) val_loss.append(batch_loss.data.numpy()) preds = np.argmax(output.data.numpy(), axis=-1) eval_preds = y_batch_v.data.numpy() == preds for k in range(index): if eval_preds[k] == False: wrong_guesses.append(w_id[idx[k]]) wrong_predictions.append(preds[k]) else: correct2 += 1 all_predictions.append(preds[k]) val_acc = correct2 / float(num_test_samples) val_cost.append(np.mean(val_loss)) if epoch % 10 == 0: print("\n Epoch %2i : Train Loss %f , Train acc %f, Valid acc %f" % ( epoch + 1, train_cost[-1], train_acc, val_acc)) return train_acc,train_cost,val_acc,val_cost, wrong_guesses, wrong_predictions, all_predictions, model