def jaccard_distance(u, v): """return jaccard distance""" u = numpy.asarray(u) v = numpy.asarray(v) return (numpy.double(numpy.bitwise_and((u != v), numpy.bitwise_or(u != 0, v != 0)).sum()) / numpy.double(numpy.bitwise_or(u != 0, v != 0).sum()))
def simulatestep(ps, pm, pv, teilchenort, teilchenmobil, number, vel): zzv = np.random.random(number) zzv2 = zzv < ps zzv3 = zzv < pm #print "ss" # print teilchen, number #zzvVel = np.random.randint(-1,2, size=number) # zzvVel2 = np.random.random(number) #zzvVel3 = (zzvVel2 < pv) * zzvVel #print zzvVel #berechne neuen Zustand für die Teilchen# bin mobil, wenn # vorher mobil und es bleibe (zzv3, pm) # oder: war nicht mobil und bleibe nicht (invertiert zu oder) mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2)))) #vel = vel + zzvVel3 * mobilneu #vel = vel * mobilneu #print vel #np.clip(vel, 0, 12, vel) #print "vel, ss", vel # wenn mobil, addiere 1 zum Ort teilchenortneu = teilchenort + vel*mobilneu return teilchenortneu, mobilneu, vel
def pattern_distance_jaccard(a, b): """ Computes a distance measure for two binary patterns based on their Jaccard-Needham distance, defined as .. math:: d_J(a,b) = 1 - J(a,b) = \\frac{|a \\cup b| - |a \\cap b|}{|a \\cup b|}. The similarity measure takes values on the closed interval [0, 1], where a value of 1 is attained for disjoint, i.e. maximally dissimilar patterns a and b and a value of 0 for the case of :math:`a=b`. Parameters ---------- a : list or array, int or bool Input pattern b : list or array, int or bool Input pattern Returns ------- dist : double Jaccard distance between `a` and `b`. """ # Note: code taken from scipy. Duplicated as only numpy references wanted for base functionality a = np.atleast_1d(a).astype(bool) b = np.atleast_1d(b).astype(bool) dist = (np.double(np.bitwise_and((a != b), np.bitwise_or(a != 0, b != 0)).sum()) / np.double(np.bitwise_or(a != 0, b != 0).sum())) return dist
def __invertibleToRGB(self, rgbVarr, varrIdx, colorLutStruct): ''' Decode an RGB image rendered in INVERTIBLE_LUT mode. The image encodes float values as colors, so the RGB value is first decoded into its represented float value and then the color table is applied. ''' w0 = np.left_shift( rgbVarr[varrIdx[0], varrIdx[1], 0].astype(np.uint32), 16) w1 = np.left_shift( rgbVarr[varrIdx[0], varrIdx[1], 1].astype(np.uint32), 8) w2 = rgbVarr[varrIdx[0], varrIdx[1], 2] value = np.bitwise_or(w0, w1) value = np.bitwise_or(value, w2) value = np.subtract(value.astype(np.int32), 1) normalized_val = np.divide(value.astype(float), 0xFFFFFE) # Map float value to color lut (use a histogram to support non-uniform # colormaps (fetch bin indices)) colorLut = colorLutStruct.lut bins = colorLutStruct.adjustedBins idx = np.digitize(normalized_val, bins) idx = np.subtract(idx, 1) valueImage = np.zeros([rgbVarr.shape[0], rgbVarr.shape[1]], dtype=np.uint32) valueImage[varrIdx[0], varrIdx[1]] = idx return colorLut[valueImage]
def valuewriter(self, imageSlice, fname, vrange): """ Takes in either a (1C) float or a RGB (3C) buffer and writes it as an image file.""" # Adjust the filename, replace png with .im baseName, ext = os.path.splitext(fname) adjustedName = baseName + self.floatExtension() dimensions = imageSlice.shape if len(dimensions) == 2 and imageSlice.dtype == numpy.float32: # Given as single channel floating point buffer. self.zwriter(imageSlice, adjustedName) elif (len(dimensions) > 2) and (dimensions[2] == 3): #if self.dontConvertValsToFloat # self.rgbwriter(imageSlice, fname) # return # Given an RGB buffer # Convert it to a floating point buffer for consistency # TODO: just one copy of this code in cinema w0 = numpy.left_shift(imageSlice[:,:,0].astype(numpy.uint32), 16) w1 = numpy.left_shift(imageSlice[:,:,1].astype(numpy.uint32), 8) w2 = imageSlice[:,:,2].astype(numpy.uint32) value = numpy.bitwise_or(w0,w1) value = numpy.bitwise_or(value,w2) value = numpy.subtract(value, 1) value = value.astype(numpy.float32) adjusted_val = numpy.divide(value, float(0xFFFFFE)) self.zwriter(adjusted_val, adjustedName) else: raise ValueError("Invalid dimensions for a value raster.")
def replace_vals(filename, replace, delim=','): ''' Replace the values in filename with specified values in replace_values Parameters ---------- filename : string Will be read into a rec array replace_values : tuple First object is value to replace and second object is what to replace it with ''' data = csv2rec(filename, delimiter=delim, missing=replace[0]) for nm in data.dtype.names: try: # Missing float isNaN = (np.isnan(data[nm])) except: isNaN = np.zeros(len(data[nm]), dtype=bool) isBlank = np.array([it == '' for it in data[nm]]) isMinusOne = (data[nm] == -1)# Missing int # Missing other isNone = np.array([i == None for i in data[nm]]) ind = np.bitwise_or(isNaN, isBlank) ind = np.bitwise_or(ind, isMinusOne) ind = np.bitwise_or(ind, isNone) data[nm][ind] = replace[1] return data
def place_ship(self, position, length, orientation): """ Return None if ship cannot be placed """ ship = None if orientation == 'H': zeros = np.zeros(self.width * self.height, dtype='int8') if (position[0] + length) > self.width: return None for i in range(length): zeros[position[1] * self.width + position[0]+i] = 1 if np.all(np.bitwise_and(self._layout, zeros) == 0): self._layout = np.bitwise_or(self._layout, zeros) ship = Ship(position, length, orientation) elif orientation == 'V': zeros = np.zeros(self.width * self.height, dtype='int8') if (position[1] + length) > self.height: return None for i in range(length): zeros[(position[1] + i) * self.width + position[0]] = 1 if np.all(np.bitwise_and(self._layout, zeros) == 0): self._layout = np.bitwise_or(self._layout, zeros) ship = Ship(position, length, orientation) if ship: self._ships.append(ship) return ship
def filter(mask, cube, header, clipMethod, threshold, rmsMode, fluxRange, verbose): err.message("Running threshold finder.") # Sanity checks of user input err.ensure( clipMethod in {"absolute", "relative"}, "Threshold finder failed. Illegal clip method: '" + str(clipMethod) + "'.") err.ensure( rmsMode in {"std", "mad", "gauss", "negative"}, "Threshold finder failed. Illegal RMS mode: '" + str(rmsMode) + "'.") err.ensure( fluxRange in {"positive", "negative", "all"}, "Threshold finder failed. Illegal flux range: '" + str(fluxRange) + "'.") # Scale threshold by RMS if requested if clipMethod == "relative": threshold *= GetRMS(cube, rmsMode=rmsMode, fluxRange=fluxRange, zoomx=1, zoomy=1, zoomz=1, verbose=verbose) # Print some information and check sign of threshold err.message(" Using threshold of " + str(threshold) + ".") err.ensure(threshold >= 0.0, "Threshold finder failed. Threshold value is negative.") # Run the threshold finder, setting bit 1 of the mask for |cube| >= |threshold|: np.bitwise_or(mask, np.greater_equal(np.absolute(cube), threshold), out=mask) return
def _combine_masks(self): """Combine multiple mask planes. Sets the detected mask bit if any image has a detection, and sets other bits only if set in all images. Returns ------- np.ndarray The combined mask plane. """ mask_arr = (exp.getMaskedImage().getMask() for exp in self.exposures) detected_mask = None mask_use = None for mask in mask_arr: mask_vals = mask.getArray() if mask_use is None: mask_use = mask_vals else: mask_use = np.bitwise_and(mask_use, mask_vals) detected_bit = mask.getPlaneBitMask('DETECTED') if detected_mask is None: detected_mask = mask_vals & detected_bit else: detected_mask = np.bitwise_or(detected_mask, (mask_vals & detected_bit)) mask_vals_return = np.bitwise_or(mask_use, detected_mask) return mask_vals_return
def read_ports(self): key = psychopy.event.getKeys(keyList=['1','2','3','k']) ports = np.asarray([False,False,False]) if key: if not key[0] in self._key_pressed: self._key_pressed.append(key[0]) if 'k' in self._key_pressed and '1' in self._key_pressed: ports = np.bitwise_or(ports,[True,False,False]) psychopy.event.clearEvents() print(self._key_pressed) self._key_pressed.remove('k') self._key_pressed.remove('1') if 'k' in self._key_pressed and '2' in self._key_pressed: ports = np.bitwise_or(ports,[False,True,False]) psychopy.event.clearEvents() print(self._key_pressed) self._key_pressed.remove('k') self._key_pressed.remove('2') if 'k' in self._key_pressed and '3' in self._key_pressed: ports = np.bitwise_or(ports,[False,False,True]) psychopy.event.clearEvents() print(self._key_pressed) self._key_pressed.remove('k') self._key_pressed.remove('3') return self.get_ports()[ports]
def reset_bad_gain(pdq, gain): """ For pixels in the gain array that are either non-positive or NaN, reset the the corresponding pixels in the pixel DQ array to NO_GAIN_VALUE and DO_NOT_USE so that they will be ignored. Parameters ---------- pdq : int, 2D array pixel dq array of input model gain : float32, 2D array gain array from reference file Returns ------- pdq : int, 2D array pixleldq array of input model, reset to NO_GAIN_VALUE and DO_NOT_USE for pixels in the gain array that are either non-positive or NaN. """ wh_g = np.where( gain <= 0.) if len(wh_g[0]) > 0: pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['NO_GAIN_VALUE'] ) pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['DO_NOT_USE'] ) wh_g = np.where( np.isnan( gain )) if len(wh_g[0]) > 0: pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['NO_GAIN_VALUE'] ) pdq[wh_g] = np.bitwise_or( pdq[wh_g], dqflags.pixel['DO_NOT_USE'] ) return pdq
def get_keyv(iarr, level): i1, i2, i3 = (v.astype("int64") for v in iarr) i1 = spread_bitsv(i1, level) i2 = spread_bitsv(i2, level) << 1 i3 = spread_bitsv(i3, level) << 2 np.bitwise_or(i1, i2, i1) np.bitwise_or(i1, i3, i1) return i1
def mas_combining4(row): rows = ['DER_mass_MMC','DER_mass_vis','DER_mass_transverse_met_lep'] frame = row[rows[0]] + row[rows[1]] + row[rows[2]] mask = np.bitwise_or(row[rows[0]]==-999, row[rows[1]]==-999.) mask = np.bitwise_or(mask, row[rows[1]]==-999.) frame[mask] = -999. frame.name = 'MNew_mc4' return frame
def partBy2(x): x = np.bitwise_and(x, 0x1f00000000ffff) x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 32)), 0x1f00000000ffff) x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 16)), 0x1f0000ff0000ff) x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 8)), 0x100f00f00f00f00f) x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 4)), 0x10c30c30c30c30c3) x = np.bitwise_and(np.bitwise_or(x, np.left_shift(x, 2)), 0x1249249249249249) return x
def __call__(self): if hasattr(self.ctx.params, 'cleaner') and self.ctx.params.cleaner != "None": #load module mod = importlib.import_module(self.ctx.params.cleaner) rfi_mask_vx, rfi_mask_vy = mod.rm_rfi(self.ctx) self.ctx.tod_vx.mask = np.bitwise_or(self.ctx.tod_vx.mask, rfi_mask_vx) self.ctx.tod_vy.mask = np.bitwise_or(self.ctx.tod_vy.mask, rfi_mask_vy)
def colRowIsOnSciencePixelList(self, col, row, padding=DEFAULT_PADDING): """similar to colRowIsOnSciencePixelList() but takes lists as input""" out = np.ones(len(col), dtype=bool) col_arr = np.array(col) row_arr = np.array(row) mask = np.bitwise_or(col_arr < 12. - padding, col_arr > 1111 + padding) out[mask] = False mask = np.bitwise_or(row_arr < 20. - padding, row_arr > 1043 + padding) out[mask] = False return out
def _numpy(self, data, weights, shape): q = self.quantity(data) self._checkNPQuantity(q, shape) self._checkNPWeights(weights, shape) weights = self._makeNPWeights(weights, shape) newentries = weights.sum() import numpy selection = numpy.isnan(q) numpy.bitwise_not(selection, selection) subweights = weights.copy() subweights[selection] = 0.0 self.nanflow._numpy(data, subweights, shape) # avoid nan warning in calculations by flinging the nans elsewhere numpy.bitwise_not(selection, selection) q = numpy.array(q, dtype=numpy.float64) q[selection] = 0.0 weights = weights.copy() weights[selection] = 0.0 if all(isinstance(v, Count) and v.transform is identity for c, v in self.bins) and numpy.all(numpy.isfinite(q)) and numpy.all(numpy.isfinite(weights)): h, _ = numpy.histogram(q, [float("-inf")] + [(c1 + c2)/2.0 for (c1, v1), (c2, v2) in zip(self.bins[:-1], self.bins[1:])] + [float("inf")], weights=weights) for hi, (c, v) in zip(h, self.bins): v.fill(None, float(hi)) else: selection = numpy.empty(q.shape, dtype=numpy.bool) selection2 = numpy.empty(q.shape, dtype=numpy.bool) for index in xrange(len(self.bins)): if index == 0: high = (self.bins[index][0] + self.bins[index + 1][0])/2.0 numpy.greater_equal(q, high, selection) elif index == len(self.bins) - 1: low = (self.bins[index - 1][0] + self.bins[index][0])/2.0 numpy.less(q, low, selection) else: low = (self.bins[index - 1][0] + self.bins[index][0])/2.0 high = (self.bins[index][0] + self.bins[index + 1][0])/2.0 numpy.less(q, low, selection) numpy.greater_equal(q, high, selection2) numpy.bitwise_or(selection, selection2, selection) subweights[:] = weights subweights[selection] = 0.0 self.bins[index][1]._numpy(data, subweights, shape) # no possibility of exception from here on out (for rollback) self.entries += float(newentries)
def color(img_on, img_off): img_off = cv2.imread(img_off) img_on = cv2.imread(img_on) img = np.array((np.array(img_on, dtype=np.int16)-np.array(img_off, dtype=np.int16)).clip(0,255), dtype=np.uint8) img = cv2.medianBlur(img,3) cv2.imshow("soustracted",cv2.resize(img, (img.shape[1]/2,img.shape[0]/2))) cv2.waitKey(0) lower = np.array([0, 0, 20], dtype=np.uint8) upper = np.array([5, 5, 255], dtype=np.uint8) mask0 = cv2.inRange(img, lower, upper) cv2.imshow("color_threshold_red",cv2.resize(mask0, (img.shape[1]/2,img.shape[0]/2))) cv2.waitKey(0) img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) lower = np.array([0, 170, 20], dtype=np.uint8) upper = np.array([10, 255, 255], dtype=np.uint8) mask1 = cv2.inRange(img_hsv, lower, upper) lower = np.array([170, 170, 20], dtype=np.uint8) upper = np.array([180, 255, 255], dtype=np.uint8) mask2 = cv2.inRange(img_hsv, lower, upper) mask = np.bitwise_or(mask1, mask2) cv2.imshow("hsv_threshold_low_brightness",cv2.resize(mask, (mask.shape[1]/2,mask.shape[0]/2))) cv2.waitKey(0) lower = np.array([80, 0, 200], dtype=np.uint8) upper = np.array([100, 255, 255], dtype=np.uint8) mask3 = cv2.inRange(img_hsv, lower, upper) cv2.imshow("hsv_threshold_high_brightness",cv2.resize(mask3, (mask3.shape[1]/2,mask3.shape[0]/2))) cv2.waitKey(0) mask = np.bitwise_or(mask, mask3) mask = np.bitwise_or(mask0, mask) mask = cv2.GaussianBlur(mask,(3,3),0) mask = cv2.inRange(mask, np.array([250]), np.array([255])) res = cv2.bitwise_and(img_on, img_on, mask=mask) tmp = np.zeros(res.shape) for line in range(res.shape[0]): moments = cv2.moments(res[line,:,2]) if(moments['m00'] != 0): tmp[line][round(moments['m01']/moments['m00'])] = [0,255,0] cv2.imshow("hsv_or_color",cv2.resize(mask, (mask.shape[1]/2,mask.shape[0]/2))) cv2.waitKey(0) cv2.imshow("result",cv2.resize(tmp, (res.shape[1]/2,res.shape[0]/2))) cv2.waitKey(0)
def gameoflife(W,H,ITER,DIST,random_state): random.setstate(random_state) LIVING_LOW= 2 LIVING_HIGH = 3 ALIVE = 3 full = np.zeros((W+2,H+2), dtype=np.long, dist=DIST) dead = np.zeros((W,H), dtype=np.long, dist=DIST) live = np.zeros((W,H), dtype=np.long, dist=DIST) live2 = np.zeros((W,H), dtype=np.long, dist=DIST) neighbors = np.zeros((W,H), dtype=np.long, dist=DIST) cells = full[1:W+1,1:H+1] ul = full[0:W, 0:H] um = full[0:W, 1:H+1] ur = full[0:W, 2:H+2] ml = full[1:W+1, 0:H] mr = full[1:W+1, 2:H+2] ll = full[2:W+2, 0:H] lm = full[2:W+2, 1:H+1] lr = full[2:W+2, 2:H+2] for i in xrange(W): for j in range(H): if random.random() > .8: cells[i][j] = 1 for i in xrange(ITER): # zero neighbors np.bitwise_and(neighbors,0,neighbors) # count neighbors neighbors += ul neighbors += um neighbors += ur neighbors += ml neighbors += mr neighbors += ll neighbors += lm neighbors += lr # extract live cells neighbors np.multiply(neighbors, cells, live) # find all living cells among the already living np.equal(live, LIVING_LOW, live2) np.equal(live, LIVING_HIGH, live) # merge living cells into 'live' np.bitwise_or(live, live2, live) # extract dead cell neighbors np.equal(cells, 0, dead) dead *= neighbors np.equal(dead,ALIVE,dead) # make sure all threads have read their values np.bitwise_or(live, dead, cells) return full
def simulatestep(ps, pm, teilchenort, teilchenmobil, number): zzv = np.random.random(number) zzv2 = zzv < ps zzv3 = zzv < pm #berechne neuen Zustand für die Teilchen # vorher mobil und es bleibe (zzv3, pm) # oder: war nicht mobil und bleibe nicht (invertiert zu oder) mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2)))) # wenn mobil, addiere 1 zum Ort teilchenortneu = teilchenort + mobilneu return teilchenortneu, mobilneu
def simulatestep(p_in, teilchenort, teilchenmobil, number, vel): psvektor = np.array([ps] * number) zzv = np.random.random(number) #bleibe stationär zzv2 = zzv < psvektor #bleibe mobil zzv3 = zzv < p_in #print "pin", p_in '''print "vektoren" print p_in print zzv print zzv2 print zzv3''' #berechne neuen Zustand für die Teilchen # bin mobil, wenn # vorher mobil und es bleibe (zzv3, bestimmt durch p_in, variable WKeiten) # oder: nicht mobil und nicht bleibe (zzv2, bestimmt durch gammavektor, fest TODO: geht das geschickter?) mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2)))) # wenn mobil, addiere vel zum Ort teilchenortneu = teilchenort + vel*mobilneu #print "mobneu", mobilneu vel = np.clip(vel, velmin, velmax) # berechne neue Geschwindigkeiten; falls mobil addiere was dazu, clip davor und danach zum Schutz velneu = (vel + ((velmax-vel)/veldivisor)) * mobilneu # wenn nicht mobil, schneller, wenn mobil #print "velneu", velneu #vel ist mindestens 1, für den neuen Ort wird mobil ja eh noch mal befragt velneu = np.clip(velneu, 0, velmax) #print max(velneu) #print "p_in", p_in #time.sleep(1) # wo ich nicht mobil bin: gamma0 -> p1: [0, g0, 0,0,0, g0, g0, 0...] p1 = np.invert(mobilneu) * gamma0 #wo ich mobil bin, erhöht sich p TODO p2: [p, 0, p,p,p,0,0,p,...] p2 = mobilneu * (gamma0 + vel/gammadivisor) #print "veldings", 1/vel #print gamma0/gammadivisor #print "p2", p2 p_out = p1 + p2 #print "pout", p_out p_out = np.clip(p_out, gamma0, 0.9999) #print velneu,'\n', p_out return teilchenortneu, mobilneu, velneu, p_out
def update_flt(raw): # Copy Pipeline flt to ir_aux_files directory # Add Gabe's extra bad pixels to FLT DQ # Flag persistence pixels (>0.6*ERR) in FLT DQ if persist.fits exists # Replace FLT data with flattened ramp from make_IMA_FLT # Add header keywords indicating when this script was run # Write fixes into original FLT to preserve WCS transformations flt = raw.replace('raw.fits', 'flt.fits') if not os.path.exists(flt): orig_flt = glob.glob('../targets/*/{}'.format(flt))[0] # print 'BEGINNING FLT UPDATE' # print '________________________________________________________' shutil.copy(orig_flt, '.') else: print 'FLT {} already in directory, skipping'.format(flt) return hdu = fits.open(flt, mode='update') orig_dq = hdu['DQ'].data new_flags = fits.getdata('/astro/pabeta/wfc3/data/badpix_spars200_Nov9.fits') if orig_dq.shape == (1014,1014): new_dq = np.bitwise_or(orig_dq,new_flags) else: hdr1 = hdu[1].header ltv1, ltv2 = abs(hdr1['LTV1']), abs(hdr1['LTV2']) naxis1, naxis2 = hdr1['NAXIS1'], hdr1['NAXIS2'] flags_sub = new_flags[ltv2:ltv2+naxis2,ltv1:ltv1+naxis1] new_dq = np.bitwise_or(orig_dq,flags_sub) today = datetime.today() date = '{}-{}-{}'.format(today.year,today.month,today.day) hdu[0].header['BPIXFLAG'] = date proposid = hdu[0].header['PROPOSID'] pers_path = '/grp/hst/wfc3a/GO_Links/{}/Visit*/Persist/{}'.format(proposid,hdu[0].header['ROOTNAME'].lower()+'_persist.fits') if os.path.exists(pers_path): err = hdu['ERR'].data pers_flags = np.zeros(err.shape, dtype=np.uint16) pers_data = fits.getdata(pers_path) pers_flags[pers_data>0.6*err] = 1024 new_dq = np.bitwise_or(new_dq,pers_flags) hdu[0].header['PERSFLAG'] = date print 'ADDING BAD PIXELS TO DQ ARRAY FOR {}'.format(flt) hdu['DQ'].data = new_dq flb = flt.replace('flt.fits','flb.fits') if os.path.exists(flb): flb_data = fits.getdata(flb,1) hdu['SCI'].data = flb_data print 'REPLACING PIPELINE FLT DATA WITH FLATTENED RAMP FOR {}'.format(flt) hdu[0].header['FLATRAMP'] = date hdu.close() return flt
def simulatestep(ps, pm, teilchenort, teilchenmobil): number = len(teilchenort) # print teilchen, number zzv = np.random.random(number) zzv2 = zzv < ps zzv3 = zzv < pm #berechne neuen Zustand für die Teilchen mobilneu = np.bitwise_or(np.bitwise_and(teilchenmobil, zzv3),(np.invert(np.bitwise_or(teilchenmobil, zzv2)))) # wenn mobil, addiere 1 zum Ort teilchenortneu = teilchenort + mobilneu return teilchenortneu, mobilneu
def apply_flat_field (science, flat ): """ Short Summary ------------- Flat fields the data and error arrays, and updates data quality array based on bad pixels in flat field arrays. Applies portion of flat field corresponding to science image subarray. Parameters ---------- science: JWST data model input science data model flat: JWST data model flat field data model Returns ------- None """ # If the input science data model is a subarray, extract the same # subarray from the flatfield model if ref_matches_sci (flat, science): flat_data = flat.data flat_dq = flat.dq else: log.info("Extracting matching subarray from flat") flat_data = get_subarray(flat.data, science) flat_dq = get_subarray(flat.dq, science) # For pixels whose flat is either NaN or NO_FLAT_FIELD, update their DQ to # indicate that no flat is applied to those pixels flat_dq[np.isnan(flat_data)] = np.bitwise_or(flat_dq[np.isnan(flat_data)], dqflags.pixel['NO_FLAT_FIELD']) # Replace NaN's in flat with 1's flat_data[np.isnan(flat_data)] = 1.0 # Reset flat values of pixels having DQ values containing NO_FLAT_FIELD # to 1.0, so that no flat fielding correction is made wh_dq = np.bitwise_and( flat_dq, dqflags.pixel['NO_FLAT_FIELD']) flat_data[ wh_dq == dqflags.pixel['NO_FLAT_FIELD'] ] = 1.0 # Flatten data and error arrays science.data /= flat_data science.err /= flat_data # Combine the science and flat DQ arrays science.dq = np.bitwise_or(science.dq, flat_dq)
def CreateTimeMatrix(DataFile): """ """ phofile = PhoAlign + os.path.sep + DataFile + '.lab' sylfile = SylAlign + os.path.sep + DataFile + '.lab' worfile = WorAlign + os.path.sep + DataFile + '.lab' phodata = htslab.read_full_lab(phofile) syldata = htslab.read_full_lab(sylfile) wordata = htslab.read_full_lab(worfile) if len(wordata[1]) != len(syldata[1]) or wordata[1][-1] != syldata[1][-1]: print "\t Unequal Length %s" % (DataFile) return 0 DataTime = np.int(syldata[1][-1]) # default, update at every frame level dataMat = np.bitwise_or(np.zeros([DataTime], dtype=np.int8), bitInfo['fra']) preSyl = '' preWor = '' for idx1 in xrange(len(syldata[0])): frameStart = syldata[0][idx1] frameEnd = syldata[1][idx1] # update the phoneme state dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['pho']) syllabel = syldata[2][idx1] worlabel = wordata[2][idx1] if syllabel != preSyl: dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['syl']) if worlabel != preWor: dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['wor']) if len(preWor)==0 or preWor == phraseSym or worlabel == phraseSym: dataMat[frameStart] = np.bitwise_or(dataMat[frameStart], bitInfo['phr']) preSyl = syllabel preWor = worlabel if CheckBinary: pholabel = phodata[2][idx1] for t in range(np.int(frameStart), np.int(frameEnd)): print "%d, %s [%s %s %s]" % (t,np.binary_repr(dataMat[t], len(bitInfo)), pholabel[0:6], syllabel[0:6], worlabel[0:6]) py_rw.write_raw_mat(dataMat, DataDir+os.path.sep+DataFile+'.bin', 'u1') return DataTime
def embedPayload(self,payload,override=False): if type(payload) is not Payload: raise TypeError("Incorrect input type") if len(payload.img.shape)==3: color=3 else: color=1 payload_size=8*payload.img.shape[0]*payload.img.shape[1]*color if len(self.img.shape)==3: color1=3 else: color1=1 carrier_size=self.img.shape[0]*self.img.shape[1] if payload_size>carrier_size*color1: raise ValueError("Carrier can not hold payload") if self.payloadExists()==True and override ==False: raise Exception("Payload exist and can not override") xml_list=numpy.ndarray(shape=(len(payload.xml),1),dtype=numpy.uint8) cnt=0 for item in payload.xml: xml_list[cnt][0]=ord(item) cnt+=1 xml_binary=numpy.unpackbits(xml_list) image=copy.deepcopy(self.img) if color1==3: reshaped_image=image.flatten() ordered_image=numpy.concatenate((reshaped_image[0::3],reshaped_image[1::3],reshaped_image[2::3])) embed_image=ordered_image[:len(xml_binary)] else: ordered_image=image.flatten() embed_image=ordered_image[:len(xml_binary)] clean_matrix=numpy.ndarray(shape=embed_image.shape,dtype=numpy.uint8) clean_matrix.fill(254) numpy.bitwise_and(clean_matrix,embed_image,embed_image) numpy.bitwise_or(embed_image,xml_binary,embed_image) reshaped_image=numpy.concatenate((embed_image,ordered_image[len(xml_binary):])) if color1==3: reshaped_image=reshaped_image.reshape(3,-1) reshaped_image=numpy.dstack(tuple(reshaped_image)) image=reshaped_image.reshape(image.shape) else: image=reshaped_image.reshape(image.shape) return image
def _get_voc_color_map(n=256): color_map = np.zeros((n, 3)) for i in xrange(n): r = b = g = 0 cid = i for j in xrange(0, 8): r = np.bitwise_or(r, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-1], 7-j)) g = np.bitwise_or(g, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-2], 7-j)) b = np.bitwise_or(b, np.left_shift(np.unpackbits(np.array([cid], dtype=np.uint8))[-3], 7-j)) cid = np.right_shift(cid, 3) color_map[i][0] = r color_map[i][1] = g color_map[i][2] = b return color_map
def labelcolormap(N=256): cmap = np.zeros((N, 3)) for i in xrange(0, N): id = i r, g, b = 0, 0, 0 for j in xrange(0, 8): r = np.bitwise_or(r, (bitget(id, 0) << 7-j)) g = np.bitwise_or(g, (bitget(id, 1) << 7-j)) b = np.bitwise_or(b, (bitget(id, 2) << 7-j)) id = (id >> 3) cmap[i, 0] = r cmap[i, 1] = g cmap[i, 2] = b cmap = cmap.astype(np.float32) / 255 return cmap
def get_training_image_segmentations(self, image_indices, tooth_indices): segmentations = [] mirrored_segmentations = [] for image_index in image_indices: combined_segmentation = np.uint8(np.zeros(self._training_segmentations[image_index][0].shape)) combined_segmentation_mirrored = np.uint8(np.zeros(self._training_segmentations[image_index][0].shape)) for tooth_index in tooth_indices: combined_segmentation = np.bitwise_or(combined_segmentation, self._training_segmentations[image_index][tooth_index]) combined_segmentation_mirrored = np.bitwise_or(combined_segmentation_mirrored, self._training_segmentations_mirrored[image_index][ tooth_index]) segmentations.append(combined_segmentation) mirrored_segmentations.append(combined_segmentation_mirrored) return segmentations, mirrored_segmentations
def get_detect(img,body_position): region_upper=int(img.shape[0]*0.3) region_lower=int(img.shape[0]*0.6) if body_position[0]<(img.shape[1]/2.0): region_left=body_position[0]+30 region_right=img.shape[1]-30 else: region_left=30 region_right=body_position[0]-30 region = img[region_upper:region_lower, region_left:region_right] edge_list=[0,0,0,0] for i in range(3): region_gray=cv2.cvtColor(region,cv2.COLOR_BGR2HSV)[:,:,i] edge_list[i]=cv2.Canny(region_gray,150,200) region_gray=cv2.cvtColor(region,cv2.COLOR_BGR2GRAY) region_gray = cv2.GaussianBlur(region_gray, (5, 5), 0) edge_list[3] = cv2.Canny(region_gray, 110, 200,apertureSize=5) edge_list[1]=np.bitwise_or(edge_list[0],edge_list[1]) edge_list[2]=np.bitwise_or(edge_list[2],edge_list[1]) edge_final=np.bitwise_or(edge_list[3],edge_list[2]) contours= cv2.findContours(edge_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[1] y=99999 for contour in contours: sorted_top=sorted(contour,key=lambda contour:contour[0][1]) if sorted_top[0][0][1]<y: raw_x = x = sorted_top[0][0][0] raw_y = y = sorted_top[0][0][1] print((int(x+ region_left), int(y+ region_upper))) mask = np.zeros((region_lower-region_upper+ 2, region_right-region_left + 2), np.uint8) mask=cv2.floodFill(region, mask, (raw_x, raw_y+16), [255,25,255])[2] cv2.circle(img, (int(x + region_left), int(y + region_upper)), 5, (255, 0, 5), -1) M = cv2.moments(mask) x = int(M["m10"] / M["m00"]) y = int(M["m01"] / M["m00"]) if y<raw_y or abs(x-raw_x)>40: x=raw_x;y=raw_y y += region_upper x += region_left y = (-abs(x-body_position[0])/pow(3,0.5)+body_position[1]) else: y += region_upper x += region_left cv2.circle(img, (int(x),int(y)), 5, (0,0,255), -1) cv2.imwrite("/home/dvaliu/temp/test_center.png",img) return [x, y]
''' x = np.linspace(10, 20, 5) print(x) # [10, 12.5, 15, 17.5, 20] y = np.linspace(10, 20, 11) # [10,11,12,13,14,15,16,17,18,19,20] ''' # 位操作 bitwise_and 对数组元素执行位 与 操作 1&1 -> 1 1&0 -> 0 0&0 -> 0 两个都是1才是1 bitwise_or 对数组元素执行位 或 操作 1|1 -> 1 1|0 -> 1 0|0 -> 0 有一个是1就是1 left_shift 向左移动二进制表示的位 0011 向左移1位,用0来补位 1100 1100000 左移1位 1000000 顶部1位丢弃掉 right_shift 向右移动二进制表示的位 1100 向右移1位,用0来补位 0110 0000011 右移1位 0000001 尾部1位丢弃掉了 ''' a = 2 # 10 b = 3 # 11 print(np.bitwise_and(a, b)) # 10 -> 1*2的1次方 + 0*2的0次方 -> 2 print(np.bitwise_or(a, b)) # 11 -> 1*2的1次方 + 1*2的0次方 -> 3 print(np.left_shift(a, 1)) # 10 左移动1位 0100 -> 4 print(np.left_shift(a, 2)) # 10 左移动2位 1000 -> 8 print(np.right_shift(a, 1)) # 10 右移动1位 01 -> 1 ''' # 字符串函数 add() 两个str或Unicode数组的逐个字符串连接 replace() 替换 split() 分割 title() 返回输入字符串的按元素标题替换版本,其中每个单词的首字母都大些 multiply() 返回按元素多重连接后的字符串 ''' print(np.char.multiply('hello ', 3)) # hello hello hello print(np.char.title('hello xfz')) # Hello Xfz ''' # 算数函数
def compute_filter_responses(self): """Compute the motion-energy filters' response to the stimuli. Parameters ---------- stimulus : 3D np.array (n, vdim, hdim) The movie frames. stimulus_fps : scalar The temporal frequency of the stimulus gabor_temporal_window : scalar, None The number of frames in one filter. If None, it defaults to floor(2/3) of `stimulus_fps` Similar to Nishimoto, 2011. quadrature_combination : function, optional Specifies how to combine the channel reponses quadratures. The function must take the sin and cos as arguments in order. Defaults to: (sin^2 + cos^2)^1/2 output_nonlinearity : function, optional Passes the channels (after `quadrature_combination`) through a non-linearity. The function input is the (`n`,`nfilters`) array. Defaults to: ln(x + 1) dozscore : bool, optional Whether to z-score the channel responses in time moten_pyramid_parameters: dict See :func:`mk_moten_pyramid_params` for details on parameters specifiying a motion-energy pyramid. Returns ------- filter_responses : np.array, (n, nfilters) """ # stimulus = stimulus.reshape(stimulus.shape[0], -1) # if gabor_temporal_window is None: # gabor_temporal_window = int(stimulus_fps*(2./3.)) # gabor_parameters = mk_moten_pyramid_params(stimulus_fps, # gabor_temporal_window, # aspect_ratio=aspect_ratio, # **moten_pyramid_parameters) channels = [] for idx, gabor_param in enumerate(self.param_list): gabor = self.make_3d_gabor((self.horizontal_dim, self.vertical_dim, self.temporal_window), *gabor_param[:-3], aspect_ratio=self.aspect_ratio) gabor0, gabor90, tgabor0, tgabor90 = gabor phase = gabor_param[7] channel_sin, channel_cos = self.dotdelay_frames(gabor0, gabor90, tgabor0, tgabor90) channel_sin, channel_cos = np.float32(channel_sin), np.float32(channel_cos) if phase == 0: channels.append(channel_sin) elif phase == 1: channels.append(channel_cos) elif phase == 2: channels.append(channel_sin**2) elif phase == 3: channels.append(channel_cos**2) channels = np.asarray(channels).T if self.zscore_each: # from scipy.stats import zscore if self._means is None: self._means = np.mean(channels, axis=0) if self._stds is None: self._stds = np.std(channels, axis=0) channels -= self._means channels /= self._stds elif self.zscore_by_type: if self._means is None or self._stds is None: usf = np.unique(self.param_list[:,9]) utf = np.unique(self.param_list[:,5]) ugg = np.unique(self.param_list[:,8]) self._means = np.zeros(len(self.param_list)) self._stds = np.zeros(len(self.param_list)) for sf in usf: for tf in utf: for gg in ugg: idx_l = np.bitwise_and(np.bitwise_and(np.bitwise_and(self.param_list[:,9] == sf, self.param_list[:,5]==tf), np.bitwise_or(self.param_list[:,7]==0, self.param_list[:,7]==1)), self.param_list[:,8]==gg) idx_q = np.bitwise_and(np.bitwise_and(np.bitwise_and(self.param_list[:,9] == sf, self.param_list[:,5]==tf), np.bitwise_or(self.param_list[:,7]==2, self.param_list[:,7]==3)), self.param_list[:,8]==gg) self._means[idx_l] = np.mean(channels[:,idx_l]) self._means[idx_q] = np.mean(channels[:,idx_q]) self._stds[idx_l] = np.sqrt(np.mean((channels[:,idx_l] - self._means[idx_l])**2)) self._stds[idx_q] = np.sqrt(np.mean((channels[:,idx_q] - self._means[idx_q])**2)) channels -= self._means channels /= self._stds return channels
def DelayFunc(self): # data analysed the old way to get delay range plot lastTag = np.max(self.ttagBuf.rawtags) firstTag = (lastTag - np.int( np.ceil(self.exptime / self.ttagBuf.resolution))).astype(np.uint64) rawPos = np.nonzero( np.bitwise_and(self.ttagBuf.rawtags > firstTag, self.ttagBuf.rawtags < lastTag))[0] rawTags = self.ttagBuf.rawtags[rawPos] rawChan = self.ttagBuf.rawchannels[rawPos] rawAll = np.vstack((rawTags, rawChan)) newAll = np.sort(rawAll, axis=0) newTags = rawAll[0] newChan = rawAll[1] # filter data to plot selDelay = self.cmbChannels.currentIndex() if selDelay > 8: selTags = newTags selChan = newChan else: if self.curTab == 0: ch1 = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2]) ch2 = np.array([3, 4, 5, 3, 4, 5, 3, 4, 5]) elif self.curTab == 1: ch1 = np.array([0, 0, 0, 1, 1, 1, -1, -1, -1]) ch2 = np.array([2, 3, -1, 2, 3, -1, 2, 3, -1]) selPos = np.nonzero( np.bitwise_or(newChan == ch1[selDelay], newChan == ch2[selDelay]))[0] selTags = newTags[selPos] selChan = newChan[selPos] # add delays to tags selTags = selTags + np.around( self.delay[selChan] / self.ttagBuf.resolution).astype(np.int64) # compute delay histogram delayEdges = np.arange( -np.around(self.delayRange / self.ttagBuf.resolution).astype( np.int32), np.around(self.delayRange / self.ttagBuf.resolution).astype( np.int32), 2) selTags = selTags.astype(np.int64) selChan = selChan.astype(np.int8) if selDelay > 8: if self.curTab == 0: chMap = np.array([0, 0, 0, 1, 1, 1]) elif self.curTab == 1: chMap = np.array([0, 0, 1, 1]) selChan = chMap[selChan] t12diff = np.diff(selTags) chdiff = np.diff(selChan) t12diff = t12diff[chdiff != 0] * np.sign(chdiff[chdiff != 0]) count, bins = np.histogram(t12diff, bins=delayEdges, range=(np.amin(delayEdges), np.amax(delayEdges))) binCenter = (bins[1:] + bins[:-1]) / 2 delays = binCenter * self.ttagBuf.resolution * 1e9 bg = pg.BarGraphItem(x=delays, height=count, width=0.1, brush='b') self.pltDelay.clear() # fit results if the fit box is checked if self.chkFit.isChecked(): gaussian = lambda x, A, x0, s: A * np.exp(-(x - x0)**2 / (2 * s**2)) p0 = [np.max(count), delays[np.argmax(count)], 0.3] popt, perr = curve_fit(gaussian, delays, count, p0=p0) x = np.linspace(np.amin(delays), np.amax(delays), 1000) yfit = gaussian(x, *popt) self.pltDelay.plot(x, yfit, pen='r') self.lblMean.setText("{:.4f}".format(popt[1])) self.lblStd.setText("{:.4f}".format(popt[2])) self.pltDelay.addItem(bg)
import numpy as np print 'Binary equivalents of 13 and 17:' a,b = 13,17 print bin(a), bin(b) print np.bitwise_and(13, 17) print np.bitwise_or(13, 17) print np.left_shift(10,2) #String functions print np.char.add(['hello'],[' xyz']) print np.char.add(['hello', 'hi'],[' abc', ' xyz']) print np.char.multiply('Hello ',3) print np.char.center('hello', 20,fillchar = '*') print np.char.capitalize('hello world') print np.char.title('hello how are you?') print np.char.lower('HELLO') print np.char.replace ('He is a good boy', 'is', 'was') a = np.char.encode('hello', 'cp500') #trignometroc functions a = np.array([0,30,45,60,90]) c=np.sin(a*np.pi/180) print np.sin(a*np.pi/180) inv = np.arcsin(a) print inv print np.degrees(inv) a = np.array([-1.7, 1.5, -0.2, 0.6, 10]) print np.floor(a)
def threshold_grad(grad, activeset, threshold=.5): absgrad = np.max(np.abs(grad), axis=0) activeset = np.bitwise_or(activeset, absgrad >= threshold*np.max(absgrad, axis=0)) grad[:, ~activeset] = 0 return (grad, activeset)
def create_combined(self, im_1_a, im_2_a): """ Short Summary ------------- Create combined image from aligned input images. In the combined image: The SCI pixel values are set by: 1. use pixels that are good (based on DQ) in image #1, else 2. for pixels that are bad in image #1, use good image #2 pixels, else 3. for pixels that are bad in both images, leave as default (0) The DQ pixel values are similarly set, by: 1. use pixels that are good in image #1, else 2. for pixels that are bad in image #1, use good image #2 pixels, else 3. for pixels that are bad in both images, add a 'DO_NOT_USE' value to the corresponding DQ value in image #1. The ERR pixel values are set by: 1. use pixels that are good in image #1, else 2. for pixels that are bad in image #1, use good image #2 pixels, else 3. for pixels that are bad in both images, leave as default (0) The WCS of the output model is set to the WCS of the 1st input Parameters ---------- im_1_a: 2D Data Model aligned image from input #1 im_2_a: 2D Data Model aligned image from input #2 Returns ------- data_comb: 2d float array combined SCI array dq_comb: 2d integer array combined DQ array err_comb: 2d float array combined ERR array wcs_1: pywcs.WCS object wcs information for the 1st input """ sci_data_1 = im_1_a.data.copy() sci_data_2 = im_2_a.data.copy() dq_data_1 = im_1_a.dq.copy() dq_data_2 = im_2_a.dq.copy() err_data_1 = im_1_a.err.copy() err_data_2 = im_2_a.err.copy() wh_1_good = (dq_data_1 == 0) # Where ALIGNED #1 pixels are good wh_1_bad_2_good = (dq_data_1 != 0) & (dq_data_2 == 0) wh_1_bad_2_bad = (dq_data_1 != 0) & (dq_data_2 != 0) # Populate combined SCI, DQ and ERR arrays data_comb = sci_data_1 * 0 # Pixels that are bad in both will stay 0 data_comb[wh_1_good] = sci_data_1[wh_1_good] data_comb[wh_1_bad_2_good] = sci_data_2[wh_1_bad_2_good] dq_comb = dq_data_1.copy() dq_comb[wh_1_bad_2_good] = dq_data_2[wh_1_bad_2_good] dq_comb[wh_1_bad_2_bad] = np.bitwise_or(dqflags.group['DO_NOT_USE'], dq_comb[wh_1_bad_2_bad]) err_comb = err_data_1.copy() err_comb[wh_1_bad_2_good] = err_data_2[wh_1_bad_2_good] err_comb[wh_1_bad_2_bad] = 0. # Use wcs of input 1 for combination wcs_1 = self.input_1.get_fits_wcs('SCI') return data_comb, dq_comb, err_comb, wcs_1
def find_bad_pix( filenames, uncal_filenames=None, jump_filenames=None, fitopt_filenames=None, clipping_sigma=5., max_clipping_iters=5, noisy_threshold=5, max_saturated_fraction=0.5, max_jump_limit=10, jump_ratio_threshold=5, early_cutoff_fraction=0.25, pedestal_sigma_threshold=5, rc_fraction_threshold=0.8, low_pedestal_fraction=0.8, high_cr_fraction=0.8, flag_values={ 'hot': ['HOT'], 'rc': ['RC'], 'low_pedestal': ['OTHER_BAD_PIXEL'], 'high_cr': ["TELEGRAPH"] }, do_not_use=['hot', 'rc', 'low_pedestal', 'high_cr'], outfile=None, plot=False): """MAIN FUNCTION Parameters ---------- filenames : list List of dark current slope files. These should be slope images. uncal_filenames : list List of uncal files. Should have a 1-to-1 correspondence to the files in ``filenames``. If None, the scipt will look in the same directory containing ``filenames``, and assume that the only difference in filename is that rate.fits is replaced with uncal.fits. Uncal files are only used when working with MIRI data. jump_filenames : list List of exposures output from the jump step of the pipeline. Should have a 1-to-1 correspondence to the files in ``filenames``. If None, the scipt will look in the same directory containing ``filenames``, and assume that the only difference in filename is that rate.fits is replaced with jump.fits fitopt_filenames : list List of exposures from the optional output from the ramp_fitting step of the pipeline. Should have a 1-to-1 correspondence to the files in ``filenames``. If None, the scipt will look in the same directory containing ``filenames``, and assume that the only difference in filename is that rate.fits is replaced with fitopt.fits clipping_sigma : int Number of sigma to use when sigma-clipping the 2D array of standard deviation values from the dark current slope files. The sigma-clipped mean and standard deviation are used to locate noisy pixels. max_clipping_iters : int Maximum number of iterations to use when sigma clipping to find the mean and standard deviation values that are used when locating noisy pixels. noisy_threshold : int Number of sigma above the mean noise (associated with the slope) to use as a threshold for identifying noisy pixels. max_saturated_fraction : float When identifying pixels that are fully saturated (in all groups of an integration), this is the fraction of integrations within which a pixel must be fully saturated before flagging it as HOT max_jump_limit : int The maximum number of jumps a pixel can have in an integration before it is flagged as a ``high jump`` pixel (which may be flagged as noisy later) jump_ratio_threshold : int Cutoff for the ratio of jumps early in the ramp to jumps later in the ramp. Pixels with a ratio greater than this value (and which also have a high total number of jumps) will be flagged as potential (I)RC pixels. early_cutoff_fraction : float Fraction of the integration to use when comparing the jump rate early in the integration to that across the entire integration. Must be <= 0.5 pedestal_sigma_threshold : int Used when searching for RC pixels via the pedestal image. Pixels with pedestal values more than ``pedestal_sigma_threshold`` above the mean are flagged as potential RC pixels rc_fraction_threshold : float Used when searching for RC pixels. This is the fraction of input files within which the pixel must be identified as an RC pixel before it will be flagged as a permanent RC pixel low_pedestal_fraction : float This is the fraction of input files within which a pixel must be identified as a low pedestal pixel before it will be flagged as a permanent low pedestal pixel high_cr_fraction : float This is the fraction of input files within which a pixel must be flagged as having a high number of jumps before it will be flagged as permanently noisy flag_values : dict This dictionary maps the types of bad pixels searched for to the flag mnemonics to use when creating the bad pixel file. Keys are the types of bad pixels searched for, and values are lists that include mnemonics recognized by the jwst calibration pipeline e.g. {'hot': ['HOT'], 'rc': ['RC'], 'low_pedestal': ['OTHER_BAD_PIXEL'], 'high_cr': ["TELEGRAPH"]} do_not_use : list List of bad pixel types to be flagged as DO_NOT_USE e.g. ['hot', 'rc', 'low_pedestal', 'high_cr'] plot : bool If True, produce plots of intermediate results. outfile : str Name of fits file to save the resulting bad pixel mask to """ # Currently the code stipulates that 5 good values of the slope are # needed in each pixel in order to determine a good stdev value. So # let's check the number of input files here and quit if there are # fewer than 5. if len(filenames) < 5: print(filenames) raise ValueError( "ERROR: >5 input files are required to find bad pixels from darks." ) # Add DO_NOT_USE to all requested types of bad pixels do_not_use = [element.lower() for element in do_not_use] for key in flag_values: if key.lower() in do_not_use: flag_values[key].append('DO_NOT_USE') # Form the outfile and outdir if outfile is None: outfile = 'badpixels_from_darks.fits' outdir = os.path.dirname(outfile) if not outdir: outdir = '.' # Read in the slope data. Strip off reference pixels. # Return a 3D array of slopes and a 3D array mapping where the # science pixels are. print('Reading slope files...') # instrument,slopes, refpix_additions = read_slope_files(filenames) instrument, slopes, indexes, refpix_additions = read_slope_integrations( filenames) shape_slope = slopes.shape # Calculate the mean and standard deviation through the stack for # each pixel. Assuming that we are looking for noisy pixels, we don't # want to do any sigma clipping on the inputs here, right? mean_slope = np.mean(slopes, axis=0) std_slope = np.std(slopes, axis=0) hdout = fits.PrimaryHDU(mean_slope) hdout.writeto('average_of_slopes.fits', overwrite=True) hdout = fits.PrimaryHDU(std_slope) hdout.writeto('sigma_of_slopes.fits', overwrite=True) # Use sigma-cliping when calculating the mean and standard deviation # of the standard deviations clipped_stdevs, cliplow, cliphigh = sigma_clip(std_slope, sigma=clipping_sigma, maxiters=max_clipping_iters, masked=False, return_bounds=True) avg_of_std = np.mean(clipped_stdevs) std_of_std = np.std(clipped_stdevs) cut_limit = avg_of_std + std_of_std * noisy_threshold # Identify noisy pixels as those with noise values more than # noisy_threshold*sigma above the average noise level # noisy = std_slope > cut_limit # not a good stat we need to remove slopes with cr hits # Plot histogram to later compare with better std_slope only containing # slopes with no jumps detected. if plot: xhigh = avg_of_std + std_of_std * noisy_threshold plot_image(std_slope, xhigh, outdir, "Pixel Standard devations", "pixel_std_withjumps.png") nbins = 5000 titleplot = 'Histogram of Pixel Slope STD with cosmic ray jumps: Clipped Ave ' + \ '{:6.4f}'.format(avg_of_std) + ' Std ' + '{:6.4f}'.format(std_of_std) plot_histogram_stats(std_slope, cut_limit, nbins, outdir, titleplot, "histo_std_withjumps.png", xaxis_log=True) # Read in the optional outputs from the ramp-fitting step, so that # we can look at the y-intercepts and the jump flags saturated = np.zeros(slopes.shape) rc_from_pedestal = np.zeros(slopes.shape) low_pedestal = np.zeros(slopes.shape) high_cr_rate = np.zeros(slopes.shape) rc_from_flags = np.zeros(slopes.shape) slope_stack = [] islope_stack = [] total_ints = 0 counter = 0 for i, filename in enumerate(filenames): # Read in the ramp and get the data and dq arrays jump_file = None if jump_filenames is not None: jump_file = jump_filenames[i] else: for suffix in RATE_FILE_SUFFIXES: if suffix in filename: slope_suffix = '{}.fits'.format(suffix) jump_file = filename.replace(slope_suffix, '_jump.fits') break if jump_file is None: raise ValueError("ERROR: Unrecognized slope filename suffix.") if not os.path.isfile(jump_file): raise FileNotFoundError( "ERROR: Jump file {} not found.".format(jump_file)) print('Opening Jump File {}'.format(jump_file)) groupdq = dq_flags.get_groupdq(jump_file, refpix_additions) cr_map = dq_flags.flag_map(groupdq, 'JUMP_DET') # Get slope data corresponding to this file by extracting the # appropriate frames from the ``slopes`` stack slope = slopes[indexes[i]:indexes[i + 1], :, :] # Read in the fitops file associated with the exposure and get # the pedestal array (y-intercept) if fitopt_filenames is not None: pedestal_file = fitopt_filenames[i] else: pedestal_file = filename.replace(slope_suffix, '_fitopt.fits') if not os.path.isfile(pedestal_file): raise FileNotFoundError( "ERROR: Pedestal file {} not found.".format(pedestal_file)) print('Opening Pedestal File {}'.format(pedestal_file)) pedestal = read_pedestal_data(pedestal_file, refpix_additions) # for MIRI the zero point of the ramp drifts with time. Adjust the # pedestal to be a relative pedestal wrt to group 2 if instrument == 'MIRI': if uncal_filenames is not None: uncal_file = uncal_filenames[i] else: uncal_file = filename.replace(slope_suffix, '_uncal.fits') if not os.path.isfile(uncal_file): raise FileNotFoundError( "ERROR: Uncal file {} not found.".format(uncal_file)) group2 = extract_group2(uncal_file, refpix_additions) pedestal_org = copy.deepcopy(pedestal) pedestal = np.fabs(group2 - pedestal) # Work one integration at a time for int_num in range(pedestal.shape[0]): # pull out the DQ of the first group. This will be use to remove # Low pedestal values that have a pedestal of 0 because they are # saturated on group 1. first_group = groupdq[int_num, 0, :, :] pedestal_int = pedestal[int_num, :, :] slope_int = slope[int_num, :, :] clipped_pedestal, cliplow, cliphigh = sigmaclip(pedestal_int, low=3., high=3.) mean_pedestal = np.mean(clipped_pedestal) std_pedestal = np.std(clipped_pedestal) rc_from_pedestal[counter, :, :] += pedestal_int > ( mean_pedestal + std_pedestal * pedestal_sigma_threshold) # Pixels with abnormally low pedestal values pedestal_low = pedestal_int < ( mean_pedestal - std_pedestal * pedestal_sigma_threshold) first_group_sat = np.bitwise_and(first_group, dqflags.pixel['SATURATED']) # do not allow pixels saturated on group 1 to be marked as low pedestal pedestal_results = np.logical_and(pedestal_low, (first_group_sat == 0)) low_pedestal[counter, :, :] += pedestal_results # Find pixels that are saturated in all groups. These will have # a pedestal value of 0 (according to the pipeline documentation). # These should end up flagged as HOT and DO_NOT_USE # Remove all the cases where ped = 0, but group 1 is not saturated # This can be dead pixels if instrument == 'MIRI': pedestal_int = pedestal_org[int_num, :, :] saturated[counter, :, :] += saturated_in_all_groups( pedestal_int, first_group_sat) # Find pixels that have an abnormally high number of jumps, as # well as those that have most of their jumps concentrated in the # early part of the integration. The latter are possibly RC or IRC # pixels many_jumps, rc_candidates, number_of_jumps =\ find_pix_with_many_jumps(cr_map[int_num, :, :, :], max_jump_limit=10, jump_ratio_threshold=5, early_cutoff_fraction=0.25) high_cr_rate[counter, :, :] += many_jumps rc_from_flags[counter, :, :] += rc_candidates # using the number_of_jumps (a per integration value) create a clean set of # pixel slopes with no cosmic rays clean_slopes, iclean_slopes = slopes_not_cr( slope_int, number_of_jumps) slope_stack.append(clean_slopes) islope_stack.append(iclean_slopes) total_ints += 1 counter += 1 # now find the mean and standard deviation of the "clean" pixel slopes clean_mean_slope, clean_std_slope, num_good = combine_clean_slopes( slope_stack, islope_stack) hdout = fits.PrimaryHDU(clean_mean_slope) hdout.writeto('average_of_slopes_nojumps.fits', overwrite=True) hdout = fits.PrimaryHDU(clean_std_slope) hdout.writeto('sigma_of_slopes_nojumps.fits', overwrite=True) num_good_slopes = num_good.astype(np.int16) hdout = fits.PrimaryHDU(num_good_slopes) hdout.writeto('number_of_slopes_nojumps.fits', overwrite=True) # Use sigma-cliping to remove large outliers to have clean stats to flag # noisy pixels. # removing nans from clean_std_slope because it causes warning messages to be print clean_std_slope_nonan = clean_std_slope[np.isfinite(clean_std_slope)] clipped_stdevs, cliplow, cliphigh = sigma_clip(clean_std_slope_nonan, sigma=clipping_sigma, maxiters=max_clipping_iters, masked=False, return_bounds=True) avg_of_std = np.mean(clipped_stdevs) std_of_std = np.std(clipped_stdevs) cut_limit = avg_of_std + std_of_std * noisy_threshold # assigning nans from clean_std_slope to very large values that will be cut # because it causes warning messages to be print values_nan = np.isnan(clean_std_slope) clean_std_slope[values_nan] = avg_of_std + std_of_std * 50 noisy = clean_std_slope > cut_limit num_noisy = len(np.where(noisy)[0]) if plot: # plot the number of good slopes per pixel max_values = np.amax(num_good) plot_image(num_good, max_values, outdir, "Number of Good slopes/pixel ", "clean_pixel_number.png") # plot the standard deviation of pixels slope after eliminating # values having jumps detectect in ramp xhigh = avg_of_std + std_of_std plot_image(clean_std_slope, xhigh, outdir, "Clean Pixel Standard devations", "clean_pixel_std.png") # plot the histogram before the clipping nbins = 5000 titleplot = 'Histogram of Clean Pixel Slope STD Average ' + \ '{:6.4f}'.format(avg_of_std) + ' Std ' + '{:6.4f}'.format(std_of_std) plot_histogram_stats(clean_std_slope, cut_limit, nbins, outdir, titleplot, "histo_clean_std.png", xaxis_log=True) # Look through the stack of saturated pixels and keep those saturated # more than N% of the time fully_saturated = np.sum(saturated, axis=0) / total_ints fully_saturated[fully_saturated < max_saturated_fraction] = 0 fully_saturated = np.ceil(fully_saturated).astype(int) fully_saturated = apply_flags(fully_saturated, flag_values['hot']) num_saturated = len(np.where(fully_saturated != 0)[0]) print('\n\nFound {} fully saturated pixels.'.format(num_saturated)) # How do we want to combine these to identify RC pixels? rc_pedestal = np.sum(rc_from_pedestal, axis=0) / total_ints rc_flags = np.sum(rc_from_flags, axis=0) / total_ints rc_from_pedestal_only = (rc_pedestal > rc_fraction_threshold).astype(int) rc_from_jumps_only = (rc_flags > rc_fraction_threshold).astype(int) num_rc_ped = len(np.where(rc_from_pedestal_only != 0)[0]) num_rc_jump = len(np.where(rc_from_jumps_only != 0)[0]) print("Found {} RC pixels from pedestal search".format(num_rc_ped)) print("Found {} RC pixels from Jump search".format(num_rc_jump)) rc = ((rc_pedestal > rc_fraction_threshold) | (rc_flags > rc_fraction_threshold)) rc = apply_flags(rc.astype(int), flag_values['rc']) num_rc = len(np.where(rc != 0)[0]) print('Found {} RC pixels.'.format(num_rc)) # Low pedestal pixels low_pedestal_vals = np.sum(low_pedestal, axis=0) / total_ints low_ped = low_pedestal_vals > low_pedestal_fraction # Pixels that are saturated on the first group will have a PEDESTAL value # of 0. Pull these out of this set (these are hot pixels) low_ped = apply_flags(low_ped.astype(int), flag_values['low_pedestal']) num_low_ped = len(np.where(low_ped != 0)[0]) print('Found {} low pedestal pixels.'.format(num_low_ped)) # Pixels with lots of CR flags should be added to the list of noisy pixels? high_cr = np.sum(high_cr_rate, axis=0) / total_ints noisy_second_pass = high_cr > high_cr_fraction combined_noisy = np.bitwise_or(noisy, noisy_second_pass) combined_noisy = apply_flags(combined_noisy.astype(int), flag_values['high_cr']) num_high_cr = len(np.where(noisy_second_pass != 0)[0]) print('Found {} pixels with a high number of jumps.'.format(num_high_cr)) print('Found {} pixels with noise above the threshold.'.format(num_noisy)) num_combined_noisy = len(np.where(combined_noisy != 0)[0]) print( 'Combining noisy and high jump pixels, found {} noisy pixels.'.format( num_combined_noisy)) # Combine the various flavors of bad pixels into a final DQ map bad_pixels = combine_bad_pixel_types(fully_saturated, rc, low_ped, combined_noisy) # Add the reference pixels back into the bad pixel map bad_pixels = add_refpix(bad_pixels, refpix_additions) # Create DQ definitions to be saved with the output file dq_def = create_dqdef() # Save the bad pixel mask to a fits file # Eventually this routine will be called as part of the dark current reference file # generator, and the bad pixel mask will be saved in the DQ extension of the # reference file h0 = fits.PrimaryHDU(fully_saturated) h0.header['EXTNAME'] = 'SATURATED' h1a = fits.ImageHDU(rc_from_pedestal_only) h1a.header['EXTNAME'] = 'RC_FROM_PED' h1b = fits.ImageHDU(rc_from_jumps_only) h1b.header['EXTNAME'] = 'RC_FROM_JUMPS' h1 = fits.ImageHDU(rc) h1.header['EXTNAME'] = 'RC' h2 = fits.ImageHDU(low_ped) h2.header['EXTNAME'] = 'LOW_PEDESTAL' h3 = fits.ImageHDU(noisy.astype(int)) h3.header['EXTNAME'] = 'NOISY' h4 = fits.ImageHDU(noisy_second_pass.astype(int)) h4.header['EXTNAME'] = 'MANY_CRS' h5 = fits.ImageHDU(combined_noisy) h5.header['EXTNAME'] = 'NOISY_AND_CRS' hlist = fits.HDUList([h0, h1a, h1b, h1, h2, h3, h4, h5]) hlist.writeto(outfile, overwrite=True) print('Multi-extension file with individual types of bad pixels saved to:') print(outfile) return bad_pixels
continue img[y][x] = 255 q.put((x + 1, y)) q.put((x - 1, y)) q.put((x, y + 1)) q.put((x, y - 1)) # check if circle was complete if not IsBounded: img[:, :] = 0 return cv2.namedWindow('Window') cv2.setMouseCallback('Window', interactive_drawing) while True: cv2.imshow('Window', np.bitwise_or(img, mask_img)) k = cv2.waitKey(1) & 0xFF if k == ord('f'): fill = True elif k == ord('d'): fill = False elif k == ord('q'): break elif k == ord('r'): mask_img[:, :] = 0 elif k == ord('s'): cv2.imshow('Mask', mask_img) cv2.imwrite("/Users/btse/Desktop/Result.png", mask_img) cv2.destroyAllWindows()
def flag_cr(sci_image, blot_image, **pars): """Masks outliers in science image. Mask blemishes in dithered data by comparing a science image with a model image and the derivative of the model image. Parameters ---------- sci_image : ImageModel the science data blot_image : ImageModel the blotted median image of the dithered science frames pars : dict the user parameters for Outlier Detection Default parameters: grow = 1 # Radius to mask [default=1 for 3x3] ctegrow = 0 # Length of CTE correction to be applied snr = "5.0 4.0" # Signal-to-noise ratio scale = "1.2 0.7" # scaling factor applied to the derivative backg = 0 # Background value """ grow = pars.get('grow', 1) ctegrow = pars.get('ctegrow', 0) # not provided by outlierpars backg = pars.get('backg', 0) snr1, snr2 = [float(val) for val in pars.get('snr', '5.0 4.0').split()] scl1, scl2 = [float(val) for val in pars.get('scale', '1.2 0.7').split()] if not sci_image.meta.background.subtracted: # Include background back into blotted image for comparison subtracted_background = sci_image.meta.background.level log.debug("Subtracted background: {}".format(subtracted_background)) if subtracted_background is None: subtracted_background = backg exptime = sci_image.meta.exposure.exposure_time sci_data = sci_image.data * exptime blot_data = blot_image.data * exptime blot_deriv = abs_deriv(blot_data) err_data = np.nan_to_num(sci_image.err) # Define output cosmic ray mask to populate cr_mask = np.zeros(sci_image.shape, dtype=np.uint8) # # # COMPUTATION PART I # # # Model the noise and create a CR mask diff_noise = np.abs(sci_data - blot_data) # ta = np.sqrt(np.abs(blot_data + subtracted_background) + rn ** 2) ta = np.sqrt(np.abs(blot_data + subtracted_background) + err_data**2) t2 = scl1 * blot_deriv + snr1 * ta tmp1 = np.logical_not(np.greater(diff_noise, t2)) # Convolve mask with 3x3 kernel kernel = np.ones((3, 3), dtype=np.uint8) tmp2 = np.zeros(tmp1.shape, dtype=np.int32) ndimage.convolve(tmp1, kernel, output=tmp2, mode='nearest', cval=0) # # # COMPUTATION PART II # # # Create a second CR Mask xt2 = scl2 * blot_deriv + snr2 * ta np.logical_not(np.greater(diff_noise, xt2) & np.less(tmp2, 9), cr_mask) # # # COMPUTATION PART III # # # Flag additional cte 'radial' and 'tail' pixels surrounding CR # pixels as CRs # In both the 'radial' and 'length' kernels below, 0=good and # 1=bad, so that upon convolving the kernels with cr_mask, the # convolution output will have low->bad and high->good from which # 2 new arrays are created having 0->bad and 1->good. These 2 new # arrays are then AND'ed to create a new cr_mask. # recast cr_mask to int for manipulations below; will recast to # Bool at end cr_mask_orig_bool = cr_mask.copy() cr_mask = cr_mask_orig_bool.astype(np.int8) # make radial convolution kernel and convolve it with original cr_mask cr_grow_kernel = np.ones((grow, grow)) cr_grow_kernel_conv = cr_mask.copy() ndimage.convolve(cr_mask, cr_grow_kernel, output=cr_grow_kernel_conv) # make tail convolution kernel and (shortly) convolve it with # original cr_mask cr_ctegrow_kernel = np.zeros((2 * ctegrow + 1, 2 * ctegrow + 1)) cr_ctegrow_kernel_conv = cr_mask.copy() # which pixels are masked by tail kernel depends on readout direction # We could put useful info in here for CTE masking if needed. Code # remains below. For now, we set to zero, which turns off CTE masking. ctedir = 0 if (ctedir == 1): cr_ctegrow_kernel[0:ctegrow, ctegrow] = 1 if (ctedir == -1): cr_ctegrow_kernel[ctegrow + 1:2 * ctegrow + 1, ctegrow] = 1 if (ctedir == 0): pass # finally do the tail convolution ndimage.convolve(cr_mask, cr_ctegrow_kernel, output=cr_ctegrow_kernel_conv) # select high pixels from both convolution outputs; then 'and' them to # create new cr_mask where_cr_grow_kernel_conv = np.where(cr_grow_kernel_conv < grow * grow, 0, 1) where_cr_ctegrow_kernel_conv = np.where(cr_ctegrow_kernel_conv < ctegrow, 0, 1) # combine masks and cast back to Bool np.logical_and(where_cr_ctegrow_kernel_conv, where_cr_grow_kernel_conv, cr_mask) cr_mask = cr_mask.astype(bool) count_sci = np.count_nonzero(sci_image.dq) count_cr = np.count_nonzero(cr_mask) log.debug("Pixels in input DQ: {}".format(count_sci)) log.debug("Pixels in cr_mask: {}".format(count_cr)) # Update the DQ array in the input image in place np.bitwise_or(sci_image.dq, np.invert(cr_mask) * CRBIT, sci_image.dq)
def find_table(img, o_img_height, o_img_width): ## find white border DoB = zc.get_DoB(img, 1, 31, method = 'Average') mask_white = zc.color_inrange(DoB, 'HSV', V_L = 10) ## find purple table (roughly) mask_table = zc.color_inrange(img, 'HSV', H_L = 130, H_U = 160, S_L = 50, V_L = 50, V_U = 220) mask_table, _ = zc.get_big_blobs(mask_table, min_area = 50) mask_table = cv2.morphologyEx(mask_table, cv2.MORPH_CLOSE, zc.generate_kernel(7, 'circular'), iterations = 1) mask_table, _ = zc.find_largest_CC(mask_table) if mask_table is None: rtn_msg = {'status': 'fail', 'message' : 'Cannot find table'} return (rtn_msg, None) mask_table_convex, _ = zc.make_convex(mask_table.copy(), app_ratio = 0.005) mask_table = np.bitwise_or(mask_table, mask_table_convex) mask_table_raw = mask_table.copy() ## fine tune the purple table based on white border mask_white = np.bitwise_and(np.bitwise_not(mask_table), mask_white) for i in range(15): mask_table = zc.expand(mask_table, 3) mask_table = np.bitwise_and(np.bitwise_not(mask_white), mask_table) if i % 4 == 3: mask_table, _ = zc.make_convex(mask_table, app_ratio = 0.01) mask_table = np.bitwise_and(np.bitwise_not(mask_white), mask_table) mask_table, _ = zc.find_largest_CC(mask_table) mask_table, hull_table = zc.make_convex(mask_table, app_ratio = 0.01) ## check if table is big enough table_area = cv2.contourArea(hull_table) table_area_percentage = float(table_area) / img.shape[0] / img.shape[1] if table_area_percentage < 0.06: rtn_msg = {'status' : 'fail', 'message' : "Detected table too small: %f" % table_area_percentage} return (rtn_msg, None) ## find top line of table hull_table = np.array(zc.sort_pts(hull_table[:,0,:], order_first = 'y')) ul = hull_table[0] ur = hull_table[1] if ul[0] > ur[0]: t = ul; ul = ur; ur = t i = 2 # the top two points in the hull are probably on the top line, but may not be the corners while i < hull_table.shape[0] and hull_table[i, 1] - hull_table[0, 1] < 80: pt_tmp = hull_table[i] if pt_tmp[0] < ul[0] or pt_tmp[0] > ur[0]: # computing the area of the part of triangle that lies inside the table triangle = np.vstack([pt_tmp, ul, ur]).astype(np.int32) mask_triangle = np.zeros_like(mask_table) cv2.drawContours(mask_triangle, [triangle], 0, 255, -1) pts = mask_table_raw[mask_triangle.astype(bool)] if np.sum(pts == 255) > 10: break if pt_tmp[0] < ul[0]: ul = pt_tmp else: ur = pt_tmp i += 1 else: break ul = [int(x) for x in ul] ur = [int(x) for x in ur] ## sanity checks about table top line detection if zc.euc_dist(ul, ur) ** 2 * 2.5 < table_area: rtn_msg = {'status' : 'fail', 'message' : "Table top line too short"} return (rtn_msg, None) if abs(zc.line_angle(ul, ur)) > 0.4: rtn_msg = {'status' : 'fail', 'message' : "Table top line tilted too much"} return (rtn_msg, None) # check if two table sides form a reasonable angle mask_table_bottom = mask_table.copy() mask_table_bottom[:-30] = 0 p_left_most = zc.get_edge_point(mask_table_bottom, (-1, 0)) p_right_most = zc.get_edge_point(mask_table_bottom, (1, 0)) if p_left_most is None or p_right_most is None: rtn_msg = {'status' : 'fail', 'message' : "Table doesn't occupy bottom part of image"} return (rtn_msg, None) left_side_angle = zc.line_angle(ul, p_left_most) right_side_angle = zc.line_angle(ur, p_right_most) angle_diff = zc.angle_dist(left_side_angle, right_side_angle, angle_range = math.pi * 2) if abs(angle_diff) > 1.8: rtn_msg = {'status' : 'fail', 'message' : "Angle between two side edge not right"} return (rtn_msg, None) ## rotate to make opponent upright, use table edge as reference pts1 = np.float32([ul,ur,[ul[0] + (ur[1] - ul[1]), ul[1] - (ur[0] - ul[0])]]) pts2 = np.float32([[0, o_img_height], [o_img_width, o_img_height], [0, 0]]) M = cv2.getAffineTransform(pts1, pts2) img[np.bitwise_not(zc.get_mask(img, rtn_type = "bool", th = 3)), :] = [3,3,3] img_rotated = cv2.warpAffine(img, M, (o_img_width, o_img_height)) ## sanity checks about rotated opponent image bool_img_rotated_valid = zc.get_mask(img_rotated, rtn_type = "bool") if float(bool_img_rotated_valid.sum()) / o_img_width / o_img_height < 0.7: rtn_msg = {'status' : 'fail', 'message' : "Valid area too small after rotation"} return (rtn_msg, None) rtn_msg = {'status' : 'success'} return (rtn_msg, (img_rotated, mask_table, M))
def getJaccardError(img1, img2): img_intersection = np.sum(np.bitwise_and(img1, img2)) img_union = np.sum(np.bitwise_or(img1, img2)) jacc_error = float(img_intersection) / float(img_union) return jacc_error
def bitwiseOrData(data1, data2): return numpy.bitwise_or(data1, data2)
def time_char_or(self): np.bitwise_or(self.c, 0, out=self.c) np.bitwise_or(0, self.c, out=self.c)
def _activity(run, baseline_activity=0., baseline_sigma=3.0, trace_type='deconvolved'): """ Get the activity levels for the temporal classifier. Parameters ---------- run : Run baseline_activity : float Scale factor of baseline activity. 'temporal-prior-baseline-activity' in classifier parameters. baseline_sigma : float Scale factor of baseline variance. 'temporal-prior-baseline-sigma' in classifier parameters. trace_type : {'deconvolved'} This only works on deconvolved data for now. Returns ------- baseline activity, variance of activity, outliers """ if trace_type != 'deconvolved': raise ValueError( 'Temporal classifier only implemented for deconvolved data.') if run.run_type == 'spontaneous' and 'sated' in run.tags: runs = run.parent.runs(run_types=['spontaneous'], tags=['sated']) spontaneous = True elif run.run_type == 'spontaneous' and 'hungry' in run.tags: runs = run.parent.runs(run_types=['spontaneous'], tags=['hungry']) spontaneous = True elif run.run_type == 'training': runs = run.parent.runs(run_types=['training']) spontaneous = False else: raise ValueError( 'Unknown run_type and tags, not sure how to calculate activity.') baseline, variance, outliers = None, None, None if spontaneous: popact, outliers = [], [] for r in runs: t2p = r.trace2p() pact = t2p.trace('deconvolved') fmin = t2p.lastonset() mask = t2p.inactivity() mask[:fmin] = False if len(popact): popact = np.concatenate([popact, pact[:, mask]], axis=1) else: popact = pact[:, mask] trs = t2p.trace('deconvolved')[:, fmin:] cellact = np.nanmean(trs, axis=1) outs = cellact > np.nanmedian(cellact) + 2 * np.std(cellact) if len(outliers) == 0: outliers = outs else: outliers = np.bitwise_or(outliers, outs) if len(popact): popact = np.nanmean(popact[np.invert(outliers), :], axis=0) baseline = np.median(popact) variance = np.std(popact) outliers = outliers else: popact = [] for r in runs: t2p = r.trace2p() ncells = t2p.ncells pact = np.nanmean(t2p.trace('deconvolved'), axis=0) skipframes = int(t2p.framerate * 4) for cs in ['plus*', 'neutral*', 'minus*', 'pavlovian*']: onsets = t2p.csonsets(cs) for ons in onsets: pact[ons:ons + skipframes] = np.nan popact = np.concatenate([popact, pact[np.isfinite(pact)]]) if len(popact): # baseline = np.median(popact) # Exclude extremes percent = 2.0 popact = np.sort(popact) trim = int(percent * popact.size / 100.) popact = popact[trim:-trim] baseline = np.median( popact) # Moved to after extreme exclusion on 190326 variance = np.std(popact) outliers = np.zeros(ncells, dtype=bool) if baseline is None: baseline, variance = 0.01, 0.08 * baseline_sigma else: baseline *= baseline_activity variance *= baseline_sigma return baseline, variance, outliers
def do_correction(input_model, ref_model): """ Short Summary ------------- Execute all tasks for saturation, including using a saturation reference file. Parameters ---------- input_model: data model object The input science data to be corrected ref_model: data model object Saturation reference file mode object Returns ------- output_model: data model object object having GROUPDQ array saturation flags set """ ramparr = input_model.data # Was IRS2 readout used? is_irs2_format = x_irs2.is_irs2(input_model) if is_irs2_format: irs2_mask = x_irs2.make_mask(input_model) # Create the output model as a copy of the input output_model = input_model.copy() groupdq = output_model.groupdq # Extract subarray from reference file, if necessary if reffile_utils.ref_matches_sci(input_model, ref_model): satmask = ref_model.data dqmask = ref_model.dq else: log.info('Extracting reference file subarray to match science data') ref_sub_model = reffile_utils.get_subarray_model( input_model, ref_model) satmask = ref_sub_model.data.copy() dqmask = ref_sub_model.dq.copy() ref_sub_model.close() # For pixels flagged in reference file as NO_SAT_CHECK, set the dq mask # and saturation mask wh_sat = np.bitwise_and(dqmask, dqflags.pixel['NO_SAT_CHECK']) dqmask[wh_sat == dqflags.pixel['NO_SAT_CHECK']] = dqflags.pixel['NO_SAT_CHECK'] satmask[wh_sat == dqflags.pixel['NO_SAT_CHECK']] = HUGE_NUM # Correct saturation values for NaNs in the ref file correct_for_NaN(satmask, dqmask) dq_flag = dqflags.group['SATURATED'] nints = ramparr.shape[0] ngroups = ramparr.shape[1] detector = input_model.meta.instrument.detector flagarray = np.zeros(ramparr.shape[-2:], dtype=groupdq.dtype) for ints in range(nints): for plane in range(ngroups): # Update the 4D groupdq array with the saturation flag. The # flag is set in the current plane and all following planes. if is_irs2_format: sci_temp = x_irs2.from_irs2(ramparr[ints, plane, :, :], irs2_mask, detector) flag_temp = np.where(sci_temp >= satmask, dq_flag, 0) # Copy flag_temp into flagarray. x_irs2.to_irs2(flagarray, flag_temp, irs2_mask, detector) else: flagarray[:, :] = np.where( ramparr[ints, plane, :, :] >= satmask, dq_flag, 0) np.bitwise_or(groupdq[ints, plane:, :, :], flagarray, groupdq[ints, plane:, :, :]) output_model.groupdq = groupdq if is_irs2_format: pixeldq_temp = x_irs2.from_irs2(output_model.pixeldq, irs2_mask, detector) pixeldq_temp = np.bitwise_or(pixeldq_temp, dqmask) x_irs2.to_irs2(output_model.pixeldq, pixeldq_temp, irs2_mask, detector) else: output_model.pixeldq = np.bitwise_or(output_model.pixeldq, dqmask) return output_model
def temporal_prior(run, pars, nan_cells=None, replace_data=None, replace_temporal_prior=None): """Given a run and trained model, classify reactivations. Parameters ---------- run : Run Run that the classifier will be applied to. model : aode.AODE A fully-trained model. pars : dict Parameters for classifier. nan_cells : np.ndarray of bool Cell mask of cells to exclude from temporal prior. Activity outliers are also automatically removed. merge_cses : optional, list List of class names. Merges the results of later values into the first one. replace_data : matrix (ncells, ntimes) Replacement data if not None replace_priors : dict of floats The prior probabilities for each category to replace that in parameters replace_temporal_prior : vector A replacement temporal prior to generate the frame priors replace_integrate_frames : int Can change the frame integration number without changing the parameters. This is useful if the data are already passed through a rolling max filter. Returns ------- dict Results dict. """ if replace_temporal_prior is not None: return replace_temporal_prior if replace_data is not None: full_traces = replace_data else: full_traces = run.trace2p().trace(pars['trace-type']) if nan_cells is None: nan_cells = np.any(np.invert(np.isfinite(full_traces)), axis=1) if pars['remove-stim'] and replace_data is None: t2p = run.trace2p() pre_pad_s = pars['classification-ms'] / 1000. / 2. post_pad_s = 0.0 + pre_pad_s pav_post_pad_s = 0.5 + pre_pad_s stim_mask = t2p.stim_mask(pre_pad_s=pre_pad_s, post_pad_s=post_pad_s, pav_post_pad_s=pav_post_pad_s) else: stim_mask = None if pars['temporal-dependent-priors']: if 'temporal-prior-fwhm-frames' not in pars: t2p = run.trace2p() pars = _convert_time_to_frames(deepcopy(pars), t2p.framerate) baseline, variance, outliers = _activity( run, baseline_activity=pars['temporal-prior-baseline-activity'], baseline_sigma=pars['temporal-prior-baseline-sigma'], trace_type=pars['trace-type']) # Make sure cells with any NaN's and outliers are removed from the # temporal prior outliers = np.bitwise_or(outliers, nan_cells) tpriorvec = aode.temporal_prior( full_traces[np.invert(outliers), :], actmn=baseline, actvar=variance, fwhm=pars['temporal-prior-fwhm-frames'], stim_mask=stim_mask) else: # NOTE: this hasn't really been tested # Default to equal probability tpriorvec = np.ones(full_traces.shape[1]) return tpriorvec
def find_overlapping_regions(true_regions, test_regions): true_boxes = [] true_classes = [] test_boxes = [] test_classes = [] test_scores = [] img_dims = true_regions['hsv_img'].shape[:2] for r in true_regions['regions']: true_boxes.append(compute_bbox(r['points'])) true_classes.append(r['label']) for r in test_regions: test_boxes.append(compute_bbox(r['contour'])) max_prob = max(r['prob'].items(), key=itemgetter(1)) test_classes.append(max_prob[0]) test_scores.append(max_prob[1]) # now we are ready to find the overlaps, we'll keep track of them with a dictionary # where the keys are the true region's index. The values will be a dictionary of # overlapping test regions, organized into 2 groups: # - matching overlaps: overlaps where the test & true region labels agree # - non-matching overlaps: overlaps where the test & true region labels differ # # Each one of those groups will be keys with a value of another list of dictionaries, # with the overlapping test region index along with the IoU value. # There are 2 other cases to cover: # - true regions with no matching overlaps (i.e. missed regions) # - test regions with no matching overlaps (i.e. false positives) overlaps = {} true_match_set = set() test_match_set = set() for i, r1 in enumerate(true_boxes): true_mask = None # reset to None, will compute as needed for j, r2 in enumerate(test_boxes): if not do_boxes_overlap(r1, r2): continue # So you're saying there's a chance? # If we get here, there is a chance for an overlap but it is not guaranteed, # we'll need to check the contours' pixels if true_mask is None: # we've never tested against this contour yet, so render it true_mask = make_boolean_mask( true_regions['regions'][i]['points'], img_dims) # and render the test contour test_mask = make_boolean_mask(test_regions[j]['contour'], img_dims) intersect_mask = np.bitwise_and(true_mask, test_mask) intersect_area = intersect_mask.sum() if not intersect_area > 0: # the bounding boxes overlapped, but the contours didn't, skip it continue union_mask = np.bitwise_or(true_mask, test_mask) true_match_set.add(i) test_match_set.add(j) if i not in overlaps: overlaps[i] = { 'true_label': true_classes[i], 'true': [], 'false': [] } test_result = { 'test_index': j, 'iou': intersect_area / union_mask.sum() } if true_classes[i] == test_classes[j]: overlaps[i]['true'].append(test_result) else: overlaps[i]['false'].append(test_result) missed_regions = true_match_set.symmetric_difference( (range(0, len(true_boxes)))) false_positives = test_match_set.symmetric_difference( (range(0, len(test_boxes)))) return { 'overlaps': overlaps, 'missed_regions': missed_regions, 'false_positives': false_positives }
def apply_filter(self, entry: DetectionAnnotation, is_empty): return np.where(np.bitwise_or(entry.x_maxs - entry.x_mins <= 0, entry.y_maxs - entry.y_mins <= 0))[0]
def tanimoto(x, y): x = np.asarray(x, np.bool) y = np.asarray(y, np.bool) return -np.log2( np.double(np.bitwise_and(x, y).sum()) / np.double(np.bitwise_or(x, y).sum()))
def jaccard(x, y): x = np.asarray(x, np.bool) y = np.asarray(y, np.bool) return 1 - np.double(np.bitwise_and(x, y).sum()) / np.double( np.bitwise_or(x, y).sum())
def clock_tec_solveMH(obs_phase, freqs, times, m0, cov, Cd_error, Ct_ratio, plot=False): '''Solves for the terms phase(CS,TEC,delay) = CS + e^2/(4pi ep0 me c) * TEC/nu + 2pi*nu*delay Assumes phase is in units of radians, freqs in is units of Hz, and times is in units of seconds with arbitrary offset obs_phase is shape (num_freqs, num_times)''' binning = 50 convergence = binning**2 * 3 def calc_phase(m, freqs): phase = np.multiply.outer(np.ones( len(freqs)), m[:, 2]) + 8.44797256e-7 * TECU * np.multiply.outer( 1. / freqs, m[:, 1]) + 2. * np.pi * np.multiply.outer( freqs, m[:, 0]) return phase def neglogL(obs_phase, phase, CdCt): L2 = obs_phase - phase L2 *= L2 L2 /= (CdCt + 1e-15) return np.sum(L2, axis=0) / 2. def sample_prior(last, cov): """Last is tau,tec,cs in matrix of size [len(times),3], return similar shaped next point""" return last + np.random.multivariate_normal( mean=[0, 0, 0], cov=cov, size=last.shape[0]) cs = m0[:, 2] tec = m0[:, 1] tau = m0[:, 0] print("Initial CS: {}".format(cs)) print("Initial TEC: {}".format(tec)) print("Initial delay: {}".format(tau)) m = m0.copy() # if plot: # plt.plot(times,cs0,label="CS0") # plt.plot(times,tec0,label="TEC0") # plt.plot(times,delay0,label="delay0") # plt.legend(frameon=False) # plt.show() Ct = (Ct_ratio * np.abs(obs_phase))**2 Cd = (Cd_error * np.pi / 180.)**2 CdCt = Cd + Ct Si = neglogL(obs_phase, calc_phase(m, freqs), CdCt) print("Initial Si: {}".format(Si)) max_iter = 100 * convergence posterior = np.zeros([convergence, len(times), 3], dtype=np.double) multiplicity = np.zeros([convergence, len(times)], dtype=np.double) posterior[0, :, :] = m minS = Si minSol = m.copy() accepted = np.ones(len(times), dtype=np.int) cov_prior = np.diag([1e-10, 1e-6, 0.5])**2 + cov iter = 1 while np.max(accepted) < convergence and iter < max_iter: #sample last = np.array( [posterior[accepted[i] - 1, i, :] for i in range(len(times))]) m_j = sample_prior(last, cov_prior) Sj = neglogL(obs_phase, calc_phase(m_j, freqs), CdCt) Lj = np.exp(-Sj) accept_mask = np.bitwise_or( Sj < Si, np.log(np.random.uniform(size=len(Sj))) < Si - Sj) #print(accept_mask) Si[accept_mask] = Sj[accept_mask] for i in range(len(times)): if accept_mask[i]: posterior[accepted[i], i, :] = m_j[i, :] multiplicity[accepted[i], i] += 1 accepted[i] += 1 else: multiplicity[accepted[i] - 1, i] += 1 if np.any(accept_mask): #print(m_j) #print("{} accepted".format(np.sum(accept_mask))) pass maxL_mask = Sj < minS minSol[maxL_mask, :] = m_j[maxL_mask] minS[maxL_mask] = Sj[maxL_mask] iter += 1 if iter != max_iter: print("Converged in {} steps with mean acceptance rate of {}".format( iter, np.mean(accepted) / iter)) posteriors = [] multiplicities = [] means = [] stds = [] maxLs = [] for i in range(len(times)): posteriors.append(posterior[:accepted[i], i, :]) multiplicities.append(multiplicity[:accepted[i], i]) means.append( np.sum(posteriors[i].T * multiplicities[i], axis=1) / np.sum(multiplicities[i])) stds.append( np.sqrt( np.sum(posteriors[i].T**2 * multiplicities[i], axis=1) / np.sum(multiplicities[i]) - means[i]**2)) maxLs.append(minSol[i, :]) print("Sol {}, (Gaussian) sol is {} +- {}".format( i, means[i], stds[i])) print(" maxL sol is {}".format(maxLs[i])) if plot: plt.hist(posteriors[0][:, 0], weights=multiplicities[0], label='tau') plt.legend(frameon=False) plt.show() plt.hist(posteriors[0][:, 1], weights=multiplicities[0], label='tec') plt.legend(frameon=False) plt.show() plt.hist(posteriors[0][:, 2], weights=multiplicities[0], label='cs') plt.legend(frameon=False) plt.show() return maxLs
import cv2 import numpy as np img1 = cv2.imread('log_3.png', 0) img2 = cv2.imread('log_4.png', 0) img3 = cv2.imread('blankl.png', 0) f, c = img1.shape for i in range(f): for j in range(c): img3[i][j] = np.bitwise_or(img1[i][j], img2[i][j]) cv2.imshow('res', img3)
del PParr, PVarr, PMarr # Make a mapping of where the particles should go. dest = na.zeros(size, dtype='int8') done = na.zeros(size, dtype='bool') for i, b in enumerate(boundaries): # We go backwards, smallest to largest grid j = len(boundaries) - i - 1 LE, RE = boundaries[j] is_inside = ( (PPall.T >= LE).all(axis=1) * \ (PPall.T < RE).all(axis=1) ) # We only change ones that haven't been assigned before. is_inside = is_inside * na.invert(done) dest += j * is_inside # Keep track of which particles have already been assigned. done = na.bitwise_or(is_inside, done) # Clean up. del is_inside, done # Find the sizes of the final files using histogram. bins = na.arange(len(boundaries) + 1) sizes, bins = na.histogram(dest, bins) # For the mass field, we need to fix the masses for particles that change # assignments. A negative value of diff means that the particle went to a # less refined grid, so it should drop in mass. And visa versa for positive # diffs. refinement = float(refinement)**3 diff = dest - source multi = na.power(refinement, diff)
def time_int_or(self): np.bitwise_or(self.i, 0, out=self.i) np.bitwise_or(0, self.i, out=self.i)
def test_ascii(): nobj = 5000 numpy.random.seed(8675309) x = numpy.random.random_sample(nobj) y = numpy.random.random_sample(nobj) z = numpy.random.random_sample(nobj) ra = numpy.random.random_sample(nobj) dec = numpy.random.random_sample(nobj) r = numpy.random.random_sample(nobj) w = numpy.random.random_sample(nobj) g1 = numpy.random.random_sample(nobj) g2 = numpy.random.random_sample(nobj) k = numpy.random.random_sample(nobj) flags = numpy.zeros(nobj).astype(int) for flag in [1, 2, 4, 8, 16]: sub = numpy.random.random_sample(nobj) < 0.1 flags[sub] = numpy.bitwise_or(flags[sub], flag) file_name = os.path.join('data', 'test.dat') with open(file_name, 'w') as fid: # These are intentionally in a different order from the order we parse them. fid.write('# ra,dec,x,y,k,g1,g2,w,flag,z,r\n') for i in range(nobj): fid.write((('%.8f ' * 10) + '%d\n') % (ra[i], dec[i], x[i], y[i], k[i], g1[i], g2[i], w[i], z[i], r[i], flags[i])) # Check basic input config = { 'x_col': 3, 'y_col': 4, 'z_col': 9, 'x_units': 'rad', 'y_units': 'rad', 'w_col': 8, 'g1_col': 6, 'g2_col': 7, 'k_col': 5, } cat1 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat1.x, x) numpy.testing.assert_almost_equal(cat1.y, y) numpy.testing.assert_almost_equal(cat1.z, z) numpy.testing.assert_almost_equal(cat1.w, w) numpy.testing.assert_almost_equal(cat1.g1, g1) numpy.testing.assert_almost_equal(cat1.g2, g2) numpy.testing.assert_almost_equal(cat1.k, k) # Check flags config['flag_col'] = 11 cat2 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat2.w[flags == 0], w[flags == 0]) numpy.testing.assert_almost_equal(cat2.w[flags != 0], 0.) # Check ok_flag config['ok_flag'] = 4 cat3 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal( cat3.w[numpy.logical_or(flags == 0, flags == 4)], w[numpy.logical_or(flags == 0, flags == 4)]) numpy.testing.assert_almost_equal( cat3.w[numpy.logical_and(flags != 0, flags != 4)], 0.) # Check ignore_flag del config['ok_flag'] config['ignore_flag'] = 16 cat4 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat4.w[flags < 16], w[flags < 16]) numpy.testing.assert_almost_equal(cat4.w[flags >= 16], 0.) # Check different units for x,y config['x_units'] = 'arcsec' config['y_units'] = 'arcsec' del config['z_col'] cat5 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180. / 3600.)) numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180. / 3600.)) config['x_units'] = 'arcmin' config['y_units'] = 'arcmin' cat5 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180. / 60.)) numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180. / 60.)) config['x_units'] = 'deg' config['y_units'] = 'deg' cat5 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat5.x, x * (pi / 180.)) numpy.testing.assert_almost_equal(cat5.y, y * (pi / 180.)) del config['x_units'] # Default is radians del config['y_units'] cat5 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat5.x, x) numpy.testing.assert_almost_equal(cat5.y, y) # Check ra,dec del config['x_col'] del config['y_col'] config['ra_col'] = 1 config['dec_col'] = 2 config['r_col'] = 10 config['ra_units'] = 'rad' config['dec_units'] = 'rad' cat6 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat6.ra, ra) numpy.testing.assert_almost_equal(cat6.dec, dec) config['ra_units'] = 'deg' config['dec_units'] = 'deg' cat6 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat6.ra, ra * (pi / 180.)) numpy.testing.assert_almost_equal(cat6.dec, dec * (pi / 180.)) config['ra_units'] = 'hour' config['dec_units'] = 'deg' cat6 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat6.ra, ra * (pi / 12.)) numpy.testing.assert_almost_equal(cat6.dec, dec * (pi / 180.)) # Check using a different delimiter, comment marker csv_file_name = os.path.join('data', 'test.csv') with open(csv_file_name, 'w') as fid: # These are intentionally in a different order from the order we parse them. fid.write('% This file uses commas for its delimiter') fid.write('% And more than one header line.') fid.write('% Plus some extra comment lines every so often.') fid.write('% And we use a weird comment marker to boot.') fid.write('% ra,dec,x,y,k,g1,g2,w,flag\n') for i in range(nobj): fid.write((('%.8f,' * 10) + '%d\n') % (ra[i], dec[i], x[i], y[i], k[i], g1[i], g2[i], w[i], z[i], r[i], flags[i])) if i % 100 == 0: fid.write('%%%% Line %d\n' % i) config['delimiter'] = ',' config['comment_marker'] = '%' cat7 = treecorr.Catalog(csv_file_name, config) numpy.testing.assert_almost_equal(cat7.ra, ra * (pi / 12.)) numpy.testing.assert_almost_equal(cat7.dec, dec * (pi / 180.)) numpy.testing.assert_almost_equal(cat7.r, r) numpy.testing.assert_almost_equal(cat7.g1, g1) numpy.testing.assert_almost_equal(cat7.g2, g2) numpy.testing.assert_almost_equal(cat7.w[flags < 16], w[flags < 16]) numpy.testing.assert_almost_equal(cat7.w[flags >= 16], 0.) # Check flip_g1, flip_g2 del config['delimiter'] del config['comment_marker'] config['flip_g1'] = True cat8 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat8.g1, -g1) numpy.testing.assert_almost_equal(cat8.g2, g2) config['flip_g2'] = 'true' cat8 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat8.g1, -g1) numpy.testing.assert_almost_equal(cat8.g2, -g2) config['flip_g1'] = 'n' config['flip_g2'] = 'yes' cat8 = treecorr.Catalog(file_name, config) numpy.testing.assert_almost_equal(cat8.g1, g1) numpy.testing.assert_almost_equal(cat8.g2, -g2) # Check overriding values with kwargs cat8 = treecorr.Catalog(file_name, config, flip_g1=True, flip_g2=False) numpy.testing.assert_almost_equal(cat8.g1, -g1) numpy.testing.assert_almost_equal(cat8.g2, g2)
def increse(d, k): d[k] -= 1 return d[k] # print("me.py file_path. eg: me.py D:/program/x86/KingRoot/adb.exe") fpath = sys.argv[1] # fpath = "D:/program/x86/KingRoot/adb.exe" print("process:%s" % fpath) fbin = np.fromfile(fpath, dtype=np.uint8) bin5 = np.resize(fbin, int(len(fbin) / 5) * 5) bin = np.reshape(bin5, (-1, 5)) xy, color_byte = bin[:, 0:2], bin[:, 2:5] xy_ = [np.uint16(np.bitwise_or(np.left_shift(i[0], 8), i[1])) for i in xy] xy_unique, xy_index, xy_counts = np.unique(xy_, return_index=True, return_counts=True) mod = np.average(xy_counts) * np.std(xy_counts) * 2 xy_unique_count = dict(zip(xy_unique, xy_counts)) xy_z = [np.array([i, increse(xy_unique_count, i)]) for i in xy_] xyz = np.array([ np.array([ np.uint8(np.right_shift(np.bitwise_and(i[0], 0xff00), 8)), np.uint8(np.bitwise_and(i[0], 0x00ff)), np.uint32(i[1] % mod) ]) for i in xy_z ]) x, y, z = xyz[:, 0:1], xyz[:, 1], xyz[:, 2] color0To1 = np.true_divide(color_byte, 256)
def to_framebuffer_or(self): self._fb1 = _np.bitwise_or(self._fb1, self._matrix) # pylint: disable=no-member
def mark_transit_cadences(time, period_days, epoch_bkjd, duration_days, num_durations=1, flags=None): """Create a logical array indicating which cadences are affected by a transit. Parameters ---------- time : array_like Numpy 1D array of cadence times. period_days : float Transit period in days. epoch_bkjd : float Transit epoch. duration_days : float Duration of transit (start to finish). If you select a duration from first to last contact, all cadences affected by transit are selected. If you only select 2 to 3 contacts, only the interior region of the transit is selected. num_durations : int How much of the lightcurve on either side of the transit to mark. Default means to mark 1/2 the transit duration on either side of the transit center. flags : array_like or `None` If set, must be an array of booleans of length ``time``. Cadences where this array is `True` are ignored in the calculation. This is useful if some of the entries of time are NaN. Returns ------- idx : array_like Array of booleans with the same length as ``time``. Element set to `True` are affected by transits. Raises ------ ValueError Invalid input. """ if flags is None: flags = np.zeros_like(time, dtype=np.bool_) good_time = time[~flags] i0 = np.floor((np.min(good_time) - epoch_bkjd) / period_days) i1 = np.ceil((np.max(good_time) - epoch_bkjd) / period_days) if not np.isfinite(i0) or not np.isfinite(i1): raise ValueError('Error bounding transit time') transit_times = epoch_bkjd + period_days * np.arange(i0, i1 + 1) big_num = sys.float_info.max # A large value that isn't NaN max_diff = 0.5 * duration_days * num_durations idx = np.zeros_like(time, dtype=np.bool8) for tt in transit_times: diff = time - tt diff[flags] = big_num if not np.all(np.isfinite(diff)): raise ValueError('NaN found in diff of transit time') idx = np.bitwise_or(idx, np.fabs(diff) < max_diff) if not np.any(idx): warnings.warn('No cadences found matching transit locations') return idx
def draw_with_fb_or(self): self._matrix = _np.bitwise_or(self._fb1, self._matrix) # pylint: disable=no-member return bytes(self)
def find_CRs(data, gdq, read_noise, rej_threshold, nframes): """ Find CRs/Jumps in each integration within the input data array. The input data array is assumed to be in units of electrons, i.e. already multiplied by the gain. We also assume that the read noise is in units of electrons. """ # Get data characteristics (nints, ngroups, nrows, ncols) = data.shape # Create array for output median slope images median_slopes = np.zeros((nints, nrows, ncols), dtype=np.float32) # Square the read noise values, for use later read_noise_2 = read_noise * read_noise # Reset saturated values in input data array to NaN, so they don't get # used in any of the subsequent calculations data[gdq == dqflags.group['SATURATED']] = np.NaN # Loop over multiple integrations for integration in range(nints): # Roll the ngroups axis of data arrays to the end, to make # memory access to the values for a given pixel faster rdata = np.rollaxis(data[integration], 0, 3) # Compute first differences of adjacent groups up the ramp first_diffs = np.diff(rdata, axis=2) nans = np.where(np.isnan(first_diffs)) first_diffs[nans] = 100000. positive_first_diffs = np.abs(first_diffs) diffsum = positive_first_diffs.sum # Make all the first diffs for saturated groups be equal to # 100,000 to put them above the good values in the sorted index matching_array = np.ones(shape=( positive_first_diffs.shape[0], positive_first_diffs.shape[1], positive_first_diffs.shape[2])) * 100000 sat_groups = (positive_first_diffs == matching_array) number_sat_groups = (sat_groups * 1).sum(axis=2) ndiffs = ngroups - 1 sort_index = np.argsort(positive_first_diffs) med_diffs = return_clipped_median(ndiffs, number_sat_groups, positive_first_diffs, sort_index) # Save initial estimate of the median slope for all pixels median_slopes[integration] = med_diffs # Compute uncertainties as the quadrature sum of the poisson noise # in the first difference signal and read noise. Because the first # differences can be biased by CRs/jumps, we use the median signal # for computing the poisson noise. Here sigma correctly has the # read noise taking into account the fact that multiple frames were # averaged into each group. poisson_noise = np.sqrt(np.abs(med_diffs)) sigma = np.sqrt(poisson_noise * poisson_noise + read_noise_2 / nframes) # Reset sigma to exclude pixels with both readnoise and signal=0 wh_sig = np.where(sigma == 0.) if len(wh_sig[0] > 0): log.debug('Twopt found %d pixels with sigma=0' % (len(wh_sig[0]))) log.debug('which will be reset so that no jump will be detected') sigma[wh_sig] = HUGE_NUM # Compute distance of each sample from the median in units of sigma; # note that the use of "abs" means we'll detect both positive and # negative outliers ratio = np.abs(first_diffs - med_diffs[:, :, np.newaxis]) / sigma[:, :, np.newaxis] # Find the group index of the max outlier in each pixel # (the RHS is identical to the previous versions: # max_index = np.nanargmax (ratio, axis=2) max_index1 = sort_index[:, :, ngroups - 2] # Get indices of highest values (may be outliers) that are above the # rejection threshold r, c = np.indices(max_index1.shape) row1, col1 = np.where( ratio[r, c, max_index1 - number_sat_groups] > rej_threshold) log.debug( 'From highest outlier Twopt found %d pixels with at least one CR' % (len(row1))) number_pixels_with_cr = len(row1) for j in range(number_pixels_with_cr): pixel_masked_diffs = first_diffs[row1[j], col1[j]] pixel_rn2 = read_noise_2[row1[j], col1[j]] pixel_sat_groups = number_sat_groups[row1[j], col1[j]] sorted_index_of_cr = max_index1[row1[j], col1[j]] - pixel_sat_groups # Create a CR mask and set 1st CR to be found # cr_mask=0 designates a CR pixel_cr_mask = np.ones(pixel_masked_diffs.shape, dtype=bool) number_CRs_found = 1 pixel_sorted_index = sort_index[row1[j], col1[j], :] pixel_cr_mask[pixel_sorted_index[ndiffs - pixel_sat_groups - 1]] = 0 new_CR_found = True # Loop over all the found CRs and see if there is more than one CR, setting the mask as you go while new_CR_found and ( (ndiffs - number_CRs_found - pixel_sat_groups) > 1): new_CR_found = False pixel_med_diff = return_clipped_median( ndiffs, number_CRs_found + pixel_sat_groups, pixel_masked_diffs, pixel_sorted_index) poisson_noise = np.sqrt(np.abs(pixel_med_diff)) sigma = np.sqrt(poisson_noise * poisson_noise + pixel_rn2 / nframes) ratio = np.abs(pixel_masked_diffs - pixel_med_diff) / sigma pixel_sorted_ratio = ratio[pixel_sorted_index[:]] # Check if largest remaining difference is above threshold if ratio[pixel_sorted_index[ndiffs - number_CRs_found - pixel_sat_groups - 1]] > rej_threshold: new_CR_found = True pixel_cr_mask[pixel_sorted_index[ndiffs - number_CRs_found - pixel_sat_groups - 1]] = 0 number_CRs_found += 1 # Found all CRs. Set CR flags in input DQ array for this pixel gdq[integration, 1:, row1[j], col1[j]] = \ np.bitwise_or(gdq[integration, 1:, row1[j], col1[j]], dqflags.group['JUMP_DET'] * np.invert(pixel_cr_mask)) # Save the CR-cleaned median slope for this pixel if not new_CR_found: # the loop ran at least one time median_slopes[integration, row1[j], col1[j]] = pixel_med_diff # Next pixel with an outlier (j loop) # Next integration (integration loop) return median_slopes