def realDataPng(realOtPath, workPath, tSampleNamePart, tModelNamePart1, tModelNamePart2): X, Y, s2n = getRealData(realOtPath, workPath, tSampleNamePart) print(X.shape) print(Y.shape) print(s2n.shape) tIdx = [ 370, 389, 419, 451, 459, 557, 559, 579, 3463, 5238, 7010, 7131, 7460, 7748, 9139, 9698, 12605, 13183, 20547, 20554, 20558, 20561, 20566, 20568, 20575, 20586, 20589, 20642, 20644, 20646, 20648, 20717, 20726, 20730, 20734, 20736, 20739, 20746, 20768, 20771, 20868, 20871, 20884, 20885 ] tIdx = np.array(tIdx) X = X[tIdx] Y = Y[tIdx] s2n = s2n[tIdx] print(X.shape) print(Y.shape) print(s2n.shape) falseImg = X falseS2n = s2n falsePred = Y print("\n\n***********************") print("model1 True, model2 False: %d" % (falseImg.shape[0])) for i in range(falseImg.shape[0]): objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" % (falsePred[i][1], falseS2n[i])) plt.show()
def getSubImages(storePath, skyListFile, otName): workPath = 'twork/%s' % (otName) os.system("rm -rf %s" % (workPath)) if not os.path.exists(workPath): os.system("mkdir -p %s" % (workPath)) size = 100 skyList = np.loadtxt(skyListFile, dtype='str', delimiter=',') for tsky in skyList: #190208,2568,19,3365.45,3507.93,G044_mon_objt_190208T12320486.fit #print(tsky) dateStr = tsky[0] cam_id = int(tsky[2]) mountNum = int(cam_id / 5) + 1 ccdNum = cam_id % 5 camName = '0%d%d' % (mountNum, ccdNum) tpath1 = '%s/%s/G00%d_%s' % (storePath, dateStr, mountNum, camName) fname = tsky[5] x = float(tsky[3]) + 20 y = float(tsky[4]) print("%s/%s" % (tpath1, fname)) #subImg = getWindowImgs(tpath1, fname, x, y, size) subImg = getWindowImgs(storePath, fname, x, y, size) subImgz = zscale_image(subImg) subImgz = scipy.ndimage.zoom(subImgz, 4, order=0) imgpre = fname.split(".")[0] tImgName = "%s.jpg" % (imgpre) savePath = "%s/%s" % (workPath, tImgName) Image.fromarray(subImgz).save(savePath)
def getWindowImgs2(self, objImg, tmpImg, resiImg, poslist, size): objPath = "%s/%s" % (self.tmpDir, objImg) tmpPath = "%s/%s" % (self.tmpDir, tmpImg) resiPath = "%s/%s" % (self.tmpDir, resiImg) objData = fits.getdata(objPath) tmpData = fits.getdata(tmpPath) resiData = fits.getdata(resiPath) i = 0 subImgs = [] for tpos in poslist: objWid = self.getWindowImg(objData, (tpos[0], tpos[1]), size) tmpWid = self.getWindowImg(tmpData, (tpos[2], tpos[3]), size) resiWid = self.getWindowImg(resiData, (tpos[4], tpos[5]), size) if len(resiWid) > 0 and len(objWid) > 0 and len(tmpWid) > 0: #self.log.debug(tpos) #cv2.circle(img, center, radius, color, thickness=1, lineType=8, shift=0) #cv2.circle(objWid,(int(size/2),int(size/2)),2,(255,0,0),1) #cv2.circle(tmpWid,(int(size/2),int(size/2)),2,(255,0,0),1) #cv2.circle(resiWid,(int(size/2),int(size/2)),2,(255,0,0),1) objWidz = zscale_image(objWid) tmpWidz = zscale_image(tmpWid) resiWidz = zscale_image(resiWid) #异常残差图像处理,如果scale失败:1)等于原diffImg;2)直接量化到255 if resiWidz.shape[0] == 21 and resiWidz.shape[1] == 21: subImgs.append([objWidz, tmpWidz, resiWidz]) #conWid = np.concatenate((objWidz, tmpWidz, resiWidz), axis=1) #plt.imshow(conWid, cmap='gray') #plt.show() i = i + 1 #if i>20: # break return subImgs
def saveImgs(imgs, tpath, zoomScale=4): if not os.path.exists(tpath): os.system("mkdir -p %s" % (tpath)) for i in range(imgs.shape[0]): X = imgs[i] tobjImg = zscale_image(X[0]) tTempImg = zscale_image(X[1]) tResiImg = zscale_image(X[2]) if zoomScale != 1: tobjImg = scipy.ndimage.zoom(tobjImg, zoomScale, order=0) tTempImg = scipy.ndimage.zoom(tTempImg, zoomScale, order=0) tResiImg = scipy.ndimage.zoom(tResiImg, zoomScale, order=0) xspace = np.ones((tobjImg.shape[0], 10), np.uint8) * 255 timg = np.concatenate((tobjImg, xspace, tTempImg, xspace, tResiImg), axis=1) savePath = "%s/%05d.jpg" % (tpath, i) Image.fromarray(timg).save(savePath)
def showImage(imgs, props, predY, label, showNum=10): maxEllip = 0.5 for i in range(imgs.shape[0]): if i>=showNum: break s2n = 1.087/props[i,12].astype(np.float) #print(props[i]) X = props[i][0] Y = props[i][1] #if math.fabs(Y-2616.9944)>5: # continue ELONGATION = props[i][5] ELLIPTICITY = props[i][6] if ELLIPTICITY>maxEllip: continue print("ELONGATION=%.2f,ELLIPTICITY=%.2f,X=%.2f,Y=%.2f"%(ELONGATION,ELLIPTICITY,X, Y)) objWidz = zscale_image(imgs[i][0]) tmpWidz = zscale_image(imgs[i][1]) resiWidz = zscale_image(imgs[i][2]) if objWidz.shape[0] == 0: objWidz = imgs[i][0] if tmpWidz.shape[0] == 0: tmpWidz = imgs[i][1] if resiWidz.shape[0] == 0: resiWidz = imgs[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(3, 3)) axes.flat[0].imshow(objWidz, interpolation = "nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation = "nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation = "nearest", cmap='gray') axes.flat[1].set_title("predicted pbb=%.2f, label=%s, s2n=%.2f,ellip=%.2f\n"%(predY[i][1],label,s2n,ELLIPTICITY)) plt.show()
def test(workPath, tModelNamePart): dataPath = "%s/data" % (workPath) tpath = "%s/test.npz" % (dataPath) print(tpath) tdata = np.load(tpath) X_test = tdata['X'] Y_test = tdata['Y'] s2n_test = tdata['s2n'] print(X_test.shape) print(Y_test.shape) print(s2n_test.shape) K.set_image_data_format('channels_first') modelName = "model_80w_20190403_dropout_train8_09.h5" #model = load_model("%s/%s"%(workPath, modelName)) model = load_model( "%s/%s" % (workPath, modelName), custom_objects={'concatenate': keras.layers.concatenate}) Y_pred = model.predict(X_test) pbb_threshold = 0.5 pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int") print(modelName) print("Correctly classified %d out of %d" % ((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0])) print("accuracy = %f" % (1. * (pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0])) TIdx = Y_test[:, 1] == 1 FIdx = Y_test[:, 1] == 0 T_pred_rst = pred_labels[TIdx] F_pred_rst = pred_labels[FIdx] print(T_pred_rst.shape) print(F_pred_rst.shape) TP = ((T_pred_rst == 1).astype(int)).sum() TN = ((F_pred_rst == 0).astype(int)).sum() FP = ((F_pred_rst == 1).astype(int)).sum() FN = ((T_pred_rst == 0).astype(int)).sum() print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" % (Y_test.shape[0], TP, TN, FP, FN)) accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP) precision = (TP) * 1.0 / (TP + FP) recall = (TP) * 1.0 / (TP + FN) f1_score = 2.0 * (precision * recall) / (precision + recall) print("accuracy=%f%%" % (accuracy * 100)) print("precision=%f%%" % (precision * 100)) print("recall=%f%%" % (recall * 100)) print("f1_score=%f%%" % (f1_score * 100)) falseImg = X_test[pred_labels != Y_test[:, 1]] falseLabel = Y_test[pred_labels != Y_test[:, 1]] falsePred = Y_pred[pred_labels != Y_test[:, 1]] falseS2n = s2n_test[pred_labels != Y_test[:, 1]] tnum1 = 0 tnum2 = 0 for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: tnum1 = tnum1 + 1 else: tnum2 = tnum2 + 1 print("total %d, miss classified %d" % (Y_test.shape[0], falseImg.shape[0])) print("True classified as False %d" % (tnum1)) print("False classified as True %d" % (tnum2)) print("\n\n***********************") print("image of True classified as False") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() print("\n\n***********************") print("image of False classified as True") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 0: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() trueImg = X_test[pred_labels == Y_test[:, 1]] trueLabel = Y_test[pred_labels == Y_test[:, 1]] truePred = Y_pred[pred_labels == Y_test[:, 1]] trueS2n = s2n_test[pred_labels == Y_test[:, 1]] showNum = 50 tnum = 0 print("\n\n***********************") print("image of True classified as True") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 1: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break tnum = 0 print("\n\n***********************") print("image of False classified as False") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 0: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break
def imgTransform(data1, data2, data3, transMethod='none'): data1[data1 < 0] = 0 data2[data2 < 0] = 0 data3[data3 < 0] = 0 tmax1 = np.max(data1) tmax2 = np.max(data2) tmax3 = np.max(data3) tmax = np.max([tmax1, tmax2, tmax3]) tminMaxValue = 50 if transMethod == 'eachMax': if tmax1 > tminMaxValue and tmax2 > tminMaxValue and tmax3 > tminMaxValue: data1 = data1 * 255.0 / tmax1 data2 = data2 * 255.0 / tmax2 data3 = data3 * 255.0 / tmax3 elif tmax1 > tminMaxValue and tmax2 <= tminMaxValue and tmax3 > tminMaxValue: data1 = data1 * 255.0 / tmax1 data2 = data2 data3 = data3 * 255.0 / tmax3 else: data1, data2, data3 = np.array([]), np.array([]), np.array([]) elif transMethod == 'unionMax': if tmax < tminMaxValue: data1, data2, data3 = np.array([]), np.array([]), np.array([]) else: data1 = data1 * 255.0 / tmax data2 = data2 * 255.0 / tmax data3 = data3 * 255.0 / tmax elif transMethod == 'zscale': tdata1 = zscale_image(data1) tdata2 = zscale_image(data2) tdata3 = zscale_image(data3) if tdata1.shape[0] == 0: tmin = np.min(data1) tmax = np.max(data1) tdata1 = (((data1 - tmin) / (tmax - tmin)) * 255).astype(np.uint8) if tdata2.shape[0] == 0: tmin = np.min(data2) tmax = np.max(data2) tdata2 = (((data2 - tmin) / (tmax - tmin)) * 255).astype(np.uint8) if tdata3.shape[0] == 0: tmin = np.min(data3) tmax = np.max(data3) tdata3 = (((data3 - tmin) / (tmax - tmin)) * 255).astype(np.uint8) data1 = tdata1 data2 = tdata2 data3 = tdata3 if transMethod == 'eachMax' or transMethod == 'unionMax' or transMethod == 'zscale': data1[data1 > 255] = 255 data2[data2 > 255] = 255 data3[data3 > 255] = 255 data1 = data1.astype(np.uint8) data2 = data2.astype(np.uint8) data3 = data3.astype(np.uint8) elif transMethod == 'none': #preMethod4 data1[data1 > 65535] = 65535 data2[data2 > 65535] = 65535 data3[data3 > 65535] = 65535 data1 = data1.astype(np.uint16) data2 = data2.astype(np.uint16) data3 = data3.astype(np.uint16) return data1, data2, data3
def doClassifyAndUpload(self, subImgPath, totFile, fotFile, fullImgPath, newImg, tmpImg, resImg, origName, serverIP, prob=0.01, maxNEllip=0.6, maxMEllip=0.5, reverse=False): self.log.debug("start new thread classifyAndUpload %s" % (origName)) starttime = datetime.now() self.pbb_threshold = prob try: nameBase = origName[:origName.index(".")] tParms1 = self.doClassifyFile(subImgPath, totFile) if tParms1.shape[0] > 0: tParms1 = tParms1[tParms1[:, 6] < maxNEllip] #tParms1 = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,17]>=prob)] if tParms1.shape[0] > 0: tflags1 = np.ones((tParms1.shape[0], 1)) #OT FLAG tParms1 = np.concatenate((tParms1, tflags1), axis=1) tParms2 = self.doClassifyFile(subImgPath, fotFile) if tParms2.shape[0] > 0: tParms2 = tParms2[(tParms2[:, 6] < maxMEllip) & (tParms2[:, 17] >= prob)] if tParms2.shape[0] > 0: tflags2 = np.zeros((tParms2.shape[0], 1)) #OT FLAG tParms2 = np.concatenate((tParms2, tflags2), axis=1) if tParms1.shape[0] > 0 and tParms2.shape[0] > 0 and tParms2.shape[ 0] < 50: tParms = np.concatenate((tParms1, tParms2), axis=0) elif tParms1.shape[0] > 0: tParms = tParms1 elif tParms2.shape[0] > 0 and tParms2.shape[0] < 25: tParms = tParms2 else: tParms = np.array([]) if tParms.shape[0] > 0: tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg, resImg, tParms, 100) if tParms.shape[0] > 0: self.log.info( "after classified, %s total get %d sub images" % (origName, tSubImgs.shape[0])) i = 1 timgNames = [] for timg in tSubImgs: objWid, tmpWid, resiWid = timg[0], timg[1], timg[2] objWidz = zscale_image(objWid) tmpWidz = zscale_image(tmpWid) resiWidz = zscale_image(resiWid) objWidz = scipy.ndimage.zoom(objWidz, 2, order=0) tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0) resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0) objWidz = objWidz.reshape(objWidz.shape[0], objWidz.shape[1], 1).repeat(3, 2) tmpWidz = tmpWidz.reshape(tmpWidz.shape[0], tmpWidz.shape[1], 1).repeat(3, 2) resiWidz = resiWidz.reshape(resiWidz.shape[0], resiWidz.shape[1], 1).repeat(3, 2) shift00 = 3 cv2.circle(objWidz, (int(objWidz.shape[0] / 2) - shift00, int(objWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(tmpWidz, (int(tmpWidz.shape[0] / 2) - shift00, int(tmpWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(resiWidz, (int(resiWidz.shape[0] / 2) - shift00, int(resiWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) spaceLine = np.ones( (objWidz.shape[0], 5), dtype=np.uint8) * 255 spaceLine = spaceLine.reshape(spaceLine.shape[0], spaceLine.shape[1], 1).repeat(3, 2) sub2Con = np.concatenate( (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz), axis=1) tImgName = "%s_%05d.jpg" % (nameBase, i) timgNames.append(tImgName) savePath = "%s/%s" % (fullImgPath, tImgName) Image.fromarray(sub2Con).save(savePath) i = i + 1 catName = tImgName = "%s.cat" % (nameBase) catPath = "%s/%s" % (fullImgPath, catName) fp0 = open(catPath, 'w') #fp0.write(self.theader) i = 0 for td in tParms: tstr = "%.4f,%.4f,%.2f,%.2f,%.2f,%.3f,%.3f,%.3f,%.2f,%.2f,%d,%.4f,%.4f,%.4f,%.4f,%f,%f,%.3f,%d,%s\n"%\ (td[0],td[1],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13], td[14],td[15],td[16],td[17],td[18], timgNames[i]) fp0.write(tstr) i = i + 1 fp0.close() self.doUpload(fullImgPath, [catName], 'diffot1', serverIP) self.doUpload(fullImgPath, timgNames, 'diffot1img', serverIP) if tParms.shape[0] == 0: self.log.info("after classified, no OT candidate left") if tParms2.shape[0] >= 50: self.log.error( "too more matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images" % (origName, tParms2.shape[0])) os.system("rm -rf %s" % (fullImgPath)) except Exception as e: self.log.error('classifyAndUpload error') self.log.error(str(e)) tstr = traceback.format_exc() self.log.error(tstr) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** classifyAndUpload %s use %d seconds" % (origName, runTime))
def simulateImage1(self, objCat, objImg, tmpCat, tmpImg, tmpOtNum=100): #对同一幅图多次仿真,只用提取一次仿真psf模板 if self.otImgNum < tmpOtNum: self.getTmpOtImgs(tmpCat, tmpImg, otNum=tmpOtNum) otAs, deltaXY = self.randomOTAPos(objCat) destImg = "%s/%s" % (self.tmpDir, objImg) with fits.open(destImg) as hdul: tflag = 1 data0 = hdul[0].data for posamag in otAs: posa = (posamag[0], posamag[1]) mag_add = posamag[2] rimgIdx = randint(0, self.otImgNum - 1) psft = self.otImgs[rimgIdx] flux_ratio = 10**((10 - mag_add) / 2.5) data0 = self.addStar(data0, psft, posa, flux_ratio=flux_ratio) if tflag == 0: ctrX = math.ceil(posa[0]) ctrY = math.ceil(posa[1]) minx = ctrX - 16 maxx = ctrX + 16 miny = ctrY - 16 maxy = ctrY + 16 plt.clf() fig, axes = plt.subplots(1, 4, figsize=(6, 8)) axes.flat[0].imshow(psft, interpolation="nearest", cmap='gray') psftz = zscale_image(psft) axes.flat[1].imshow(psftz, interpolation="nearest", cmap='gray') widImg = data0[miny:maxy, minx:maxx] axes.flat[2].imshow(widImg, interpolation="nearest", cmap='gray') widImgz = zscale_image(widImg) axes.flat[3].imshow(widImgz, interpolation="nearest", cmap='gray') plt.show() tflag = 1 outpre = objImg.split(".")[0] regfile = "%s_simaddstar1_ds9.reg" % (outpre) posfile = "%s_simaddstar1_pos.cat" % (outpre) outfile = "%s_simaddstar1.fit" % (outpre) regPath = "%s/%s" % (self.tmpDir, regfile) posPath = "%s/%s" % (self.tmpDir, posfile) outPath = "%s/%s" % (self.tmpDir, outfile) hdul_new = fits.HDUList(hdul) hdul_new.writeto(outPath, overwrite=True) hdul_new.close() print("output simulated fits file to %s " % (outfile)) fp1 = open(regPath, 'w') fp2 = open(posPath, 'w') for posamag in otAs: posa = (posamag[0], posamag[1]) mag_add = posamag[2] fp1.write( "image;circle(%.2f,%.2f,%.2f) # color=green width=1 text={%.2f} font=\"times 7\"\n" % (posa[0], posa[1], 10.0, mag_add)) fp2.write("%6.2f %6.2f %2.1f\n" % (posa[0], posa[1], mag_add)) fp1.close() fp2.close() print("simulateImageByAddStar1 done.") return outfile, posfile, deltaXY, self.otImgs
def doClassifyAndUpload(self, subImgPath, totFile, fotFile, fullImgPath, newImg, tmpImg, resImg, origName, serverIP, runName, skyName, tcatParm, prob=0.0000001, maxNEllip=0.6, maxMEllip=0.5, reverse=False): self.log.info("start new thread classifyAndUpload %s" % (origName)) print("start new thread classifyAndUpload %s" % (origName)) minFwhm = 1.5 maxFwhm = 3.0 starttime = datetime.now() self.pbb_threshold = prob try: nameBase = origName[:origName.index(".")] dateStr = nameBase.split('_')[3][:6] camName = nameBase[:4] cmbNum = 5 #tidx = nameBase.index('_c')+2 #cmbNum = nameBase[tidx:tidx+3] #'G021_tom_objt_190109T13531492_c005.fit' crossTaskName = "%s_%s_%s_C%03d" % (dateStr, camName, skyName, cmbNum) if runName != 'p1': crossTaskName = "%s_%s" % (crossTaskName, runName) #self.log.info("crossTaskName %s"%(crossTaskName)) #print("crossTaskName %s"%(crossTaskName)) tParms1t = np.array([]) tParms2t = np.array([]) tParms1, obsUtc1 = self.doClassifyFile(subImgPath, totFile) #print("doClassifyAndUpload 001") if tParms1.shape[0] > 0: tParms1t = tParms1[(tParms1[:, 6] < maxNEllip) & (tParms1[:, 9] < maxFwhm) & (tParms1[:, 9] > minFwhm)] #tParms1t = tParms1[(tParms1[:,6]<maxMEllip) & (tParms1[:,15]>=prob)] if tParms1t.shape[0] > 0: tflags1t = np.ones((tParms1t.shape[0], 1)) #OT FLAG tParms1t = np.concatenate((tParms1t, tflags1t), axis=1) #print("doClassifyAndUpload 002") tParms2, obsUtc2 = self.doClassifyFile(subImgPath, fotFile) if tParms2.shape[0] > 0: #tParms2t = tParms2[(tParms2[:,6]<maxMEllip) & (tParms2[:,15]>=prob)&(tParms2[:,9]<maxFwhm)&(tParms2[:,9]>minFwhm)] tParms2t = tParms2[(tParms2[:, 6] < maxMEllip) & (tParms2[:, 9] < maxFwhm) & (tParms2[:, 9] > minFwhm)] if tParms2t.shape[0] > 0: tflags2t = np.zeros((tParms2t.shape[0], 1)) #OT FLAG tParms2t = np.concatenate((tParms2t, tflags2t), axis=1) ''' tParms3t = np.array([]) badotFile = fotFile.replace('fotimg', 'badimg2') tParms3, obsUtc3 = self.doClassifyFile(subImgPath, badotFile) if tParms3.shape[0]>0: tParms3t = tParms3[(tParms3[:,6]<maxMEllip) & (tParms3[:,15]>=prob)] if tParms3t.shape[0]>0: tflags3t = np.zeros((tParms3t.shape[0],1)) #OT FLAG tParms3t = np.concatenate((tParms3t, tflags3t), axis=1) ''' print("classify result: %d tot(%d), %d fot(%d)" % (tParms1t.shape[0], tParms1.shape[0], tParms2t.shape[0], tParms2.shape[0])) #print("doClassifyAndUpload 003") if tParms1t.shape[0] > 0 and tParms1t.shape[ 0] < 100 and tParms2t.shape[0] > 0 and tParms2t.shape[ 0] < 100: tParms = np.concatenate((tParms1t, tParms2t), axis=0) elif tParms1t.shape[0] > 0 and tParms1t.shape[0] < 100: tParms = tParms1t elif tParms2t.shape[0] > 0 and tParms2t.shape[0] < 100: tParms = tParms2t else: tParms = np.array([]) #print("doClassifyAndUpload 004") if tParms.shape[0] == 0: #print("doClassifyAndUpload 005") self.log.info("after classified, no OT candidate left") elif tParms.shape[0] > 0: #print("doClassifyAndUpload 006") tSubImgs, tParms = getWindowImgs(fullImgPath, newImg, tmpImg, resImg, tParms, 100) if tParms.shape[0] > 0: self.log.info( "after classified, %s total get %d sub images" % (origName, tSubImgs.shape[0])) #print("after classified, %s total get %d sub images"%(origName, tSubImgs.shape[0])) i = 1 timgNames = [] for timg in tSubImgs: objWid, tmpWid, resiWid = timg[0], timg[1], timg[2] objWidz = zscale_image(objWid) tmpWidz = zscale_image(tmpWid) resiWidz = zscale_image(resiWid) objWidz = scipy.ndimage.zoom(objWidz, 2, order=0) tmpWidz = scipy.ndimage.zoom(tmpWidz, 2, order=0) resiWidz = scipy.ndimage.zoom(resiWidz, 2, order=0) objWidz = objWidz.reshape(objWidz.shape[0], objWidz.shape[1], 1).repeat(3, 2) tmpWidz = tmpWidz.reshape(tmpWidz.shape[0], tmpWidz.shape[1], 1).repeat(3, 2) resiWidz = resiWidz.reshape(resiWidz.shape[0], resiWidz.shape[1], 1).repeat(3, 2) shift00 = 3 cv2.circle(objWidz, (int(objWidz.shape[0] / 2) - shift00, int(objWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(tmpWidz, (int(tmpWidz.shape[0] / 2) - shift00, int(tmpWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) cv2.circle(resiWidz, (int(resiWidz.shape[0] / 2) - shift00, int(resiWidz.shape[1] / 2) - shift00), 10, (0, 255, 0), thickness=2) spaceLine = np.ones( (objWidz.shape[0], 5), dtype=np.uint8) * 255 spaceLine = spaceLine.reshape(spaceLine.shape[0], spaceLine.shape[1], 1).repeat(3, 2) sub2Con = np.concatenate( (objWidz, spaceLine, tmpWidz, spaceLine, resiWidz), axis=1) tstrs = tmpImg.split( '_') #G044_mon_objt_190128T10264248.fit if len(tstrs) == 4: tmpDateTime = tstrs[3] cv2.putText( sub2Con, #numpy array on which text is written tmpDateTime[:tmpDateTime.index('.')], #text (objWidz.shape[1] + 25, objWidz.shape[0] - 10), #position at which writing has to start cv2.FONT_HERSHEY_SIMPLEX, #font family 0.5, #font size (0, 255, 0), #font color 2) tImgName = "%s_%05d.jpg" % (nameBase, i) timgNames.append(tImgName) savePath = "%s/%s" % (fullImgPath, tImgName) Image.fromarray(sub2Con).save(savePath) i = i + 1 catName = tImgName = "%s.cat" % (nameBase) catPath = "%s/%s" % (fullImgPath, catName) fp0 = open(catPath, 'w') fp0.write(self.theader2) t2oX = tcatParm[-2] t2oY = tcatParm[-1] tx = tParms[:, 0] ty = tParms[:, 1] ox = t2oX(tx, ty) oy = t2oY(tx, ty) i = 0 for td in tParms: #print(td) #print(td[15]) tstr = self.catFormate%\ (ox[i],oy[i],td[2],td[3],td[4],td[5],td[6],td[7],td[8],td[9],td[10],td[11],td[12],td[13], td[14],td[15],td[16], timgNames[i], obsUtc1, td[0],td[1]) fp0.write(tstr) i = i + 1 fp0.close() self.doUpload(fullImgPath, [catName], 'crossOTList', serverIP, crossTaskName) self.doUpload(fullImgPath, timgNames, 'crossOTStamp', serverIP, crossTaskName) if tParms1.shape[0] >= 100 or tParms2.shape[0] >= 100: self.log.error( "too more unmatched or matched OT candidate, skip upload matched to db: after classified, %s total get %d matchend sub images" % (origName, tParms2.shape[0])) os.system("rm -rf %s" % (fullImgPath)) except Exception as e: self.log.error('classifyAndUpload error') self.log.error(str(e)) tstr = traceback.format_exc() self.log.error(tstr) endtime = datetime.now() runTime = (endtime - starttime).seconds self.log.info("********** classifyAndUpload %s use %d seconds" % (origName, runTime))
def test(): tpath2 = "/home/xy/Downloads/myresource/deep_data2/gwac_ot2_apart" tpath3 = "/home/xy/Downloads/myresource/deep_data2/simot/rest_data_0929" dateStr = datetime.strftime(datetime.now(), "%Y%m%d") workPath = "/home/xy/Downloads/myresource/deep_data2/simot/train_%s" % ( dateStr) if not os.path.exists(workPath): os.system("mkdir %s" % (workPath)) print("work path is %s" % (workPath)) X, Y, s2n = getData3(tpath3, tpath2, workPath) print(X.shape) print(Y.shape) print(s2n.shape) N_data = X.shape[0] N_train = int(N_data * 0.9) print("train: %d, test: %d" % (N_train, N_data - N_train)) X_train, Y_train, s2n_train = X[:N_train], Y[:N_train], s2n[:N_train] X_test, Y_test, s2n_test = X[N_train:], Y[N_train:], s2n[N_train:] from keras.models import load_model model = load_model("%s/model_128_5_RealFOT.h5" % (workPath)) Y_pred = model.predict(X_test) pbb_threshold = 0.5 pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int") print("Correctly classified %d out of %d" % ((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0])) print("accuracy = %f" % (1. * (pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0])) TIdx = Y_test[:, 1] == 1 FIdx = Y_test[:, 1] == 0 T_pred_rst = pred_labels[TIdx] F_pred_rst = pred_labels[FIdx] print(T_pred_rst.shape) print(F_pred_rst.shape) TP = ((T_pred_rst == 1).astype(int)).sum() TN = ((F_pred_rst == 0).astype(int)).sum() FP = ((F_pred_rst == 1).astype(int)).sum() FN = ((T_pred_rst == 0).astype(int)).sum() print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" % (Y_test.shape[0], TP, TN, FP, FN)) accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP) precision = (TP) * 1.0 / (TP + FP) recall = (TP) * 1.0 / (TP + FN) f1_score = 2.0 * (precision * recall) / (precision + recall) print("accuracy=%f%%" % (accuracy * 100)) print("precision=%f%%" % (precision * 100)) print("recall=%f%%" % (recall * 100)) print("f1_score=%f%%" % (f1_score * 100)) falseImg = X_test[pred_labels != Y_test[:, 1]] falseLabel = Y_test[pred_labels != Y_test[:, 1]] falsePred = Y_pred[pred_labels != Y_test[:, 1]] falseS2n = s2n_test[pred_labels != Y_test[:, 1]] tnum1 = 0 tnum2 = 0 for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: tnum1 = tnum1 + 1 else: tnum2 = tnum2 + 1 print("total %d, miss classified %d" % (Y_test.shape[0], falseImg.shape[0])) print("True classified as False %d" % (tnum1)) print("False classified as True %d" % (tnum2)) print("\n\n***********************") print("image of True classified as False") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(3, 3)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() print("\n\n***********************") print("image of False classified as True") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 0: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(3, 3)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() trueImg = X_test[pred_labels == Y_test[:, 1]] trueLabel = Y_test[pred_labels == Y_test[:, 1]] truePred = Y_pred[pred_labels == Y_test[:, 1]] trueS2n = s2n_test[pred_labels == Y_test[:, 1]] showNum = 50 tnum = 0 print("\n\n***********************") print("image of True classified as True") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 1: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(3, 3)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break tnum = 0 print("\n\n***********************") print("image of False classified as False") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 0: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(3, 3)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break
def outMisMatch(workPath, imgsFile, destName, tModelNamePart): dataPath = "%s/data" % (workPath) tpath = "%s/%s.npz" % (dataPath, imgsFile) print(tpath) destPath = "%s/%s/%s" % (dataPath, destName, imgsFile) if not os.path.exists(destPath): os.system("mkdir -p %s" % (destPath)) destPath0 = "%s/0" % (destPath) if not os.path.exists(destPath0): os.system("mkdir %s" % (destPath0)) destPath1 = "%s/1" % (destPath) if not os.path.exists(destPath1): os.system("mkdir %s" % (destPath1)) tdata = np.load(tpath) X_test = tdata['X'] Y_test = tdata['Y'] s2n_test = tdata['s2n'] print(X_test.shape) print(Y_test.shape) print(s2n_test.shape) K.set_image_data_format('channels_first') modelName = "model_80w_20190403_dropout_train11_09.h5" model = load_model("%s/%s" % (workPath, modelName)) Y_pred = model.predict(X_test) pbb_threshold = 0.5 pred_labels = np.array((Y_pred[:, 1] >= pbb_threshold), dtype="int") print("Correctly classified %d out of %d" % ((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0])) print("accuracy = %f" % (1. * (pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0])) TIdx = Y_test[:, 1] == 1 FIdx = Y_test[:, 1] == 0 T_pred_rst = pred_labels[TIdx] F_pred_rst = pred_labels[FIdx] print(T_pred_rst.shape) print(F_pred_rst.shape) TP = ((T_pred_rst == 1).astype(int)).sum() TN = ((F_pred_rst == 0).astype(int)).sum() FP = ((F_pred_rst == 1).astype(int)).sum() FN = ((T_pred_rst == 0).astype(int)).sum() print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" % (Y_test.shape[0], TP, TN, FP, FN)) for i in range(X_test.shape[0]): try: ty = Y_test[i][1] if ty != pred_labels[i]: objWidz = zscale_image(X_test[i][0]) tmpWidz = zscale_image(X_test[i][1]) resiWidz = zscale_image(X_test[i][2]) if objWidz.shape[0] == 0: objWidz = X_test[i][0] if tmpWidz.shape[0] == 0: tmpWidz = X_test[i][1] if resiWidz.shape[0] == 0: resiWidz = X_test[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[1].set_title("pbb = " + str(np.round(Y_pred[i][1], 2)) + ", label = " + str(Y_pred[i, 1]) + ", s2n = " + str(s2n_test[i])) #plt.show() savePath = "%s/%d/%06d.jpg" % (destPath, ty, i) plt.savefig(savePath, bbox_inches='tight') #break except Exception as e: tstr = traceback.format_exc() print(tstr)
def test(totpath, fotpath, workPath, tSampleNamePart, tModelNamePart): X, Y, s2n = getData2(totpath, fotpath, workPath, tSampleNamePart) print(X.shape) print(Y.shape) print(s2n.shape) N_data = X.shape[0] N_train = int(N_data * 0.9) print("train: %d, test: %d" % (N_train, N_data - N_train)) X_train, Y_train, s2n_train = X[:N_train], Y[:N_train], s2n[:N_train] X_test, Y_test, s2n_test = X[N_train:], Y[N_train:], s2n[N_train:] from keras.models import load_model modelName = "model_RealFOT_%s.h5" % (tModelNamePart) model = load_model("%s/%s" % (workPath, modelName)) Y_pred = model.predict(X_test) pbb_threshold = 0.5 pred_labels = np.array((Y_pred[:, 1] > pbb_threshold), dtype="int") print("Correctly classified %d out of %d" % ((pred_labels == Y_test[:, 1].astype(int)).sum(), Y_test.shape[0])) print("accuracy = %f" % (1. * (pred_labels == Y_test[:, 1].astype(int)).sum() / Y_test.shape[0])) TIdx = Y_test[:, 1] == 1 FIdx = Y_test[:, 1] == 0 T_pred_rst = pred_labels[TIdx] F_pred_rst = pred_labels[FIdx] print(T_pred_rst.shape) print(F_pred_rst.shape) TP = ((T_pred_rst == 1).astype(int)).sum() TN = ((F_pred_rst == 0).astype(int)).sum() FP = ((F_pred_rst == 1).astype(int)).sum() FN = ((T_pred_rst == 0).astype(int)).sum() print("total=%d,TP=%d,TN=%d,FP=%d,FN=%d" % (Y_test.shape[0], TP, TN, FP, FN)) accuracy = (TP + TN) * 1.0 / (TP + FN + TN + FP) precision = (TP) * 1.0 / (TP + FP) recall = (TP) * 1.0 / (TP + FN) f1_score = 2.0 * (precision * recall) / (precision + recall) print("accuracy=%f%%" % (accuracy * 100)) print("precision=%f%%" % (precision * 100)) print("recall=%f%%" % (recall * 100)) print("f1_score=%f%%" % (f1_score * 100)) falseImg = X_test[pred_labels != Y_test[:, 1]] falseLabel = Y_test[pred_labels != Y_test[:, 1]] falsePred = Y_pred[pred_labels != Y_test[:, 1]] falseS2n = s2n_test[pred_labels != Y_test[:, 1]] tnum1 = 0 tnum2 = 0 for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: tnum1 = tnum1 + 1 else: tnum2 = tnum2 + 1 print("total %d, miss classified %d" % (Y_test.shape[0], falseImg.shape[0])) print("True classified as False %d" % (tnum1)) print("False classified as True %d" % (tnum2)) print("\n\n***********************") print("image of True classified as False") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 1: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() print("\n\n***********************") print("image of False classified as True") for i in range(falseImg.shape[0]): if falseLabel[i, 1] == 0: objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(falsePred[i][1], 2)) + ", label = " + str(falseLabel[i, 1]) + ", s2n = " + str(falseS2n[i])) plt.show() trueImg = X_test[pred_labels == Y_test[:, 1]] trueLabel = Y_test[pred_labels == Y_test[:, 1]] truePred = Y_pred[pred_labels == Y_test[:, 1]] trueS2n = s2n_test[pred_labels == Y_test[:, 1]] showNum = 50 tnum = 0 print("\n\n***********************") print("image of True classified as True") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 1: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break tnum = 0 print("\n\n***********************") print("image of False classified as False") for i in range(trueImg.shape[0]): if trueLabel[i, 1] == 0: objWidz = zscale_image(trueImg[i][0]) tmpWidz = zscale_image(trueImg[i][1]) resiWidz = zscale_image(trueImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[2].set_title("predicted pbb = " + str(np.round(truePred[i][1], 2)) + ", label = " + str(trueLabel[i, 1]) + ", s2n = " + str(trueS2n[i])) plt.show() tnum = tnum + 1 if tnum > showNum: break
def realDataTest(realOtPath, workPath, tSampleNamePart, tModelNamePart1, tModelNamePart2): X, Y, s2n = getRealData(realOtPath, workPath, tSampleNamePart) subNum = 1000 X = X[:subNum] Y = Y[:subNum] s2n = s2n[:subNum] print(X.shape) print(Y.shape) print(s2n.shape) X_test, Y_test, s2n_test = X, Y, s2n y = np.zeros(X_test.shape[0]) Y_test = np.array([np.logical_not(y), y]).transpose() from keras.models import load_model modelName1 = "model_RealFOT_%s.h5" % (tModelNamePart1) model = load_model("%s/%s" % (workPath, modelName1)) Y_pred1 = model.predict(X_test) modelName2 = "model_RealFOT_%s.h5" % (tModelNamePart2) model = load_model("%s/%s" % (workPath, modelName2)) Y_pred2 = model.predict(X_test) pbb_threshold = 0.5 pred_labels1 = np.array((Y_pred1[:, 1] > pbb_threshold), dtype="int") pred_labels2 = np.array((Y_pred2[:, 1] > pbb_threshold), dtype="int") model1True = pred_labels1[pred_labels1 == Y_test[:, 1]] model2True = pred_labels2[pred_labels2 == Y_test[:, 1]] print("model1 correct=%d" % (model1True.shape[0])) print("model2 correct=%d" % (model2True.shape[0])) model1True = pred_labels1[pred_labels1 == 1] model2True = pred_labels2[pred_labels2 == 1] print("model1 True=%d" % (model1True.shape[0])) print("model2 True=%d" % (model2True.shape[0])) diffIdx = pred_labels1 != pred_labels2 m1t = (pred_labels1 == 1) & diffIdx m1f = (pred_labels1 == 0) & diffIdx m2t = (pred_labels2 == 1) & diffIdx m2f = (pred_labels2 == 0) & diffIdx m1tImg = X_test[m1t] m1fImg = X_test[m1f] m2tImg = X_test[m2t] m2fImg = X_test[m2f] m1tS2n = s2n_test[m1t] m1fS2n = s2n_test[m1f] m2tS2n = s2n_test[m2t] m2fS2n = s2n_test[m2f] m1tPred = Y_pred1[m1t] m1fPred = Y_pred1[m1f] m2tPred = Y_pred2[m2t] m2fPred = Y_pred2[m2f] print("m1t=%d" % (m1tImg.shape[0])) print("m1f=%d" % (m1fImg.shape[0])) print("m2t=%d" % (m2tImg.shape[0])) print("m2f=%d" % (m2fImg.shape[0])) #totImgPath = 'totimg' #for i, timg in enumerate(m1tImg): # savePath = "%s/%05d.png"%(totImgPath, i) showNum = 100 falseImg = m1tImg[:showNum] falseS2n = m1tS2n[:showNum] falsePred = m1tPred[:showNum] print("\n\n***********************") print("model1 True, model2 False: %d" % (falseImg.shape[0])) for i in range(falseImg.shape[0]): objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" % (falsePred[i][1], falseS2n[i])) plt.show() falseImg = m1fImg[:showNum] falseS2n = m1fS2n[:showNum] falsePred = m1fPred[:showNum] print("\n\n***********************") print("model1 True, model2 False: %d" % (falseImg.shape[0])) for i in range(falseImg.shape[0]): objWidz = zscale_image(falseImg[i][0]) tmpWidz = zscale_image(falseImg[i][1]) resiWidz = zscale_image(falseImg[i][2]) if objWidz.shape[0] == 0: objWidz = falseImg[i][0] if tmpWidz.shape[0] == 0: tmpWidz = falseImg[i][1] if resiWidz.shape[0] == 0: resiWidz = falseImg[i][2] plt.clf() fig, axes = plt.subplots(1, 3, figsize=(6, 2)) axes.flat[0].imshow(objWidz, interpolation="nearest", cmap='gray') axes.flat[1].imshow(tmpWidz, interpolation="nearest", cmap='gray') axes.flat[2].imshow(resiWidz, interpolation="nearest", cmap='gray') axes.flat[1].set_title("pbb=%.2f,s2n=%.2f" % (falsePred[i][1], falseS2n[i])) plt.show()
def getRealData(destPath): from gwac_util import zscale_image ot2path = '/home/xy/Downloads/myresource/deep_data2/simot/ot2_img_collection_180906_20180907' ot2listFile = '%s/list.txt' % (ot2path) ot2list = np.genfromtxt(ot2listFile, dtype='str') print(ot2list.shape) print(ot2list[:3]) X = [] badImgNum = 0 badIdx = [] #timgbinPath = '%s/gwac_otimg_nozscale.npz'%(destPath) timgbinPath = '%s/gwac_otimg.npz' % (destPath) if os.path.exists(timgbinPath): print("bin file exist, read from %s" % (timgbinPath)) timgbin = np.load(timgbinPath) X = timgbin['ot'] ot2prop = timgbin['ot2prop'] else: for i in range(ot2list.shape[0]): tot2 = ot2list[i] objPath = "%s/%s" % (ot2path, tot2[1]) refPath = "%s/%s" % (ot2path, tot2[2]) diffPath = "%s/%s" % (ot2path, tot2[3]) objImg = readStampFromFits(objPath) refImg = readStampFromFits(refPath) diffImg = readStampFromFits(diffPath) if objImg.shape[0] == 0 or refImg.shape[0] == 0 or diffImg.shape[ 0] == 0: continue objImgz = zscale_image(objImg) refImgz = zscale_image(refImg) diffImgz = zscale_image(diffImg) ''' objImgz = objImg refImgz = refImg diffImgz = diffImg ''' #异常残差图像处理,如果scale失败:1)等于原diffImg;2)直接量化到255 if diffImgz.shape[0] != 12 or diffImgz.shape[1] != 12: print("%d, %s" % (i, tot2[1])) badIdx.append(i) tmin = np.min(diffImg) tmax = np.max(diffImg) diffImgz = (((diffImg - tmin) / (tmax - tmin)) * 255).astype( np.uint8) #diffImgz = diffImg badImgNum = badImgNum + 1 X.append([objImgz, refImgz, diffImgz]) X = np.array(X) ''' for i in range(X.shape[0]): tot2 = ot2list[i] tlabel = tot2[4] ttype = tot2[5] if ttype=='2': fig, axes = plt.subplots(1, 3, figsize=(3, 1)) axes.flat[0].imshow(X[i][0], cmap='gray') axes.flat[1].imshow(X[i][1], cmap='gray') axes.flat[2].imshow(X[i][2], cmap='gray') axes.flat[1].set_title("%d, %s, look=%s, type=%s"%(i, tot2[1][:14], tlabel, ttype)) plt.show() ''' ot2prop = ot2list print("save bin fiel to %s" % (timgbinPath)) np.savez_compressed(timgbinPath, ot=X, ot2prop=ot2prop) print("bad image %d" % (badImgNum)) return X, ot2prop