def bm4d(data,sigma):
tmpName = utils.absPath("0123456789_TMP.tiff")
tmpName2 = utils.absPath("0123456789_OUT_TMP.tiff")
imgtools.write3dTiff(data,tmpName)
subprocess.call([utils.absPath("cxx_code/bm4d/bm4d.sh"),tmpName,tmpName2,str(sigma)], stdout=subprocess.PIPE)
out = imgtools.read3dTiff(tmpName2)
# os.remove(tmpName)
# os.remove(tmpName2)
return out
def bm4d(data,sigma):
tmpName = utils.absPath("0123456789_TMP.tiff")
tmpName2 = utils.absPath("0123456789_OUT_TMP.tiff")
imgtools.write3dTiff(data,tmpName)
subprocess.call([utils.absPath("cxx_code/bm4d/bm4d.sh"),tmpName,tmpName2,str(sigma)], stdout=subprocess.PIPE)
out = imgtools.read3dTiff(tmpName2)
# os.remove(tmpName)
# os.remove(tmpName2)
return out
def nlm3(data, fSize, bSize, sigma, dev = None):
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
dtypes_kernels = {np.float32:"nlm3_float",
np.uint16:"nlm3_short"}
if not dtype in dtypes_kernels.keys():
print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
return
proc = OCLProcessor(dev,utils.absPath("kernels/nlmeans3d.cl"))
img = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
buf = dev.createBuffer(data.size,dtype = dtype)
dev.writeImage(img,data)
print img.shape
proc.runKernel(dtypes_kernels[dtype],img.shape,None,img,buf,
np.int32(img.width),np.int32(img.height),
np.int32(fSize), np.int32(bSize),np.float32(sigma))
return dev.readBuffer(buf,dtype=dtype).reshape(data.shape)
def bilateral3(data, fSize, sigmaX, dev = None):
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
dtypes_kernels = {np.float32:"run3_float",
np.uint16:"run3_short"}
if not dtype in dtypes_kernels.keys():
print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
return
proc = OCLProcessor(dev,utils.absPath("kernels/bilateral.cl"))
clImg = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
clBuf = dev.createBuffer(data.size, dtype= dtype)
dev.writeImage(clImg,data)
proc.runKernel(dtypes_kernels[dtype],clImg.shape,None,clImg,clBuf,
np.int32(clImg.shape[0]),np.int32(clImg.shape[1]),
np.int32(fSize),np.float32(sigmaX))
return dev.readBuffer(clBuf,dtype=dtype).reshape(data.shape)
def nlm3(data, fSize, bSize, sigma, dev = None):
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
dtypes_kernels = {np.float32:"nlm3_float",
np.uint16:"nlm3_short"}
if not dtype in dtypes_kernels.keys():
print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
return
proc = OCLProcessor(dev,utils.absPath("kernels/nlmeans3d.cl"))
img = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
buf = dev.createBuffer(data.size,dtype = dtype)
dev.writeImage(img,data)
print img.shape
proc.runKernel(dtypes_kernels[dtype],img.shape,None,img,buf,
np.int32(img.width),np.int32(img.height),
np.int32(fSize), np.int32(bSize),np.float32(sigma))
return dev.readBuffer(buf,dtype=dtype).reshape(data.shape)
def bilateral3(data, fSize, sigmaX, dev = None):
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
dtypes_kernels = {np.float32:"run3_float",
np.uint16:"run3_short"}
if not dtype in dtypes_kernels.keys():
print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
return
proc = OCLProcessor(dev,utils.absPath("kernels/bilateral.cl"))
clImg = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
clBuf = dev.createBuffer(data.size, dtype= dtype)
dev.writeImage(clImg,data)
proc.runKernel(dtypes_kernels[dtype],clImg.shape,None,clImg,clBuf,
np.int32(clImg.shape[0]),np.int32(clImg.shape[1]),
np.int32(fSize),np.float32(sigmaX))
return dev.readBuffer(clBuf,dtype=dtype).reshape(data.shape)
def testReadUrlParamFile(self):
filePath = utils.absPath("tests/data/header.txt")
actUrl,actParamDto,err =req.Req().readUrlParamFile(filePath)
self.assertIsNone(err)
expParamDto={
"redirectURL": "https%3A%2F%2Fwww.taobao.com%2F"
}
expUrl="/member/login.jhtml"
self.assertEqual(actUrl,expUrl)
self.assertEqual(actParamDto,expParamDto)
def nlm3_thresh(dev, data, FS,BS, sigma, thresh= 0, mean = False):
dtype = data.dtype.type
if not dtype in [np.float32,np.uint16]:
print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
return
if dtype == np.float32:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
else:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i"%(FS,BS))
img = dev.createImage_like(data)
buf = dev.createBuffer(data.size,dtype = dtype)
dev.writeImage(img,data)
proc.runKernel("nlm3_thresh",img.shape,None,img,buf,
np.int32(img.width),np.int32(img.height),
np.float32(sigma),np.float32(thresh))
return dev.readBuffer(buf,dtype = dtype).reshape(data.shape)
def nlm3_thresh(dev, data, FS,BS, sigma, thresh= 0, mean = False):
dtype = data.dtype.type
if not dtype in [np.float32,np.uint16]:
print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
return
if dtype == np.float32:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
else:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i"%(FS,BS))
img = dev.createImage_like(data)
buf = dev.createBuffer(data.size,dtype = dtype)
dev.writeImage(img,data)
proc.runKernel("nlm3_thresh",img.shape,None,img,buf,
np.int32(img.width),np.int32(img.height),
np.float32(sigma),np.float32(thresh))
return dev.readBuffer(buf,dtype = dtype).reshape(data.shape)
def testReadHeaderFile(self):
filePath = utils.absPath("tests/data/header.txt")
actDict,err =req.Req().readHeaderFile(filePath)
del actDict["Cookie"]
self.assertIsNone(err)
expDict={
"Host": "login.taobao.com",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
"Accept-Encoding": "gzip, deflate, br",
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": "2786",
"Origin": "https://login.taobao.com",
"Connection": "keep-alive",
"Referer": "https://login.taobao.com/member/login.jhtml?redirectURL=https%3A%2F%2Fwww.taobao.com%2F",
"Upgrade-Insecure-Requests": "1",
"Pragma": "no-cache",
"Cache-Control": "no-cache"
}
self.assertEqual(actDict,expDict)
def tv3_gpu(data,weight,Niter=50, Ncut = 1, dev = None):
"""
chambolles tv regularized denoising on the gpu
weight should be around 2+1.5*noise_sigma
"""
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
proc = OCLProcessor(dev,utils.absPath("kernels/tv_chambolle.cl"))
if Ncut ==1:
inImg = dev.createImage(data.shape[::-1],dtype = np.float32)
pImgs = [ dev.createImage(data.shape[::-1],
mem_flags = cl.mem_flags.READ_WRITE,
dtype= np.float32,
channel_order = cl.channel_order.RGBA)
for i in range(2)]
outImg = dev.createImage(data.shape[::-1],
dtype = np.float32,
mem_flags = cl.mem_flags.READ_WRITE)
dev.writeImage(inImg,data.astype(np.float32));
dev.writeImage(pImgs[0],np.zeros((4,)+data.shape,dtype=np.float32));
dev.writeImage(pImgs[1],np.zeros((4,)+data.shape,dtype=np.float32));
for i in range(Niter):
proc.runKernel("div_step",inImg.shape,None,
inImg,pImgs[i%2],outImg)
proc.runKernel("grad_step",inImg.shape,None,
outImg,pImgs[i%2],pImgs[1-i%2],
np.float32(weight))
return dev.readImage(outImg,dtype=np.float32)
else:
res = np.empty_like(data,dtype=np.float32)
Nz,Ny,Nx = data.shape
# a heuristic guess: Npad = Niter means perfect
Npad = 1+Niter/2
for i0,(i,j,k) in enumerate(product(range(Ncut),repeat=3)):
print "calculating box %i/%i"%(i0+1,Ncut**3)
sx = slice(i*Nx/Ncut,(i+1)*Nx/Ncut)
sy = slice(j*Ny/Ncut,(j+1)*Ny/Ncut)
sz = slice(k*Nz/Ncut,(k+1)*Nz/Ncut)
sx1,sx2 = utils._extended_slice(sx,Nx,Npad)
sy1,sy2 = utils._extended_slice(sy,Ny,Npad)
sz1,sz2 = utils._extended_slice(sz,Nz,Npad)
data_sliced = data[sz1,sy1,sx1].copy()
_res = tv3_gpu(dev,data_sliced,weight,Niter,Ncut = 1)
res[sz,sy,sx] = _res[sz2,sy2,sx2]
return res
def nlm3_fast(data,FS,BS,sigma,dev = None, proc = None):
"""for noise level (and FS,BS = 2,3) of sigma_0, choose sigma = 1.1*sigma_0
"""
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
if not dtype in [np.float32,np.uint16]:
print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
return
if dtype == np.float32:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
else:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i "%(FS,BS))
Nz,Ny, Nx = data.shape
inImg = dev.createImage_like(data,
mem_flags = "READ_ONLY")
distImg = dev.createImage_like(data,
mem_flags = "READ_WRITE")
tmpImg = dev.createImage_like(data,
mem_flags = "READ_WRITE")
tmp2Img = dev.createImage_like(data,
mem_flags = "READ_WRITE")
accBuf = dev.createBuffer(Nx*Ny*Nz,
mem_flags = cl.mem_flags.READ_WRITE,
dtype = np.float32)
weightBuf = dev.createBuffer(Nx*Ny*Nz,
mem_flags = cl.mem_flags.READ_WRITE,
dtype = np.float32)
dev.writeImage(inImg,data);
dev.writeBuffer(weightBuf,np.zeros((Ny,Nx,Nz),dtype=np.float32));
tdist = 0
tconv = 0
tcomp =0
from time import time
for dx in range(BS+1):
for dy in range(-BS,BS+1):
for dz in range(-BS,BS+1):
t = time()
proc.runKernel("dist",(Nx,Ny,Nz),None,inImg,tmpImg,np.int32(dx),np.int32(dy),np.int32(dz)).wait()
tdist += time()-t
t = time()
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,tmp2Img,np.int32(1))
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmp2Img,tmpImg,np.int32(2))
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,distImg,np.int32(4)).wait()
tconv += time()-t
t = time()
proc.runKernel("computePlus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
np.int32(Nx),np.int32(Ny),np.int32(Nz),
np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma))
if any([dx,dy,dz]):
proc.runKernel("computeMinus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
np.int32(Nx),np.int32(Ny),np.int32(Nz),
np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma)).wait()
tcomp += time()-t
# print tdist,tconv, tcomp
acc = dev.readBuffer(accBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
weights = dev.readBuffer(weightBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
return acc/weights
def tv3_gpu(data,weight,Niter=50, Ncut = 1, dev = None):
"""
chambolles tv regularized denoising on the gpu
weight should be around 2+1.5*noise_sigma
"""
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
proc = OCLProcessor(dev,utils.absPath("kernels/tv_chambolle.cl"))
if Ncut ==1:
inImg = dev.createImage(data.shape[::-1],dtype = np.float32)
pImgs = [ dev.createImage(data.shape[::-1],
mem_flags = cl.mem_flags.READ_WRITE,
dtype= np.float32,
channel_order = cl.channel_order.RGBA)
for i in range(2)]
outImg = dev.createImage(data.shape[::-1],
dtype = np.float32,
mem_flags = cl.mem_flags.READ_WRITE)
dev.writeImage(inImg,data.astype(np.float32));
dev.writeImage(pImgs[0],np.zeros((4,)+data.shape,dtype=np.float32));
dev.writeImage(pImgs[1],np.zeros((4,)+data.shape,dtype=np.float32));
for i in range(Niter):
proc.runKernel("div_step",inImg.shape,None,
inImg,pImgs[i%2],outImg)
proc.runKernel("grad_step",inImg.shape,None,
outImg,pImgs[i%2],pImgs[1-i%2],
np.float32(weight))
return dev.readImage(outImg,dtype=np.float32)
else:
res = np.empty_like(data,dtype=np.float32)
Nz,Ny,Nx = data.shape
# a heuristic guess: Npad = Niter means perfect
Npad = 1+Niter/2
for i0,(i,j,k) in enumerate(product(range(Ncut),repeat=3)):
print "calculating box %i/%i"%(i0+1,Ncut**3)
sx = slice(i*Nx/Ncut,(i+1)*Nx/Ncut)
sy = slice(j*Ny/Ncut,(j+1)*Ny/Ncut)
sz = slice(k*Nz/Ncut,(k+1)*Nz/Ncut)
sx1,sx2 = utils._extended_slice(sx,Nx,Npad)
sy1,sy2 = utils._extended_slice(sy,Ny,Npad)
sz1,sz2 = utils._extended_slice(sz,Nz,Npad)
data_sliced = data[sz1,sy1,sx1].copy()
_res = tv3_gpu(dev,data_sliced,weight,Niter,Ncut = 1)
res[sz,sy,sx] = _res[sz2,sy2,sx2]
return res
def nlm3_fast(data,FS,BS,sigma,dev = None, proc = None):
"""for noise level (and FS,BS = 2,3) of sigma_0, choose sigma = 1.1*sigma_0
"""
if dev is None:
dev = imgtools.__DEFAULT_OPENCL_DEVICE__
if dev is None:
raise ValueError("no OpenCLDevice found...")
dtype = data.dtype.type
if not dtype in [np.float32,np.uint16]:
print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
return
if dtype == np.float32:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
else:
proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i "%(FS,BS))
Nz,Ny, Nx = data.shape
inImg = dev.createImage_like(data,
mem_flags = "READ_ONLY")
distImg = dev.createImage_like(data,
mem_flags = "READ_WRITE")
tmpImg = dev.createImage_like(data,
mem_flags = "READ_WRITE")
tmp2Img = dev.createImage_like(data,
mem_flags = "READ_WRITE")
accBuf = dev.createBuffer(Nx*Ny*Nz,
mem_flags = cl.mem_flags.READ_WRITE,
dtype = np.float32)
weightBuf = dev.createBuffer(Nx*Ny*Nz,
mem_flags = cl.mem_flags.READ_WRITE,
dtype = np.float32)
dev.writeImage(inImg,data);
dev.writeBuffer(weightBuf,np.zeros((Ny,Nx,Nz),dtype=np.float32));
tdist = 0
tconv = 0
tcomp =0
from time import time
for dx in range(BS+1):
for dy in range(-BS,BS+1):
for dz in range(-BS,BS+1):
t = time()
proc.runKernel("dist",(Nx,Ny,Nz),None,inImg,tmpImg,np.int32(dx),np.int32(dy),np.int32(dz)).wait()
tdist += time()-t
t = time()
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,tmp2Img,np.int32(1))
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmp2Img,tmpImg,np.int32(2))
proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,distImg,np.int32(4)).wait()
tconv += time()-t
t = time()
proc.runKernel("computePlus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
np.int32(Nx),np.int32(Ny),np.int32(Nz),
np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma))
if any([dx,dy,dz]):
proc.runKernel("computeMinus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
np.int32(Nx),np.int32(Ny),np.int32(Nz),
np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma)).wait()
tcomp += time()-t
# print tdist,tconv, tcomp
acc = dev.readBuffer(accBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
weights = dev.readBuffer(weightBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
return acc/weights
def testAbsPath(self):
print(utils.absPath("tests/data"))
def testReadParamPostFile(self):
filePath = utils.absPath("tests/data/param.txt")
actDict =req.Req().readParamPostFile(filePath)
expDict={
"TPL_username": "",
"TPL_password": "",
"ncoSig": "",
"ncoSessionid": "",
"ncoToken": "81ba8ac206cdbe98463d8af3ba1cd3ec6ecb82b3",
"slideCodeShow": "false",
"useMobile": "false",
"lang": "zh_CN",
"loginsite": "0",
"newlogin": "******",
"TPL_redirect_url": "https://www.taobao.com/",
"from": "tb",
"fc": "default",
"style": "default",
"css_style": "",
"keyLogin": "******",
"qrLogin": "******",
"newMini": "false",
"newMini2": "false",
"tid": "",
"loginType": "3",
"minititle": "",
"minipara": "",
"pstrong": "",
"sign": "",
"need_sign": "",
"isIgnore": "",
"full_redirect": "",
"sub_jump": "",
"popid": "",
"callback": "",
"guf": "",
"not_duplite_str": "",
"need_user_id": "",
"poy": "",
"gvfdcname": "10",
"gvfdcre": "68747470733A2F2F6C6F67696E2E74616F62616F2E636F6D2F6D656D6265722F6C6F676F75742E6A68746D6C3F73706D3D613231626F2E323031372E3735343839343433372E372E356166393131643939314943676426663D746F70266F75743D7472756526726564697265637455524C3D68747470732533412532462532467777772E74616F62616F2E636F6D253246",
"from_encoding": "",
"sub": "",
"TPL_password_2": "",
"loginASR": "1",
"loginASRSuc": "1",
"allp": "",
"oslanguage": "zh-CN",
"sr": "1920*1080",
"osVer": "",
"naviVer": "firefox|71",
"osACN": "Mozilla",
"osAV": "5.0 (Windows)",
"osPF": "Win32",
"miserHardInfo": "",
"appkey": "00000000",
"nickLoginLink": "",
"mobileLoginLink": "https://login.taobao.com/member/login.jhtml?redirectURL=https://www.taobao.com/&useMobile=true",
"showAssistantLink": "",
"um_token": "T90A4D3B40710C978FF908CBCBA0F4420FCB7483FE19BC35EC493E234B5",
"ua": ""
}
self.assertEqual(actDict,expDict)