Esempio n. 1
0
def bm4d(data,sigma):
    tmpName = utils.absPath("0123456789_TMP.tiff")
    tmpName2 = utils.absPath("0123456789_OUT_TMP.tiff")
    imgtools.write3dTiff(data,tmpName)
    subprocess.call([utils.absPath("cxx_code/bm4d/bm4d.sh"),tmpName,tmpName2,str(sigma)], stdout=subprocess.PIPE)

    out = imgtools.read3dTiff(tmpName2)
    # os.remove(tmpName)
    # os.remove(tmpName2)
    return out
Esempio n. 2
0
def bm4d(data,sigma):
    tmpName = utils.absPath("0123456789_TMP.tiff")
    tmpName2 = utils.absPath("0123456789_OUT_TMP.tiff")
    imgtools.write3dTiff(data,tmpName)
    subprocess.call([utils.absPath("cxx_code/bm4d/bm4d.sh"),tmpName,tmpName2,str(sigma)], stdout=subprocess.PIPE)

    out = imgtools.read3dTiff(tmpName2)
    # os.remove(tmpName)
    # os.remove(tmpName2)
    return out
Esempio n. 3
0
def nlm3(data, fSize, bSize, sigma, dev = None):

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")

    dtype = data.dtype.type
    dtypes_kernels = {np.float32:"nlm3_float",
                        np.uint16:"nlm3_short"}

    if not dtype in dtypes_kernels.keys():
        print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
        return

    proc = OCLProcessor(dev,utils.absPath("kernels/nlmeans3d.cl"))

    img = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
    buf = dev.createBuffer(data.size,dtype = dtype)
    dev.writeImage(img,data)


    print img.shape
    proc.runKernel(dtypes_kernels[dtype],img.shape,None,img,buf,
                     np.int32(img.width),np.int32(img.height),
                     np.int32(fSize), np.int32(bSize),np.float32(sigma))

    return dev.readBuffer(buf,dtype=dtype).reshape(data.shape)
Esempio n. 4
0
def bilateral3(data, fSize, sigmaX, dev = None):

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")


    dtype = data.dtype.type
    dtypes_kernels = {np.float32:"run3_float",
                        np.uint16:"run3_short"}

    if not dtype in dtypes_kernels.keys():
        print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
        return

    proc = OCLProcessor(dev,utils.absPath("kernels/bilateral.cl"))

    clImg = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
    clBuf = dev.createBuffer(data.size, dtype= dtype)
    dev.writeImage(clImg,data)

    proc.runKernel(dtypes_kernels[dtype],clImg.shape,None,clImg,clBuf,
                     np.int32(clImg.shape[0]),np.int32(clImg.shape[1]),
                     np.int32(fSize),np.float32(sigmaX))

    return dev.readBuffer(clBuf,dtype=dtype).reshape(data.shape)
Esempio n. 5
0
def nlm3(data, fSize, bSize, sigma, dev = None):

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")

    dtype = data.dtype.type
    dtypes_kernels = {np.float32:"nlm3_float",
                        np.uint16:"nlm3_short"}

    if not dtype in dtypes_kernels.keys():
        print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
        return

    proc = OCLProcessor(dev,utils.absPath("kernels/nlmeans3d.cl"))

    img = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
    buf = dev.createBuffer(data.size,dtype = dtype)
    dev.writeImage(img,data)


    print img.shape
    proc.runKernel(dtypes_kernels[dtype],img.shape,None,img,buf,
                     np.int32(img.width),np.int32(img.height),
                     np.int32(fSize), np.int32(bSize),np.float32(sigma))

    return dev.readBuffer(buf,dtype=dtype).reshape(data.shape)
Esempio n. 6
0
def bilateral3(data, fSize, sigmaX, dev = None):

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")


    dtype = data.dtype.type
    dtypes_kernels = {np.float32:"run3_float",
                        np.uint16:"run3_short"}

    if not dtype in dtypes_kernels.keys():
        print "data type %s not supported yet, please convert to:"%dtype,dtypes_kernels.keys()
        return

    proc = OCLProcessor(dev,utils.absPath("kernels/bilateral.cl"))

    clImg = dev.createImage(data.shape[::-1],channel_type = cl_datatype_dict[dtype])
    clBuf = dev.createBuffer(data.size, dtype= dtype)
    dev.writeImage(clImg,data)

    proc.runKernel(dtypes_kernels[dtype],clImg.shape,None,clImg,clBuf,
                     np.int32(clImg.shape[0]),np.int32(clImg.shape[1]),
                     np.int32(fSize),np.float32(sigmaX))

    return dev.readBuffer(clBuf,dtype=dtype).reshape(data.shape)
Esempio n. 7
0
 def testReadUrlParamFile(self):
     filePath = utils.absPath("tests/data/header.txt")
     actUrl,actParamDto,err =req.Req().readUrlParamFile(filePath)
     self.assertIsNone(err)
     expParamDto={
         "redirectURL": "https%3A%2F%2Fwww.taobao.com%2F"
     }
     expUrl="/member/login.jhtml"
     self.assertEqual(actUrl,expUrl)
     self.assertEqual(actParamDto,expParamDto)
Esempio n. 8
0
def nlm3_thresh(dev, data, FS,BS, sigma, thresh= 0, mean = False):

    dtype = data.dtype.type

    if not dtype in [np.float32,np.uint16]:
        print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
        return

    if dtype == np.float32:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
    else:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i"%(FS,BS))


    img = dev.createImage_like(data)
    buf = dev.createBuffer(data.size,dtype = dtype)
    dev.writeImage(img,data)


    proc.runKernel("nlm3_thresh",img.shape,None,img,buf,
                   np.int32(img.width),np.int32(img.height),
                   np.float32(sigma),np.float32(thresh))

    return dev.readBuffer(buf,dtype = dtype).reshape(data.shape)
Esempio n. 9
0
def nlm3_thresh(dev, data, FS,BS, sigma, thresh= 0, mean = False):

    dtype = data.dtype.type

    if not dtype in [np.float32,np.uint16]:
        print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
        return

    if dtype == np.float32:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
    else:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm3_thresh.cl"),options="-D FS=%i -D BS=%i"%(FS,BS))


    img = dev.createImage_like(data)
    buf = dev.createBuffer(data.size,dtype = dtype)
    dev.writeImage(img,data)


    proc.runKernel("nlm3_thresh",img.shape,None,img,buf,
                   np.int32(img.width),np.int32(img.height),
                   np.float32(sigma),np.float32(thresh))

    return dev.readBuffer(buf,dtype = dtype).reshape(data.shape)
Esempio n. 10
0
    def testReadHeaderFile(self):
        filePath = utils.absPath("tests/data/header.txt")
        actDict,err =req.Req().readHeaderFile(filePath)
        del actDict["Cookie"]
        self.assertIsNone(err)
        expDict={
            "Host": "login.taobao.com",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
            "Accept-Encoding": "gzip, deflate, br",

            "Content-Type": "application/x-www-form-urlencoded",
            "Content-Length": "2786",
            "Origin": "https://login.taobao.com",
            "Connection": "keep-alive",
            "Referer": "https://login.taobao.com/member/login.jhtml?redirectURL=https%3A%2F%2Fwww.taobao.com%2F",
            
            "Upgrade-Insecure-Requests": "1",
            "Pragma": "no-cache",
            "Cache-Control": "no-cache"
        }
        self.assertEqual(actDict,expDict)
Esempio n. 11
0
def tv3_gpu(data,weight,Niter=50, Ncut = 1, dev = None):
    """
    chambolles tv regularized denoising on the gpu

    weight should be around  2+1.5*noise_sigma
    """

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")


    proc = OCLProcessor(dev,utils.absPath("kernels/tv_chambolle.cl"))

    if Ncut ==1:
        inImg = dev.createImage(data.shape[::-1],dtype = np.float32)

        pImgs = [ dev.createImage(data.shape[::-1],
                                  mem_flags = cl.mem_flags.READ_WRITE,
                                  dtype= np.float32,
                                  channel_order = cl.channel_order.RGBA)
                                  for i in range(2)]

        outImg = dev.createImage(data.shape[::-1],
                                 dtype = np.float32,
                                 mem_flags = cl.mem_flags.READ_WRITE)


        dev.writeImage(inImg,data.astype(np.float32));
        dev.writeImage(pImgs[0],np.zeros((4,)+data.shape,dtype=np.float32));
        dev.writeImage(pImgs[1],np.zeros((4,)+data.shape,dtype=np.float32));


        for i in range(Niter):
            proc.runKernel("div_step",inImg.shape,None,
                           inImg,pImgs[i%2],outImg)
            proc.runKernel("grad_step",inImg.shape,None,
                           outImg,pImgs[i%2],pImgs[1-i%2],
                           np.float32(weight))
        return dev.readImage(outImg,dtype=np.float32)

    else:
        res = np.empty_like(data,dtype=np.float32)
        Nz,Ny,Nx = data.shape
        # a heuristic guess: Npad = Niter means perfect
        Npad = 1+Niter/2
        for i0,(i,j,k) in enumerate(product(range(Ncut),repeat=3)):
            print "calculating box  %i/%i"%(i0+1,Ncut**3)
            sx = slice(i*Nx/Ncut,(i+1)*Nx/Ncut)
            sy = slice(j*Ny/Ncut,(j+1)*Ny/Ncut)
            sz = slice(k*Nz/Ncut,(k+1)*Nz/Ncut)
            sx1,sx2 = utils._extended_slice(sx,Nx,Npad)
            sy1,sy2 = utils._extended_slice(sy,Ny,Npad)
            sz1,sz2 = utils._extended_slice(sz,Nz,Npad)

            data_sliced = data[sz1,sy1,sx1].copy()
            _res = tv3_gpu(dev,data_sliced,weight,Niter,Ncut = 1)
            res[sz,sy,sx] = _res[sz2,sy2,sx2]

        return res
Esempio n. 12
0
def nlm3_fast(data,FS,BS,sigma,dev = None, proc = None):
    """for noise level (and FS,BS = 2,3) of sigma_0, choose sigma = 1.1*sigma_0
    """
    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")

    dtype = data.dtype.type

    if not dtype in [np.float32,np.uint16]:
        print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
        return

    if dtype == np.float32:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
    else:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i "%(FS,BS))

    Nz,Ny, Nx = data.shape

    inImg = dev.createImage_like(data,
                             mem_flags = "READ_ONLY")


    distImg = dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    tmpImg =  dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    tmp2Img =  dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    accBuf = dev.createBuffer(Nx*Ny*Nz,
                             mem_flags = cl.mem_flags.READ_WRITE,
                             dtype = np.float32)

    weightBuf = dev.createBuffer(Nx*Ny*Nz,
                             mem_flags = cl.mem_flags.READ_WRITE,
                             dtype = np.float32)


    dev.writeImage(inImg,data);
    dev.writeBuffer(weightBuf,np.zeros((Ny,Nx,Nz),dtype=np.float32));

    tdist = 0
    tconv = 0
    tcomp =0

    from time import time
    for dx in range(BS+1):
        for dy in range(-BS,BS+1):
            for dz in range(-BS,BS+1):
                t = time()
                proc.runKernel("dist",(Nx,Ny,Nz),None,inImg,tmpImg,np.int32(dx),np.int32(dy),np.int32(dz)).wait()
                tdist += time()-t

                t = time()

                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,tmp2Img,np.int32(1))
                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmp2Img,tmpImg,np.int32(2))
                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,distImg,np.int32(4)).wait()
                tconv += time()-t

                t = time()

                proc.runKernel("computePlus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
                               np.int32(Nx),np.int32(Ny),np.int32(Nz),
                               np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma))

                if any([dx,dy,dz]):
                    proc.runKernel("computeMinus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
                               np.int32(Nx),np.int32(Ny),np.int32(Nz),
                               np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma)).wait()

                tcomp += time()-t

    # print tdist,tconv, tcomp

    acc  = dev.readBuffer(accBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
    weights  = dev.readBuffer(weightBuf,dtype=np.float32).reshape((Nz,Ny,Nx))

    return acc/weights
Esempio n. 13
0
def tv3_gpu(data,weight,Niter=50, Ncut = 1, dev = None):
    """
    chambolles tv regularized denoising on the gpu

    weight should be around  2+1.5*noise_sigma
    """

    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")


    proc = OCLProcessor(dev,utils.absPath("kernels/tv_chambolle.cl"))

    if Ncut ==1:
        inImg = dev.createImage(data.shape[::-1],dtype = np.float32)

        pImgs = [ dev.createImage(data.shape[::-1],
                                  mem_flags = cl.mem_flags.READ_WRITE,
                                  dtype= np.float32,
                                  channel_order = cl.channel_order.RGBA)
                                  for i in range(2)]

        outImg = dev.createImage(data.shape[::-1],
                                 dtype = np.float32,
                                 mem_flags = cl.mem_flags.READ_WRITE)


        dev.writeImage(inImg,data.astype(np.float32));
        dev.writeImage(pImgs[0],np.zeros((4,)+data.shape,dtype=np.float32));
        dev.writeImage(pImgs[1],np.zeros((4,)+data.shape,dtype=np.float32));


        for i in range(Niter):
            proc.runKernel("div_step",inImg.shape,None,
                           inImg,pImgs[i%2],outImg)
            proc.runKernel("grad_step",inImg.shape,None,
                           outImg,pImgs[i%2],pImgs[1-i%2],
                           np.float32(weight))
        return dev.readImage(outImg,dtype=np.float32)

    else:
        res = np.empty_like(data,dtype=np.float32)
        Nz,Ny,Nx = data.shape
        # a heuristic guess: Npad = Niter means perfect
        Npad = 1+Niter/2
        for i0,(i,j,k) in enumerate(product(range(Ncut),repeat=3)):
            print "calculating box  %i/%i"%(i0+1,Ncut**3)
            sx = slice(i*Nx/Ncut,(i+1)*Nx/Ncut)
            sy = slice(j*Ny/Ncut,(j+1)*Ny/Ncut)
            sz = slice(k*Nz/Ncut,(k+1)*Nz/Ncut)
            sx1,sx2 = utils._extended_slice(sx,Nx,Npad)
            sy1,sy2 = utils._extended_slice(sy,Ny,Npad)
            sz1,sz2 = utils._extended_slice(sz,Nz,Npad)

            data_sliced = data[sz1,sy1,sx1].copy()
            _res = tv3_gpu(dev,data_sliced,weight,Niter,Ncut = 1)
            res[sz,sy,sx] = _res[sz2,sy2,sx2]

        return res
Esempio n. 14
0
def nlm3_fast(data,FS,BS,sigma,dev = None, proc = None):
    """for noise level (and FS,BS = 2,3) of sigma_0, choose sigma = 1.1*sigma_0
    """
    if dev is None:
        dev = imgtools.__DEFAULT_OPENCL_DEVICE__

    if dev is None:
        raise ValueError("no OpenCLDevice found...")

    dtype = data.dtype.type

    if not dtype in [np.float32,np.uint16]:
        print "data type %s not supported yet, please convert to:"%[np.float32,np.uint16]
        return

    if dtype == np.float32:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i -D FLOAT"%(FS,BS))
    else:
        proc = OCLProcessor(dev,utils.absPath("kernels/nlm_fast3.cl"),options="-D FS=%i -D BS=%i "%(FS,BS))

    Nz,Ny, Nx = data.shape

    inImg = dev.createImage_like(data,
                             mem_flags = "READ_ONLY")


    distImg = dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    tmpImg =  dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    tmp2Img =  dev.createImage_like(data,
                             mem_flags = "READ_WRITE")

    accBuf = dev.createBuffer(Nx*Ny*Nz,
                             mem_flags = cl.mem_flags.READ_WRITE,
                             dtype = np.float32)

    weightBuf = dev.createBuffer(Nx*Ny*Nz,
                             mem_flags = cl.mem_flags.READ_WRITE,
                             dtype = np.float32)


    dev.writeImage(inImg,data);
    dev.writeBuffer(weightBuf,np.zeros((Ny,Nx,Nz),dtype=np.float32));

    tdist = 0
    tconv = 0
    tcomp =0

    from time import time
    for dx in range(BS+1):
        for dy in range(-BS,BS+1):
            for dz in range(-BS,BS+1):
                t = time()
                proc.runKernel("dist",(Nx,Ny,Nz),None,inImg,tmpImg,np.int32(dx),np.int32(dy),np.int32(dz)).wait()
                tdist += time()-t

                t = time()

                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,tmp2Img,np.int32(1))
                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmp2Img,tmpImg,np.int32(2))
                proc.runKernel("convolve",(Nx,Ny,Nz),None,tmpImg,distImg,np.int32(4)).wait()
                tconv += time()-t

                t = time()

                proc.runKernel("computePlus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
                               np.int32(Nx),np.int32(Ny),np.int32(Nz),
                               np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma))

                if any([dx,dy,dz]):
                    proc.runKernel("computeMinus",(Nx,Ny,Nz),None,inImg,distImg,accBuf,weightBuf,
                               np.int32(Nx),np.int32(Ny),np.int32(Nz),
                               np.int32(dx),np.int32(dy),np.int32(dz),np.float32(sigma)).wait()

                tcomp += time()-t

    # print tdist,tconv, tcomp

    acc  = dev.readBuffer(accBuf,dtype=np.float32).reshape((Nz,Ny,Nx))
    weights  = dev.readBuffer(weightBuf,dtype=np.float32).reshape((Nz,Ny,Nx))

    return acc/weights
Esempio n. 15
0
 def testAbsPath(self):
     print(utils.absPath("tests/data"))
Esempio n. 16
0
 def testReadParamPostFile(self):
     filePath = utils.absPath("tests/data/param.txt")
     actDict =req.Req().readParamPostFile(filePath)
     expDict={
         "TPL_username": "", 
         "TPL_password": "", 
         "ncoSig": "", 
         "ncoSessionid": "", 
         "ncoToken": "81ba8ac206cdbe98463d8af3ba1cd3ec6ecb82b3", 
         "slideCodeShow": "false", 
         "useMobile": "false", 
         "lang": "zh_CN", 
         "loginsite": "0", 
         "newlogin": "******", 
         "TPL_redirect_url": "https://www.taobao.com/", 
         "from": "tb", 
         "fc": "default", 
         "style": "default", 
         "css_style": "", 
         "keyLogin": "******", 
         "qrLogin": "******", 
         "newMini": "false", 
         "newMini2": "false", 
         "tid": "", 
         "loginType": "3", 
         "minititle": "", 
         "minipara": "", 
         "pstrong": "", 
         "sign": "", 
         "need_sign": "", 
         "isIgnore": "", 
         "full_redirect": "", 
         "sub_jump": "", 
         "popid": "", 
         "callback": "", 
         "guf": "", 
         "not_duplite_str": "", 
         "need_user_id": "", 
         "poy": "", 
         "gvfdcname": "10", 
         "gvfdcre": "68747470733A2F2F6C6F67696E2E74616F62616F2E636F6D2F6D656D6265722F6C6F676F75742E6A68746D6C3F73706D3D613231626F2E323031372E3735343839343433372E372E356166393131643939314943676426663D746F70266F75743D7472756526726564697265637455524C3D68747470732533412532462532467777772E74616F62616F2E636F6D253246", 
         "from_encoding": "", 
         "sub": "", 
         "TPL_password_2": "", 
         "loginASR": "1", 
         "loginASRSuc": "1", 
         "allp": "", 
         "oslanguage": "zh-CN", 
         "sr": "1920*1080", 
         "osVer": "", 
         "naviVer": "firefox|71", 
         "osACN": "Mozilla", 
         "osAV": "5.0 (Windows)", 
         "osPF": "Win32", 
         "miserHardInfo": "", 
         "appkey": "00000000", 
         "nickLoginLink": "", 
         "mobileLoginLink": "https://login.taobao.com/member/login.jhtml?redirectURL=https://www.taobao.com/&useMobile=true", 
         "showAssistantLink": "", 
         "um_token": "T90A4D3B40710C978FF908CBCBA0F4420FCB7483FE19BC35EC493E234B5", 
         "ua": ""
     }
     self.assertEqual(actDict,expDict)