def Weiner(f,s,n,cut,p):
w=zeros(f.shape[0])
#print(w.shape)
p = int(4)
inds = [i for i,nu in enumerate(f) if npabs(nu)<cut]
w[inds] = s*nppower(cos(pi/2. * f[inds] / cut) , p)
return w/(w+n)
def pow(x, p):
r"""Power
Usage:
q = pow(x, p) := x ^ p
Arguments:
x = base
p = exponent
"""
return nppower(x, p)
def TSSECluster(j, Cluster, ImageColumn, ImageIn, InitialCluster, NumberOfBands, NumberOfClusters): CountTemporalUnstablePixel = 0 TsseCluster = zeros((1, NumberOfClusters)) for k in range(0, ImageColumn): FlagSwitch = int(max(Cluster[k, :])) #store SSE of related to each pixel if FlagSwitch == 0: CountTemporalUnstablePixel = CountTemporalUnstablePixel + 1 else: TsseCluster[0,FlagSwitch] = TsseCluster[0,FlagSwitch] + npsum(nppower((squeeze(ImageIn[k, 0:NumberOfBands]) - transpose(InitialCluster[FlagSwitch, :])),2)) return([CountTemporalUnstablePixel,TsseCluster,j])
def EuclideanDistance(j, ImageColumn, ImageIn, ImageRow, InitialCluster, NumberOfBands, NumberOfClusters): Cluster = zeros((1, ImageColumn, NumberOfClusters)) CountClusterPixels = zeros((NumberOfClusters, 1)) MeanCluster = zeros((NumberOfClusters, NumberOfBands)) EuclideanDistanceResultant = zeros((1, ImageColumn, NumberOfClusters)) for k in range(0, ImageColumn): temp = ImageIn[k, 0:NumberOfBands] EuclideanDistanceResultant[0, k, :] = npsqrt(npsum(nppower((matlib.repmat(temp, NumberOfClusters, 1)) - InitialCluster, 2), axis=1)) DistanceNearestCluster = min(EuclideanDistanceResultant[0, k, :]) for l in range(0, NumberOfClusters): if DistanceNearestCluster != 0: if DistanceNearestCluster == EuclideanDistanceResultant[0, k, l]: CountClusterPixels[l] = CountClusterPixels[l] + 1 for m in range(0, NumberOfBands): MeanCluster[l, m] = MeanCluster[l, m] + ImageIn[k, m] Cluster[0, k, l] = l return([Cluster,CountClusterPixels,EuclideanDistanceResultant,MeanCluster,j])
def simulate_cb(signal_ft,noise_ft,freqs,times,retardations,transmissions,intensity=1.,photonenergy=600.,angle=0.,amplitude=50.):
collection = nparray([0,1,2],dtype=float)
'''
Same as below for simulate_tof()
Just this time we are inputing the alread built collection_ft etc.
input also the angle of streaking and amplitude of streaking
imput also the retardations and transmissions as vectors
'''
angles = np.arange(retardations.shape[0])*2*pi/float(transmissions.shape[0])
nphotos = 50*gamma(transmissions*nppower(cos(angles),int(2))*intensity).astype(int)
npistars = 50*gamma(transmissions*intensity).astype(int)
nsigstars = 50*gamma(transmissions*intensity).astype(int)
for i in range(retardations.shape[0]):
evec = fillcollection(e_photon = photonenergy,nphotos=nphotos[i],npistars=npistars[i],nsigstars=nsigstars[i])
# d1-3 based on CookieBoxLayout_v2.3.dxf
d1 = 7.6/2.
d2 = 17.6/2.
d3 = 58.4/2.
d3 -= d2
d2 -= d1
sim_times = energy2time(evec,r=retardations[i],d1=d1,d2=d2,d3=d3)
sim_times = append(sim_times,0.) # adds a prompt
signal_colinds = choice(signal_ft.shape[1],sim_times.shape[0])
noise_colinds = choice(noise_ft.shape[1],sim_times.shape[0])
v_simsum_ft = zeros(signal_ft.shape[0],dtype=complex)
for i,t in enumerate(sim_times):
v_simsum_ft += signal_ft[:,signal_colinds[i]] * fourier_delay(freqs,t)
v_simsum_ft += noise_ft[:,noise_colinds[i]]
v_simsum = real(IFFT(v_simsum_ft,axis=0))
if collection.shape[0] < v_simsum.shape[0]:
collection = times
collection = column_stack((collection,v_simsum))
return collection
def pow(x, p):
return nppower(x, p)
def fillimpulseresponses(printfiles = True,samplefiles = False):
(s_collection_ft,n_collection_ft) = (nparray([0,0,0],dtype=complex),nparray([0,0,0],dtype=complex))
filepath = '../data_fs/ave1/'
filematch = filepath + 'C1--LowPulseHighRes-in-100-out1700-an2100--*.txt'
filelist = glob.glob(filematch)
print('filling impulse response files\n\tnum files = %i' % len(filelist))
for i,f in enumerate(filelist):
## processing images
## samplefiles = False
m = re.search('(.+).txt$',f)
if (i%10 == 0 and samplefiles):
outname_spect = m.group(1) + '.spect.dat'
outname_time = m.group(1) + '.time.dat'
outname_simTOF = m.group(1) + '.simTOF.dat'
fi = open(f, "r")
for passline in range(6):
headline = '# ' + fi.readline()
(t,v) = fi.readline().split()
v_vec=nparray(float(v),dtype=float)
t_vec=nparray(float(t)*1.e9,dtype=float)
for line in fi:
(t,v) = line.split()
v_vec = row_stack((v_vec,float(v)))
t_vec = row_stack((t_vec,float(t)*1.e9))
fi.close()
#Get the mean time-step for sake of frequencies
dt = mean(diff(t_vec,n=1,axis=0))
#FFT the vector
v_vec_ft = FFT(v_vec,axis=0)
f = FREQ(v_vec_ft.shape[0],dt)
m_extend = 10
f_extend = FREQ(v_vec_ft.shape[0]*m_extend,dt)
t_extend = arange(0,((t_vec[-1]-t_vec[0])+dt)*m_extend,dt)
# deep copy for the noise extimation
n_vec_ft = npcopy(v_vec_ft)
# find indices where there is only noise in the power, and indices with predominantly signal
# replace the signal elements in the noise vector with a random sampling from the noise portion
chooseinds = nparray([i for i,nu in enumerate(f) if (npabs(nu)> 6.5 and npabs(nu)<(20))])
replaceinds = nparray([i for i,nu in enumerate(f) if npabs(nu)< 6.5])
values = choice(n_vec_ft[chooseinds,0],len(replaceinds))
n_vec_ft[replaceinds,0] = values
## build noise vector and add to n_collection_ft
# sort inds for f and use for interp to extend noise in fourier domain
inds = argsort(f)
n_vec_extend_ft_r = interp(f_extend,f[inds],npabs(n_vec_ft[inds,0]))
n_vec_extend_ft_phi = choice(npangle(n_vec_ft[:,0]),f_extend.shape[0])
n_vec_extend_ft = nprect(n_vec_extend_ft_r,n_vec_extend_ft_phi)
n_vec_extend_ft.shape = (n_vec_extend_ft.shape[0],1)
if n_collection_ft.shape[0] < n_vec_extend_ft.shape[0]:
n_collection_ft = npcopy(n_vec_extend_ft)
# s_collection_ft.shape = (s_collection_ft.shape[0],1)
else:
n_collection_ft = column_stack((n_collection_ft,n_vec_extend_ft))
## build signal vector and add to n_collection_ft
noiseamp = nppower(mean(npabs(values)),int(2))
sigamp = nppower(mean(nparray([i for i,nu in enumerate(f) if npabs(nu)< 1.0])),int(2))
s_vec_ft = npcopy(v_vec_ft)
s_vec_ft[:,0] *= Weiner(f,sigamp,noiseamp,cut = 5,p = 4) * fourier_delay(f,-40) ## Weiner filter and dial back by 40 ns
if samplefiles:
out = column_stack((f,npabs(v_vec_ft),npabs(n_vec_ft),npabs(s_vec_ft)))
savetxt(outname_spect,out,fmt='%.4f')
s_vec = real(IFFT(s_vec_ft,axis=0))
s_vec_extend = zeros((f_extend.shape[0],1),dtype=float)
s_vec_extend[:s_vec.shape[0],0] = s_vec[:,0]
s_vec_extend_ft = FFT(s_vec_extend,axis=0)
if s_collection_ft.shape[0] < s_vec_extend_ft.shape[0]:
s_collection_ft = npcopy(s_vec_extend_ft)
# s_collection_ft.shape = (s_collection_ft.shape[0],1)
else:
s_collection_ft = column_stack((s_collection_ft,s_vec_extend_ft))
# first sum all the Weiner filtered and foureir_delay() signals, then add the single noise vector back
if printfiles:
outpath = '../data_fs/extern/'
filename = outpath + 'signal_collection_ft'
npsave(filename,s_collection_ft)
filename = outpath + 'noise_collection_ft'
npsave(filename,n_collection_ft)
filename = outpath + 'frequencies_collection'
npsave(filename,f_extend)
filename = outpath + 'times_collection'
npsave(filename,t_extend)
return (s_collection_ft,n_collection_ft,f_extend,t_extend)
def Chip_Classify(ImageLocation,SaveLocation,ImageFile,NumberOfClusters,InitialCluster):
ticOverall = time.time()
#sleep(random.beta(1,1)*30)
# Reshape InitialCluster
InitialCluster = array(InitialCluster).reshape((NumberOfClusters,-1))
ImageIn = imread(ImageFile)
with rio.open(ImageFile) as gtf_img:
Info = gtf_img.profile
Info.update(dtype=rio.int8)
#print(time.time()-tic)
ImageRow, ImageColumn, NumberOfBands = ImageIn.shape
if NumberOfBands > 8:
NumberOfBands = NumberOfBands - 1
# prealocate
Cluster = zeros((ImageRow, ImageColumn, NumberOfClusters))
CountClusterPixels = zeros((NumberOfClusters, 1))
MeanCluster = zeros((NumberOfClusters, NumberOfBands))
EuclideanDistanceResultant = zeros((ImageRow, ImageColumn, NumberOfClusters))
#os.mkdir('local/larry.leigh.temp/')
directory = '/tmp/ChipS'
if not os.path.exists(directory):
os.makedirs(directory)
print('starting big loop')
tic = time.time()
for j in range(0,ImageRow):
# if(j % 10 == 0):
# progbar(j, ImageRow)
for k in range(0, ImageColumn):
temp = ImageIn[j, k, 0:NumberOfBands]
#EuclideanDistanceResultant[j, k, :] = np.npsqrt(np.npsum(np.nppower(np.subtract(np.matlib.repmat(temp, NumberOfClusters, 1), InitialCluster[: ,:]), 2), axis = 1))
EuclideanDistanceResultant[j, k, :] = npsqrt(npsum(nppower((matlib.repmat(temp, NumberOfClusters, 1)) - InitialCluster, 2), axis=1))
DistanceNearestCluster = min(EuclideanDistanceResultant[j, k, :])
#print(str(j) +" "+ str(k))
for l in range(0, NumberOfClusters):
if DistanceNearestCluster != 0:
if DistanceNearestCluster == EuclideanDistanceResultant[j, k, l]:
CountClusterPixels[l] = CountClusterPixels[l] + 1
for m in range(0, NumberOfBands):
MeanCluster[l, m] = MeanCluster[l, m] + ImageIn[j, k, m]
Cluster[j, k, l] = l
# progbar(ImageRow, ImageRow)
print('\n')
# print(Cluster.shape)
# print(CountClusterPixels.shape)
# print(EuclideanDistanceResultant.shape)
# print(MeanCluster.shape)
print('\nfinished big loop')
ImageDisplay = npsum(Cluster, axis = 2)
print("Execution time: " + str(time.time() - tic))
#print(globals())
#shelver("big.loop",['Cluster','CountClusterPixels','EuclideanDistanceResultant','MeanCluster'])
savez("big.loop.serial",Cluster=Cluster,
CountClusterPixels=CountClusterPixels,
EuclideanDistanceResultant=EuclideanDistanceResultant,
MeanCluster=MeanCluster)
ClusterPixelCount = count_nonzero(Cluster, axis = 2)
print("Non-zero cluster pixels: " + str(ClusterPixelCount))
#Calculate TSSE within clusters
TsseCluster = zeros((1, NumberOfClusters))
CountTemporalUnstablePixel = 0
# TSSECluster Serial
print("Starting TSSE Cluster computation (Serial version)\n")
tic = time.time()
for j in range(0, ImageRow):
for k in range(0, ImageColumn):
FlagSwitch = int(max(Cluster[j, k, :]))
#print(Cluster[j, k, :]) #This prints to the log
#store SSE of related to each pixel
if FlagSwitch == 0:
CountTemporalUnstablePixel = CountTemporalUnstablePixel + 1
else:
#Might be TsseCluster[0,FlagSwitch-1]
#TsseCluster[0,FlagSwitch - 1] = TsseCluster[0,FlagSwitch - 1] + np.sum(np.power(np.subtract(np.squeeze(ImageIn[j, k, 0:NumberOfBands - 1]), np.transpose(InitialCluster[FlagSwitch - 1, :])),2), axis = 0)
TsseCluster[0,FlagSwitch] = TsseCluster[0,FlagSwitch] + npsum(nppower((squeeze(ImageIn[j, k, 0:NumberOfBands]) - transpose(InitialCluster[FlagSwitch, :])),2))
#count the number of pixels in each cluster
#Collected_ClusterPixelCount[FlagSwitch] = Collected_ClusterPixelCount[FlagSwitch] + 1
Totalsse = npsum(TsseCluster)
print("Execution time: " + str(time.time() - tic))
savez("small.loop.serial",CountTemporalUnstablePixel=CountTemporalUnstablePixel,TsseCluster=TsseCluster)
#get data for final stats....
#calculate the spatial mean and standard deviation of each cluster
ClusterMeanAllBands = zeros((NumberOfClusters, NumberOfBands))
ClusterSdAllBands = zeros((NumberOfClusters, NumberOfBands))
print('finished small loop')
#print(time.time()-tic)
# Cluster Summary Serial
tic = time.time()
FinalClusterMean = zeros(NumberOfBands)
FinalClusterSd = zeros(NumberOfBands)
for i in range(0, NumberOfClusters):
Temp = Cluster[:, :, i]
Temp[Temp == i] = 1
MaskedClusterAllBands = Temp[:,:,None]*ImageIn[:, :, 0:NumberOfBands]
for j in range(0, NumberOfBands):
#Mean = MaskedClusterAllBands(:,:,j)
Temp = MaskedClusterAllBands[:, :, j]
TempNonZero = Temp[npnonzero(Temp)]
TempNonzeronan = TempNonZero[~npisnan(TempNonZero)]
#TempNonan = Temp[!np.isnan(Temp)]
with warnings.catch_warnings():
warnings.filterwarnings('error')
try:
FinalClusterMean[j] = npmean(TempNonZero)
FinalClusterSd[j] = npstd(TempNonZero)
except RuntimeWarning:
FinalClusterMean[j] = 0
FinalClusterSd[j] = 0
ClusterMeanAllBands[i, :] = FinalClusterMean[:]
ClusterSdAllBands[i, :] = FinalClusterSd[:]
print("Execution time: " + str(time.time() - tic))
savez("cluster.summary.serial",ClusterMeanAllBands=ClusterMeanAllBands,ClusterSdAllBands=ClusterSdAllBands)
filename = str(SaveLocation) + 'ImageDisplay_' + ImageFile[len(ImageFile)-32:len(ImageFile)-3] + 'mat'
print('Got filename. Now save the data')
print(filename)
save(filename, ImageDisplay)
filename = str(SaveLocation) + 'ClusterCount' + str(NumberOfClusters) + '_' + ImageFile[len(ImageFile)-32:len(ImageFile)-4] + '.tif'
#geotiffwrite(filename, int8(ImageDisplay), Info.RefMatrix);
with rio.open(filename, 'w', **Info) as dst:
dst.write(int8(ImageDisplay), 1)
filename = str(SaveLocation) + 'Stats_' + ImageFile[len(ImageFile)-32:len(ImageFile)-3] + 'mat'
savez(filename, [MeanCluster, CountClusterPixels, ClusterPixelCount, ClusterMeanAllBands, ClusterSdAllBands, Totalsse])
print('done!')
print(time.time()-ticOverall)