def __init__(self, dataset_size, batch_size, num_batches, rng=None):
if rng is not None:
raise ValueError("non-None rng argument not supported for "
"sequential batch iteration")
assert num_batches is None or num_batches >= 0
self._dataset_size = dataset_size
if batch_size is None:
if num_batches is not None:
batch_size = int(np.ceil(self._dataset_size / num_batches))
else:
raise ValueError("need one of batch_size, num_batches "
"for sequential batch iteration")
elif batch_size is not None:
if num_batches is not None:
max_num_batches = np.ceil(self._dataset_size / batch_size)
if num_batches > max_num_batches:
raise ValueError("dataset of %d examples can only provide "
"%d batches with batch_size %d, but %d "
"batches were requested" %
(self._dataset_size, max_num_batches,
batch_size, num_batches))
else:
num_batches = np.ceil(self._dataset_size / batch_size)
self._batch_size = batch_size
self._num_batches = num_batches
self._next_batch_no = 0
self._idx = 0
self._batch = 0
def remlplen_ichige(fp, fs, dp, ds):
"""Determine the length of the low pass filter with passband frequency
fp, stopband frequency fs, passband ripple dp, and stopband ripple ds.
fp and fs must be normalized with respect to the sampling frequency.
Note that the filter order is one less than the filter length.
References
----------
K. Ichige, M. Iwaki, and R. Ishii, Accurate Estimation of Minimum
Filter Length for Optimum FIR Digital Filters, IEEE Transactions on
Circuits and Systems, 47(10):1008-1017, October 2000.
"""
dF = fs-fp
v = lambda dF,dp:2.325*((-log10(dp))**-0.445)*dF**(-1.39)
g = lambda fp,dF,d:(2.0/pi)*arctan(v(dF,dp)*(1.0/fp-1.0/(0.5-dF)))
h = lambda fp,dF,c:(2.0/pi)*arctan((c/dF)*(1.0/fp-1.0/(0.5-dF)))
Nc = ceil(1.0+(1.101/dF)*(-log10(2.0*dp))**1.1)
Nm = (0.52/dF)*log10(dp/ds)*(-log10(dp))**0.17
N3 = ceil(Nc*(g(fp,dF,dp)+g(0.5-dF-fp,dF,dp)+1.0)/3.0)
DN = ceil(Nm*(h(fp,dF,1.1)-(h(0.5-dF-fp,dF,0.29)-1.0)/2.0))
N4 = N3+DN
return int(N4)
def __cqt_filter_fft(sr, fmin, n_bins, bins_per_octave, tuning,
filter_scale, norm, sparsity, hop_length=None,
window='hann'):
'''Generate the frequency domain constant-Q filter basis.'''
basis, lengths = filters.constant_q(sr,
fmin=fmin,
n_bins=n_bins,
bins_per_octave=bins_per_octave,
tuning=tuning,
filter_scale=filter_scale,
norm=norm,
pad_fft=True,
window=window)
# Filters are padded up to the nearest integral power of 2
n_fft = basis.shape[1]
if (hop_length is not None and
n_fft < 2.0**(1 + np.ceil(np.log2(hop_length)))):
n_fft = int(2.0 ** (1 + np.ceil(np.log2(hop_length))))
# re-normalize bases with respect to the FFT window length
basis *= lengths[:, np.newaxis] / float(n_fft)
# FFT and retain only the non-negative frequencies
fft_basis = fft.fft(basis, n=n_fft, axis=1)[:, :(n_fft // 2)+1]
# sparsify the basis
fft_basis = util.sparsify_rows(fft_basis, quantile=sparsity)
return fft_basis, n_fft, lengths
def __init__(self, dataset_size, batch_size, num_batches=None, rng=None):
self._rng = make_np_rng(rng, which_method=["random_integers",
"shuffle"])
assert num_batches is None or num_batches >= 0
self._dataset_size = dataset_size
if batch_size is None:
if num_batches is not None:
batch_size = int(np.ceil(self._dataset_size / num_batches))
else:
raise ValueError("need one of batch_size, num_batches "
"for sequential batch iteration")
elif batch_size is not None:
if num_batches is not None:
max_num_batches = np.ceil(self._dataset_size / batch_size)
if num_batches > max_num_batches:
raise ValueError("dataset of %d examples can only provide "
"%d batches with batch_size %d, but %d "
"batches were requested" %
(self._dataset_size, max_num_batches,
batch_size, num_batches))
else:
num_batches = np.ceil(self._dataset_size / batch_size)
self._batch_size = batch_size
self._num_batches = int(num_batches)
self._next_batch_no = 0
self._idx = 0
self._batch_order = list(range(self._num_batches))
self._rng.shuffle(self._batch_order)
def _filter_ridge_lines(cwt, ridge_lines, window_size=None, min_length=None,
min_snr=1, noise_perc=10):
"""
Filter ridge lines according to prescribed criteria. Intended
to be used for finding relative maxima.
Parameters
-------------
cwt : 2-D ndarray
Continuous wavelet transform from which
the ridge_lines were defined
ridge_lines: 1-D sequence
Each element should contain 2 sequences, the rows and columns
of the ridge line (respectively)
window_size: int, optional
Size of window to use to calculate noise floor.
Default is `cwt`.shape[1]/20
min_length: int, optional
Minimum length a ridge line needs to be acceptable.
Default is `cwt`.shape[0]/4, ie 1/4th the number of widths.
min_snr: float, optional
Minimum SNR ratio. Default 0. The signal is the value of
the cwt matrix at the shortest length scale (`cwt`[0,loc]), the noise is
the `noise_perc`th percentile of datapoints contained within
a window of `window_size` around `cwt`[0,loc]
noise_perc: float,optional
When calculating the noise floor, percentile of data points
examined below which to consider noise. Calculated using
scipy.stats.scoreatpercentile.
References
----------
Bioinformatics (2006) 22 (17): 2059-2065. doi: 10.1093/bioinformatics/btl355
http://bioinformatics.oxfordjournals.org/content/22/17/2059.long
"""
num_points = cwt.shape[1]
if min_length is None:
min_length = np.ceil(cwt.shape[0] / 4)
if window_size is None:
window_size = np.ceil(num_points / 20)
hf_window = window_size / 2
#Filter based on SNR
row_one = cwt[0, :]
noises = np.zeros_like(row_one)
for ind, val in enumerate(row_one):
window = np.arange(max([ind - hf_window, 0]), min([ind + hf_window, num_points]))
window = window.astype(int)
noises[ind] = scoreatpercentile(row_one[window], per=noise_perc)
def filt_func(line):
if len(line[0]) < min_length:
return False
snr = abs(cwt[line[0][0], line[1][0]] / noises[line[1][0]])
if snr < min_snr:
return False
return True
return filter(filt_func, ridge_lines)
def _setup_paganin(self, height, width):
micron = 10**(-6)
keV = 1000.0
distance = self.parameters['Distance']
energy = self.parameters['Energy'] * keV
resolution = self.parameters['Resolution'] * micron
wavelength = (1240.0 / energy) * 10.0**(-9)
ratio = self.parameters['Ratio']
height1 = height + 2 * self.parameters['Padtopbottom']
width1 = width + 2 * self.parameters['Padleftright']
centery = np.ceil(height1 / 2.0) - 1.0
centerx = np.ceil(width1 / 2.0) - 1.0
# Define the paganin filter
dpx = 1.0 / (width1 * resolution)
dpy = 1.0 / (height1 * resolution)
pxlist = (np.arange(width1) - centerx) * dpx
pylist = (np.arange(height1) - centery) * dpy
pxx = np.zeros((height1, width1), dtype=np.float32)
pxx[:, 0:width1] = pxlist
pyy = np.zeros((height1, width1), dtype=np.float32)
pyy[0:height1, :] = np.reshape(pylist, (height1, 1))
pd = (pxx * pxx + pyy * pyy) * wavelength * distance * math.pi
filter1 = 1.0 + ratio * pd
self.filtercomplex = filter1 + filter1 * 1j
def max_lm(baselines, wavelengths, uwidth, vwidth=0.0):
"""Get the maximum (l,m) that a baseline is sensitive to.
Parameters
----------
baselines : np.ndarray
An array of baselines.
wavelengths : np.ndarray
An array of wavelengths.
uwidth : np.ndarray
Width of the receiver in the u-direction.
vwidth : np.ndarray
Width of the receiver in the v-direction.
Returns
-------
lmax, mmax : array_like
"""
umax = (np.abs(baselines[:, 0]) + uwidth) / wavelengths
vmax = (np.abs(baselines[:, 1]) + vwidth) / wavelengths
mmax = np.ceil(2 * np.pi * umax).astype(np.int64)
lmax = np.ceil((mmax**2 + (2*np.pi*vmax)**2)**0.5).astype(np.int64)
return lmax, mmax
def spectrum(self, shape, surface_point, bound):
"""Returns the counts histogram (bins,counts) for """
wavelengths = []
key = shape.surface_identifier(surface_point)
if not self.store.has_key(key):
return None
entries = self.store[key]
if len(entries) == 0:
return None
for entry in entries:
if entry[2] == bound:
wavelengths.append(float(entry[1]))
if len(wavelengths) is 0:
return None
wavelengths = np.array(wavelengths)
min = wavelengths.min()
max = wavelengths.max()
if len(wavelengths) is 1:
bins = np.arange(np.floor( wavelengths[0] - 1), np.ceil(wavelengths[0] + 2))
freq, bins = np.histogram(wavelengths, bins=bins)
else:
bins = np.arange(np.floor( wavelengths.min()-1), np.ceil(wavelengths.max()+2))
freq, bins = np.histogram(wavelengths, bins=bins)
return Spectrum(bins[0:-1], freq)
def affine_grid(self,Hz,rhoz,Lam):
"""
Get data on regular spatial grid
"""
#First find dimensionless density params
Om0 = 8*pi*rhoz[0]/(3*Hz[0]**2)
OL0 = Lam/(3*Hz[0]**2)
Ok0 = 1-Om0-OL0
#Get t0
t0 = self.get_age(Om0,Ok0,OL0,Hz[0])
#Set affine parameter vals
dvo = uvs(self.z,1/(self.uz**2*Hz),k=3,s=0.0)
vzo = dvo.antiderivative()
vz = vzo(self.z)
vz[0] = 0.0
#Compute grid sizes that gives num error od err
NJ = int(ceil(vz[-1]/sqrt(self.err) + 1))
NI = int(ceil(3.0*(NJ - 1)*(t0 - self.tmin)/vz[-1] + 1))
#Get functions on regular grid
v = linspace(0,vz[-1],NJ)
delv = (v[-1] - v[0])/(NJ-1)
if delv > sqrt(self.err):
print 'delv > sqrt(err)'
Ho = uvs(vz,Hz,s=0.0,k=3)
H = Ho(v)
rhoo = uvs(vz,rhoz,s=0.0,k=3)
rho = rhoo(v)
uo = uvs(vz,self.uz,s=0.0,k=3)
u = uo(v)
u[0] = 1.0
return v,vzo,H,rho,u,NJ,NI,delv,Om0,OL0,Ok0,t0
def _search_fine(sino, srad, step, init_cen, ratio, drop):
"""
Fine search for finding the rotation center.
"""
Nrow, Ncol = sino.shape
centerfliplr = (Ncol + 1.0) / 2.0 - 1.0
# Use to shift the sinogram 2 to the raw CoR.
shiftsino = np.int16(2 * (init_cen - centerfliplr))
_copy_sino = np.roll(np.fliplr(sino[1:]), shiftsino, axis=1)
lefttake = 0
righttake = Ncol - 1
if init_cen <= centerfliplr:
lefttake = np.ceil(srad + 1)
righttake = np.floor(2 * init_cen - srad - 1)
else:
lefttake = np.ceil(
init_cen - (Ncol - 1 - init_cen) + srad + 1)
righttake = np.floor(Ncol - 1 - srad - 1)
Ncol1 = righttake - lefttake + 1
mask = _create_mask(2 * Nrow - 1, Ncol1, 0.5 * ratio * Ncol, drop)
numshift = np.int16((2 * srad + 1.0) / step)
listshift = np.linspace(-srad, srad, num=numshift)
listmetric = np.zeros(len(listshift), dtype='float32')
num1 = 0
for i in listshift:
_sino = ndimage.interpolation.shift(
_copy_sino, (0, i), prefilter=False)
sinojoin = np.vstack((sino, _sino))
listmetric[num1] = np.sum(np.abs(np.fft.fftshift(
pyfftw.interfaces.numpy_fft.fft2(
sinojoin[:, lefttake:righttake + 1]))) * mask)
num1 = num1 + 1
minpos = np.argmin(listmetric)
return init_cen + listshift[minpos] / 2.0
def setupFakePulsar(nodes=range(1, 9), fpgaclk=360e6, frqs=cfs, sideband=-1):
n = np.arange(8)
clk = fpgaclk
if frqs is None:
frqs = (
18e9
- (np.ceil(150e6 / (clk * 4 / 1024.0)) * clk * 4 / 1024.0)
+ ((clk * 2) * (2 * n + 1))
- ((np.ceil(150e6 / (clk * 4 / 1024.0)) * clk * 4 / 1024.0) * n)
)
frqd = dict(zip(n + 1, frqs))
esr = fpgaclk * 8 # effective sample rate
pfb_rate = sideband * esr / (2 * 1024.0)
for node in nodes:
vsd[node].setParams(
EFSAMPFR=esr,
NCHAN=1024,
EXPOSURE=1e-6,
SUB0FREQ=frqd[node],
OBSFREQ=frqd[node],
CHAN_BW=pfb_rate,
FPGACLK=fpgaclk,
) # exposure should be ~0 to get every single spectrum
pass
def interp(pic,flow):
ys=np.arange(pic.shape[0]*pic.shape[1])/pic.shape[1]
ud=(flow[:,:,0].reshape(-1)+ys)%pic.shape[0]
xs=np.arange(pic.shape[0]*pic.shape[1])%pic.shape[1]
lr=(flow[:,:,1].reshape(-1)+xs)%pic.shape[1]
u=np.int32(np.floor(ud))
d=np.int32(np.ceil(ud))%pic.shape[0]
udiffs=ud-u
udiffs=np.dstack((udiffs,udiffs,udiffs))
l=np.int32(np.floor(lr))
r=np.int32(np.ceil(lr))%pic.shape[1]
ldiffs=lr-l
ldiffs=np.dstack((ldiffs,ldiffs,ldiffs))
ul=pic[u,l,:]
ur=pic[u,r,:]
dl=pic[d,l,:]
dr=pic[d,r,:]
udl=ul*(1-udiffs)+dl*udiffs
udr=ur*(1-udiffs)+dr*udiffs
ans=np.zeros(pic.shape)
ans[ys,xs,:]=udl*(1-ldiffs)+udr*ldiffs
return ans
def qwtCanvasClip(canvas, canvasRect):
x1 = np.ceil(canvasRect.left())
x2 = np.floor(canvasRect.right())
y1 = np.ceil(canvasRect.top())
y2 = np.floor(canvasRect.bottom())
r = QRect(x1, y1, x2-x1-1, y2-y1-1)
return canvas.borderPath(r)
def plot_rand(txyxidata, b,X, outfile):
""" Plot stochastic forces and response of x """
me = "LE_Plot.plot_rand: "
if os.path.isfile(outfile): return me+"skip"
t0 = time.time()
showplot = False
t, x, eta, xi = txyxidata
del txyxidata
tmax = np.ceil(t.max())
## Plot walk
fs = 25
winsize = int(tmax/80)
fig, (ax1, ax2, ax3) = plt.subplots(nrows=3, sharex=True)
fig.suptitle(outfile)#+"\n"+str(argv)[1:-1])
envelope_plot(t, xi, winsize, ax=ax1)
ax1.set_ylabel("$\\xi$",fontsize=fs)
envelope_plot(t, eta, winsize, ax=ax2)
ax2.set_ylabel("$\eta$",fontsize=fs)
envelope_plot(t, x, winsize, ax=ax3)
ax3.plot([0,t.max()],[X,X],"k--"); ax3.plot([0,t.max()],[-X,-X],"k--")
ax3.set_xlabel("$t$",fontsize=fs);ax3.set_ylabel("$x$",fontsize=fs)
etalim = np.ceil(abs(eta).max()) ## Not perfect
#fig.tight_layout()
plt.savefig(outfile)
print me+"Plot saved as",outfile
print me+"Plotting random data:",round(time.time()-t0,1),"seconds"
if showplot: plt.show()
plt.close(fig)
return
def allowable_ref_dividers(self, f_ref, require_integer_n, require_fractional_n):
"""
given a reference frequency and whether an integer-N solution is needed,
return a qualified list of reference dividers that are within the limits
of the PLL
"""
# first, we establish a minimum and maximum reference divider modulus....
# if we are forcing an integer-N solution, use those phase-detector frequency limits
int_n_nref_max = (np.floor(f_ref / self.f_pfd_limits_integer_n[0])).astype(int)
int_n_nref_min = (np.ceil(f_ref / self.f_pfd_limits_integer_n[1])).astype(int)
frac_n_nref_max = (np.floor(f_ref / self.f_pfd_limits_fractional_n[0])).astype(int)
frac_n_nref_min = (np.ceil(f_ref / self.f_pfd_limits_fractional_n[1])).astype(int)
if require_integer_n:
ref_divider_min = int_n_nref_min
ref_divider_max = int_n_nref_max
elif require_fractional_n:
ref_divider_min = frac_n_nref_min
ref_divider_max = frac_n_nref_max
else:
ref_divider_min = min(int_n_nref_min, frac_n_nref_min)
ref_divider_max = max(int_n_nref_max, frac_n_nref_max)
n_ref_allowed, n_ref_digital_codes = zip(*self.n_ref_data)
# now, make a list of all the divider moduli in that range from min to max
# making sure that the PLL can do each one
allowable_divs = []
for n_ref_modulus in range(ref_divider_min, ref_divider_max+1):
if n_ref_modulus in n_ref_allowed:
allowable_divs.append(n_ref_modulus)
return allowable_divs
def read_power(file, datadir='data/'):
"""
29-apr-2009/dintrans: coded
t,dat=read_power(name_power_file)
Read a power spectra file like 'data/poweru.dat'
"""
filename = path.join(datadir, file)
infile = open(filename, 'r')
lines = infile.readlines()
infile.close()
#
# find the number of blocks (t,power) that should be read
#
dim=read_dim(datadir=datadir)
nblock=int(len(lines)/int(N.ceil(dim.nxgrid/2/8.)+1))
#
with open(filename, 'r') as infile:
t=N.zeros(1, dtype='Float32')
data=N.zeros(1, dtype='Float32')
for i in range(nblock):
st=infile.readline()
t=N.append(t, float(st))
for ii in range(int(N.ceil(dim.nxgrid/2/8.))):
st=infile.readline()
data=N.append(data, N.asarray(st.split()).astype('f'))
t=t[1:] ; data=data[1:]
nt=len(t) ; nk=int(len(data)/nt)
data=data.reshape(nt, nk)
return t, data
def _scale_to_res(self):
"""Change self._A and _extent to render an image whose
resolution is matched to the eventual rendering."""
ax = self.axes
ext = ax.transAxes.transform([1, 1]) - ax.transAxes.transform([0, 0])
xlim, ylim = ax.get_xlim(), ax.get_ylim()
dx, dy = xlim[1] - xlim[0], ylim[1] - ylim[0]
y0 = max(self.miny, ylim[0] - 5)
y1 = min(self._full_res.shape[0] + self.miny, ylim[1] + 5)
x0 = max(self.minx, xlim[0] - 5)
x1 = min(self._full_res.shape[1] + self.minx, xlim[1] + 5)
y0, y1, x0, x1 = map(int, [y0, y1, x0, x1])
sy = int(max(1, min((y1 - y0) / 5., np.ceil(dy / ext[1]))))
sx = int(max(1, min((x1 - x0) / 5., np.ceil(dx / ext[0]))))
# have we already calculated what we need?
if sx == self._sx and sy == self._sy and \
x0 == self._bounds[0] and x1 == self._bounds[1] and \
y0 == self._bounds[2] and y1 == self._bounds[3]:
return
self._A = self._full_res[y0 - self.miny:y1 - self.miny:sy,
x0 - self.minx:x1 - self.minx:sx]
x1 = x0 + self._A.shape[1] * sx
y1 = y0 + self._A.shape[0] * sy
self.set_extent([x0 - .5, x1 - .5, y0 - .5, y1 - .5])
self._sx = sx
self._sy = sy
self._bounds = (x0, x1, y0, y1)
self.changed()
def plot_scatter_with_histograms(xvals, yvals, colour='k', oneToOneLine=True, xlabel=None, ylabel=None, title=None):
gs = gridspec.GridSpec(5, 5)
xmin = np.floor(min(xvals))
xmax = np.ceil(max(xvals))
ymin = np.floor(min(yvals))
ymax = np.ceil(max(yvals))
plt.subplot(gs[1:, 0:4])
plt.plot(xvals, yvals, 'o', color=colour)
if xlabel is not None:
plt.xlabel(xlabel)
if ylabel is not None:
plt.ylabel(ylabel)
if oneToOneLine:
oneToOneMax = max([max(xvals),max(yvals)])
plt.plot([0,oneToOneMax],[0,oneToOneMax],'b--')
plt.xlim(xmin,xmax)
plt.ylim(ymin,ymax)
plt.subplot(gs[0, 0:4])
plt.hist(xvals, np.linspace(xmin,xmax,50))
plt.axis('off')
plt.subplot(gs[1:,4])
plt.hist(yvals, np.linspace(ymin,ymax,50), orientation='horizontal')
plt.axis('off')
if title is not None:
plt.suptitle(title)
def extract_matched_slices(ax, shape):
"""Determine the slice parameters to use, matched to the screen.
:param ax: Axes object to query. It's extent and pixel size
determine the slice parameters
:param shape: Tuple of the full image shape to slice into. Upper
boundaries for slices will be cropped to fit within
this shape.
:rtype: tulpe of x0, x1, sx, y0, y1, sy
Indexing the full resolution array as array[y0:y1:sy, x0:x1:sx] returns
a view well-matched to the axes' resolution and extent
"""
ext = ax.transAxes.transform([1, 1]) - ax.transAxes.transform([0, 0])
xlim, ylim = ax.get_xlim(), ax.get_ylim()
dx, dy = xlim[1] - xlim[0], ylim[1] - ylim[0]
y0 = int(max(0, ylim[0] - 5))
y1 = int(min(shape[0], ylim[1] + 5))
x0 = int(max(0, xlim[0] - 5))
x1 = int(min(shape[1], xlim[1] + 5))
sy = int(max(1, min((y1 - y0) / 5., np.ceil(dy / ext[1]))))
sx = int(max(1, min((x1 - x0) / 5., np.ceil(dx / ext[0]))))
return x0, x1, sx, y0, y1, sy
def get_spectral_magnitude(y_data,time_data, fs):
tStep = np.max(time_data)/len(time_data)
timeV = np.arange(0, np.max(time_data), tStep)
numsamp = 512 #timeDomainVectorLength(timeV)
if (len(y_data) < numsamp):
y_data = np.resize(y_data, (numsamp,))
window = hann(numsamp)
## setup the fft spectrum arrays
mag_spectrum = np.zeros([numsamp,int(np.ceil(float(len(timeV))/numsamp))])
#print 'time.len= %d, numsamp=%d, loop:%d' % (len(timeV), numsamp, int(np.ceil(float(len(timeV))/numsamp)))
for k in range(0,int(np.ceil(float(len(timeV))/numsamp))):
slice_dat = y_data[k*numsamp:numsamp*(k+1)]
if (len(slice_dat) < numsamp):
if (len(slice_dat) < numsamp/2): # WE DISCARDS LAST SLICE POINTS IF < NUMSAMP/2
break;
slice_dat = np.resize(slice_dat,(numsamp,))
#multiply it with the window and transform it into frequency domain
spectrum_dat = fft(slice_dat*window);
#get the spectrum mag @ each of the 256 frequency points and store it
#print 'k:',k,' spectrum_dat.len:',len(spectrum_dat)
mag_spectrum[:,k]= 20 * np.log10(abs(spectrum_dat))
mag_spectrum[:,k]= abs(spectrum_dat)
#print "fs= %.4g, NFFT= %ld, y_data.shape= %d, mag_spectrum= %dx%d" % (fs, numsamp,np.shape(y_data)[0], np.shape(mag_spectrum)[0],np.shape(mag_spectrum)[1])
## DOUBLE CHECK THE SIZE OF THE MATRIX
avg_fft_foreach = np.mean(mag_spectrum, axis=1)
# print "np.shape(avg_fft_foreach):", np.shape(avg_fft_foreach)
return avg_fft_foreach
def fetch_raster(self, projection, extent, target_resolution):
"""
Fetch SRTM elevation for the given projection and approximate extent.
"""
if not self.validate_projection(projection):
raise ValueError(
'Unsupported projection for the SRTM{} source.'.format(
self._resolution))
min_x, max_x, min_y, max_y = extent
min_x, min_y = np.floor([min_x, min_y])
nx = int(np.ceil(max_x) - min_x)
ny = int(np.ceil(max_y) - min_y)
skip = False
if nx > self._max_tiles[0]:
warnings.warn(
'Required SRTM{} tile count ({}) exceeds maximum ({}). '
'Increase max_nx limit.'.format(self._resolution, nx,
self._max_tiles[0]))
skip = True
if ny > self._max_tiles[1]:
warnings.warn(
'Required SRTM{} tile count ({}) exceeds maximum ({}). '
'Increase max_ny limit.'.format(self._resolution, ny,
self._max_tiles[1]))
skip = True
if skip:
return []
else:
img, _, extent = self.combined(min_x, min_y, nx, ny)
return [LocatedImage(np.flipud(img), extent)]
def check_orbits(p1, t1, p2, t2, tmn, tmx, tol):
n1 = t1 + p1 * np.arange(np.floor((tmn-t1)/p1), np.ceil((tmx-t1)/p1))
n1 = n1[(tmn <= n1) * (n1 <= tmx)]
n2 = t2 + p2 * np.arange(np.floor((tmn-t2)/p2), np.ceil((tmx-t2)/p2))
n2 = n2[(tmn <= n2) * (n2 <= tmx)]
delta = np.fabs(n1[:, None] - n2[None, :])
return max(len(n1), len(n2)) == np.sum(delta < tol)
def each_SASA(sasas,sort_keys,kcat_cut=30,plot=True,meta=None):
num_sims=len(sort_keys)
labels=label_maker(sasas,kcat_cut=kcat_cut,name_list=sort_keys)
base_size = 20.
wide_factor = 1.5
color_dict={True:'r', False:'g', 'maybe':'b', 'wt':'m'}
ncols = int(np.ceil(np.sqrt(num_sims)))
nrows = int(np.ceil(float(num_sims)/ncols))
fig = plt.figure(figsize=(base_size,base_size*(float(nrows)/ncols)/wide_factor))
gs = gridspec.GridSpec(nrows,ncols,hspace=0.65,wspace=0.8)
axes = [plt.subplot(gs[plot_num/ncols,plot_num%ncols]) for plot_num in range(num_sims)]
max_SASA=0;ts_scaling=0.02
for plot_num,ax in enumerate(axes):
SASA=sasas[sort_keys[plot_num]]
ts_sasa=np.sum([SASA['base_sasa'][res]['sasa_vals'] for res in SASA['base_sasa']],axis=0)
name=SASA['name'];activity=labels[plot_num]
ts = np.array(range(len(ts_sasa)))*ts_scaling
ax.plot(ts,ts_sasa,color=color_dict[activity])
ax.set_title(name)
ax.tick_params(axis='y',which='both',left='off',right='off',labelleft='on')
ax.tick_params(axis='x',which='both',bottom='off',top='off',labelbottom='on')
max_SASA=max(max_SASA,max(ts_sasa))
min_SASA=0
if meta:
meta['kcat cut']=kcat_cut
meta['max sasa']=max_sasa
meta['ts scaling']=ts_scaling
else: meta={'kcat cut':kcat_cut,'max sasa':max_SASA,'ts scaling':ts_scaling}
for plot_num,ax in enumerate(axes):
ax.set_ylim(min_SASA,max_SASA)
if plot:
plt.show(block=False)
else: picturesave('fig.each-%s'%plotname,work.plotdir,backup=False,version=True,meta=meta)
def init_log_binned_fx_buckets(self):
# initializes the refex_log_binned_buckets with the vertical log bin values,
# computed based on p and the number of vertices in the graph
max_fx_value = np.ceil(np.log2(self.no_of_vertices) + self.TOLERANCE) # fixing value of p = 0.5,
# In our experiments, we found p = 0.5 to be a sensible choice:
# with each bin containing the bottom half of the remaining nodes.
log_binned_fx_keys = [value for value in xrange(0, int(max_fx_value))]
fx_bucket_size = []
starting_bucket_size = self.no_of_vertices
for idx in np.arange(0.0, max_fx_value):
starting_bucket_size *= self.p
fx_bucket_size.append(int(np.ceil(starting_bucket_size)))
total_slots_in_all_buckets = sum(fx_bucket_size)
if total_slots_in_all_buckets > self.no_of_vertices:
fx_bucket_size[0] -= (total_slots_in_all_buckets - self.no_of_vertices)
log_binned_buckets_dict = dict(zip(log_binned_fx_keys, fx_bucket_size))
for binned_value in sorted(log_binned_buckets_dict.keys()):
for count in xrange(0, log_binned_buckets_dict[binned_value]):
self.refex_log_binned_buckets.append(binned_value)
if len(self.refex_log_binned_buckets) != self.no_of_vertices:
raise Exception("Vertical binned bucket size not equal to the number of vertices!")
def _drawGraticules(self,m,gd):
par = np.arange(np.ceil(gd.ymin),np.floor(gd.ymax)+1,1.0)
mer = np.arange(np.ceil(gd.xmin),np.floor(gd.xmax)+1,1.0)
merdict = m.drawmeridians(mer,labels=[0,0,0,1],fontsize=10,
linewidth=0.5,color='gray',zorder=GRATICULE_ZORDER)
pardict = m.drawparallels(par,labels=[1,0,0,0],fontsize=10,
linewidth=0.5,color='gray',zorder=GRATICULE_ZORDER)
#loop over meridian and parallel dicts, change/increase font, draw ticks
xticks = []
for merkey,mervalue in merdict.items():
merline,merlablist = mervalue
merlabel = merlablist[0]
merlabel.set_family('sans-serif')
merlabel.set_fontsize(12.0)
xticks.append(merline[0].get_xdata()[0])
yticks = []
for parkey,parvalue in pardict.items():
parline,parlablist = parvalue
parlabel = parlablist[0]
parlabel.set_family('sans-serif')
parlabel.set_fontsize(12.0)
yticks.append(parline[0].get_ydata()[0])
#plt.tick_params(axis='both',color='k',direction='in')
plt.xticks(xticks,())
plt.yticks(yticks,())
m.ax.tick_params(direction='out')
def create_mask(Nx,Ny,frac,
rmin = 0.5,
rmax = 2):
"""
create a mask Nx by Ny pixels
frac: 0 <= frac <= 1: fraction of pixels to be covered
"""
mask = numpy.ones((Nx,Ny))
ncovered = 0
goal = frac*Nx*Ny
while ncovered < goal:
x = Nx*numpy.random.random()
y = Ny*numpy.random.random()
r = rmin + numpy.random.random()*(rmax-rmin)
xmin = max(0,int(numpy.floor(x-r)))
xmax = min(Nx,int(numpy.ceil(x+r)))
ymin = max(0,int(numpy.floor(y-r)))
ymax = min(Ny,int(numpy.ceil(y+r)))
for ix in range(xmin,xmax):
for iy in range(ymin,ymax):
if (x-ix)**2 + (y-iy)**2 < r**2:
ncovered += mask[ix,iy]
mask[ix,iy] = 0
return mask
def sample_size_necessary_under_cph(power, ratio_of_participants, p_exp, p_con,
postulated_hazard_ratio, alpha=0.05):
"""
This computes the sample size for needed power to compare two groups under a Cox
Proportional Hazard model.
References:
https://cran.r-project.org/web/packages/powerSurvEpi/powerSurvEpi.pdf
Parameters:
power: power to detect the magnitude of the hazard ratio as small as that specified by postulated_hazard_ratio.
ratio_of_participants: ratio of participants in experimental group over control group.
p_exp: probability of failure in experimental group over period of study.
p_con: probability of failure in control group over period of study
postulated_hazard_ratio: the postulated hazard ratio
alpha: type I error rate
Returns:
n_exp, n_con: the samples sizes need for the experiment and control group, respectively, to achieve desired power
"""
z = lambda p: stats.norm.ppf(p)
m = 1.0 / ratio_of_participants \
* ((ratio_of_participants * postulated_hazard_ratio + 1.0) / (postulated_hazard_ratio - 1.0)) ** 2 \
* (z(1. - alpha / 2.) + z(power)) ** 2
n_exp = m * ratio_of_participants / (ratio_of_participants * p_exp + p_con)
n_con = m / (ratio_of_participants * p_exp + p_con)
return int(np.ceil(n_exp)), int(np.ceil(n_con))
def return_unit_round_neighborhood(self, row, col, radius):
"""Return a list with (row, col, distance) of the units around a unit. This version uses a circle as radius, all the element inside the radius are taken as neighborood.
@param row index of the unit
@param col the column index of the unit
@param radius the radius of the distance to consider
"""
output_list = list()
if(radius <= 0): output_list.append((row, col, 0)); return output_list #return empty if radius=0
#Finding the square around the unit
#with wide=radius using the ceil of radius
row_range_min = row - int(np.ceil(radius))
if(row_range_min < 0): row_range_min = 0
row_range_max = row + int(np.ceil(radius))
if(row_range_max >= self._matrix_size): row_range_max = self._matrix_size - 1
col_range_min = col - int(np.ceil(radius))
if(col_range_min < 0): col_range_min = 0
col_range_max = col + int(np.ceil(radius))
if(col_range_max >= self._matrix_size): col_range_max = self._matrix_size - 1
for row_iter in range(row_range_min, row_range_max+1):
for col_iter in range(col_range_min, col_range_max+1):
#Finding the distances from the BMU
col_distance = np.abs(col - col_iter)
row_distance = np.abs(row - row_iter)
#Pitagora's Theorem to estimate distance
distance = np.sqrt( np.power(col_distance,2) + np.power(row_distance,2) )
#Store the unit only if the distance is
#less than the radius
if(distance <= radius): output_list.append((row_iter, col_iter, distance))
return output_list
def dispims_color(M, border=0, bordercolor=[0.0, 0.0, 0.0], savePath=None, *imshow_args, **imshow_keyargs):
""" Display an array of rgb images.
The input array is assumed to have the shape numimages x numpixelsY x numpixelsX x 3
"""
bordercolor = numpy.array(bordercolor)[None, None, :]
numimages = len(M)
M = M.copy()
for i in range(M.shape[0]):
M[i] -= M[i].flatten().min()
M[i] /= M[i].flatten().max()
height, width, three = M[0].shape
assert three == 3
n0 = numpy.int(numpy.ceil(numpy.sqrt(numimages)))
n1 = numpy.int(numpy.ceil(numpy.sqrt(numimages)))
im = numpy.array(bordercolor)*numpy.ones(
((height+border)*n1+border,(width+border)*n0+border, 1),dtype='<f8')
for i in range(n0):
for j in range(n1):
if i*n1+j < numimages:
im[j*(height+border)+border:(j+1)*(height+border)+border,
i*(width+border)+border:(i+1)*(width+border)+border,:] = numpy.concatenate((
numpy.concatenate((M[i*n1+j,:,:,:],
bordercolor*numpy.ones((height,border,3),dtype=float)), 1),
bordercolor*numpy.ones((border,width+border,3),dtype=float)
), 0)
imshow_keyargs["interpolation"]="nearest"
pylab.imshow(im, *imshow_args, **imshow_keyargs)
if savePath == None:
pylab.show()
else:
pylab.savefig(savePath)
def calc_slit_box_aps_1id(slit_box_corners, inclip=(1, 10, 1, 10)):
"""
Calculate the clip box based on given slip corners.
Parameters
----------
slit_box_corners : np.ndarray
Four corners of the slit box as a 4x2 matrix
inclip : tuple, optional
Extra inclipping to avoid clipping artifacts
Returns
-------
Tuple:
Cliping indices as a tuple of four
(clipFromTop, clipToBottom, clipFromLeft, clipToRight)
"""
return (
np.floor(slit_box_corners[:, 0].min()).astype(
int) + inclip[0], # clip top row
np.ceil(slit_box_corners[:, 0].max()).astype(
int) - inclip[1], # clip bottom row
np.floor(slit_box_corners[:, 1].min()).astype(
int) + inclip[2], # clip left col
np.ceil(slit_box_corners[:, 1].max()).astype(
int) - inclip[3], # clip right col
)
def test_stumpi_self_join_egress():
m = 3
zone = int(np.ceil(m / 4))
seed = np.random.randint(100000)
np.random.seed(seed)
n = 30
T = np.random.rand(n)
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
np.random.seed(seed)
T = np.random.rand(n)
T = pd.Series(T)
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
def evaluate(self, v):
"""
Evaluate coefficients in standard 3D coordinate basis from those in 3D FB basis
:param v: A coefficient vector (or an array of coefficient vectors) in FB basis
to be evaluated. The first dimension must equal `self.count`.
:return x: The evaluation of the coefficient vector(s) `x` in standard 3D
coordinate basis. This is an array whose first three dimensions equal
`self.sz` and the remaining dimensions correspond to dimensions two and
higher of `v`.
"""
# make should the first dimension of v is self.count
v, sz_roll = unroll_dim(v, 2)
v = m_reshape(v, (self.count, -1))
# get information on polar grids from precomputed data
n_theta = np.size(self._precomp['ang_theta_wtd'], 0)
n_phi = np.size(self._precomp['ang_phi_wtd_even'][0], 0)
n_r = np.size(self._precomp['radial_wtd'], 0)
# number of 3D image samples
n_data = np.size(v, 1)
u_even = np.zeros((n_r, int(2 * self.ell_max + 1), n_data,
int(np.floor(self.ell_max / 2) + 1)),
dtype=v.dtype)
u_odd = np.zeros((n_r, int(2 * self.ell_max + 1), n_data,
int(np.ceil(self.ell_max / 2))),
dtype=v.dtype)
# go through each basis function and find corresponding coefficient
# evaluate the radial parts
for ell in range(0, self.ell_max + 1):
k_max_ell = self.k_max[ell]
radial_wtd = self._precomp['radial_wtd'][:, 0:k_max_ell, ell]
ind = self._indices['ells'] == ell
v_ell = m_reshape(v[ind, :], (k_max_ell, (2 * ell + 1) * n_data))
v_ell = radial_wtd @ v_ell
v_ell = m_reshape(v_ell, (n_r, 2 * ell + 1, n_data))
if np.mod(ell, 2) == 0:
u_even[:,
int(self.ell_max - ell):int(self.ell_max + ell + 1), :,
int(ell / 2)] = v_ell
else:
u_odd[:,
int(self.ell_max - ell):int(self.ell_max + ell + 1), :,
int((ell - 1) / 2)] = v_ell
u_even = np.transpose(u_even, (3, 0, 1, 2))
u_odd = np.transpose(u_odd, (3, 0, 1, 2))
w_even = np.zeros((n_phi, n_r, n_data, 2 * self.ell_max + 1),
dtype=v.dtype)
w_odd = np.zeros((n_phi, n_r, n_data, 2 * self.ell_max + 1),
dtype=v.dtype)
# evaluate the phi parts
for m in range(0, self.ell_max + 1):
ang_phi_wtd_m_even = self._precomp['ang_phi_wtd_even'][m]
ang_phi_wtd_m_odd = self._precomp['ang_phi_wtd_odd'][m]
n_even_ell = np.size(ang_phi_wtd_m_even, 1)
n_odd_ell = np.size(ang_phi_wtd_m_odd, 1)
if m == 0:
sgns = (1, )
else:
sgns = (1, -1)
for sgn in sgns:
end = np.size(u_even, 0)
u_m_even = u_even[end - n_even_ell:end, :,
self.ell_max + sgn * m, :]
end = np.size(u_odd, 0)
u_m_odd = u_odd[end - n_odd_ell:end, :,
self.ell_max + sgn * m, :]
u_m_even = m_reshape(u_m_even, (n_even_ell, n_r * n_data))
u_m_odd = m_reshape(u_m_odd, (n_odd_ell, n_r * n_data))
w_m_even = ang_phi_wtd_m_even @ u_m_even
w_m_odd = ang_phi_wtd_m_odd @ u_m_odd
w_m_even = m_reshape(w_m_even, (n_phi, n_r, n_data))
w_m_odd = m_reshape(w_m_odd, (n_phi, n_r, n_data))
w_even[:, :, :, self.ell_max + sgn * m] = w_m_even
w_odd[:, :, :, self.ell_max + sgn * m] = w_m_odd
w_even = np.transpose(w_even, (3, 0, 1, 2))
w_odd = np.transpose(w_odd, (3, 0, 1, 2))
u_even = w_even
u_odd = w_odd
u_even = m_reshape(u_even,
(2 * self.ell_max + 1, n_phi * n_r * n_data))
u_odd = m_reshape(u_odd, (2 * self.ell_max + 1, n_phi * n_r * n_data))
# evaluate the theta parts
w_even = self._precomp['ang_theta_wtd'] @ u_even
w_odd = self._precomp['ang_theta_wtd'] @ u_odd
pf = w_even + 1j * w_odd
pf = m_reshape(pf, (n_theta * n_phi * n_r, n_data))
# perform inverse non-uniformly FFT transformation back to 3D rectangular coordinates
freqs = m_reshape(self._precomp['fourier_pts'],
(3, n_r * n_theta * n_phi, -1))
x = np.zeros((self.sz[0], self.sz[1], self.sz[2], n_data),
dtype=v.dtype)
for isample in range(0, n_data):
x[..., isample] = np.real(anufft3(pf[:, isample], freqs, self.sz))
# return the x with the first three dimensions of self.sz
x = roll_dim(x, sz_roll)
return x
def get_pool_and_conv_props(spacing, patch_size, min_feature_map_size, max_numpool):
"""
:param spacing:
:param patch_size:
:param min_feature_map_size: min edge length of feature maps in bottleneck
:return:
"""
dim = len(spacing)
current_spacing = deepcopy(list(spacing))
current_size = deepcopy(list(patch_size))
pool_op_kernel_sizes = []
conv_kernel_sizes = []
num_pool_per_axis = [0] * dim
while True:
# This is a problem because sometimes we have spacing 20, 50, 50 and we want to still keep pooling.
# Here we would stop however. This is not what we want! Fixed in get_pool_and_conv_propsv2
min_spacing = min(current_spacing)
valid_axes_for_pool = [i for i in range(dim) if current_spacing[i] / min_spacing < 2]
axes = []
for a in range(dim):
my_spacing = current_spacing[a]
partners = [i for i in range(dim) if current_spacing[i] / my_spacing < 2 and my_spacing / current_spacing[i] < 2]
if len(partners) > len(axes):
axes = partners
conv_kernel_size = [3 if i in axes else 1 for i in range(dim)]
# exclude axes that we cannot pool further because of min_feature_map_size constraint
#before = len(valid_axes_for_pool)
valid_axes_for_pool = [i for i in valid_axes_for_pool if current_size[i] >= 2*min_feature_map_size]
#after = len(valid_axes_for_pool)
#if after == 1 and before > 1:
# break
valid_axes_for_pool = [i for i in valid_axes_for_pool if num_pool_per_axis[i] < max_numpool]
if len(valid_axes_for_pool) == 0:
break
#print(current_spacing, current_size)
other_axes = [i for i in range(dim) if i not in valid_axes_for_pool]
pool_kernel_sizes = [0] * dim
for v in valid_axes_for_pool:
pool_kernel_sizes[v] = 2
num_pool_per_axis[v] += 1
current_spacing[v] *= 2
current_size[v] = np.ceil(current_size[v] / 2)
for nv in other_axes:
pool_kernel_sizes[nv] = 1
pool_op_kernel_sizes.append(pool_kernel_sizes)
conv_kernel_sizes.append(conv_kernel_size)
#print(conv_kernel_sizes)
must_be_divisible_by = get_shape_must_be_divisible_by(num_pool_per_axis)
patch_size = pad_shape(patch_size, must_be_divisible_by)
# we need to add one more conv_kernel_size for the bottleneck. We always use 3x3(x3) conv here
conv_kernel_sizes.append([3]*dim)
return num_pool_per_axis, pool_op_kernel_sizes, conv_kernel_sizes, patch_size, must_be_divisible_by
def get_pool_and_conv_props_v2(spacing, patch_size, min_feature_map_size, max_numpool):
"""
:param spacing:
:param patch_size:
:param min_feature_map_size: min edge length of feature maps in bottleneck
:return:
"""
dim = len(spacing)
current_spacing = deepcopy(list(spacing))
current_size = deepcopy(list(patch_size))
pool_op_kernel_sizes = []
conv_kernel_sizes = []
num_pool_per_axis = [0] * dim
kernel_size = [1] * dim
while True:
# exclude axes that we cannot pool further because of min_feature_map_size constraint
valid_axes_for_pool = [i for i in range(dim) if current_size[i] >= 2*min_feature_map_size]
if len(valid_axes_for_pool) < 1:
break
spacings_of_axes = [current_spacing[i] for i in valid_axes_for_pool]
# find axis that are within factor of 2 within smallest spacing
min_spacing_of_valid = min(spacings_of_axes)
valid_axes_for_pool = [i for i in valid_axes_for_pool if current_spacing[i] / min_spacing_of_valid < 2]
# max_numpool constraint
valid_axes_for_pool = [i for i in valid_axes_for_pool if num_pool_per_axis[i] < max_numpool]
if len(valid_axes_for_pool) == 1:
if current_size[valid_axes_for_pool[0]] >= 3 * min_feature_map_size:
pass
else:
break
if len(valid_axes_for_pool) < 1:
break
# now we need to find kernel sizes
# kernel sizes are initialized to 1. They are successively set to 3 when their associated axis becomes within
# factor 2 of min_spacing. Once they are 3 they remain 3
for d in range(dim):
if kernel_size[d] == 3:
continue
else:
if spacings_of_axes[d] / min(current_spacing) < 2:
kernel_size[d] = 3
other_axes = [i for i in range(dim) if i not in valid_axes_for_pool]
pool_kernel_sizes = [0] * dim
for v in valid_axes_for_pool:
pool_kernel_sizes[v] = 2
num_pool_per_axis[v] += 1
current_spacing[v] *= 2
current_size[v] = np.ceil(current_size[v] / 2)
for nv in other_axes:
pool_kernel_sizes[nv] = 1
pool_op_kernel_sizes.append(pool_kernel_sizes)
conv_kernel_sizes.append(deepcopy(kernel_size))
#print(conv_kernel_sizes)
must_be_divisible_by = get_shape_must_be_divisible_by(num_pool_per_axis)
patch_size = pad_shape(patch_size, must_be_divisible_by)
# we need to add one more conv_kernel_size for the bottleneck. We always use 3x3(x3) conv here
conv_kernel_sizes.append([3]*dim)
return num_pool_per_axis, pool_op_kernel_sizes, conv_kernel_sizes, patch_size, must_be_divisible_by
# 指定搜索的超参数的范围===============
weight_decay = 10 ** np.random.uniform(-8, -4)
lr = 10 ** np.random.uniform(-6, -2)
# ================================================
val_acc_list, train_acc_list = __train(lr, weight_decay)
print("val acc:" + str(val_acc_list[-1]) + " | lr:" + str(lr) + ", weight decay:" + str(weight_decay))
key = "lr:" + str(lr) + ", weight decay:" + str(weight_decay)
results_val[key] = val_acc_list
results_train[key] = train_acc_list
# 绘制图形========================================================
print("=========== Hyper-Parameter Optimization Result ===========")
graph_draw_num = 20
col_num = 5
row_num = int(np.ceil(graph_draw_num / col_num))
i = 0
for key, val_acc_list in sorted(results_val.items(), key=lambda x:x[1][-1], reverse=True):
print("Best-" + str(i+1) + "(val acc:" + str(val_acc_list[-1]) + ") | " + key)
plt.subplot(row_num, col_num, i+1)
plt.title("Best-" + str(i+1))
plt.ylim(0.0, 1.0)
if i % 5: plt.yticks([])
plt.xticks([])
x = np.arange(len(val_acc_list))
plt.plot(x, val_acc_list)
plt.plot(x, results_train[key], "--")
i += 1
expt = Experiment(path=r'C:\_Lib\python\slab\instruments\awg\chase',
config_file='config.json')
expt.plotter.clear('dac1')
expt.plotter.clear('dac2')
dac = DAx22000('dac', '1')
im = InstrumentManager()
trig = im['trig']
print(dac.initialize(ext_clk_ref=True))
print(dac.set_clk_freq(freq=0.5e9))
xpts = np.arange(6400)
ypts = np.ceil(2047.5 + 2047.5 * np.sin(2.0 * np.pi * xpts / (32)))
print(ypts)
print(dac.create_single_segment(1, 0, 2047, 2047, ypts, 1))
print(dac.place_mrkr2(1))
print(dac.set_ext_trig(ext_trig=True))
numsegs = 65
segs = []
waveforms = []
def waveform_compressor(waveform):
print("compressing")
for ii in range(numsegs):
def _get_xy_bounding_box(vertex, padding):
"""Returns the xy bounding box of the environment."""
min_ = np.floor(np.min(vertex[:, :2], axis=0) - padding).astype(np.int)
max_ = np.ceil(np.max(vertex[:, :2], axis=0) + padding).astype(np.int)
return min_, max_
def test_stumpi_constant_subsequence_self_join_egress():
m = 3
zone = int(np.ceil(m / 4))
seed = np.random.randint(100000)
np.random.seed(seed)
T = np.concatenate(
(np.zeros(20, dtype=np.float64), np.ones(10, dtype=np.float64)))
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
np.random.seed(seed)
T = np.concatenate(
(np.zeros(20, dtype=np.float64), np.ones(10, dtype=np.float64)))
T = pd.Series(T)
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
def test_stumpi_init_nan_inf_self_join_egress(substitute,
substitution_locations):
m = 3
zone = int(np.ceil(m / 4))
seed = np.random.randint(100000)
# seed = 58638
for substitution_location in substitution_locations:
np.random.seed(seed)
n = 30
T = np.random.rand(n)
if substitution_location == -1:
substitution_location = T.shape[0] - 1
T[substitution_location] = substitute
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
np.random.seed(seed)
T = np.random.rand(n)
if substitution_location == -1:
substitution_location = T.shape[0] - 1
T[substitution_location] = substitute
T = pd.Series(T)
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
def test_stumpi_identical_subsequence_self_join_egress():
m = 3
zone = int(np.ceil(m / 4))
seed = np.random.randint(100000)
np.random.seed(seed)
identical = np.random.rand(8)
T = np.random.rand(20)
T[1:1 + identical.shape[0]] = identical
T[11:11 + identical.shape[0]] = identical
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P,
comp_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P,
comp_left_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P,
comp_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P,
comp_left_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
np.random.seed(seed)
identical = np.random.rand(8)
T = np.random.rand(20)
T[1:1 + identical.shape[0]] = identical
T[11:11 + identical.shape[0]] = identical
T = pd.Series(T)
ref_mp = naive.stumpi_egress(T, m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T, m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P,
comp_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P,
comp_left_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_left_I, comp_left_I)
for i in range(34):
t = np.random.rand()
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P,
comp_P,
decimal=config.STUMPY_TEST_PRECISION)
# npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P,
comp_left_P,
decimal=config.STUMPY_TEST_PRECISION)
def readBED(basefilename,
useMAFencoding=False,
blocksize=1,
start=0,
nSNPs=SP.inf,
startpos=None,
endpos=None,
order='F',
standardizeSNPs=False,
ipos=2,
bim=None,
fam=None):
'''
read [basefilename].bed,[basefilename].bim,[basefilename].fam
--------------------------------------------------------------------------
Input:
basefilename : string of the basename of [basename].bed, [basename].bim,
and [basename].fam
blocksize : load blocksize SNPs at a time (default 1)
start : index of the first SNP to be loaded from the .bed-file
(default 0)
nSNPs : load nSNPs from the .bed file (default SP.inf, meaning all)
startpos : starting position of the loaded genomic region[chr,bpdist]
endpos : end-position of the loaded genomic region [chr,bpdist]
order : memory layout of the returned SNP array (default 'F')
'F' : Fortran-style column-major array (SNP-major)
'C' : C-style row-major array (individual-major)
standardizeSNPs : bool indeicator if the resulting SNP array is supposed to
be zero-mean and unit-vatiance with mean imputed missing
values (default False)
ipos : the index of the position index to use (default 2)
1 : genomic distance
2 : base-pair distance
useMAFencoding : if set to one, the minor allele is encoded with 2, the major allele with 0.
otherwise, the plink coding is used (default False).
--------------------------------------------------------------------------
Output dictionary:
'rs' : [S] array rs-numbers
'pos' : [S*3] array of positions [chromosome, genetic dist, basepair dist]
'snps' : [N*S] array of snp-data
'iid' : [N*2] array of family IDs and individual IDs
--------------------------------------------------------------------------
'''
if bim is None: bim = readBIM(basefilename, usecols=(0, 1, 2, 3))
if fam is None: fam = readFAM(basefilename, usecols=(0, 1))
rs = bim[:, 1]
pos = SP.array(bim[:, (0, 2, 3)], dtype='float')
if startpos is not None:
#pdb.set_trace()
i_c = pos[:, 0] == startpos[0]
i_largerbp = pos[:, ipos] >= startpos[ipos]
start = which(i_c * i_largerbp)
while (start - 1 >= 0 and pos[start - 1, ipos] == startpos[ipos]):
start = start - 1
i_c = pos[:, 0] == endpos[0]
i_smallerbp = pos[:, ipos] >= endpos[ipos]
end = which(i_c * i_smallerbp)
while (end + 1 < pos.shape[0] and pos[end + 1, ipos] == endpos[ipos]):
end = end + 1
nSNPs = end - start
if (nSNPs <= 0) or (end == 0) or (start <= 0):
ret = {
'pos': SP.zeros((0, 3)),
'rs': SP.zeros((0)),
'iid': fam,
'snps': SP.zeros((fam.shape[0], 0))
}
return ret
pass
N = fam.shape[0]
S = bim.shape[0]
S_res = min(S, start + nSNPs)
nSNPs = min(S - start, nSNPs)
#if startpos is not None:
#print("start: " + str(start))
#print("end: " + str(end))
#print("S_res: " + str(S_res))
#print("nSNPs: " + str(nSNPs))
if nSNPs <= 0:
ret = {
'rs': rs[start:start],
'pos': pos[start:start, :],
#'snps' :SNPs[0:N,start:start],
'snps': SP.zeros((N, 0)),
'iid': fam
}
return ret
SNPs = SP.zeros(((SP.ceil(0.25 * N) * 4), nSNPs), order=order)
bed = basefilename + '.bed'
with open(bed, "rb") as f:
mode = f.read(2)
if mode != b'l\x1b':
raise Exception('No valid binary PED file')
mode = f.read(1) #\x01 = SNP major \x00 = individual major
if mode != b'\x01':
raise Exception('only SNP-major is implemented')
startbit = SP.ceil(0.25 * N) * start + 3
f.seek(int(startbit))
for blockStart in SP.arange(0, nSNPs, blocksize, dtype=int):
blockEnd = int(min(S, blockStart + blocksize))
Sblock = min(nSNPs - blockStart, blocksize)
nbyte = int(SP.ceil(0.25 * N) * Sblock)
bytes = SP.array(bytearray(f.read(nbyte))).reshape(
(SP.ceil(0.25 * N), Sblock), order='F')
SNPs[3::4, blockStart:blockEnd][bytes >= 64] = SP.nan
SNPs[3::4, blockStart:blockEnd][bytes >= 128] = 1
SNPs[3::4, blockStart:blockEnd][bytes >= 192] = 2
bytes = SP.mod(bytes, 64)
SNPs[2::4, blockStart:blockEnd][bytes >= 16] = SP.nan
SNPs[2::4, blockStart:blockEnd][bytes >= 32] = 1
SNPs[2::4, blockStart:blockEnd][bytes >= 48] = 2
bytes = SP.mod(bytes, 16)
SNPs[1::4, blockStart:blockEnd][bytes >= 4] = SP.nan
SNPs[1::4, blockStart:blockEnd][bytes >= 8] = 1
SNPs[1::4, blockStart:blockEnd][bytes >= 12] = 2
bytes = SP.mod(bytes, 4)
SNPs[0::4, blockStart:blockEnd][bytes >= 1] = SP.nan
SNPs[0::4, blockStart:blockEnd][bytes >= 2] = 1
SNPs[0::4, blockStart:blockEnd][bytes >= 3] = 2
if 0: #the binary format as described in the documentation (seems wrong)
SNPs[3::4][bytes >= 128] = SP.nan
SNPs[3::4][bytes >= 192] = 1
bytes = SP.mod(bytes, 128)
SNPs[3::4][bytes >= 64] += 1
bytes = SP.mod(bytes, 64)
SNPs[2::4][bytes >= 32] = SP.nan
SNPs[2::4][bytes >= 48] = 1
bytes = SP.mod(bytes, 32)
SNPs[2::4][bytes >= 16] += 1
bytes = SP.mod(bytes, 16)
SNPs[1::4][bytes >= 8] = SP.nan
SNPs[1::4][bytes >= 12] = 1
bytes = SP.mod(bytes, 8)
SNPs[1::4][bytes >= 4] += 1
bytes = SP.mod(bytes, 4)
SNPs[0::4][bytes >= 2] = SP.nan
SNPs[0::4][bytes >= 3] = 1
bytes = SP.mod(bytes, 2)
SNPs[0::4][bytes >= 1] += 1
snps = SNPs[0:N, :]
if useMAFencoding:
imaf = SP.sum(snps == 2, axis=0) > SP.sum(snps == 0, axis=0)
snps[:, imaf] = 2 - snps[:, imaf]
if standardizeSNPs:
snps = standardize(snps)
ret = {
'rs': rs[start:S_res],
'pos': pos[start:S_res, :],
'snps': snps,
'iid': fam
}
return ret
def test_stumpi_stream_nan_inf_self_join_egress(substitute,
substitution_locations):
m = 3
zone = int(np.ceil(m / 4))
seed = np.random.randint(100000)
for substitution_location in substitution_locations:
np.random.seed(seed)
T = np.random.rand(64)
n = 30
ref_mp = naive.stumpi_egress(T[:n], m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T[:n], m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
if substitution_location == -1:
substitution_location = T[30:].shape[0] - 1
T[n:][substitution_location] = substitute
for t in T[n:]:
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
np.random.seed(seed)
T = np.random.rand(64)
ref_mp = naive.stumpi_egress(T[:n], m)
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
stream = stumpi(T[:n], m, egress=True)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
if substitution_location == -1:
substitution_location = T[n:].shape[0] - 1
T[n:][substitution_location] = substitute
for t in T[n:]:
ref_mp.update(t)
stream.update(t)
comp_P = stream.P_.copy()
comp_I = stream.I_
comp_left_P = stream.left_P_.copy()
comp_left_I = stream.left_I_
ref_P = ref_mp.P_.copy()
ref_I = ref_mp.I_
ref_left_P = ref_mp.left_P_.copy()
ref_left_I = ref_mp.left_I_
naive.replace_inf(ref_P)
naive.replace_inf(ref_left_P)
naive.replace_inf(comp_P)
naive.replace_inf(comp_left_P)
npt.assert_almost_equal(ref_P, comp_P)
npt.assert_almost_equal(ref_I, comp_I)
npt.assert_almost_equal(ref_left_P, comp_left_P)
npt.assert_almost_equal(ref_left_I, comp_left_I)
def trunc_and_sum_inplace(function=None, function_uses='points',
sigma=None, points=None, bins=None, center=None, weights=1,
limit=3, method='auto'):
"""Wrap a simple broadening function, evaluate within a consistent range and sum to spectrum
center1 center2
: :
: :
* :
* * :
* * **
* |-| sigma1 + * * + ...
* * * |--| sigma2
* * . * * .
..: * * :.. ..:: * * ::..
-------------------------- -----------------------
|----| |----|
limit * max(sigma) limit * max(sigma)
A spectrum is returned corresponding to the entire input set of
bins/points, but this is zeroed outside of a restricted range (``limit * sigma``).
The range is the same for all peaks, so max(sigma) is used for all.
The purpose of this is performance optimisation compared to evaluating over the whole range.
There is an artefact associated with this: a vertical step down to zero at the range cutoff.
The function will be evaluated with the input points or bins
for each set of (center, sigma) and summed to a single
spectrum. Two technical approaches are provided: a python for-loop
which iterates over the functions and sums/writes to the
appropriate regions of the output array, and a np.histogram dump of
all the frequency/intensity values. The for-loop approach has a
more consistent speed but performance ultimately depends on the array sizes.
:param function: broadening function; this should accept named arguments "center", "sigma" and either "bins" or "points".
:type function: python function
:param function_uses: 'points' or 'bins'; select which type of x data is passed to "function"
:param sigma: widths of broadening functions (passed to "sigma" argument of function)
:type sigma: float or Nx1 array
:param bins: sample bins for function evaluation. This _must_ be evenly-spaced.
:type bins: 1-D array
:param points: regular grid of points for which function should be evaluated.
:type points: 1-D array
:param center: centers of broadening functions
:type center: float or Nx1 array
:param weights: weights of peaks for summation
:type weights: float or array corresponding to "center"
:param limit: range (as multiple of sigma) for cutoff
:type limit: float
:param method:
'auto', 'histogram' or 'forloop'; select between implementations for summation at appropriate
frequencies. 'auto' uses 'forloop' when there are > 1500 frequencies
:type method: str
:returns: (points, spectrum)
:returntype: (1D array, 1D array)
"""
# Select method for summation
# Histogram seems to be faster below 1500 points; tested on a macbook pro and a Xeon workstation.
# This threshold may change depending on updates to hardware, Python and Numpy...
# For now we hard-code the number. It would ideally live in abins.parameters but is very specific to this function.
if method == 'auto' and points.size < 1500:
sum_method = 'histogram'
elif method == 'auto':
sum_method = 'forloop'
else:
sum_method = method
bin_width = bins[1] - bins[0]
if not np.isclose(points[1] - points[0], bin_width):
raise ValueError("Bin spacing and point spacing are not consistent")
freq_range = limit * max(sigma)
nrows = len(center)
ncols = int(np.ceil((freq_range * 2) / bin_width))
if ncols > len(points):
raise ValueError("Kernel is wider than evaluation region; "
"use a smaller cutoff limit or a larger range of points/bins")
# Work out locations of frequency blocks. As these should all be the same size,
# blocks which would exceed array size are "justified" into the array
#
# Start by calculating ideal start positions (allowed to exceed bounds) and
# corresponding frequencies
#
# s-|-x---- |
# | s---x---- |
# | s---x---- |
# | s---x--|-
#
#
start_indices = np.asarray((center - points[0] - freq_range + (bin_width / 2)) // bin_width,
int)
# Values exceeding calculation range would have illegally large "initial guess" indices so clip those to final point
# | | s---x---- -> | s|--x----
start_indices[start_indices > len(points)] = -1
start_freqs = points[start_indices]
# Next identify points which will overshoot left: x lies to left of freq range
# s-|-x-:-- |
# | s-:-x---- |
# | s:--x---- |
# | : s---x--|-
left_justified = center < freq_range
# "Left-justify" these points by setting start position to low limit
# |sx------- |
# | s---x---- |
# | s---x---- |
# | s---x--|-
start_freqs[left_justified] = points[0]
start_indices[left_justified] = 0
# Apply same reasoning to fix regions overshooting upper bound
# Note that the center frequencies do not move: only the grid on which they are evaluated
# |sx------: | |sx------- |
# | s---x--:- | ---> | s---x---- |
# | s---x-:-- | | s---x---- |
# | s--:x--|- | s-----x--|
right_justified = center > (points[-1] - freq_range)
start_freqs[right_justified] = points[-1] - (ncols * bin_width)
start_indices[right_justified] = len(points) - ncols
# freq_matrix is not used in (bins, forloop) mode so only generate if needed
if (function_uses == 'points') or (function_uses == 'bins' and sum_method == 'histogram'):
freq_matrix = start_freqs.reshape(nrows, 1) + np.arange(0, 2 * freq_range, bin_width)
# Dispatch kernel generation depending on x-coordinate scheme
if function_uses == 'points':
kernels = function(sigma=sigma,
points=freq_matrix,
center=center)
elif function_uses == 'bins':
bin_matrix = start_freqs.reshape(nrows, 1) + np.arange(-bin_width / 2, 2 * freq_range + bin_width / 2, bin_width)
kernels = function(sigma=sigma,
bins=bin_matrix,
center=center)
else:
raise ValueError('x-basis "{}" for broadening function is unknown.'.format(function_uses))
# Sum spectrum using selected method
if sum_method == 'histogram':
spectrum, bin_edges = np.histogram(np.ravel(freq_matrix),
bins,
weights=np.ravel(weights * kernels),
density=False)
elif sum_method == 'forloop':
spectrum = np.zeros_like(points)
for start, kernel, weight in zip(start_indices.flatten(), kernels, np.asarray(weights).flatten()):
scaled_kernel = kernel * weight
spectrum[start:start+ncols] += scaled_kernel
else:
raise ValueError('Summation method "{}" is unknown.', format(method))
return points, spectrum
def __init__(self, parent,config,video,shuffle,Dataframe,scorer,savelabeled):
# Settting the GUI size and panels design
displays = (wx.Display(i) for i in range(wx.Display.GetCount())) # Gets the number of displays
screenSizes = [display.GetGeometry().GetSize() for display in displays] # Gets the size of each display
index = 0 # For display 1.
screenWidth = screenSizes[index][0]
screenHeight = screenSizes[index][1]
self.gui_size = (screenWidth*0.7,screenHeight*0.85)
wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = 'DeepLabCut2.0 - Manual Outlier Frame Extraction',
size = wx.Size(self.gui_size), pos = wx.DefaultPosition, style = wx.RESIZE_BORDER|wx.DEFAULT_FRAME_STYLE|wx.TAB_TRAVERSAL )
self.statusbar = self.CreateStatusBar()
self.statusbar.SetStatusText("")
self.SetSizeHints(wx.Size(self.gui_size)) # This sets the minimum size of the GUI. It can scale now!
###################################################################################################################################################
# Spliting the frame into top and bottom panels. Bottom panels contains the widgets. The top panel is for showing images and plotting!
topSplitter = wx.SplitterWindow(self)
self.image_panel = ImagePanel(topSplitter, config,video,shuffle,Dataframe,self.gui_size)
self.widget_panel = WidgetPanel(topSplitter)
topSplitter.SplitHorizontally(self.image_panel, self.widget_panel,sashPosition=self.gui_size[1]*0.83)#0.9
topSplitter.SetSashGravity(1)
sizer = wx.BoxSizer(wx.VERTICAL)
sizer.Add(topSplitter, 1, wx.EXPAND)
self.SetSizer(sizer)
###################################################################################################################################################
# Add Buttons to the WidgetPanel and bind them to their respective functions.
widgetsizer = wx.WrapSizer(orient=wx.HORIZONTAL)
self.load_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help_button_sizer = wx.BoxSizer(wx.VERTICAL)
self.help = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Help")
self.help_button_sizer.Add(self.help , 1, wx.ALL, 15)
# widgetsizer.Add(self.help , 1, wx.ALL, 15)
self.help.Bind(wx.EVT_BUTTON, self.helpButton)
widgetsizer.Add(self.help_button_sizer,1,wx.ALL,0)
self.grab = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Grab Frames")
widgetsizer.Add(self.grab , 1, wx.ALL, 15)
self.grab.Bind(wx.EVT_BUTTON, self.grabFrame)
self.grab.Enable(True)
widgetsizer.AddStretchSpacer(5)
self.slider = wx.Slider(self.widget_panel, id=wx.ID_ANY, value = 0, minValue=0, maxValue=1,size=(200, -1), style=wx.SL_HORIZONTAL | wx.SL_AUTOTICKS | wx.SL_LABELS )
widgetsizer.Add(self.slider,1, wx.ALL,5)
self.slider.Bind(wx.EVT_SLIDER, self.OnSliderScroll)
widgetsizer.AddStretchSpacer(5)
self.start_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.end_frames_sizer = wx.BoxSizer(wx.VERTICAL)
self.start_frames_sizer.AddSpacer(15)
# self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1), min=0, max=120)
self.startFrame = wx.SpinCtrl(self.widget_panel, value='0', size=(100, -1))#,style=wx.SP_VERTICAL)
self.startFrame.Enable(False)
self.start_frames_sizer.Add(self.startFrame,1, wx.EXPAND|wx.ALIGN_LEFT,15)
start_text = wx.StaticText(self.widget_panel, label='Start Frame Index')
self.start_frames_sizer.Add(start_text,1, wx.EXPAND|wx.ALIGN_LEFT,15)
self.checkBox = wx.CheckBox(self.widget_panel, id=wx.ID_ANY,label = 'Range of frames')
self.checkBox.Bind(wx.EVT_CHECKBOX,self.activate_frame_range)
self.start_frames_sizer.Add(self.checkBox,1, wx.EXPAND|wx.ALIGN_LEFT,15)
#
self.end_frames_sizer.AddSpacer(15)
self.endFrame = wx.SpinCtrl(self.widget_panel, value='1', size=(160, -1))#, min=1, max=120)
self.endFrame.Enable(False)
self.end_frames_sizer.Add(self.endFrame,1, wx.EXPAND|wx.ALIGN_LEFT,15)
end_text = wx.StaticText(self.widget_panel, label='Number of Frames')
self.end_frames_sizer.Add(end_text,1, wx.EXPAND|wx.ALIGN_LEFT,15)
self.updateFrame = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Update")
self.end_frames_sizer.Add(self.updateFrame,1, wx.EXPAND|wx.ALIGN_LEFT,15)
self.updateFrame.Bind(wx.EVT_BUTTON, self.updateSlider)
self.updateFrame.Enable(False)
widgetsizer.Add(self.start_frames_sizer,1,wx.ALL,0)
widgetsizer.AddStretchSpacer(5)
widgetsizer.Add(self.end_frames_sizer,1,wx.ALL,0)
widgetsizer.AddStretchSpacer(15)
self.quit = wx.Button(self.widget_panel, id=wx.ID_ANY, label="Quit")
widgetsizer.Add(self.quit , 1, wx.ALL, 15)
self.quit.Bind(wx.EVT_BUTTON, self.quitButton)
self.quit.Enable(True)
self.widget_panel.SetSizer(widgetsizer)
self.widget_panel.SetSizerAndFit(widgetsizer)
# Variables initialization
self.numberFrames = 0
self.currFrame = 0
self.figure = Figure()
self.axes = self.figure.add_subplot(111)
self.drs = []
self.extract_range_frame = False
self.firstFrame = 0
# self.cropping = False
# Read confing file
self.cfg = auxiliaryfunctions.read_config(config)
self.Task = self.cfg['Task']
self.start = self.cfg['start']
self.stop = self.cfg['stop']
self.date = self.cfg['date']
self.trainFraction = self.cfg['TrainingFraction']
self.trainFraction = self.trainFraction[0]
self.videos = self.cfg['video_sets'].keys()
self.bodyparts = self.cfg['bodyparts']
self.colormap = plt.get_cmap(self.cfg['colormap'])
self.colormap = self.colormap.reversed()
self.markerSize = self.cfg['dotsize']
self.alpha = self.cfg['alphavalue']
self.iterationindex=self.cfg['iteration']
self.cropping = self.cfg['cropping']
self.video_names = [Path(i).stem for i in self.videos]
self.config_path = Path(config)
self.video_source = Path(video).resolve()
self.shuffle = shuffle
self.Dataframe = Dataframe
self.scorer = scorer
self.savelabeled = savelabeled
# Read the video file
self.vid = cv2.VideoCapture(str(self.video_source))
self.videoPath = os.path.dirname(self.video_source)
self.filename = Path(self.video_source).name
self.numberFrames = int(self.vid.get(cv2.CAP_PROP_FRAME_COUNT))
self.strwidth = int(np.ceil(np.log10(self.numberFrames)))
# Set the values of slider and range of frames
self.startFrame.SetMax(self.numberFrames-1)
self.slider.SetMax(self.numberFrames-1)
self.endFrame.SetMax(self.numberFrames-1)
self.startFrame.Bind(wx.EVT_SPINCTRL,self.updateSlider)#wx.EVT_SPIN
# Set the status bar
self.statusbar.SetStatusText('Working on video: {}'.format(os.path.split(str(self.video_source))[-1]))
# Adding the video file to the config file.
if not (str(self.video_source.stem) in self.video_names) :
add.add_new_videos(self.config_path,[self.video_source])
self.filename = Path(self.video_source).name
self.update()
self.plot_labels()
self.widget_panel.Layout()
def __getitem__(self, index):
if self.training:
index_ratio = int(self.ratio_index[index])
else:
index_ratio = index
# get the anchor index for current sample index
# here we set the anchor index to the last one
# sample in this group
minibatch_db = [self._roidb[index_ratio]]
blobs = get_minibatch(minibatch_db, self._num_classes,
self.cfg.TRAIN.batch_size, self.cfg.TRAIN.SCALES,
self.cfg.TRAIN.USE_ALL_GT,
self.cfg.PIXEL_MEANS, self.cfg.TRAIN.MAX_SIZE)
data = torch.from_numpy(blobs['data'])
im_info = torch.from_numpy(blobs['im_info'])
# we need to random shuffle the bounding box.
data_height, data_width = data.size(1), data.size(2)
if self.training:
np.random.shuffle(blobs['gt_boxes'])
gt_boxes = torch.from_numpy(blobs['gt_boxes'])
########################################################
# padding the input image to fixed size for each group #
########################################################
# NOTE1: need to cope with the case where a group cover both conditions. (done)
# NOTE2: need to consider the situation for the tail samples. (no worry)
# NOTE3: need to implement a parallel data loader. (no worry)
# get the index range
# if the image need to crop, crop to the target size.
ratio = self.ratio_list_batch[index]
if self._roidb[index_ratio]['need_crop']:
if ratio < 1:
# this means that data_width << data_height, we need to crop the
# data_height
min_y = int(torch.min(gt_boxes[:, 1]))
max_y = int(torch.max(gt_boxes[:, 3]))
trim_size = int(np.floor(data_width / ratio))
if trim_size > data_height:
trim_size = data_height
box_region = max_y - min_y + 1
if min_y == 0:
y_s = 0
else:
if (box_region - trim_size) < 0:
y_s_min = max(max_y - trim_size, 0)
y_s_max = min(min_y, data_height - trim_size)
if y_s_min == y_s_max:
y_s = y_s_min
else:
y_s = np.random.choice(range(y_s_min, y_s_max))
else:
y_s_add = int((box_region - trim_size) / 2)
if y_s_add == 0:
y_s = min_y
else:
y_s = np.random.choice(range(min_y, min_y + y_s_add))
# crop the image
data = data[:, y_s:(y_s + trim_size), :, :]
# shift y coordiante of gt_boxes
gt_boxes[:, 1] = gt_boxes[:, 1] - float(y_s)
gt_boxes[:, 3] = gt_boxes[:, 3] - float(y_s)
# update gt bounding box according the trip
gt_boxes[:, 1].clamp_(0, trim_size - 1)
gt_boxes[:, 3].clamp_(0, trim_size - 1)
else:
# this means that data_width >> data_height, we need to crop the
# data_width
min_x = int(torch.min(gt_boxes[:, 0]))
max_x = int(torch.max(gt_boxes[:, 2]))
trim_size = int(np.ceil(data_height * ratio))
if trim_size > data_width:
trim_size = data_width
box_region = max_x - min_x + 1
if min_x == 0:
x_s = 0
else:
if (box_region - trim_size) < 0:
x_s_min = max(max_x - trim_size, 0)
x_s_max = min(min_x, data_width - trim_size)
if x_s_min == x_s_max:
x_s = x_s_min
else:
x_s = np.random.choice(range(x_s_min, x_s_max))
else:
x_s_add = int((box_region - trim_size) / 2)
if x_s_add == 0:
x_s = min_x
else:
x_s = np.random.choice(range(min_x, min_x + x_s_add))
# crop the image
data = data[:, :, x_s:(x_s + trim_size), :]
# shift x coordiante of gt_boxes
gt_boxes[:, 0] = gt_boxes[:, 0] - float(x_s)
gt_boxes[:, 2] = gt_boxes[:, 2] - float(x_s)
# update gt bounding box according the trip
gt_boxes[:, 0].clamp_(0, trim_size - 1)
gt_boxes[:, 2].clamp_(0, trim_size - 1)
# based on the ratio, padding the image.
if ratio < 1:
# this means that data_width < data_height
trim_size = int(np.floor(data_width / ratio))
padding_data = torch.FloatTensor(int(np.ceil(data_width / ratio)), data_width, 3).zero_()
padding_data[:data_height, :, :] = data[0]
# update im_info
im_info[0, 0] = padding_data.size(0)
elif ratio > 1:
# this means that data_width > data_height
# if the image need to crop.
padding_data = torch.FloatTensor(data_height, int(np.ceil(data_height * ratio)), 3).zero_()
padding_data[:, :data_width, :] = data[0]
im_info[0, 1] = padding_data.size(1)
else:
trim_size = min(data_height, data_width)
padding_data = torch.FloatTensor(trim_size, trim_size, 3).zero_()
padding_data = data[0][:trim_size, :trim_size, :]
gt_boxes[:, :4].clamp_(0, trim_size)
im_info[0, 0] = trim_size
im_info[0, 1] = trim_size
# check the bounding box:
not_keep = (gt_boxes[:, 0] == gt_boxes[:, 2]) | (gt_boxes[:, 1] == gt_boxes[:, 3])
keep = torch.nonzero(not_keep == 0).view(-1)
gt_boxes_padding = torch.FloatTensor(self.max_num_box, gt_boxes.size(1)).zero_()
if keep.numel() != 0:
gt_boxes = gt_boxes[keep]
num_boxes = min(gt_boxes.size(0), self.max_num_box)
gt_boxes_padding[:num_boxes, :] = gt_boxes[:num_boxes]
else:
num_boxes = 0
# permute trim_data to adapt to downstream processing
padding_data = padding_data.permute(2, 0, 1).contiguous()
im_info = im_info.view(3)
return padding_data, im_info, gt_boxes_padding, num_boxes
else:
data = data.permute(0, 3, 1, 2).contiguous().view(3, data_height, data_width)
im_info = im_info.view(3)
gt_boxes = torch.FloatTensor([1, 1, 1, 1, 1])
num_boxes = 0
return data, im_info, gt_boxes, num_boxes
def interpolated_broadening(sigma=None, points=None, bins=None,
center=None, weights=1, is_hist=False, limit=3,
function='gaussian', spacing='sqrt2'):
"""Return a fast estimate of frequency-dependent broadening
Consider a spectrum of two peaks, in the case where (as in indirect-geometry INS) the peak width
increases with frequency.
|
| |
| |
-----------------
In the traditional scheme we broaden each peak individually and combine:
* | *
* | + | * = * *
* * | | * * * * * *
----------------- ----------------- -----------------
Instead of summing over broadening kernels evaluated at each peak, the approximate obtains
a spectrum corresponding to the following scheme:
- For each sigma value, the entire spectrum is convolved with an
appropriate-width broadening function
- At each frequency, the final spectrum value is drawn from the spectrum
broadened with corresponding sigma.
Compared to a summation over broadened peaks, this method introduces an
asymmetry to the spectrum about each peak.
* * * *
* * * * * * * * --> ** *
* * * * * * * * * * * * * * *
----------------- , ----------------- , ----------------- -----------------
This asymmetry should be tolerable as long as the broadening function
varies slowly in frequency relative to its width.
The benefit of this scheme is that we do not need to evaluate the
convolution at every sigma value; nearby spectra can be interpolated.
Trial-and-error finds that with optimal mixing the error of a Gaussian
approximated by mixing a wider and narrower Gaussian is ~ 5% when the sigma
range is factor of 2, and ~ 1% when the sigma range is a factor of sqrt(2).
A pre-optimised transfer function can be used for a fixed ratio between the
reference functions.
:param sigma: widths of broadening functions (passed to "sigma" argument of function)
:type sigma: float or Nx1 array
:param bins: sample bins for function evaluation. This _must_ be evenly-spaced.
:type bins: 1-D array
:param points: regular grid of points for which function should be evaluated.
:type points: 1-D array
:param center: centers of broadening functions
:type center: float or Nx1 array
:param weights: weights of peaks for summation
:type weights: float or array corresponding to "center"
:param is_hist:
If "weights" is already a histogram corresponding to evenly-spaced
frequencies, set this to True to avoid a redundant binning operation.
:type is_hist: bool
:param function: broadening function; currently only 'gaussian' is accepted
:type function: str
:param limit: range (as multiple of sigma) for cutoff
:type limit: float
:param spacing:
Spacing factor between Gaussian samples on log scale. This is not a
free parameter as a pre-computed curve is used for interpolation.
Allowed values: '2', 'sqrt2', with error ~5% and ~1% respectively.
:type spacing: str
:returns: (points, spectrum)
:returntype: (1D array, 1D array)
"""
mix_functions = {'gaussian': {'2': {'lower': [-0.1873, 1.464, -4.079, 3.803],
'upper': [0.2638, -1.968, 5.057, -3.353]},
'sqrt2': {'lower': [-0.6079, 4.101, -9.632, 7.139],
'upper': [0.7533, -4.882, 10.87, -6.746]}}}
log_bases = {'2': 2, 'sqrt2': np.sqrt(2)}
log_base = log_bases[spacing]
# Sample on appropriate log scale: log_b(x) = log(x) / log(b)
n_kernels = int(np.ceil(np.log(max(sigma) / min(sigma)) / np.log(log_base))) + 1
if n_kernels == 1: # Special case: same width everywhere, only need one kernel
sigma_samples = np.array([min(sigma)])
else:
sigma_samples = log_base**np.arange(n_kernels) * min(sigma)
bin_width = bins[1] - bins[0]
# Get set of convolved spectra for interpolation
if is_hist:
hist = weights
else:
hist, _ = np.histogram(center, bins=bins, weights=weights, density=False)
freq_range = 3 * max(sigma)
kernel_npts_oneside = np.ceil(freq_range / bin_width)
if function == 'gaussian':
kernels = mesh_gaussian(sigma=sigma_samples[:, np.newaxis],
points=np.arange(-kernel_npts_oneside, kernel_npts_oneside + 1, 1) * bin_width,
center=0)
else:
raise ValueError('"{}" kernel not supported for "interpolate" broadening method.'.format(function))
spectra = np.array([convolve(hist, kernel, mode='same') for kernel in kernels])
# Interpolate with parametrised relationship
sigma_locations = np.searchsorted(sigma_samples, sigma) # locations in sampled values of points from sigma
spectrum = np.zeros_like(points)
# Samples with sigma == min(sigma) are a special case: copy directly from spectrum
spectrum[sigma_locations==0] = spectra[0, sigma_locations==0]
for i in range(1, len(sigma_samples)):
masked_block = (sigma_locations == i)
sigma_factors = sigma[masked_block] / sigma_samples[i - 1]
lower_mix = np.polyval(mix_functions[function][spacing]['lower'], sigma_factors)
upper_mix = np.polyval(mix_functions[function][spacing]['upper'], sigma_factors)
spectrum[masked_block] = (lower_mix * spectra[i-1, masked_block]
+ upper_mix * spectra[i, masked_block])
return points, spectrum
def main(args):
device = "cuda:0"
with open(DETECTION_JSON) as f:
dataset_dict = json.load(f)
# count the number of valid face detections
n_valid_dets = 0
for synset in sorted(dataset_dict['synsets'].keys()):
for image in sorted(dataset_dict['synsets'][synset]['images'].keys()):
for face in dataset_dict['synsets'][synset]['images'][image][
'faces']:
if face['score'] > DETECTION_THRESHOLD:
n_valid_dets += 1
# prepare the apparent gender model
gender_model = torch.load(
GENDER_MODEL_PTH_FILE) # 0 for female, 1 for male
gender_model.eval()
gender_model.to(device)
n_batches = int(np.ceil(n_valid_dets / float(INFERENCE_BATCH_SIZE)))
batch_no = 1
batch_list = []
for synset in sorted(dataset_dict['synsets'].keys()):
for image in sorted(dataset_dict['synsets'][synset]['images'].keys()):
for face in dataset_dict['synsets'][synset]['images'][image][
'faces']:
if face['score'] > DETECTION_THRESHOLD: # TODO - should we reject any small faces ('w' or 'h' < 20)??
filepath = os.path.join(TRAINING_ROOT,
os.path.join(synset, image))
batch_list.append((filepath, face))
if len(batch_list) == INFERENCE_BATCH_SIZE:
print('Batch', batch_no, 'of', n_batches)
image_batch = read_image_batch(batch_list)
image_batch = torch.from_numpy(image_batch).type(
torch.FloatTensor)
image_batch = image_batch.to(device)
gender_outputs = gender_model(image_batch)
gender_preds = expectation(gender_outputs.cpu())
for gender_pred, batch_item in zip(
gender_preds, batch_list):
filepath = batch_item[0]
synset_filename = filepath.split(TRAINING_ROOT)[1]
synset, filename = synset_filename.split('/')
if 'gender_preds' not in dataset_dict['synsets'][
synset]['images'][filename]:
dataset_dict['synsets'][synset]['images'][
filename]['gender_preds'] = []
dataset_dict['synsets'][synset]['images'][
filename]['gender_preds'].append(gender_pred)
batch_no += 1
batch_list = []
del image_batch
if len(batch_list) > 0:
print('Batch', batch_no, 'of', n_batches)
image_batch = read_image_batch(batch_list)
image_batch = torch.from_numpy(image_batch).type(torch.FloatTensor)
image_batch = image_batch.to(device)
gender_outputs = gender_model(image_batch)
gender_preds = expectation(gender_outputs.cpu())
for gender_pred, batch_item in zip(gender_preds, batch_list):
filepath = batch_item[0]
synset_filename = filepath.split(TRAINING_ROOT)[1]
synset, filename = synset_filename.split('/')
if 'gender_preds' not in dataset_dict['synsets'][synset]['images'][
filename]:
dataset_dict['synsets'][synset]['images'][filename][
'gender_preds'] = []
dataset_dict['synsets'][synset]['images'][filename][
'gender_preds'].append(gender_pred)
del image_batch
out_json = json.dumps(dataset_dict)
with open(OUTFILE, 'w') as f:
f.write(out_json)
return
def __len__(self):
return int(np.ceil(len(self.indexes) / self.batchSize))
import matplotlib.pyplot as p
# print(os.listdir(os.path.dirname(pywt.__file__)))
import cv2 as cv
if __name__ == '__main__':
print("python dwt-for-images.py read.jpg out.jpg outputPrefix")
img=cv.imread(sys.argv[1])
outPref=sys.argv[2].strip()
cv.imwrite("A.jpg",img)
p.show()
dwt=np.ndarray((4,np.ceil(img.shape[0]/2).astype(np.int32),\
np.ceil(img.shape[1]/2).astype(np.int32),img.shape[2]))
for c in range(img.shape[2]):
coeffs = pywt.dwt2(img[:,:,c], 'haar')
for cc in range(4):
if cc==0:
dwt[cc,:,:,c]=coeffs[cc]
else:
dwt[cc, :, :, c] = coeffs[0][cc-1]
# newImg[:,:,c]=coeffs[0]
print(dwt.shape)
newImg=np.ndarray(shape=img.shape,dtype=np.uint8)
zero=np.zeros(dwt[1,:,:,c].shape)
pttr_time = []
for p in pttr[0]:
time = librosa.core.frames_to_time(np.array(
[subbeats[p - pttr[1]], subbeats[p - 1]]),
sr=sr,
n_fft=fft_size,
hop_length=hop_size)
pttr_time.append(time)
pattern_time.append(pttr_time)
pattern_bpm = []
for pt_time in pattern_time:
pt_bpm = []
for p in pt_time:
p_b_start = np.floor((p[0] / 60.0) * audio_test['info'][1])
p_b_end = np.ceil((p[1] / 60.0) * audio_test['info'][1])
pt_bpm.append([
p_b_start + audio_test['info'][2][0],
p_b_end + audio_test['info'][2][0]
])
pt_bpm = sorted(pt_bpm, key=lambda pt: pt[0])
pattern_bpm.append(pt_bpm)
csv_path = file_path + song_list[ind] + '/polyphonic/csv/'
csv_file_path = glob.glob(csv_path + '*')
csv_file = open(csv_file_path[0], 'r')
cs = list(csv.reader(csv_file))
cs_float = [[float(_c) for _c in c[:2]] for c in cs]
csv_file.close()
found = open(
nValid = int(np.floor(0.01 * nTrain))
xValid = xTrain[0:nValid, :, :]
yValid = yTrain[0:nValid, :, :]
xTrain = xTrain[nValid:, :, :]
yTrain = yTrain[nValid:, :, :]
nTrain = xTrain.shape[0]
# Declaring the optimizers for each architectures
optimizer = optim.Adam(model.parameters(), lr=learningRate)
if nTrain < batchSize:
nBatches = 1
batchSize = [nTrain]
elif nTrain % batchSize != 0:
nBatches = np.ceil(nTrain / batchSize).astype(np.int64)
batchSize = [batchSize] * nBatches
while sum(batchSize) != nTrain:
batchSize[-1] -= 1
else:
nBatches = np.int(nTrain / batchSize)
batchSize = [batchSize] * nBatches
batchIndex = np.cumsum(batchSize).tolist()
batchIndex = [0] + batchIndex
epoch = 0 # epoch counter
# Store the training...
lossTrain = []
costTrain = []
lossValid = []
def _make_recordset(self, series, start_time='', end_time='', wavelength='',
segment='', primekey={}, **kwargs):
"""
Take the query arguments and build a record string.
All the primekeys are now stored in primekey dict, including Time and Wavelength
which were passed through pre-defined attributes. The following piece of code,
extracts the passed prime-keys and arranges it in the order as it appears in the
JSOC database.
`pkeys_isTime` is a Pandas DataFrame, whose index values are the Prime-key names
and the column stores a boolean value, identifying whether the prime-key is a
Time-type prime-key or not. Since, time-type prime-keys exist by different names,
we made it uniform in the above piece of code, by storing the time-type primekey
with a single name `TIME`.
Considering an example, if the primekeys that exist for a given series are
['HARPNUM', 'T_OBS', 'WAVELNTH'], we will consider three different cases of the
passed primekeys.
pkeys_isTime.index.values = ['HARPNUM', 'T_OBS', 'WAVELNTH']
Case 1
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI',
'HARPNUM' : '4864',
'WAVELNTH': '605'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{4864}{2014.01.01_00:00:45_TAI}{605}'
Case 2
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI',
'WAVELNTH': '605'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{}{2014.01.01_00:00:45_TAI}{605}'
Case 3
------
primekey = {'T_OBS' : , '2014.01.01_00:00:45_TAI'}
If the primekey dict is as above, then pkstr should be as:
pkstr = '{}{2014.01.01_00:00:45_TAI}'
The idea behind this should be clear. We build up the `pkstr` string
containing the values of the prime-keys passed in the same order as
it occurs in the list `pkeys_isTime.index.values`, i.e. how it is stored
in the online database. Any missing prime-keys should be compensated by
an empty {}, if it occurs before any passed prime-key. Any empty curly braces
that is present at last of the pkstr, can be skipped.
"""
# Extract and format segment
# Convert list of segments into a comma-separated string
if segment:
if isinstance(segment, list):
segment = str(segment)[1:-1].replace(' ', '').replace("'", '')
segment = f'{{{segment}}}'
# Extract and format sample
sample = kwargs.get('sample', '')
if sample:
sample = f'@{sample}s'
# Populate primekeys dict with Time and Wavelength values
if start_time and end_time:
# Check whether any primekey listed in PKEY_LIST_TIME has been passed through
# PrimeKey() attribute. If yes, raise an error, since Time can only be passed
# either through PrimeKey() attribute or Time() attribute.
if not any(x in PKEY_LIST_TIME for x in primekey):
timestr = '{start}-{end}{sample}'.format(
start=start_time.tai.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
end=end_time.tai.strftime("%Y.%m.%d_%H:%M:%S_TAI"),
sample=sample)
else:
error_message = "Time attribute has been passed both as a Time()"\
" and PrimeKey(). Please provide any one of them"\
" or separate them by OR operator."
raise ValueError(error_message)
else:
# This is executed when Time has not been passed through Time() attribute.
# `match` stores all the time-type prime-keys that has been passed through
# PrimeKey() attribute. The length of `match` won't ever be greater than 1,
# but it is a good idea to keep a check.
match = set(primekey.keys()) & PKEY_LIST_TIME
if len(match) > 1:
error_message = "Querying of series, having more than 1 Time-type "\
"prime-keys is not yet supported. Alternative is to "\
"use only one of the primekey to query for series data."
raise ValueError(error_message)
if match:
timestr = '{}'.format(primekey.pop(list(match)[0], ''))
else:
timestr = ''
if wavelength != '':
if not primekey.get('WAVELNTH', ''):
if isinstance(wavelength, list):
wavelength = [int(np.ceil(wave.to(u.AA).value)) for wave in wavelength]
wavelength = str(wavelength)
else:
wavelength = '{}'.format(int(np.ceil(wavelength.to(u.AA).value)))
else:
# This is executed when wavelength has been passed both through PrimeKey()
# and Wavelength().
error_message = "Wavelength attribute has been passed both as a Wavelength()"\
" and PrimeKey(). Please provide any one of them"\
" or separate them by OR operator."
raise ValueError(error_message)
else:
# This is executed when wavelength has been passed through PrimeKey().
wavelength = '{}'.format(primekey.pop('WAVELNTH', ''))
# Populate primekey dict with formatted Time and Wavlength.
if timestr:
primekey['TIME'] = timestr
if wavelength != '':
primekey['WAVELNTH'] = wavelength
# Extract and format primekeys
pkstr = ''
c = drms.Client()
si = c.info(series)
pkeys_isTime = si.keywords.loc[si.primekeys].is_time
for pkey in pkeys_isTime.index.values:
# The loop is iterating over the list of prime-keys existing for the given series.
if len(primekey) > 0:
if pkeys_isTime[pkey]:
pkstr += '[{}]'.format(primekey.pop('TIME', ''))
else:
pkstr += '[{}]'.format(primekey.pop(pkey, ''))
else:
break
# break because we can skip adding {} at the end of pkstr, if the primekey
# dict is empty.
if not pkstr:
# pkstr cannot be totally empty
#
# Note that whilst it is technically posisble to just search by series,
# this is not allowed here, because some of these would be very large
# searches that would make JSOC sad
raise ValueError("Time, Wavelength or an explicit PrimeKey must be specified.")
dataset = '{series}{primekey}{segment}'.format(series=series,
primekey=pkstr,
segment=segment)
return dataset
PI.piezo = 200
print('--------- start acquisition --------')
AI.start_acquisition()
print('--------- start piezo output --------')
for i in range(10):
PI.piezo = i + 2 * 100
status = AI.status
print("status after running acquisition:")
for elem in status:
print(elem, ': ', AI.status[elem])
number_of_reads = int(
np.ceil(1.0 * parameters_Acq['data_length'] /
parameters_Acq['block_size']))
print('number_of_reads', number_of_reads)
data_AI1 = np.zeros((number_of_reads, parameters_Acq['block_size']))
elements_left = -1. * np.ones(number_of_reads)
for i in range(number_of_reads):
fifo_data = AI.data_queue.get()
data_AI1[i, :] = np.array(fifo_data[0])
elements_left[i] = int(fifo_data[2])
print('finished')
print('elements left after each block read from FIFO')
print(elements_left)
vx = np.multiply(bf1, vx1) + np.multiply(bf2, vx2)
vy = np.multiply(bf1, vy1) + np.multiply(bf2, vy2)
bf = np.sqrt(np.power(vx, 2) + np.power(vy, 2))
print('B field min = %.2e max = %.2e' % (bf[np.nonzero(bf)].min(),
bf.max()))
vx, vy = np.divide(vx, bf), np.divide(vy, bf)
fig, ax = plt.subplots(figsize=(3,3))
ax.plot(mesh.posx, mesh.posy, '.k',
marker='.', markersize=3,
color='black', linestyle='None')
# Alternatively, you can manually set the levels
# and the norm:
lev_exp = np.arange(np.floor(np.log10(bf[np.nonzero(bf)].min())),
np.ceil(np.log10(bf.max())), 0.05)
levs = np.power(10, lev_exp)
#levs = np.linspace(bf.min(), bf.max(), 50)
cs = ax.contour(mesh.posx, mesh.posy, bf, levs, norm=colors.LogNorm())
#ax.clabel(cs, cs.levels)
#fig.colorbar(cs)
#ax.quiver(mesh.posx, mesh.posy, vx, vy)
#ax.plot(pos1[0], pos1[1],
# color='red', marker='o', markersize=15)
#ax.plot(pos2[0], pos2[1],
# color='red', marker='o', markersize=15)
fig, ax = plt.subplots(2, 1, figsize=(6,6))
ax[0].plot(mesh.posx[int(nx/2), :], bf[int(nx/2), :])
ax[1].plot(mesh.posy[:, int(ny/2)], bf[:, int(ny/2)])
def createDatabaseFromFeatureset(database,
featureFile,
length,
threshold=20,
mode=None,
label_type=None,
classes=None,
n_classes=None,
oversample=False,
sample_factor=50,
pbc=(1, 1, 1),
noProtons=False):
featureFile = h5py.File(featureFile)
indexes = np.arange(length)
np.random.shuffle(indexes)
labels = []
for i in indexes:
labels.append(featureFile[str(i) + '/label'].value)
hist = np.histogram(labels, 25)
maxdist = []
for i in indexes:
print(i)
# Add Ligand
atom_list = []
property_list = []
ligcoords = featureFile[str(i) + '/ligcoords'].value
ligAtNum = np.char.decode(featureFile[str(i) + '/lignum'].value)
ligFeatures = featureFile[str(i) + '/lig'].value
x = ligcoords[:, 0].mean()
y = ligcoords[:, 1].mean()
z = ligcoords[:, 2].mean()
mean = np.array([x, y, z])
for j in range(len(ligcoords)):
maxdist.append(np.linalg.norm(ligcoords[j] - mean))
if noProtons:
if ligAtNum[j] != 'H':
atom_list.append(ase.Atom(ligAtNum[j], ligcoords[j]))
property_list.append(ligFeatures[j])
else:
atom_list.append(ase.Atom(ligAtNum[j], ligcoords[j]))
property_list.append(ligFeatures[j])
#Add Protein-Atoms in Cutoff-Range
protcoords = featureFile[str(i) + '/protcoords'].value
protAtNum = np.char.decode(featureFile[str(i) + '/protnum'].value)
protFeatures = featureFile[str(i) + '/prot'].value
for j in range(len(protcoords)):
dist = np.linalg.norm(protcoords[j] - mean)
if dist <= threshold:
if noProtons:
if protAtNum[j] != 'H':
atom_list.append(
ase.Atom(protAtNum[j], protcoords[j]))
property_list.append(protFeatures[j])
else:
atom_list.append(ase.Atom(protAtNum[j], protcoords[j]))
property_list.append(protFeatures[j])
# Create Complex
complexe = [ase.Atoms(atom_list, pbc=pbc)]
label = featureFile[str(i) + '/label'].value
affi = PreprocessingSchnet.classLabel(label,
mode,
label_type,
classes=classes,
n_classes=n_classes,
min_v=np.min(labels),
max_v=np.max(labels))
affi[0]['props'] = np.array(property_list)
if not oversample:
database.add_systems(complexe, affi)
else:
classn = np.zeros(25)
for j in range(len(hist[1]) - 1):
if j == len(hist[1]) - 2:
if hist[1][j] <= labels[i] <= hist[1][j + 1]:
classn[j] = 1
else:
if hist[1][j] <= labels[i] < hist[1][j + 1]:
classn[j] = 1
if np.unique(classn, return_counts=True)[1][1] != 1:
print('warning -> Onehot is more than one')
print(classn)
ind = np.argmax(classn)
if hist[0][ind] == 0:
print('Warning -> zero-sample')
continue
n_sampling = int(
np.ceil((1 / hist[0][ind]) * sample_factor * 25))
print(i, len(indexes), n_sampling, ind)
for _ in range(n_sampling):
database.add_systems(complexe, affi)
print(np.max(maxdist))
def load_streaks_from_xml(self,
dataset,
settings,
image_shape_WH,
use_pickle=True,
verbose=True):
'''
Function to load and store the output from the physical simulator.
Streak data is stored in a dictionary. Key: ID, Value: data
'''
print('Reading particles file {}'.format(self.streaks_path_xml))
pickle_version = '1.0'
# Compute the simulation file hash
hasher = hashlib.md5()
with open(self.streaks_path_xml, 'rb') as afile:
buf = afile.read()
hasher.update(buf)
sim_hash = hasher.hexdigest()
pickle_path = self.streaks_path_xml + '.pkl'
if use_pickle and os.path.exists(pickle_path):
print(' loading from pickle')
input = open(pickle_path, 'rb')
pickle_data = pickle.load(input)
# If sim_hash did not change, and image shape is identical too
if 'version' in pickle_data and pickle_data[
'version'] == pickle_version and pickle_data[
'sim_hash'] == sim_hash and np.all(
pickle_data['image_shapeWH'] == image_shape_WH):
self.streaks_simulator = pickle_data['streaks']
input.close()
return
else:
print('Pickle out-dated. Regenerate.')
input.close()
if not os.path.exists(self.streaks_path_xml):
my_utils.print_error("No existing path for XML file (" +
self.streaks_path_xml + ")")
exit(-1)
simulation = parse(self.streaks_path_xml).getroot()
if verbose:
my_utils.print_progress_bar(0, len(simulation))
try:
for fix, frame in enumerate(simulation):
f = Frame()
f.id = int(frame.attrib['id'])
f.exposure_time = int(frame.attrib['t'])
f.starting_time = int(frame.attrib['d'])
f.streaks_count = int(frame.attrib['rs'])
f.streaks = {}
for drop in frame:
s = Streak()
s.pid = int(drop.attrib["pid"])
s.world_position_start = np.array(
drop.attrib["wp1"][1:-1].split(';'), dtype=float)
s.world_position_end = np.array(
drop.attrib["wp2"][1:-1].split(';'), dtype=float)
s.world_diameter_start = float(drop.attrib['wd1'])
s.world_diameter_end = float(drop.attrib['wd2'])
s.image_position_start = np.array(
drop.attrib["ip1"][1:-1].split(';'),
dtype=float) / settings["render_scale"] # x,y
s.image_position_end = np.array(
drop.attrib["ip2"][1:-1].split(';'),
dtype=float) / settings["render_scale"] # x,y
s.image_diameter_start = float(
drop.attrib['iw1']) / settings["render_scale"]
s.image_diameter_end = float(
drop.attrib['iw2']) / settings["render_scale"]
if dataset == 'nuscenes_gan':
# in case the simulation and the rendering are not at the same resolution
r = np.mean((image_shape_WH[0] / 1600,
image_shape_WH[1] / 900))
s.image_position_start = np.array(
drop.attrib["ip1"][1:-1].split(';'),
dtype=float) * r # x,y
s.image_position_end = np.array(
drop.attrib["ip2"][1:-1].split(';'),
dtype=float) * r # x,y
s.image_diameter_start = float(drop.attrib['iw1']) * r
s.image_diameter_end = float(drop.attrib['iw2']) * r
s.image_position_start[
1] = image_shape_WH[1] - s.image_position_start[1]
s.image_position_end[
1] = image_shape_WH[1] - s.image_position_end[1]
s.world_position_start[2] *= -1
s.world_position_end[2] *= -1
diff = abs(s.image_position_start - s.image_position_end)
s.max_width = int(
max(s.image_diameter_start, s.image_diameter_end))
dir1 = np.array([0, -1])
dir2 = diff / np.linalg.norm(diff)
dir2[1] = -dir2[1]
cos_theta = np.dot(dir1, dir2)
actual_length = diff[1] / cos_theta
s.ratio = s.max_width / actual_length
s.image_position_end = s.image_position_end.round().astype(
int)
s.image_position_start = s.image_position_start.round(
).astype(int)
s.length = np.ceil(
np.linalg.norm(s.image_position_start -
s.image_position_end)).astype(int)
s.drop_type = self.classify_drop(s.max_width)
if s.max_width >= 1 and s.length >= 1:
f.streaks.update({s.pid: s})
self.streaks_simulator.update({f.id: f})
if verbose:
my_utils.print_progress_bar(fix + 1, len(simulation))
except Exception as e:
ex_type, ex, tb = sys.exc_info()
my_utils.print_error('Error while parsing XML file.\n\tFile: ' +
self.streaks_path_xml)
traceback.print_tb(tb)
exit(-1)
def create_neuropil_basis(ops, Ly, Lx):
"""
computes neuropil basis functions
Parameters
----------
ops:
ratio_neuropil, tile_factor, diameter, neuropil_type
Ly: int
Lx: int
Returns
-------
S:
basis functions (pixels x nbasis functions)
"""
if 'ratio_neuropil' in ops:
ratio_neuropil = ops['ratio_neuropil']
else:
ratio_neuropil = 6.
if 'tile_factor' in ops:
tile_factor = ops['tile_factor']
else:
tile_factor = 1.
diameter = ops['diameter']
ntilesY = 1+2*int(np.ceil(tile_factor * Ly / (ratio_neuropil * diameter[0]/2))/2)
ntilesX = 1+2*int(np.ceil(tile_factor * Lx / (ratio_neuropil * diameter[1]/2))/2)
ntilesY = np.maximum(2,ntilesY)
ntilesX = np.maximum(2,ntilesX)
yc = np.linspace(1, Ly, ntilesY)
xc = np.linspace(1, Lx, ntilesX)
ys = np.arange(0,Ly)
xs = np.arange(0,Lx)
Kx = np.ones((Lx, ntilesX), 'float32')
Ky = np.ones((Ly, ntilesY), 'float32')
if 1:
# basis functions are fourier modes
for k in range(int((ntilesX-1)/2)):
Kx[:,2*k+1] = np.sin(2*math.pi * (xs+0.5) * (1+k)/Lx)
Kx[:,2*k+2] = np.cos(2*math.pi * (xs+0.5) * (1+k)/Lx)
for k in range(int((ntilesY-1)/2)):
Ky[:,2*k+1] = np.sin(2*math.pi * (ys+0.5) * (1+k)/Ly)
Ky[:,2*k+2] = np.cos(2*math.pi * (ys+0.5) * (1+k)/Ly)
else:
for k in range(ntilesX):
Kx[:,k] = np.cos(math.pi * (xs+0.5) * k/Lx)
for k in range(ntilesY):
Ky[:,k] = np.cos(math.pi * (ys+0.5) * k/Ly)
S = np.zeros((ntilesY, ntilesX, Ly, Lx), np.float32)
for kx in range(ntilesX):
for ky in range(ntilesY):
S[ky,kx,:,:] = np.outer(Ky[:,ky], Kx[:,kx])
S = np.reshape(S,(ntilesY*ntilesX, Ly*Lx))
S = S / np.reshape(np.sum(S**2,axis=-1)**0.5, (-1,1))
S = np.transpose(S, (1, 0)).copy()
S = np.reshape(S, (Ly, Lx, -1))
return S
def plot_images(images, targets, paths=None, fname='images.jpg', names=None, max_size=640, max_subplots=16):
# Plot image grid with labels
if isinstance(images, torch.Tensor):
images = images.cpu().float().numpy()
if isinstance(targets, torch.Tensor):
targets = targets.cpu().numpy()
# un-normalise
if np.max(images[0]) <= 1:
images *= 255
tl = 3 # line thickness
tf = max(tl - 1, 1) # font thickness
bs, _, h, w = images.shape # batch size, _, height, width
bs = min(bs, max_subplots) # limit plot images
ns = np.ceil(bs ** 0.5) # number of subplots (square)
# Check if we should resize
scale_factor = max_size / max(h, w)
if scale_factor < 1:
h = math.ceil(scale_factor * h)
w = math.ceil(scale_factor * w)
colors = color_list() # list of colors
mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init
for i, img in enumerate(images):
if i == max_subplots: # if last batch has fewer images than we expect
break
block_x = int(w * (i // ns))
block_y = int(h * (i % ns))
img = img.transpose(1, 2, 0)
if scale_factor < 1:
img = cv2.resize(img, (w, h))
mosaic[block_y:block_y + h, block_x:block_x + w, :] = img
if len(targets) > 0:
image_targets = targets[targets[:, 0] == i]
boxes = xywh2xyxy(image_targets[:, 2:6]).T
classes = image_targets[:, 1].astype('int')
labels = image_targets.shape[1] == 6 # labels if no conf column
conf = None if labels else image_targets[:, 6] # check for confidence presence (label vs pred)
if boxes.shape[1]:
if boxes.max() <= 1.01: # if normalized with tolerance 0.01
boxes[[0, 2]] *= w # scale to pixels
boxes[[1, 3]] *= h
elif scale_factor < 1: # absolute coords need scale if image scales
boxes *= scale_factor
boxes[[0, 2]] += block_x
boxes[[1, 3]] += block_y
for j, box in enumerate(boxes.T):
cls = int(classes[j])
color = colors[cls % len(colors)]
cls = names[cls] if names else cls
if labels or conf[j] > 0.25: # 0.25 conf thresh
label = '%s' % cls if labels else '%s %.1f' % (cls, conf[j])
plot_one_box(box, mosaic, label=label, color=color, line_thickness=tl)
# Draw image filename labels
if paths:
label = Path(paths[i]).name[:40] # trim to 40 char
t_size = cv2.getTextSize(label, 0, fontScale=tl / 3, thickness=tf)[0]
cv2.putText(mosaic, label, (block_x + 5, block_y + t_size[1] + 5), 0, tl / 3, [220, 220, 220], thickness=tf,
lineType=cv2.LINE_AA)
# Image border
cv2.rectangle(mosaic, (block_x, block_y), (block_x + w, block_y + h), (255, 255, 255), thickness=3)
if fname:
r = min(1280. / max(h, w) / ns, 1.0) # ratio to limit image size
mosaic = cv2.resize(mosaic, (int(ns * w * r), int(ns * h * r)), interpolation=cv2.INTER_AREA)
# cv2.imwrite(fname, cv2.cvtColor(mosaic, cv2.COLOR_BGR2RGB)) # cv2 save
Image.fromarray(mosaic).save(fname) # PIL save
return mosaic
def sourcery(mov: np.ndarray, ops):
change_codes = True
i0 = time.time()
if isinstance(ops['diameter'], int):
ops['diameter'] = [ops['diameter'], ops['diameter']]
ops['diameter'] = np.array(ops['diameter'])
ops['spatscale_pix'] = ops['diameter'][1]
ops['aspect'] = ops['diameter'][0] / ops['diameter'][1]
ops, U,sdmov, u = getSVDdata(mov=mov, ops=ops) # get SVD components
S, StU , StS = getStU(ops, U)
Lyc, Lxc,nsvd = U.shape
ops['Lyc'] = Lyc
ops['Lxc'] = Lxc
d0 = ops['diameter']
sig = np.ceil(d0 / 4) # smoothing constant
# make array of radii values of size (2*d0+1,2*d0+1)
rs,dy,dx = circleMask(d0)
nsvd = U.shape[-1]
nbasis = S.shape[-1]
codes = np.zeros((0, nsvd), np.float32)
LtU = np.zeros((0, nsvd), np.float32)
LtS = np.zeros((0, nbasis), np.float32)
L = np.zeros((Lyc, Lxc, 0), np.float32)
# regress maps onto basis functions and subtract neuropil contribution
neu = np.linalg.solve(StS, StU).astype('float32')
Ucell = U - (S.reshape((-1,nbasis))@neu).reshape(U.shape)
it = 0
ncells = 0
refine = -1
# initialize
ypix,xpix,lam = [], [], []
while 1:
if refine<0:
V, us = getVmap(Ucell, sig)
if it==0:
vrem = morphOpen(V, rs<=1.)
V = V - vrem # make V more uniform
if it==0:
V = V.astype('float64')
# find indices of all maxima in +/- 1 range
maxV = filters.maximum_filter(V, footprint= np.ones((3,3)), mode='reflect')
imax = V > (maxV - 1e-10)
peaks = V[imax]
# use the median of these peaks to decide if ROI is accepted
thres = ops['threshold_scaling'] * np.median(peaks[peaks>1e-4])
ops['Vcorr'] = V
V = np.minimum(V, ops['Vcorr'])
# add extra ROIs here
n = ncells
while n<ncells+200:
ind = np.argmax(V)
i,j = np.unravel_index(ind, V.shape)
if V[i,j] < thres:
break;
yp, xp, la, ix, code = iter_extend(i, j, Ucell, us[i,j,:], change_codes=change_codes)
codes = np.append(codes, np.expand_dims(code,axis=0), axis=0)
ypix.append(yp)
xpix.append(xp)
lam.append(la)
Ucell[ypix[n], xpix[n], :] -= np.outer(lam[n], codes[n,:])
yp, xp = extendROI(yp,xp,Lyc,Lxc, int(np.mean(d0)))
V[yp, xp] = 0
n += 1
newcells = len(ypix) - ncells
if it==0:
Nfirst = newcells
L = np.append(L, np.zeros((Lyc, Lxc, newcells), 'float32'), axis =-1)
LtU = np.append(LtU, np.zeros((newcells, nsvd), 'float32'), axis = 0)
LtS = np.append(LtS, np.zeros((newcells, nbasis), 'float32'), axis = 0)
for n in range(ncells, len(ypix)):
L[ypix[n],xpix[n], n] = lam[n]
LtU[n,:] = lam[n] @ U[ypix[n],xpix[n],:]
LtS[n,:] = lam[n] @ S[ypix[n],xpix[n], :]
ncells +=newcells
# regression with neuropil
LtL = L.reshape((-1, ncells)).transpose() @ L.reshape((-1, ncells))
cellcode = np.concatenate((LtL,LtS), axis=1)
neucode = np.concatenate((LtS.transpose(),StS), axis=1)
codes = np.concatenate((cellcode, neucode), axis=0)
Ucode = np.concatenate((LtU, StU),axis=0)
codes = np.linalg.solve(codes + 1e-3*np.eye((codes.shape[0])), Ucode).astype('float32')
neu = codes[ncells:,:]
codes = codes[:ncells,:]
Ucell = U - (S.reshape((-1,nbasis))@neu + L.reshape((-1,ncells))@codes).reshape(U.shape)
# reestimate masks
n,k = 0,0
while n < len(ypix):
Ucell[ypix[n], xpix[n], :] += np.outer(lam[n], codes[k,:])
ypix[n], xpix[n], lam[n], ix, codes[n,:] = iter_extend(ypix[n], xpix[n], Ucell,
codes[k,:], refine, change_codes=change_codes)
k+=1
if ix.sum()==0:
print('dropped ROI with no pixels')
del ypix[n], xpix[n], lam[n]
continue;
Ucell[ypix[n], xpix[n], :] -= np.outer(lam[n], codes[n,:])
n+=1
codes = codes[:n, :]
ncells = len(ypix)
L = np.zeros((Lyc,Lxc, ncells), 'float32')
LtU = np.zeros((ncells, nsvd), 'float32')
LtS = np.zeros((ncells, nbasis), 'float32')
for n in range(ncells):
L[ypix[n],xpix[n],n] = lam[n]
if refine<0:
LtU[n,:] = lam[n] @ U[ypix[n],xpix[n],:]
LtS[n,:] = lam[n] @ S[ypix[n],xpix[n],:]
err = (Ucell**2).mean()
print('ROIs: %d, cost: %2.4f, time: %2.4f'%(ncells, err, time.time()-i0))
it += 1
if refine ==0:
break
if refine==2:
# good place to get connected regions
stat = [{'ypix':ypix[n], 'lam':lam[n], 'xpix':xpix[n]} for n in range(ncells)]
stat = connected_region(stat, ops)
# good place to remove ROIs that overlap, change ncells, codes, ypix, xpix, lam, L
#stat, ix = remove_overlaps(stat, ops, Lyc, Lxc)
#print('removed %d overlapping ROIs'%(len(ypix)-len(ix)))
ypix = [stat[n]['ypix'] for n in range(len(stat))]
xpix = [stat[n]['xpix'] for n in range(len(stat))]
lam = [stat[n]['lam'] for n in range(len(stat))]
#L = L[:,:,ix]
#codes = codes[ix, :]
ncells = len(ypix)
if refine>0:
Ucell = Ucell + (S.reshape((-1,nbasis))@neu).reshape(U.shape)
if refine<0 and (newcells<Nfirst/10 or it==ops['max_iterations']):
refine = 3
U, sdmov = getSVDproj(mov, ops, u)
Ucell = U
if refine>=0:
StU = S.reshape((Lyc*Lxc,-1)).transpose() @ Ucell.reshape((Lyc*Lxc,-1))
#StU = np.reshape(S, (Lyc*Lxc,-1)).transpose() @ np.reshape(Ucell, (Lyc*Lxc, -1))
neu = np.linalg.solve(StS, StU).astype('float32')
refine -= 1
Ucell = U - (S.reshape((-1,nbasis))@neu).reshape(U.shape)
sdmov = np.reshape(sdmov, (Lyc, Lxc))
ops['sdmov'] = sdmov
stat = [{'ypix':ypix[n], 'lam':lam[n]*sdmov[ypix[n], xpix[n]], 'xpix':xpix[n]} for n in range(ncells)]
stat = postprocess(ops, stat, Ucell, codes)
return ops, stat
def createDatabase(database,
threshold=20,
data_path='../Data/train/',
index_path='../Data/index/INDEX_refined_data.2016',
ligand_end='_ligand.sdf',
alt_ligand_end='_ligand.pdb',
prot_end='_pocket.pdb',
mode=None,
label_type=None,
classes=None,
n_classes=None,
oversample=False,
sample_factor=50,
pbc=(1, 1, 1)):
ligandPaths = PreprocessingSchnet.getAllMolPaths(data_path, ligand_end)
ligandPaths2 = PreprocessingSchnet.getAllMolPaths(
data_path, alt_ligand_end)
proteinPaths = PreprocessingSchnet.getAllMolPaths(data_path, prot_end)
labels = PreprocessingSchnet.getLabels(data_path, index_path)
indexes = np.arange(len(proteinPaths[0]))
np.random.shuffle(indexes)
# For oversampling, the data is split into 25 equally parts. Proteins with labels,
# which are low represented are taken more ofter, to simulate a equal distributed label distribution
hist = np.histogram(labels, 25)
for i in indexes:
atom_list = []
# Reads the ligand and creates an list of ligand-atoms.
# Sometimes the sdf is not working so it uses the alternative pdb.
try:
atoms2 = read(ligandPaths[0][i], format='sdf')
for at in atoms2:
atom_list.append(at)
x = atoms2.positions[:, 0].mean()
y = atoms2.positions[:, 1].mean()
z = atoms2.positions[:, 2].mean()
except:
try:
atoms3 = read(ligandPaths2[0][i], format='proteindatabank')
x = atoms3.positions[:, 0].mean()
y = atoms3.positions[:, 1].mean()
z = atoms3.positions[:, 2].mean()
for at in atoms3:
atom_list.append(at)
except:
print('Does not work')
continue
# Generate the labels in the necessary format for schnetpack
affi = PreprocessingSchnet.classLabel(labels[i],
mode,
label_type,
classes=classes,
n_classes=n_classes,
min_v=np.min(labels),
max_v=np.max(labels))
mean = np.array([x, y, z])
atoms = read(proteinPaths[0][i], format='proteindatabank')
# Create protein-atom list
for at in atoms:
dist = np.linalg.norm(at.position - mean)
if dist <= threshold:
atom_list.append(at)
# concat the protein and ligand atoms
complexe = [ase.Atoms(atom_list, pbc=pbc)]
if not oversample:
# Add the complex to the database
database.add_systems(complexe, affi)
else:
# Calculate, how often a protein have to be added to a database,
# to simulate a equal distributed label distribution
classn = np.zeros(25)
for j in range(len(hist[1]) - 1):
if j == len(hist[1]) - 2:
if hist[1][j] <= labels[i] <= hist[1][j + 1]:
classn[j] = 1
else:
if hist[1][j] <= labels[i] < hist[1][j + 1]:
classn[j] = 1
if np.unique(classn, return_counts=True)[1][1] != 1:
print('warning -> Onehot is more than one')
print(classn)
ind = np.argmax(classn)
if hist[0][ind] == 0:
print('Warning -> zero-sample')
continue
n_sampling = int(
np.ceil((1 / hist[0][ind]) * sample_factor * 25))
print(i, len(indexes), n_sampling, ind)
for _ in range(n_sampling):
database.add_systems(complexe, affi)