def _aces_proxy_OECF(value, bit_depth='10 Bit'):
"""
Defines the *ACESproxy* colourspace opto-electronic conversion function.
Parameters
----------
value : numeric or array_like
Value.
bit_depth : unicode, optional
**{'10 Bit', '12 Bit'}**,
*ACESproxy* bit depth.
Returns
-------
numeric or ndarray
Companded value.
"""
value = np.asarray(value)
constants = ACES_PROXY_CONSTANTS.get(bit_depth)
CV_min = np.resize(constants.CV_min, value.shape)
CV_max = np.resize(constants.CV_max, value.shape)
float_2_cv = lambda x: np.maximum(CV_min, np.minimum(CV_max, np.round(x)))
output = np.where(value > 2 ** -9.72,
float_2_cv((np.log2(value) + constants.mid_log_offset) *
constants.steps_per_stop +
constants.mid_CV_offset),
np.resize(CV_min, value.shape))
return output
def feature_normalization(train, test):
"""Rescale the data so that each feature in the training set is in
the interval [0,1], and apply the same transformations to the test
set, using the statistics computed on the training set.
Args:
train - training set, a 2D numpy array of size (num_instances, num_features)
test - test set, a 2D numpy array of size (num_instances, num_features)
Returns:
train_normalized - training set after normalization
test_normalized - test set after normalization
"""
## TODONE
trainMin = np.resize(train.min(axis=0), (100, 48))
trainMax = np.resize(train.max(axis=0), (100, 48))
topTrain = train - trainMin
bottomTrain = trainMax - trainMin
outputTrain = topTrain / bottomTrain
topTest = test - trainMin
bottomTest = bottomTrain
outputTest = topTest / bottomTest
return outputTrain, outputTest
def get_spectral_magnitude(y_data,time_data, fs):
tStep = np.max(time_data)/len(time_data)
timeV = np.arange(0, np.max(time_data), tStep)
numsamp = 512 #timeDomainVectorLength(timeV)
if (len(y_data) < numsamp):
y_data = np.resize(y_data, (numsamp,))
window = hann(numsamp)
## setup the fft spectrum arrays
mag_spectrum = np.zeros([numsamp,int(np.ceil(float(len(timeV))/numsamp))])
#print 'time.len= %d, numsamp=%d, loop:%d' % (len(timeV), numsamp, int(np.ceil(float(len(timeV))/numsamp)))
for k in range(0,int(np.ceil(float(len(timeV))/numsamp))):
slice_dat = y_data[k*numsamp:numsamp*(k+1)]
if (len(slice_dat) < numsamp):
if (len(slice_dat) < numsamp/2): # WE DISCARDS LAST SLICE POINTS IF < NUMSAMP/2
break;
slice_dat = np.resize(slice_dat,(numsamp,))
#multiply it with the window and transform it into frequency domain
spectrum_dat = fft(slice_dat*window);
#get the spectrum mag @ each of the 256 frequency points and store it
#print 'k:',k,' spectrum_dat.len:',len(spectrum_dat)
mag_spectrum[:,k]= 20 * np.log10(abs(spectrum_dat))
mag_spectrum[:,k]= abs(spectrum_dat)
#print "fs= %.4g, NFFT= %ld, y_data.shape= %d, mag_spectrum= %dx%d" % (fs, numsamp,np.shape(y_data)[0], np.shape(mag_spectrum)[0],np.shape(mag_spectrum)[1])
## DOUBLE CHECK THE SIZE OF THE MATRIX
avg_fft_foreach = np.mean(mag_spectrum, axis=1)
# print "np.shape(avg_fft_foreach):", np.shape(avg_fft_foreach)
return avg_fft_foreach
def genImg(self,xml,compressionLevel):
xml = re.split('\n',xml)
img_data = re.split('"',xml[1])
type = img_data[1]
size = [int(x) for x in img_data[3].split(',')]
compressed = img_data[5]
pixels = xml[2]
pixels = base64.b64decode(pixels) # decode base 64 encoding
if compressed == 'True':
pixels = zlib.decompress(pixels) # if data was compressed, decompress it
pixels = list(pixels) # converting byte data into a list, which will give us the actual numbers of ndarray(i.e. the image)
if type == 'Gray': # Based on image type, reconstruct numpy.ndarray
return numpy.resize(pixels,tuple(size))
else:
r = pixels[:size[0]*size[1]]
g = pixels[size[0]*size[1]:2*size[0]*size[1]]
b = pixels[2*size[0]*size[1]:3*size[0]*size[1]]
image = []
for i in range(size[0] * size[1]):
image.append(r[i])
image.append(g[i])
image.append(b[i])
size.append(3)
return numpy.resize(image,tuple(size))
def aabut (source, *args):
"""
Like the |Stat abut command. It concatenates two arrays column-wise
and returns the result. CAUTION: If one array is shorter, it will be
repeated until it is as long as the other.
Usage: aabut (source, args) where args=any # of arrays
Returns: an array as long as the LONGEST array past, source appearing on the
'left', arrays in <args> attached on the 'right'.
"""
if len(source.shape)==1:
width = 1
source = N.resize(source,[source.shape[0],width])
else:
width = source.shape[1]
for addon in args:
if len(addon.shape)==1:
width = 1
addon = N.resize(addon,[source.shape[0],width])
else:
width = source.shape[1]
if len(addon) < len(source):
addon = N.resize(addon,[source.shape[0],addon.shape[1]])
elif len(source) < len(addon):
source = N.resize(source,[addon.shape[0],source.shape[1]])
source = N.concatenate((source,addon),1)
return source
def twopoint_spidx_bootstrap(freq, flux, flux_err, niter=10000):
"""
Quick bootstrap for spectral index calulcation
freq: 2 array
flux: 2 or 2xN array
flux_err: 2 or 2xN array
N is the number of sources
"""
# calculate spidx assuming [iter,source,freq_point] shapes
def spidx(freq, flux):
return np.log10(flux[:,:,0]/flux[:,:,1])/np.log10(freq[:,:,0]/freq[:,:,1])
freq = np.array(freq).astype(float)
flux = np.array(flux).astype(float)
flux_err = np.array(flux_err).astype(float)
# if only 1 source, add degenerate axis
if flux.shape == (2,): flux = np.expand_dims(flux, axis=1)
if flux_err.shape == (2,): flux_err = np.expand_dims(flux_err, axis=1)
flux = flux.T
flux_err = flux_err.T
nsource = flux.shape[0]
results = np.zeros(shape=(niter,nsource))
random_flux = np.resize(flux, (niter, nsource, 2)) + np.resize(flux_err, (niter, nsource, 2)) * np.random.randn(niter, nsource, 2)
random_flux[random_flux <= 0] = np.nan # remove negative, this create a bias
freq = np.resize(freq, (niter, nsource, 2))
results = spidx(freq, random_flux)
mean = np.nanmean(results,axis=0)
err = np.nanstd(results,axis=0)
return mean, err
def batchsd(trace, batches=5):
"""
Calculates the simulation standard error, accounting for non-independent
samples. The trace is divided into batches, and the standard deviation of
the batch means is calculated.
"""
if len(np.shape(trace)) > 1:
dims = np.shape(trace)
# ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:]))))
ttrace = np.transpose([t.ravel() for t in trace])
return np.reshape([batchsd(t, batches) for t in ttrace], dims[1:])
else:
if batches == 1:
return np.std(trace) / np.sqrt(len(trace))
try:
batched_traces = np.resize(trace, (batches, len(trace) / batches))
except ValueError:
# If batches do not divide evenly, trim excess samples
resid = len(trace) % batches
batched_traces = np.resize(trace[:-resid],
(batches, len(trace[:-resid]) / batches))
means = np.mean(batched_traces, 1)
return np.std(means) / np.sqrt(batches)
def draw_imf_samples(**kwargs):
''' Draw samples for power-law model
Parameters
----------
**kwargs: string
Keyword arguments as model parameters and number of samples
Returns
-------
array
The first mass
array
The second mass
'''
alpha_salpeter = kwargs.get('alpha', -2.35)
nsamples = kwargs.get('nsamples', 1)
min_mass = kwargs.get('min_mass', 5.)
max_mass = kwargs.get('max_mass', 95.)
max_mtotal = min_mass + max_mass
a = (max_mass/min_mass)**(alpha_salpeter + 1.0) - 1.0
beta = 1.0 / (alpha_salpeter + 1.0)
k = nsamples * int(1.5 + log(1 + 100./nsamples))
aa = min_mass * (1.0 + a * np.random.random(k))**beta
bb = np.random.uniform(min_mass, aa, k)
idx = np.where(aa + bb < max_mtotal)
m1, m2 = (np.maximum(aa, bb))[idx], (np.minimum(aa, bb))[idx]
return np.resize(m1, nsamples), np.resize(m2, nsamples)
def draw_lnm_samples(**kwargs):
''' Draw samples for uniform-in-log model
Parameters
----------
**kwargs: string
Keyword arguments as model parameters and number of samples
Returns
-------
array
The first mass
array
The second mass
'''
#PDF doesnt match with sampler
nsamples = kwargs.get('nsamples', 1)
min_mass = kwargs.get('min_mass', 5.)
max_mass = kwargs.get('max_mass', 95.)
max_mtotal = min_mass + max_mass
lnmmin = log(min_mass)
lnmmax = log(max_mass)
k = nsamples * int(1.5 + log(1 + 100./nsamples))
aa = np.exp(np.random.uniform(lnmmin, lnmmax, k))
bb = np.exp(np.random.uniform(lnmmin, lnmmax, k))
idx = np.where(aa + bb < max_mtotal)
m1, m2 = (np.maximum(aa, bb))[idx], (np.minimum(aa, bb))[idx]
return np.resize(m1, nsamples), np.resize(m2, nsamples)
def JacobiMatrix(F, x, h = 1e-05):
J = np.matrix([[],[]])
np.resize((len(F), len(F)))
for i in range(0, len(F)):
for j in range(0, len(F)):
J[i][j] = derivative(F[i], x, j, h)
return J
def mc_error(x, batches=5):
"""
Calculates the simulation standard error, accounting for non-independent
samples. The trace is divided into batches, and the standard deviation of
the batch means is calculated.
:Arguments:
x : Numpy array
An array containing MCMC samples
batches : integer
Number of batchas
"""
if x.ndim > 1:
dims = np.shape(x)
#ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:]))))
trace = np.transpose([t.ravel() for t in x])
return np.reshape([mc_error(t, batches) for t in trace], dims[1:])
else:
if batches == 1: return np.std(x)/np.sqrt(len(x))
try:
batched_traces = np.resize(x, (batches, len(x)/batches))
except ValueError:
# If batches do not divide evenly, trim excess samples
resid = len(x) % batches
batched_traces = np.resize(x[:-resid], (batches, len(x)/batches))
means = np.mean(batched_traces, 1)
return np.std(means)/np.sqrt(batches)
def _evaluatePart(self, expr, part):
"""Evaluate expression expr for part part.
Returns True if succeeded
"""
# replace dataset names with calls
newexpr = substituteDatasets(self.document.data, expr, part)[0]
comp = self.document.evaluate.compileCheckedExpression(
newexpr, origexpr=expr)
if comp is None:
return False
# set up environment to evaluate expressions in
environment = self.document.evaluate.context.copy()
# create dataset using parametric expression
if self.parametric:
p = self.parametric
if p[2] >= 2:
deltat = (p[1]-p[0]) / (p[2]-1)
t = N.arange(p[2])*deltat + p[0]
else:
t = N.array([p[0]])
environment['t'] = t
# this fn gets called to return the value of a dataset
environment['_DS_'] = self.evaluateDataset
# actually evaluate the expression
try:
result = eval(comp, environment)
evalout = N.array(result, N.float64)
if len(evalout.shape) > 1:
raise RuntimeError("Number of dimensions is not 1")
except Exception as ex:
self.document.log(
_("Error evaluating expression: %s\n"
"Error: %s") % (self.expr[part], cstr(ex)) )
return False
# make evaluated error expression have same shape as data
if part != 'data':
data = self.evaluated['data']
if evalout.shape == ():
# zero dimensional - expand to data shape
evalout = N.resize(evalout, data.shape)
else:
# 1-dimensional - make it right size and trim
oldsize = evalout.shape[0]
evalout = N.resize(evalout, data.shape)
evalout[oldsize:] = N.nan
else:
if evalout.shape == ():
# zero dimensional - make a single point
evalout = N.resize(evalout, 1)
self.evaluated[part] = evalout
return True
def __init__(self, name, geometry, order, init_context=True):
LagrangeSimplexPolySpace.__init__(self, name, geometry, order,
init_context=False)
nodes, nts, node_coors = self.nodes, self.nts, self.node_coors
shape = [nts.shape[0] + 1, 2]
nts = nm.resize(nts, shape)
nts[-1,:] = [3, 0]
shape = [nodes.shape[0] + 1, nodes.shape[1]]
nodes = nm.resize(nodes, shape)
# Make a 'hypercubic' (cubic in 2D) node.
nodes[-1,:] = 1
n_v = self.geometry.n_vertex
tmp = nm.ones((n_v,), nm.int32)
node_coors = nm.vstack((node_coors,
nm.dot(tmp, self.geometry.coors) / n_v))
self.nodes, self.nts = nodes, nts
self.node_coors = nm.ascontiguousarray(node_coors)
self.bnode = nodes[-1:,:]
self.n_nod = self.nodes.shape[0]
if init_context:
self.eval_ctx = self.create_context(None, 0, 1e-15, 100, 1e-8,
tdim=n_v - 1)
else:
self.eval_ctx = None
def build_data_dict(data1, data2, match):
"""Build a dictionary of zeros like the union of the two data
dictionaries, but with 'length' equal to the shorter dict.
"""
data = {}
keys1 = [key for key in data1.keys() if key != match]
keys2 = [key for key in data2.keys() if key != match]
nfinal = min(np.shape(data1[match])[0],
np.shape(data2[match])[0])
if nfinal == np.shape(data1[match])[0]:
data[match] = np.zeros_like(data1[match])
else:
data[match] = np.zeros_like(data2[match])
for k in keys1:
data[k] = np.zeros_like(data1[k])
shp = list(np.shape(data1[k]))
shp[0] = nfinal
data[k] = np.resize(data[k], tuple(shp))
for k in keys2:
data[k] = np.zeros_like(data2[k])
shp = list(np.shape(data2[k]))
shp[0] = nfinal
data[k] = np.resize(data[k], tuple(shp))
return data, keys1, keys2
def test(npoints):
xx = numpy.arange(npoints)
xx=numpy.resize(xx,(npoints,1))
#yy = 1000.0 * exp (- 0.5 * (xx * xx) /15)+ 2.0 * xx + 10.5
yy = gauss([10.5,2,1000.0,20.,15],xx)
yy=numpy.resize(yy,(npoints,1))
sy = numpy.sqrt(abs(yy))
sy=numpy.resize(sy,(npoints,1))
data = numpy.concatenate((xx, yy, sy),1)
parameters = [0.0,1.0,900.0, 25., 10]
stime = time.time()
if 0:
#old fashion
fittedpar, chisq, sigmapar = LeastSquaresFit(gauss,parameters,data)
else:
#easier to handle
fittedpar, chisq, sigmapar = LeastSquaresFit(gauss,parameters,
xdata=xx.reshape((-1,)),
ydata=yy.reshape((-1,)),
sigmadata=sy.reshape((-1,)))
etime = time.time()
print("Took ",etime - stime, "seconds")
print("chi square = ",chisq)
print("Fitted pars = ",fittedpar)
print("Sigma pars = ",sigmapar)
def abut(source, *args):
# comment: except for the repetition, this is equivalent to hstack.
"""\nLike the |Stat abut command. It concatenates two arrays column-wise
and returns the result. CAUTION: If one array is shorter, it will be
repeated until it is as long as the other.
Format: abut (source, args) where args=any # of arrays
Returns: an array as long as the LONGEST array past, source appearing on the
'left', arrays in <args> attached on the 'right'.\n"""
source = asarray(source)
if len(source.shape) == 1:
width = 1
source = np.resize(source,[source.shape[0],width])
else:
width = source.shape[1]
for addon in args:
if len(addon.shape) == 1:
width = 1
addon = np.resize(addon,[source.shape[0],width])
else:
width = source.shape[1]
if len(addon) < len(source):
addon = np.resize(addon,[source.shape[0],addon.shape[1]])
elif len(source) < len(addon):
source = np.resize(source,[addon.shape[0],source.shape[1]])
source = np.concatenate((source,addon),1)
return source
def _assignInitialPoints(cvmat,S):
h,w,c = cvmat.shape
# Compute the max grid assignment
nx = w/S
ny = h/S
# Compute the super pixel x,y grid
xgrid = np.arange(nx).reshape(1,nx)*np.ones(ny,dtype=np.int).reshape(ny,1)
ygrid = np.arange(ny).reshape(ny,1)*np.ones(nx,dtype=np.int).reshape(1,nx)
# compute an x,y lookup to a label look up
label_map = nx*ygrid + xgrid
# Compute the x groups in pixel space
tmp = np.arange(nx)
tmp = np.resize(tmp,(w,))
xgroups = tmp[tmp.argsort()]
# Compute the y groups in pixel space
tmp = np.arange(ny)
tmp = np.resize(tmp,(h,))
ygroups = tmp[tmp.argsort()]
labels = np.zeros((h,w),dtype=np.int)
for x in range(w):
for y in range(h):
labels[y,x] = label_map[ygroups[y],xgroups[x]]
return label_map,xgroups,ygroups,labels
def preprocessForTwoPhenotypeAsso(self, snpData, which_phenotype_ls, data_matrix_phen):
"""
2009-3-20
refactored out of run(), easy for MpiAssociation.py to call
"""
if len(which_phenotype_ls) < 2:
sys.stderr.write("Error: Require to specify 2 phenotypes in order to carry out this test type.\n")
sys.exit(3)
snpData.data_matrix = numpy.vstack((snpData.data_matrix, snpData.data_matrix))
# stack the two phenotypes, fake a data_matrix_phen, which_phenotype_ls
no_of_strains = len(strain_acc_list)
which_phenotype1, which_phenotype2 = which_phenotype_ls[:2]
phenotype1 = data_matrix_phen[:, which_phenotype1]
phenotype1 = numpy.resize(
phenotype1, [no_of_strains, 1]
) # phenotype1.resize([10,1]) doesn't work. ValueError: 'resize only works on single-segment arrays'
phenotype2 = data_matrix_phen[:, which_phenotype2]
phenotype2 = numpy.resize(phenotype2, [no_of_strains, 1])
data_matrix_phen = numpy.vstack((phenotype1, phenotype2))
which_phenotype_index_ls = [0]
# stack the PC_matrix as well
if PC_matrix is not None:
PC_matrix = numpy.vstack((PC_matrix, PC_matrix))
# create an environment variable
a = numpy.zeros(no_of_strains)
a.resize([no_of_strains, 1])
b = numpy.ones(no_of_strains)
b.resize([no_of_strains, 1])
environment_matrix = numpy.vstack((a, b))
return which_phenotype_index_ls, environment_matrix, data_matrix_phen
def convert_to_batched_episodes(self, episodes, max_length=None):
"""Convert batch-major list of episodes to time-major batch of episodes."""
lengths = [len(ep[-2]) for ep in episodes]
max_length = max_length or max(lengths)
new_episodes = []
for ep, length in zip(episodes, lengths):
initial, observations, actions, rewards, terminated = ep
observations = [np.resize(obs, [max_length + 1] + list(obs.shape)[1:])
for obs in observations]
actions = [np.resize(act, [max_length + 1] + list(act.shape)[1:])
for act in actions]
pads = np.array([0] * length + [1] * (max_length - length))
rewards = np.resize(rewards, [max_length]) * (1 - pads)
new_episodes.append([initial, observations, actions, rewards,
terminated, pads])
(initial, observations, actions, rewards,
terminated, pads) = zip(*new_episodes)
observations = [np.swapaxes(obs, 0, 1)
for obs in zip(*observations)]
actions = [np.swapaxes(act, 0, 1)
for act in zip(*actions)]
rewards = np.transpose(rewards)
pads = np.transpose(pads)
return (initial, observations, actions, rewards, terminated, pads)
def __init__(self, name, geometry, order):
LagrangeSimplexPolySpace.__init__(self, name, geometry, order)
nodes, nts, node_coors = self.nodes, self.nts, self.node_coors
shape = [nts.shape[0] + 1, 2]
nts = nm.resize(nts, shape)
nts[-1,:] = [3, 0]
shape = [nodes.shape[0] + 1, nodes.shape[1]]
nodes = nm.resize(nodes, shape)
# Make a 'hypercubic' (cubic in 2D) node.
nodes[-1,:] = 1
n_v = self.geometry.n_vertex
tmp = nm.ones((n_v,), nm.int32)
node_coors = nm.vstack((node_coors,
nm.dot(tmp, self.geometry.coors) / n_v))
self.nodes, self.nts = nodes, nts
self.node_coors = nm.ascontiguousarray(node_coors)
self.bnode = nodes[-1:,:]
self.n_nod = self.nodes.shape[0]
def partial_autocorr(series, lag):
#-------------------------------------------------------------------------------
"""Partial autocorrelation function, using Durbin (1960) recursive algorithm
"""
# Initialize matrices of phi and rho
phi = resize(0.0, (lag, lag))
rho = resize(0.0, lag)
if isinstance(series, list):
series = numpy.array(series)
# \phi_{1,1} = \rho_1
phi[0, 0] = rho[0] = autocorr(series, 1)
for k in range(1, lag):
# Calculate autocorrelation for current lag
rho[k] = autocorr(series, k + 1)
for j in range(k - 1):
# \phi_{k+1,j} = \phi_{k,j} - \phi_{k+1,k+1} * \phi_{k,k+1-j}
phi[k - 1, j] = phi[k - 2, j] - phi[k - 1, k - 1] * phi[k - 2, k - 2 - j]
# Numerator: \rho_{k+1} - \sum_{j=1}^k \phi_{k,j}\rho_j
phi[k, k] = rho[k] - sum([phi[k - 1, j] * rho[k - 1 - j] for j in range(k)])
# Denominator: 1 - \sum_{j=1}^k \phi_{k,j}\rho_j
phi[k, k] /= 1 - sum([phi[k - 1, j] * rho[j] for j in range(k)])
# Return partial autocorrelation value
return phi[lag - 1, lag - 1]
def eulertrap(ode, vardict, soln, h, relerr):
"""
Implementation of the Euler-Trapezoidal method.
"""
eqnum = len(ode)
dim = [eqnum, 3]
dim.extend(soln[0][0].shape)
dim = tuple(dim)
if numpy.iscomplexobj(soln[0]):
aux = numpy.resize([0. + 0j], dim)
else:
aux = numpy.resize([0.], dim)
dim = soln[0][0].shape
for vari in range(eqnum):
vardict.update({'y_{}'.format(vari): soln[vari][-1]})
for vari in range(eqnum):
aux[vari][0] = numpy.resize(seval(ode[vari], **vardict), dim)
for vari in range(eqnum):
aux[vari][1] = numpy.resize(seval(ode[vari], **vardict) * h[0] + soln[vari][-1], dim)
for vari in range(eqnum):
vardict.update({'y_{}'.format(vari): aux[vari][1]})
vardict.update({'t': vardict['t'] + h[0]})
for vari in range(eqnum):
aux[vari][2] = numpy.resize(seval(ode[vari], **vardict), dim)
for vari in range(eqnum):
vardict.update({"y_{}".format(vari): soln[vari][-1] + h[0] * (aux[vari][0] + aux[vari][2])})
pt = soln[vari]
kt = numpy.array([vardict['y_{}'.format(vari)]])
soln[vari] = numpy.concatenate((pt, kt))
def test_QuaternionClass(self):
v1 = np.array([0.2, 0.2, 0.4])
v2 = np.array([1, 0, 0])
q1 = Quaternion.q_exp(v1)
q2 = Quaternion.q_exp(v2)
v=np.array([1, 2, 3])
# Testing Mult and rotate
np.testing.assert_almost_equal(Quaternion.q_rotate(Quaternion.q_mult(q1,q2),v), Quaternion.q_rotate(q1,Quaternion.q_rotate(q2,v)), decimal=7)
np.testing.assert_almost_equal(Quaternion.q_rotate(q1,v2), np.resize(Quaternion.q_toRotMat(q1),(3,3)).dot(v2), decimal=7)
# Testing Boxplus, Boxminus, Log and Exp
np.testing.assert_almost_equal(Quaternion.q_boxPlus(q1,Quaternion.q_boxMinus(q2,q1)), q2, decimal=7)
np.testing.assert_almost_equal(Quaternion.q_log(q1), v1, decimal=7)
# Testing Lmat and Rmat
np.testing.assert_almost_equal(Quaternion.q_mult(q1,q2), Quaternion.q_Lmat(q1).dot(q2), decimal=7)
np.testing.assert_almost_equal(Quaternion.q_mult(q1,q2), Quaternion.q_Rmat(q2).dot(q1), decimal=7)
# Testing ypr and quat
roll = 0.2
pitch = -0.5
yaw = 2.5
q_test = Quaternion.q_mult(np.array([np.cos(0.5*pitch), 0, np.sin(0.5*pitch), 0]),np.array([np.cos(0.5*yaw), 0, 0, np.sin(0.5*yaw)]))
q_test = Quaternion.q_mult(np.array([np.cos(0.5*roll), np.sin(0.5*roll), 0, 0]),q_test)
np.testing.assert_almost_equal(Quaternion.q_toYpr(q_test), np.array([roll, pitch, yaw]), decimal=7)
# Testing Jacobian of Ypr
for i in np.arange(0,3):
dv1 = np.array([0.0, 0.0, 0.0])
dv1[i] = 1.0
epsilon = 1e-6
ypr1 = Quaternion.q_toYpr(q1)
ypr1_dist = Quaternion.q_toYpr(Quaternion.q_boxPlus(q1,dv1*epsilon))
dypr1_1 = (ypr1_dist-ypr1)/epsilon
J = np.resize(Quaternion.q_toYprJac(q1),(3,3))
dypr1_2 = J.dot(dv1)
np.testing.assert_almost_equal(dypr1_1,dypr1_2, decimal=5)
def append(self, value):
# convert the appended object to an array if it starts as something else
if type(value) is not np.ndarray:
value = np.array(value)
# add the data
if value.ndim == 1:
# adding a single row of data
n = self.__n
if n + 1 > self.__N:
# need to allocate more memory
self.__N += self.__n_grow
self.__data = np.resize(self.__data, (self.__N, self.__cols))
self.__data[n] = value
self.__n = n + 1
elif value.ndim == 2:
# adding multiple rows of data
# avoid loops for appending large arrays
n = self.__n
L = value.shape[0]
N_needed = n + L - self.__N
if N_needed > 0:
# need to allocate more memory
self.__N += (N_needed / self.__n_grow + 1) * self.__n_grow
self.__data = np.resize(self.__data, (self.__N, self.__cols))
self.__data[n:n+L] = value
self.__n += L
def initialize(dx, x_shore, eta_shore, eta_toe, S_d, S_b, S, Q_w, B0):
"""Initialize the variables for the simulation.
Args:
Returns:
eta_b : array of z-coordinate of the basement at every node. Has the
same size as dx. [ L ]
Comments:
"""
# First thing we should do is specify our computational domain.
N, dx_shore = nodes_in_domain(x_shore, dx)
N_old = N.copy()
dx = init_domain(N, dx_shore, dx)
x_shore = x(dx)[-1]
# Then we compute the basement elevation and the location of the delta toe.
eta_b = np.resize(eta_toe, N)
eta_b, x_toe = init_basement(dx, eta_shore, eta_toe, eta_b, S_d, S_b)
# Next, we compute the initial fluvial profile
eta, S = init_flumen(dx, eta_shore, S)
# Instantiate a water-depth array for the domain.
H = np.zeros_like(dx)
# Compute unit flow rate
qw = unit_flowrate(Q_w, B0) # [ L**2 / T ]
# Compute critical depth for the section.
Hc = critical_flow(qw)
# Redefine B0 as a vector, to have that information on every node.
B0 = np.resize(B0, N)
# Instantiate a sediment transport capacity array for the domain
qt = np.zeros_like(dx)
return (N, N_old, dx_shore, dx, x_shore, eta_b, x_toe, eta, S, H, Hc, qw,
B0, qt)
def cube():
"""
Build vertices for a colored cube.
V is the vertices
I1 is the indices for a filled cube (use with GL_TRIANGLES)
I2 is the indices for an outline cube (use with GL_LINES)
"""
vtype = [('a_position', np.float32, 3),
('a_normal' , np.float32, 3),
('a_color', np.float32, 4)]
# Vertices positions
v = [ [ 1, 1, 1], [-1, 1, 1], [-1,-1, 1], [ 1,-1, 1],
[ 1,-1,-1], [ 1, 1,-1], [-1, 1,-1], [-1,-1,-1] ]
# Face Normals
n = [ [ 0, 0, 1], [ 1, 0, 0], [ 0, 1, 0] ,
[-1, 0, 1], [ 0,-1, 0], [ 0, 0,-1] ]
# Vertice colors
c = [ [0, 1, 1, 1], [0, 0, 1, 1], [0, 0, 0, 1], [0, 1, 0, 1],
[1, 1, 0, 1], [1, 1, 1, 1], [1, 0, 1, 1], [1, 0, 0, 1] ];
V = np.array([(v[0],n[0],c[0]), (v[1],n[0],c[1]), (v[2],n[0],c[2]), (v[3],n[0],c[3]),
(v[0],n[1],c[0]), (v[3],n[1],c[3]), (v[4],n[1],c[4]), (v[5],n[1],c[5]),
(v[0],n[2],c[0]), (v[5],n[2],c[5]), (v[6],n[2],c[6]), (v[1],n[2],c[1]),
(v[1],n[3],c[1]), (v[6],n[3],c[6]), (v[7],n[3],c[7]), (v[2],n[3],c[2]),
(v[7],n[4],c[7]), (v[4],n[4],c[4]), (v[3],n[4],c[3]), (v[2],n[4],c[2]),
(v[4],n[5],c[4]), (v[7],n[5],c[7]), (v[6],n[5],c[6]), (v[5],n[5],c[5]) ],
dtype = vtype)
I1 = np.resize( np.array([0,1,2,0,2,3], dtype=np.uint32), 6*(2*3))
I1 += np.repeat( 4*np.arange(2*3), 6)
I2 = np.resize( np.array([0,1,1,2,2,3,3,0], dtype=np.uint32), 6*(2*4))
I2 += np.repeat( 4*np.arange(6), 8)
return V, I1, I2
def _wrap(vector, pad_tuple, iaxis, kwargs):
'''
Private function to calculate the before/after vectors for pad_wrap.
Parameters
----------
vector : ndarray
Input vector that already includes empty padded values.
pad_tuple : tuple
This tuple represents the (before, after) width of the padding
along this particular iaxis.
iaxis : int
The axis currently being looped across. Not used in _wrap.
kwargs : keyword arguments
Keyword arguments. Not used in _wrap.
Return
------
_wrap : ndarray
Padded vector
'''
if pad_tuple[1] == 0:
after_vector = vector[pad_tuple[0]:None]
else:
after_vector = vector[pad_tuple[0]:-pad_tuple[1]]
before_vector = np.resize(after_vector[::-1], pad_tuple[0])[::-1]
after_vector = np.resize(after_vector, pad_tuple[1])
return _create_vector(vector, pad_tuple, before_vector, after_vector)
def explicitmidpoint(ode, vardict, soln, h, relerr):
"""
Implementation of the Explicit Midpoint method.
"""
eqnum = len(ode)
dim = [eqnum, 2]
dim.extend(soln[0][0].shape)
dim = tuple(dim)
if numpy.iscomplexobj(soln[0]):
aux = numpy.resize([0. + 0j], dim)
else:
aux = numpy.resize([0.], dim)
dim = soln[0][0].shape
for vari in range(eqnum):
vardict.update({'y_{}'.format(vari): soln[vari][-1]})
for vari in range(eqnum):
aux[vari][0] = numpy.resize([seval(ode[vari], **vardict) * h[0] + soln[vari][-1]], dim)
for vari in range(eqnum):
vardict.update({"y_{}".format(vari): aux[vari][0]})
vardict.update({'t': vardict['t'] + 0.5 * h[0]})
for vari in range(eqnum):
aux[vari][0] = numpy.resize([seval(ode[vari], **vardict)], dim)
for vari in range(eqnum):
vardict.update({"y_{}".format(vari): numpy.array(soln[vari][-1] + h[0] * aux[vari][0])})
pt = soln[vari]
kt = numpy.array([vardict['y_{}'.format(vari)]])
soln[vari] = numpy.concatenate((pt, kt))
vardict.update({'t': vardict['t'] + 0.5 * h[0]})
def plot_image(self, image, nb_repeat=40, show_plot=True):
"""Plot augmented variations of an image.
This method takes an image and plots it by default in 40 differently
augmented versions.
This method is intended to visualize the strength of your chosen
augmentations (so for debugging).
Args:
image: The image to plot.
nb_repeat: How often to plot the image. Each time it is plotted,
the chosen augmentation will be different. (Default: 40).
show_plot: Whether to show the plot. False makes sense if you
don't have a graphical user interface on the machine.
(Default: True)
Returns:
The figure of the plot.
Use figure.savefig() to save the image.
"""
if len(image.shape) == 2:
images = np.resize(image, (nb_repeat, image.shape[0], image.shape[1]))
else:
images = np.resize(image, (nb_repeat, image.shape[0], image.shape[1],
image.shape[2]))
return self.plot_images(images, True, show_plot=show_plot)
def sympforeuler(ode, vardict, soln, h, relerr):
"""
Implementation of the Symplectic Euler method.
"""
eqnum = len(ode)
dim = [eqnum, 1]
dim.extend(soln[0][0].shape)
dim = tuple(dim)
if numpy.iscomplexobj(soln[0]):
aux = numpy.resize([0. + 0j], dim)
else:
aux = numpy.resize([0.], dim)
dim = soln[0][0].shape
for vari in range(eqnum):
vardict.update({'y_{}'.format(vari): soln[vari][-1]})
for vari in range(eqnum):
aux[vari][0] = numpy.resize((seval(ode[vari], **vardict) * h[0] + soln[vari][-1]), dim)
if vari % 2 == 0:
vardict.update({"y_{}".format(vari): aux[vari][0]})
for vari in range(eqnum):
vardict.update({"y_{}".format(vari): aux[vari][0]})
pt = soln[vari]
kt = numpy.array([vardict['y_{}'.format(vari)]])
soln[vari] = numpy.concatenate((pt, kt))
vardict.update({'t': vardict['t'] + h[0]})
def main(argv):
# --------------------- Initializations ----------------------- #
now = datetime.datetime.now()
date = str('{:04d}'.format(now.year)) + str('{:02d}'.format(
now.month)) + str('{:02d}'.format(now.day))
# -------- Flags ---------- #
chunk_time = FLAGS.chunk_time
db = FLAGS.use_database
expID = FLAGS.expID
patience = FLAGS.patience
percentage_test = FLAGS.percentage_test
percentage_validation = FLAGS.percentage_validation
use_class_weights = FLAGS.use_class_weights
target_id = FLAGS.ini_spk
# ---------- NN ------------ #
restart_ID_model = FLAGS.restart_ID_model
filters = eval(FLAGS.num_filters)
filter_sizes = eval(FLAGS.filter_sizes)
# ---- Training options ---- #
batch_size = FLAGS.batch_size
earlystop = FLAGS.earlystop
epochs = FLAGS.num_epochs
ini_spk = FLAGS.ini_spk
nusr_train = FLAGS.nusr_train
n_spks = FLAGS.n_spks
pool_factor = FLAGS.pool_factor
random_shuffles = FLAGS.random_shuffles
if not os.path.exists('../models'):
os.makedirs('../models')
# -------------------------- Main ----------------------------- #
print('=-=-=-=-=-==-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-')
print(bcolors.OKBLUE + 'Chunk_Time:' + bcolors.ENDC, chunk_time)
if db == 'PulseID':
fs = 200
path_df = './DataFrame_PulseID.pkl'
signal = 'PPG_GDF'
df, num_subjects, num_segments = split_db_pulseID(
path_df=path_df,
fs=fs,
chunk_time=chunk_time,
percentage_validation=percentage_validation,
percentage_test=percentage_test,
nusr_train=nusr_train,
target=target_id)
elif db == 'TROIKA':
fs = 125
signal = 'PPG'
path_df = './DataFrame_TROIKA.pkl'
df, num_subjects, num_segments = split_db_TROIKA(
path_df=path_df,
fs=fs,
chunk_time=chunk_time,
percentage_validation=percentage_validation,
percentage_test=percentage_test,
nusr_train=nusr_train,
target=target_id)
#print(df.shape)
data_train, labels_train, data_validation, labels_validation, data_test, labels_test, data_testIDs, labels_testIDs, data_zscore = atomic_test(
df=df,
signal=signal,
num_subjects=num_subjects,
num_segments=num_segments,
target=target_id,
percentage_test=percentage_test
) # data_train & labels_train are BALANCED
#print(data_train.shape[1])
data_train = data_train.reshape(data_train.shape[0],
1) # Reshaping necessary to train
if restart_ID_model != True:
model = get_model(filters=filters,
kernel_size=filter_sizes,
x_train=data_train,
cnn_stride=1,
pool_factor=pool_factor)
print(bcolors.WARNING +
'Keeping previous trained parameters from previous models' +
bcolors.ENDC)
for i in range(ini_spk,
n_spks + 1): # round(num_subjects * percentage_test) + 1):
# Different model for each subject ID
if restart_ID_model == True:
model = get_model(filters=filters,
kernel_size=filter_sizes,
x_train=data_train,
cnn_stride=1,
pool_factor=pool_factor)
print(
bcolors.WARNING + 'Reseting NN model per userID ' +
bcolors.ENDC, i)
sgd = SGD(lr=0.0001, momentum=0.0, decay=0.0, nesterov=True)
model.compile(optimizer=sgd,
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
print('*****************************************')
print(bcolors.OKGREEN + 'ID:' + bcolors.ENDC, i)
target_id = i
labels_validation = []
labels_test = []
score_validation = []
score_test = []
# For the Circular Queue:
buffer_auc = collections.deque(maxlen=patience +
1) #circular queue declaration
buffer_models = collections.deque(maxlen=patience + 1)
buffer_score_validation = collections.deque(maxlen=patience + 1)
buffer_score_test = collections.deque(maxlen=patience + 1)
buffer_score_testIDs = collections.deque(maxlen=patience + 1)
buffer_labels_validation = collections.deque(maxlen=patience + 1)
buffer_labels_test = collections.deque(maxlen=patience + 1)
buffer_labels_testIDs = collections.deque(maxlen=patience + 1)
for i in range(patience + 1):
buffer_auc.append(0.0)
print(bcolors.WARNING + 'Entering at Random_Shuffle for' +
bcolors.ENDC)
for r in range(1, random_shuffles + 1):
print('------------------------------------------')
data_train, labels_train, data_validation, labels_validation, data_test, labels_test, data_testIDs, labels_testIDs, data_zscore = atomic_test(
df=df,
signal=signal,
num_subjects=num_subjects,
num_segments=num_segments,
target=target_id,
percentage_test=percentage_test)
data_train = data_train.reshape(data_train.shape[0], 1, 1)
data_test = data_test.reshape(data_test.shape[0], 1, 1)
data_testIDs = data_testIDs.reshape(data_testIDs.shape[0], 1, 1)
data_validation = data_validation.reshape(data_validation.shape[0],
1, 1)
#up=np.array(labels_validation,dtype='int')
develop_weights = class_weight.compute_class_weight(
"balanced", (np.unique(labels_validation)),
(labels_validation))
develop_weights_vec = []
#develop_weights=develop_weights.reshape(2,1,1)
#print((develop_weights[0]))
for j in range(0, len(labels_validation)):
if labels_validation[j] == 0:
develop_weights_vec.append(develop_weights[0])
elif labels_validation[j] == 1:
develop_weights_vec.append(develop_weights[1])
develop_weights_vec = np.array(develop_weights_vec)
class_weights = {
0: 1.,
1: 150.
} # Weights in case of unbalanced validation data
# 1 appearence of class 0 equals 150 appearences of class 1
#data_validation=[0]*19
data_train = np.resize(data_train, (19, 19, 1))
labels_train = labels_train[19:]
#print(batch_size)
data_validation = np.resize(data_validation, (2, 19, 1))
#labels_train=np.reshape(1,19)
labels_train = np.reshape(labels_train, (19, 1))
if use_class_weights == True:
history = model.fit(x=data_train,
y=labels_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
callbacks=None,
validation_split=0.0,
validation_data=(data_validation,
labels_validation,
develop_weights_vec),
shuffle=True,
class_weight=class_weights,
sample_weight=None,
initial_epoch=0) # default values
else:
"""history = model.fit(x=data_train, y=labels_train, batch_size=batch_size, epochs=epochs, verbose=0,
validation_split=0.0,
validation_data=(data_validation, labels_validation, develop_weights_vec),
shuffle=True,
class_weight=None,
sample_weight=None,
initial_epoch=0)""" # default values
#print(data_test.shape)
data_test = np.resize(data_test, (7, 19, 1))
data_testIDs = np.resize(data_testIDs, (7, 19, 1))
score_validation = model.predict(x=data_validation,
batch_size=1,
verbose=0)
score_test = model.predict(x=data_test, batch_size=1, verbose=0)
score_testIDs = model.predict(x=data_testIDs,
batch_size=1,
verbose=0)
score_validation = score_validation.reshape(
score_validation.shape[0])
score_test = score_test.reshape(score_test.shape[0])
score_testIDs = score_testIDs.reshape(score_testIDs.shape[0])
labels_validation = labels_validation.reshape(
labels_validation.shape[0])
labels_test = labels_test.reshape(labels_test.shape[0])
labels_testIDs = labels_testIDs.reshape(labels_testIDs.shape[0])
roc_auc = roc_auc_score(labels_validation, score_validation)
roc_auc_test = roc_auc_score(labels_test, score_test)
print('random shuffle iteration: ', r, 'Validation AUC: ', roc_auc,
'Test AUC: ', roc_auc_test)
model_copy = clone_model(
model
) # necesary to clone to create a copy of the whole model. If we do not do this,
# the model is passed as pointer by default
model_copy.compile(optimizer=sgd,
loss='binary_crossentropy',
metrics=['accuracy'])
print(
"\n---------------------------------------\nmodel_copy summary: \n"
)
model_copy.summary()
print("\n---------------------------------------\n")
# Circular queue
# If the AUC decreases 'patience' consecutive stop training and save the model & scores for the last 'best' model
# (actual model -patience steps)
buffer_auc.append(roc_auc) #buffer_auc[0 --> patience]
buffer_models.append(model_copy)
buffer_score_validation.append(score_validation)
buffer_score_test.append(score_test)
buffer_score_testIDs.append(score_testIDs)
buffer_labels_validation.append(labels_validation)
buffer_labels_test.append(labels_test)
buffer_labels_testIDs.append(labels_testIDs)
# Boundaries to set the maximum FPR value
fprbound1 = 0.2 # more restrictive
fprbound2 = 0.3 # "checkpoint" value
if (buffer_auc.index(max(buffer_auc)) == 0
or r == random_shuffles) and earlystop == True:
if buffer_auc.index(max(buffer_auc)) == 0:
index_modelok = 0
elif r == random_shuffles:
index_modelok = -1
if not os.path.exists('../models/' + db + '/' + date + '/'):
os.makedirs('../models/' + db + '/' + date + '/')
buffer_models[index_modelok].save('../models/' + db + '/' +
date + '/model_' + db +
'_ES' + str(patience) + '_' +
expID + '.h5')
if not os.path.exists('../results/' + db + '/' + 'scores_ES' +
str(patience) + '_' + date + '/'):
os.makedirs('../results/' + db + '/' + 'scores_ES' +
str(patience) + '_' + date + '/')
filename = '../results/' + db + '/' + 'scores_ES' + str(
patience) + '_' + date + '/' + expID + '_scores'
# Computing the optimal threshold for a given scores
validation_metrics, test_metrics, testIDs_metrics, threshold = optimal_threshold(
buffer_score_validation[index_modelok],
buffer_score_test[index_modelok],
buffer_score_testIDs[index_modelok],
buffer_labels_validation[index_modelok],
buffer_labels_test[index_modelok],
buffer_labels_testIDs[index_modelok], fprbound1, fprbound2)
# Saving scores in a .json file
scores = {
'score_validation':
buffer_score_validation[index_modelok].tolist(),
'score_test':
buffer_score_test[index_modelok].tolist(),
'score_testIDs':
buffer_score_testIDs[index_modelok].tolist(),
'labels_validation':
buffer_labels_validation[index_modelok].tolist(),
'labels_test':
buffer_labels_test[index_modelok].tolist(),
'labels_testIDs':
buffer_labels_testIDs[index_modelok].tolist(),
'threshold':
threshold.tolist()
}
with open(filename + '.json', 'w') as fp:
json.dump(scores, fp)
# pickle.dump((buffer_score_validation[index_modelok], buffer_score_test[index_modelok], buffer_labels_validation[index_modelok],
# buffer_predict_labels_validation[index_modelok], buffer_labels_test[index_modelok]),open(filename,"wb"))
break
if earlystop == False:
if not os.path.exists('../models/' + db + '/' + date + '/'):
os.makedirs('../models/' + db + '/' + date + '/')
model.save('../models/' + db + '/' + date + '/model_' + db + '_' +
expID + '.h5')
if not os.path.exists('../results/' + db + '/' + 'scores_' + date +
'/'):
os.makedirs('../results/' + db + '/' + 'scores_' + date + '/')
filename = '../results/' + db + '/' + 'scores_' + date + '/' + expID + '_scores'
# Computing the optimal threshold for a given scores
validation_metrics, test_metrics, testIDs_metrics, threshold = optimal_threshold(
score_validation, score_test, score_testIDs, labels_validation,
labels_test, labels_testIDs, fprbound1, fprbound2)
# Saving the scores in a .json file
scores = {
'score_validation': score_validation.tolist(),
'score_test': score_test.tolist(),
'score_testIDs': score_testIDs.tolist(),
'labels_validation': labels_validation.tolist(),
'labels_test': labels_test.tolist(),
'labels_testIDs': labels_testIDs.tolist(),
'threshold': threshold.tolist()
}
with open(filename + '.json', 'w') as fp:
json.dump(scores, fp)
# Unpacking variables
fpr_validation = validation_metrics[0]
tpr_validation = validation_metrics[1]
roc_auc = validation_metrics[2]
report_validation = validation_metrics[3]
cnf_matrix_validation = validation_metrics[4]
fpr_test = test_metrics[0]
tpr_test = test_metrics[1]
roc_auc_test = test_metrics[2]
report_test = test_metrics[3]
cnf_matrix_test = test_metrics[4]
fpr_testIDs = testIDs_metrics[0]
tpr_testIDs = testIDs_metrics[1]
roc_auc_testIDs = testIDs_metrics[2]
report_testIDs = testIDs_metrics[3]
cnf_matrix_testIDs = testIDs_metrics[4]
print('FINAL ITERATION -> Validation AUC: ', roc_auc, 'Test AUC: ',
roc_auc_test, 'TestIDs AUC: ', roc_auc_testIDs)
plot_ROC_and_Report(db, expID, date, 'validation', False, filter_sizes,
filters, fpr_validation, tpr_validation, roc_auc,
report_validation, cnf_matrix_validation,
threshold)
plot_ROC_and_Report(db, expID, date, 'test', False, filter_sizes,
filters, fpr_test, tpr_test, roc_auc_test,
report_test, cnf_matrix_test, threshold)
plot_ROC_and_Report(db, expID, date, 'testIDs', False, filter_sizes,
filters, fpr_testIDs, tpr_testIDs, roc_auc_testIDs,
report_testIDs, cnf_matrix_testIDs, threshold)
def saveToCenter(i, rList, jList, bufferArray, summaryLength, h_size, sess,
mainQN, time_per_step):
with open('./Center/log.csv', 'a') as myfile:
state_display = (np.zeros([1, h_size]), np.zeros([1, h_size]))
imagesS = []
for idx, z in enumerate(np.vstack(bufferArray[:, 0])):
img, state_display = sess.run(
[mainQN.salience, mainQN.rnn_state],
feed_dict={
mainQN.scalarInput:
np.reshape(bufferArray[idx, 0], [1, 21168]) / 255.0,
mainQN.trainLength:
1,
mainQN.state_in:
state_display,
mainQN.batch_size:
1
})
imagesS.append(img)
imagesS = (imagesS - np.main(imagesS)) / (np.max(imagesS) -
np.min(imagesS))
imagesS = np.vstack(imagesS)
imagesS = np.resize(imagesS, [len(imagesS), 84, 84, 3])
luminance = np.max(imagesS, 3)
imagesS = np.multiply(np.ones([len(imagesS), 84, 84, 3]),
np.reshape(luminance, [len(imagesS), 84, 84, 1]))
make_gif(np.ones([len(imagesS), 84, 84, 3]),
'./Center/frames/sal' + str(i) + '.gif',
duration=len(imagesS) * time_per_step,
true_image=False,
salience=True,
salIMGS=luminance)
images = list(zip(bufferArray[:, 0]))
images.append(bufferArray[-1, 3])
images = np.vstack(images)
images = np.resize(images, [len(images), 84, 84, 3])
make_gif(images,
'./Center/frames/image' + str(i) + '.gif',
duration=len(images) * time_per_step,
true_image=True,
salience=False)
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
wr.writerow([
i, np,
mean(jList[-100:0]),
np.mean(rList[-summaryLength:]),
'./frames/image' + str(i) + '.gif',
'./frames/log' + str(i) + '.csv', './frames/sal' + str(i) + '.gif'
])
myfile.close()
with open('./Center/frames/log' + str(i) + '.csv', 'w') as myfile:
state_train = (np.zeros([1, h_size]), np.zeros([1, h_size]))
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
wr.writerow(["ACTION", "REWARD", "A0", "A1", 'A2', 'A3', 'V'])
a, v = sess.run(
[mainQN.Advantage, mainQN.Value],
feed_dict={
mainQN.scalarInput: np.vstack(bufferArray[:, 0]) / 255.0,
mainQN.trainLength: len(bufferArray),
mainQN.state_in: state_train,
mainQN.batch_size: 1
})
wr.writerows(
zip(bufferArray[:, 1], bufferArray[:, 2], a[:, 0], a[:, 1],
a[:, 2], a[:, 3], v[:, 0]))
def train_neural_network(x):
prediction = neural_network_model(x)
cost = tf.reduce_sum(tf.square(y - prediction))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
#wr.writerow([ "Run Number", "Training Accuracy", "Testing Accuracy" ])
with tf.Session() as sess:
##########################
# TRAINING FUNCTIONALITY #
##########################
sess.run(tf.global_variables_initializer())
# This loop will run of the number of total run required.
for i in range(numOfRuns):
accuracy = 0
for epoch in range(hm_epochs):
epoch_loss = 0
accuracy = 0
np.random.shuffle(xy_dataTraining)
data_x = xy_dataTraining[0:trainingDataSize, 0:36]
data_y = xy_dataTraining[0:trainingDataSize, 36:38]
data_y = np.resize(data_y, (328, 2))
for piece in range(len(data_x)):
input_x = [data_x[piece]]
expected_y = [data_y[piece]]
_, c, predict = sess.run(
[optimizer, cost, prediction],
feed_dict={
x: input_x,
y: expected_y,
keep_probL1: dropOutRateL1,
keep_probL2: dropOutRateL2
})
epoch_loss += c
if np.argmax(predict[0]) == np.argmax(expected_y[0]):
accuracy += 1
accuracyPercentage = (accuracy / len(data_x))
print("Epoch: ", epoch, ", accuracy_standing: ",
accuracyPercentage)
runLabel = "Accuracy for run " + str(i) + ":"
######################
# IMAGE TESTING BULK #
######################
accuracy = 0
accuracyPercentage2 = 0
data_x = xy_dataTesting[0:testingDataSize, 0:36]
data_y = xy_dataTesting[0:testingDataSize, 36:38]
for piece in range(len(data_x)):
input_x = [data_x[piece]]
expected_y = [data_y[piece]]
predict = sess.run([prediction],
feed_dict={
x: input_x,
y: expected_y,
keep_probL1: 1.0,
keep_probL2: 1.0
})
if np.argmax(predict[0]) == np.argmax(expected_y[0]):
accuracy += 1
accuracyPercentage2 = (accuracy / len(data_x))
trialLabel = "Testing Accuracy on Entire Set: "
print(trialLabel, accuracyPercentage2)
# wr.writerow([i, accuracyPercentage, accuracyPercentage2])
# Play a sound to signal the numOfRuns complete
winsound.PlaySound("SystemExit", winsound.SND_ALIAS)
########################
# AD HOC IMAGE TESTING #
########################
# Preload cmu
dG = dataGenerator.dataScriptGenerator()
continueProcessing = "Y"
while (continueProcessing == "Y" or continueProcessing == "y"):
# Call data script generator on new image and generate csv
dG.adHocData()
# Pass new csv data into network and print out prediction
data_x = np.genfromtxt(sys.path[0] + '\\aClean.csv', delimiter=',')
if data_x.ndim == 1:
data_x = np.resize(data_x, (1, 36))
data_x[data_x < 0] = 0
if (len(data_x)) > 0:
for piece in range(len(data_x)):
input_x = [data_x[piece]]
predict = sess.run([prediction],
feed_dict={
x: input_x,
keep_probL1: 1.0,
keep_probL2: 1.0
})
print("Network Output: ", predict[0])
if (np.argmax(predict[0]) == 0):
print("SQUATTING")
elif (np.argmax(predict[0]) == 1):
print("STANDING")
continueProcessing = input("Continue processing? ('Y' or 'N'): ")
cfg=cfg)
if args.cuda:
newmodel.cuda()
start_mask = torch.ones(3)
layer_id_in_cfg = 0
linear_number = 1
end_mask = cfg_mask[layer_id_in_cfg]
for [m0, m1] in zip(model.modules(), newmodel.modules()):
if isinstance(m0, nn.Conv2d):
idx0 = np.squeeze(np.argwhere(np.asarray(start_mask.cpu().numpy())))
idx1 = np.squeeze(np.argwhere(np.asarray(end_mask.cpu().numpy())))
print('In shape: {:d}, Out shape {:d}.'.format(idx0.size, idx1.size))
if idx0.size == 1:
idx0 = np.resize(idx0, (1, ))
if idx1.size == 1:
idx1 = np.resize(idx1, (1, ))
w1 = m0.weight.data[:, idx0.tolist(), :, :].clone()
w1 = w1[idx1.tolist(), :, :, :].clone()
m1.weight.data = w1.clone()
elif isinstance(m0, nn.Linear):
if layer_id_in_cfg == len(cfg_mask):
idx0 = np.squeeze(
np.argwhere(np.asarray(cfg_mask[-1].cpu().numpy())))
if idx0.size == 1:
idx0 = np.resize(idx0, (1, ))
m1.weight.data = m0.weight.data[:, idx0].clone()
m1.bias.data = m0.bias.data.clone()
layer_id_in_cfg += 1
continue
cv2.imshow("Legend", legend)
#----------------------循环主体-------------------------------------------
time1 = time()
while True:
ret, frame = cap.read()
#ret是否有读取到----------frame帧图像--------
#input_image = cap.read(args.input_vedio, 1).astype(np.float32)
input_image = frame
input_image = cv2.resize(input_image, (input_shape[3], input_shape[2]))
input_image = input_image.transpose((2, 0, 1))
input_image = np.asarray([input_image])
out = net.forward_all(**{net.inputs[0]: input_image})
prediction = net.blobs['deconv6_0_0'].data[0].argmax(axis=0)
prediction = np.squeeze(prediction)
prediction = np.resize(prediction, (3, input_shape[2], input_shape[3]))
prediction = prediction.transpose(1, 2, 0).astype(np.uint8)
prediction_rgb = np.zeros(prediction.shape, dtype=np.uint8)
label_colours_bgr = label_colours[..., ::-1]
cv2.LUT(prediction, label_colours_bgr, prediction_rgb)
frame = cv2.resize(frame, (1280, 720),
interpolation=cv2.INTER_CUBIC) #修改大小
prediction_rgb = cv2.resize(prediction_rgb, (1280, 720),
interpolation=cv2.INTER_CUBIC)
output = ((0.3 * frame) + (0.7 * prediction_rgb)).astype("uint8")
cv2.imshow("input", frame)
#cv2.imshow("ENet", prediction_rgb)
cv2.imshow("ENet", output)
# if writer is None:
# initialize our video writer
#------- fourcc = cv2.VideoWriter_fourcc(*"MPEG")
# <b>resize() only works if no otehr references to array, else error</b> # In[158]: ar2 = ar # In[159]: ar.resize((8, )) # <b>Workaround is to use numpy.ndarray.resize() instead</b> # In[160]: np.resize(ar, (8, )) # In[161]: ar # <h2>Adding a dimension</h2> # In[162]: ar = np.array([14, 15, 16]) ar.shape # In[163]: ar
def test_check_different_dimensions():
""" Ensure an error is raised if the dimensions are different. """
XA = np.resize(np.arange(45), (5, 9))
XB = np.resize(np.arange(32), (4, 8))
assert_raises(ValueError, check_pairwise_arrays, XA, XB)
img = Image.open(os.path.join(PATH_TO_TEST_IMAGES_DIR, PATH_TO_TEST_IMAGE))
image.imsave(os.path.join(PATH_TO_INPUT_IMAGE, '{}.png'.format(0)), img)
for i in range(conv1.shape[3]):
# Get the 9x9x1 filter:
extracted_filter = conv1[:, :, :, i]
# Get rid of the last dimension (hence get 9x9):
extracted_filter = np.squeeze(extracted_filter)
image.imsave(os.path.join(PATH_TO_CONV1_KERNEL, '{}.png'.format(i)), extracted_filter, vmin=-0.6064218, vmax=0.24946211)
img = Image.open(os.path.join(PATH_TO_TEST_IMAGES_DIR, PATH_TO_TEST_IMAGE))
arr2 = np.array(img.getdata(), dtype=np.uint8)
arr2 = np.resize(arr2, (img.size[1], img.size[0], 4))
arr2 = arr2[:, :, 1]
conv = signal.correlate2d(arr2, extracted_filter, 'valid')
conv = conv + conv1_bias[i]
conv = np.maximum(0, conv)
output[:, :, i] = conv
image.imsave(os.path.join(PATH_TO_RELU, '{}.png'.format(i)), conv, cmap=CMAP, vmin=0, vmax=255)
for i in range(primary_caps.shape[3]):
# 9,9,256,256
extracted_filter = primary_caps[:, :, :, i]
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights,
times, height, order, r_0, beta, outQueue):
"""
Fits a screen to given phase values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of phase values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
real_fit_white_all = np.zeros((N_times, N_sources, N_stations))
imag_fit_white_all = np.zeros((N_times, N_sources, N_stations))
phase_fit_white_all = np.zeros((N_times, N_sources, N_stations))
real_residual_all = np.zeros((N_times, N_sources, N_stations))
imag_residual_all = np.zeros((N_times, N_sources, N_stations))
phase_residual_all = np.zeros((N_times, N_sources, N_stations))
# Change phase to real/imag
rr_real = np.cos(rr)
rr_imag = np.sin(rr)
for k in range(N_times):
try:
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], U[:, :order])),
rcond=1e-3)
# Calculate real screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr_real[:, k]))
real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
real_fit_white_all[k, :, :] = real_fit.reshape(
(N_sources, N_stations))
residual = rr_real - np.dot(C, real_fit)[:, newaxis]
real_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
# Calculate imag screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr_imag[:, k]))
imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
imag_fit_white_all[k, :, :] = imag_fit.reshape(
(N_sources, N_stations))
residual = rr_imag - np.dot(C, imag_fit)[:, newaxis]
imag_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
# Calculate phase screen
phase_fit = np.dot(
pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit)))
phase_fit_white_all[k, :, :] = phase_fit.reshape(
(N_sources, N_stations))
residual = rr - np.dot(C, phase_fit)[:, newaxis]
phase_residual_all[k, :, :] = residual.reshape(
(N_sources, N_stations))
except:
# Set screen to zero if fit did not work
logging.debug('Screen fit failed for timeslot {}'.format(k))
real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
real_residual_all[k, :, :] = np.ones((N_sources, N_stations))
imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
imag_residual_all[k, :, :] = np.ones((N_sources, N_stations))
phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations))
phase_residual_all[k, :, :] = np.ones((N_sources, N_stations))
outQueue.put([
real_fit_white_all, real_residual_all, imag_fit_white_all,
imag_residual_all, phase_fit_white_all, phase_residual_all, times
])
u_exact = x_grid**2
u_pred_err = u_exact - u_pred
#print X_star
#np.savetxt('inp.out', X_star, delimiter=',')
#print u_pred
np.savetxt('uPred.out', u_pred, delimiter=',')
np.savetxt('uPredErr.out', u_pred_err, delimiter=',')
np.savetxt('uExact.out', u_pred_err, delimiter=',')
np.savetxt('f_uPred.out', f_uPred, delimiter=',')
error_u = (np.linalg.norm(u_exact - u_pred, 2) /
np.linalg.norm(u_exact, 2))
print('Relative error u: %e' % (error_u))
# #plot the solution u_comp
u_pred = np.resize(u_pred, [nPred, nPred])
CS = plt.contour(xPred, yPred, u_pred, 20, linewidths=0.5, colors='k')
CS = plt.contourf(xPred, yPred, u_pred, 20, cmap=plt.cm.jet)
plt.colorbar() # draw colorbar
plt.title('$u_{comp}$')
plt.show()
#plot the error u_ex - u_comp
u_pred_err = np.resize(u_pred_err, [nPred, nPred])
CS2 = plt.contour(xPred, yPred, u_pred_err, 20, linewidths=0.5, colors='k')
CS2 = plt.contourf(xPred, yPred, u_pred_err, 20, cmap=plt.cm.jet)
plt.colorbar() # draw colorbar
plt.title('$u_{ex}-u_{comp}$')
plt.show()
#
import numpy as np
from sys import argv
from keras.preprocessing import image
from keras.models import load_model
img = argv[1]
# Loads the image
test_image = image.load_img(img, target_size=(28, 28, 1))
# Transform the img into a array
test_image = image.img_to_array(test_image)
test_image = np.resize(test_image, (28, 28, 1))
test_image = np.expand_dims(test_image, axis=0)
test_image = test_image.astype('float32')
test_image /= 255
# Loads the classifier model
classifier = load_model('results/models/cnn_30e.h5')
# Predicts the label
res = classifier.predict_classes(test_image)
print(res)
print(classifier.predict(test_image))
def _fit_tec_screen(station_names, source_names, pp, airmass, rr, weights,
times, height, order, r_0, beta, outQueue):
"""
Fits a screen to given TEC values using Karhunen-Lo`eve base vectors
Parameters
----------
station_names: array
Array of station names
source_names: array
Array of source names
pp: array
Array of piercepoint locations
airmass: array
Array of airmass values (note: not currently used)
rr: array
Array of TEC values to fit screen to
weights: array
Array of weights
times: array
Array of times
height: float
Height of screen (m)
order: int
Order of screen (i.e., number of KL base vectors to keep)
r_0: float
Scale size of phase fluctuations (m)
beta: float
Power-law index for phase structure function (5/3 => pure Kolmogorov
turbulence)
"""
import numpy as np
from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye
logging.info('Fitting screens...')
# Initialize arrays
N_stations = len(station_names)
N_sources = len(source_names)
N_times = len(times)
N_piercepoints = N_sources * N_stations
tec_fit_white_all = np.zeros((N_times, N_sources, N_stations))
tec_residual_all = np.zeros((N_times, N_sources, N_stations))
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / r_0**2)**(beta / 2.0) / 2.0
pinvC = pinv(C, rcond=1e-3)
U, S, V = svd(C)
invU = pinv(np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], U[:, :order])),
rcond=1e-3)
# Calculate screen
rr1 = np.dot(np.transpose(U[:, :order]),
np.dot(weights[:, :, k], rr[:, k]))
tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1)))
tec_fit_white_all[k, :, :] = tec_fit.reshape((N_sources, N_stations))
residual = rr - np.dot(C, tec_fit)[:, newaxis]
tec_residual_all[k, :, :] = residual.reshape((N_sources, N_stations))
outQueue.put([tec_fit_white_all, tec_residual_all, times])
labels=one_hot(org_y))
# Run optimization op (backprop) and cost op (to get loss value)
_, c = sess.run([train_op, loss_op], {X: batch_x, Y: batch_y})
# Compute average loss
avg_cost += c / batch_size
# Display logs per epoch s1tep
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch + 1),
"cost={:.9f}".format(avg_cost))
saver.save(sess, './model/TensorFlow/model.ckpt')
print("Optimization Finished!")
print("Predicting sample from train set!")
random_index = np.random.randint(0, 5000)
sample_x = org_X[random_index]
logits = np.argmax(predict(np.resize(sample_x, (1, 400))).eval()[0]) + 1
print("Predicted X:", logits)
print("True X:", org_y[random_index][0])
print("Prediction finished!")
p = []
logits = predict(org_X)
for i in logits.eval():
p.append(np.argmax(i) + 1)
p = np.asarray(p)
c = 0
for i in range(org_y.shape[0]):
if p[i] == org_y[i][0]:
c += 1
print("Accuracy: %.6f" % (c / org_y.shape[0]))
"""""" """""" """""" """"""
""" INTEGRATOR """
"""""" """""" """""" """"""
# Variable extraction
xAcc = DataSet["Denoised + Offset Removed X Accel. (g)"]
yAcc = DataSet["Denoised + Offset Removed Y Accel. (g)"]
zAcc = DataSet["Denoised + Offset Removed Z Accel. (g)"]
time = DataSet["Time (all)"]
# Integrating acceleration vectors, producing velocities
xVel = it.cumtrapz(xAcc, time)
yVel = it.cumtrapz(yAcc, time)
zVel = it.cumtrapz(zAcc, time)
# Plotting Velocities in X, Y, and Z
time = np.resize(time, time.size - 1)
plt.figure(figsize=(10, 6))
plt.plot(time, xVel, linewidth=2.25, alpha=0.6, label='x', color='r')
plt.plot(time, yVel, linewidth=2.25, alpha=0.6, label='y', color='g')
plt.plot(time, zVel, linewidth=2.25, alpha=0.6, label='z', color='b')
plt.title("Velocities in X, Y, and Z")
plt.xlabel("Time (s)")
plt.ylabel("Velocity (m/s)")
plt.grid()
plt.xlim()
plt.ylim()
plt.legend(loc='upper left')
plt.show()
# Adding new dictionary entries
DataSet["X Velocity"] = xVel
def calculate_pattern(length, repeats):
return np.resize(np.repeat(np.array((0, 1, 0, -1)), repeats),
length + 1)[1:length + 1]
def my_run_simulation(self, initial_run=False, update_plots=True):
"""
Runs the simulation.
Args:
self(PyBERT): Reference to an instance of the *PyBERT* class.
initial_run(Bool): If True, don't update the eye diagrams, since
they haven't been created, yet. (Optional; default = False.)
update_plots(Bool): If True, update the plots, after simulation
completes. This option can be used by larger scripts, which
import *pybert*, in order to avoid graphical back-end
conflicts and speed up this function's execution time.
(Optional; default = True.)
"""
num_sweeps = self.num_sweeps
sweep_num = self.sweep_num
start_time = clock()
self.status = 'Running channel...(sweep %d of %d)' % (sweep_num, num_sweeps)
self.run_count += 1 # Force regeneration of bit stream.
# Pull class variables into local storage, performing unit conversion where necessary.
t = self.t
t_ns = self.t_ns
f = self.f
w = self.w
bits = self.bits
symbols = self.symbols
ffe = self.ffe
nbits = self.nbits
nui = self.nui
bit_rate = self.bit_rate * 1.e9
eye_bits = self.eye_bits
eye_uis = self.eye_uis
nspb = self.nspb
nspui = self.nspui
rn = self.rn
pn_mag = self.pn_mag
pn_freq = self.pn_freq * 1.e6
Vod = self.vod
Rs = self.rs
Cs = self.cout * 1.e-12
RL = self.rin
CL = self.cac * 1.e-6
Cp = self.cin * 1.e-12
R0 = self.R0
w0 = self.w0
Rdc = self.Rdc
Z0 = self.Z0
v0 = self.v0 * 3.e8
Theta0 = self.Theta0
l_ch = self.l_ch
pattern_len = self.pattern_len
rx_bw = self.rx_bw * 1.e9
peak_freq = self.peak_freq * 1.e9
peak_mag = self.peak_mag
ctle_offset = self.ctle_offset
ctle_mode = self.ctle_mode
delta_t = self.delta_t * 1.e-12
alpha = self.alpha
ui = self.ui
n_taps = self.n_taps
gain = self.gain
n_ave = self.n_ave
decision_scaler = self.decision_scaler
n_lock_ave = self.n_lock_ave
rel_lock_tol = self.rel_lock_tol
lock_sustain = self.lock_sustain
bandwidth = self.sum_bw * 1.e9
rel_thresh = self.thresh
mod_type = self.mod_type[0]
try:
# Calculate misc. values.
fs = bit_rate * nspb
Ts = t[1]
ts = Ts
# Generate the ideal over-sampled signal.
#
# Duo-binary is problematic, in that it requires convolution with the ideal duobinary
# impulse response, in order to produce the proper ideal signal.
x = repeat(symbols, nspui)
self.x = x
if(mod_type == 1): # Handle duo-binary case.
duob_h = array(([0.5] + [0.] * (nspui - 1)) * 2)
x = convolve(x, duob_h)[:len(t)]
self.ideal_signal = x
# Find the ideal crossing times, for subsequent jitter analysis of transmitted signal.
ideal_xings = find_crossings(t, x, decision_scaler, min_delay=(ui / 2.), mod_type=mod_type)
self.ideal_xings = ideal_xings
# Calculate the channel output.
#
# Note: We're not using 'self.ideal_signal', because we rely on the system response to
# create the duobinary waveform. We only create it explicitly, above,
# so that we'll have an ideal reference for comparison.
chnl_h = self.calc_chnl_h()
chnl_out = convolve(self.x, chnl_h)[:len(x)]
self.channel_perf = nbits * nspb / (clock() - start_time)
split_time = clock()
self.status = 'Running Tx...(sweep %d of %d)' % (sweep_num, num_sweeps)
except Exception:
self.status = 'Exception: channel'
raise
self.chnl_out = chnl_out
self.chnl_out_H = fft(chnl_out)
# Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the Tx.
try:
if(self.tx_use_ami):
# Note: Within the PyBERT computational environment, we use normalized impulse responses,
# which have units of (V/ts), where 'ts' is the sample interval. However, IBIS-AMI models expect
# units of (V/s). So, we have to scale accordingly, as we transit the boundary between these two worlds.
tx_cfg = self._tx_cfg # Grab the 'AMIParamConfigurator' instance for this model.
# Get the model invoked and initialized, except for 'channel_response', which
# we need to do several different ways, in order to gather all the data we need.
tx_param_dict = tx_cfg.input_ami_params
tx_model_init = AMIModelInitializer(tx_param_dict)
tx_model_init.sample_interval = ts # Must be set, before 'channel_response'!
tx_model_init.channel_response = [1. / ts] + [0.] * (len(chnl_h) - 1) # Start with a delta function, to capture the model's impulse response.
tx_model_init.bit_time = ui
tx_model = AMIModel(self.tx_dll_file)
tx_model.initialize(tx_model_init)
self.log("Tx IBIS-AMI model initialization results:\nInput parameters: {}\nOutput parameters: {}\nMessage: {}".format(
tx_model.ami_params_in, tx_model.ami_params_out, tx_model.msg))
if(tx_cfg.fetch_param_val(['Reserved_Parameters', 'Init_Returns_Impulse'])):
tx_h = array(tx_model.initOut) * ts
elif(not tx_cfg.fetch_param_val(['Reserved_Parameters', 'GetWave_Exists'])):
error("ERROR: Both 'Init_Returns_Impulse' and 'GetWave_Exists' are False!\n \
I cannot continue.\nThis condition is supposed to be caught sooner in the flow.")
self.status = "Simulation Error."
return
elif(not self.tx_use_getwave):
error("ERROR: You have elected not to use GetWave for a model, which does not return an impulse response!\n \
I cannot continue.\nPlease, select 'Use GetWave' and try again.", 'PyBERT Alert')
self.status = "Simulation Error."
return
if(self.tx_use_getwave):
# For GetWave, use a step to extract the model's native properties.
# Position the input edge at the center of the vector, in
# order to minimize high frequency artifactual energy
# introduced by frequency domain processing in some models.
half_len = len(chnl_h) // 2
tx_s = tx_model.getWave(array([0.] * half_len + [1.] * half_len))
# Shift the result back to the correct location, extending the last sample.
tx_s = pad(tx_s[half_len:], (0, half_len), 'edge')
tx_h = diff(concatenate((array([0.0]), tx_s))) # Without the leading 0, we miss the pre-tap.
tx_out = tx_model.getWave(self.x)
else: # Init()-only.
tx_s = tx_h.cumsum()
tx_out = convolve(tx_h, self.x)[:len(self.x)]
else:
# - Generate the ideal, post-preemphasis signal.
# To consider: use 'scipy.interp()'. This is what Mark does, in order to induce jitter in the Tx output.
ffe_out = convolve(symbols, ffe)[:len(symbols)]
self.rel_power = mean(ffe_out ** 2) # Store the relative average power dissipated in the Tx.
tx_out = repeat(ffe_out, nspui) # oversampled output
# - Calculate the responses.
# - (The Tx is unique in that the calculated responses aren't used to form the output.
# This is partly due to the out of order nature in which we combine the Tx and channel,
# and partly due to the fact that we're adding noise to the Tx output.)
tx_h = array(sum([[x] + list(zeros(nspui - 1)) for x in ffe], [])) # Using sum to concatenate.
tx_h.resize(len(chnl_h))
tx_s = tx_h.cumsum()
temp = tx_h.copy()
temp.resize(len(w))
tx_H = fft(temp)
tx_H *= tx_s[-1] / abs(tx_H[0])
# - Generate the uncorrelated periodic noise. (Assume capacitive coupling.)
# - Generate the ideal rectangular aggressor waveform.
pn_period = 1. / pn_freq
pn_samps = int(pn_period / Ts + 0.5)
pn = zeros(pn_samps)
pn[pn_samps // 2:] = pn_mag
pn = resize(pn, len(tx_out))
# - High pass filter it. (Simulating capacitive coupling.)
(b, a) = iirfilter(2, gFc/(fs/2), btype='highpass')
pn = lfilter(b, a, pn)[:len(pn)]
# - Add the uncorrelated periodic and random noise to the Tx output.
tx_out += pn
tx_out += normal(scale=rn, size=(len(tx_out),))
# - Convolve w/ channel.
tx_out_h = convolve(tx_h, chnl_h)[:len(chnl_h)]
temp = tx_out_h.copy()
temp.resize(len(w))
tx_out_H = fft(temp)
rx_in = convolve(tx_out, chnl_h)[:len(tx_out)]
self.tx_s = tx_s
self.tx_out = tx_out
self.rx_in = rx_in
self.tx_out_s = tx_out_h.cumsum()
self.tx_out_p = self.tx_out_s[nspui:] - self.tx_out_s[:-nspui]
self.tx_H = tx_H
self.tx_h = tx_h
self.tx_out_H = tx_out_H
self.tx_out_h = tx_out_h
self.tx_perf = nbits * nspb / (clock() - split_time)
split_time = clock()
self.status = 'Running CTLE...(sweep %d of %d)' % (sweep_num, num_sweeps)
except Exception:
self.status = 'Exception: Tx'
raise
# Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the CTLE.
try:
if(self.rx_use_ami):
rx_cfg = self._rx_cfg # Grab the 'AMIParamConfigurator' instance for this model.
# Get the model invoked and initialized, except for 'channel_response', which
# we need to do several different ways, in order to gather all the data we need.
rx_param_dict = rx_cfg.input_ami_params
rx_model_init = AMIModelInitializer(rx_param_dict)
rx_model_init.sample_interval = ts # Must be set, before 'channel_response'!
rx_model_init.channel_response = tx_out_h / ts
rx_model_init.bit_time = ui
rx_model = AMIModel(self.rx_dll_file)
rx_model.initialize(rx_model_init)
self.log("Rx IBIS-AMI model initialization results:\nInput parameters: {}\nMessage: {}\nOutput parameters: {}".format(
rx_model.ami_params_in, rx_model.msg, rx_model.ami_params_out))
if(rx_cfg.fetch_param_val(['Reserved_Parameters', 'Init_Returns_Impulse'])):
ctle_out_h = array(rx_model.initOut) * ts
elif(not rx_cfg.fetch_param_val(['Reserved_Parameters', 'GetWave_Exists'])):
error("ERROR: Both 'Init_Returns_Impulse' and 'GetWave_Exists' are False!\n \
I cannot continue.\nThis condition is supposed to be caught sooner in the flow.")
self.status = "Simulation Error."
return
elif(not self.rx_use_getwave):
error("ERROR: You have elected not to use GetWave for a model, which does not return an impulse response!\n \
I cannot continue.\nPlease, select 'Use GetWave' and try again.", 'PyBERT Alert')
self.status = "Simulation Error."
return
if(self.rx_use_getwave):
if(False):
ctle_out, clock_times = rx_model.getWave(rx_in, 32)
else:
ctle_out, clock_times = rx_model.getWave(rx_in, len(rx_in))
self.log(rx_model.ami_params_out)
ctle_H = fft(ctle_out * signal.hann(len(ctle_out))) / fft(rx_in * signal.hann(len(rx_in)))
ctle_h = real(ifft(ctle_H)[:len(chnl_h)])
ctle_out_h = convolve(ctle_h, tx_out_h)[:len(chnl_h)]
else: # Init() only.
ctle_out_h_padded = pad(ctle_out_h, (nspb, len(rx_in) - nspb - len(ctle_out_h)),
'linear_ramp', end_values=(0., 0.))
tx_out_h_padded = pad(tx_out_h, (nspb, len(rx_in) - nspb - len(tx_out_h)),
'linear_ramp', end_values=(0., 0.))
ctle_H = fft(ctle_out_h_padded) / fft(tx_out_h_padded)
ctle_h = real(ifft(ctle_H)[:len(chnl_h)])
ctle_out = convolve(rx_in, ctle_h)[:len(tx_out)]
ctle_s = ctle_h.cumsum()
else:
if(self.use_ctle_file):
ctle_h = import_channel(self.ctle_file, ts)
if(max(abs(ctle_h)) < 100.): # step response?
ctle_h = diff(ctle_h) # impulse response is derivative of step response.
else:
ctle_h *= ts # Normalize to (V/sample)
ctle_h.resize(len(t))
ctle_H = fft(ctle_h)
ctle_H *= sum(ctle_h) / ctle_H[0]
else:
w_dummy, ctle_H = make_ctle(rx_bw, peak_freq, peak_mag, w, ctle_mode, ctle_offset)
ctle_h = real(ifft(ctle_H))[:len(chnl_h)]
ctle_h *= abs(ctle_H[0]) / sum(ctle_h)
ctle_out = convolve(rx_in, ctle_h)[:len(tx_out)]
ctle_out -= mean(ctle_out) # Force zero mean.
if(self.ctle_mode == 'AGC'): # Automatic gain control engaged?
ctle_out *= 2. * decision_scaler / ctle_out.ptp()
ctle_s = ctle_h.cumsum()
ctle_out_h = convolve(tx_out_h, ctle_h)[:len(tx_out_h)]
self.ctle_s = ctle_s
ctle_out_h_main_lobe = where(ctle_out_h >= max(ctle_out_h) / 2.)[0]
if(len(ctle_out_h_main_lobe)):
conv_dly_ix = ctle_out_h_main_lobe[0]
else:
conv_dly_ix = self.chnl_dly / Ts
conv_dly = t[conv_dly_ix]
ctle_out_s = ctle_out_h.cumsum()
temp = ctle_out_h.copy()
temp.resize(len(w))
ctle_out_H = fft(temp)
# - Store local variables to class instance.
self.ctle_out_s = ctle_out_s
# Consider changing this; it could be sensitive to insufficient "front porch" in the CTLE output step response.
self.ctle_out_p = self.ctle_out_s[nspui:] - self.ctle_out_s[:-nspui]
self.ctle_H = ctle_H
self.ctle_h = ctle_h
self.ctle_out_H = ctle_out_H
self.ctle_out_h = ctle_out_h
self.ctle_out = ctle_out
self.conv_dly = conv_dly
self.conv_dly_ix = conv_dly_ix
self.ctle_perf = nbits * nspb / (clock() - split_time)
split_time = clock()
self.status = 'Running DFE/CDR...(sweep %d of %d)' % (sweep_num, num_sweeps)
except Exception:
self.status = 'Exception: Rx'
raise
# Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the DFE.
try:
if(self.use_dfe):
dfe = DFE(n_taps, gain, delta_t, alpha, ui, nspui, decision_scaler, mod_type,
n_ave=n_ave, n_lock_ave=n_lock_ave, rel_lock_tol=rel_lock_tol, lock_sustain=lock_sustain,
bandwidth=bandwidth, ideal=self.sum_ideal)
else:
dfe = DFE(n_taps, 0., delta_t, alpha, ui, nspui, decision_scaler, mod_type,
n_ave=n_ave, n_lock_ave=n_lock_ave, rel_lock_tol=rel_lock_tol, lock_sustain=lock_sustain,
bandwidth=bandwidth, ideal=True)
(dfe_out, tap_weights, ui_ests, clocks, lockeds, clock_times, bits_out) = dfe.run(t, ctle_out)
bits_out = array(bits_out)
auto_corr = 1. * correlate(bits_out[(nbits - eye_bits):], bits[(nbits - eye_bits):], mode='same') \
/ sum(bits[(nbits - eye_bits):])
auto_corr = auto_corr[len(auto_corr) // 2 :]
self.auto_corr = auto_corr
bit_dly = where(auto_corr == max(auto_corr))[0][0]
bits_ref = bits[(nbits - eye_bits) :]
bits_tst = bits_out[(nbits + bit_dly - eye_bits) :]
if(len(bits_ref) > len(bits_tst)):
bits_ref = bits_ref[: len(bits_tst)]
elif(len(bits_tst) > len(bits_ref)):
bits_tst = bits_tst[: len(bits_ref)]
bit_errs = where(bits_tst ^ bits_ref)[0]
self.bit_errs = len(bit_errs)
dfe_h = array([1.] + list(zeros(nspb - 1)) + sum([[-x] + list(zeros(nspb - 1)) for x in tap_weights[-1]], []))
dfe_h.resize(len(ctle_out_h))
temp = dfe_h.copy()
temp.resize(len(w))
dfe_H = fft(temp)
self.dfe_s = dfe_h.cumsum()
dfe_out_H = ctle_out_H * dfe_H
dfe_out_h = convolve(ctle_out_h, dfe_h)[:len(ctle_out_h)]
dfe_out_s = dfe_out_h.cumsum()
self.dfe_out_p = dfe_out_s - pad(dfe_out_s[:-nspui], (nspui,0), 'constant', constant_values=(0,0))
self.dfe_H = dfe_H
self.dfe_h = dfe_h
self.dfe_out_H = dfe_out_H
self.dfe_out_h = dfe_out_h
self.dfe_out_s = dfe_out_s
self.dfe_out = dfe_out
self.dfe_perf = nbits * nspb / (clock() - split_time)
split_time = clock()
self.status = 'Analyzing jitter...(sweep %d of %d)' % (sweep_num, num_sweeps)
except Exception:
self.status = 'Exception: DFE'
raise
# Save local variables to class instance for state preservation, performing unit conversion where necessary.
self.adaptation = tap_weights
self.ui_ests = array(ui_ests) * 1.e12 # (ps)
self.clocks = clocks
self.lockeds = lockeds
self.clock_times = clock_times
# Analyze the jitter.
self.thresh_tx = array([])
self.jitter_ext_tx = array([])
self.jitter_tx = array([])
self.jitter_spectrum_tx = array([])
self.jitter_ind_spectrum_tx = array([])
self.thresh_ctle = array([])
self.jitter_ext_ctle = array([])
self.jitter_ctle = array([])
self.jitter_spectrum_ctle = array([])
self.jitter_ind_spectrum_ctle = array([])
self.thresh_dfe = array([])
self.jitter_ext_dfe = array([])
self.jitter_dfe = array([])
self.jitter_spectrum_dfe = array([])
self.jitter_ind_spectrum_dfe = array([])
self.f_MHz_dfe = array([])
self.jitter_rejection_ratio = array([])
try:
if(mod_type == 1): # Handle duo-binary case.
pattern_len *= 2 # Because, the XOR pre-coding can invert every other pattern rep.
if(mod_type == 2): # Handle PAM-4 case.
if(pattern_len % 2):
pattern_len *= 2 # Because, the bits are taken in pairs, to form the symbols.
# - channel output
actual_xings = find_crossings(t, chnl_out, decision_scaler, mod_type=mod_type)
(jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \
thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \
hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh)
self.t_jitter = t_jitter
self.isi_chnl = isi
self.dcd_chnl = dcd
self.pj_chnl = pj
self.rj_chnl = rj
self.thresh_chnl = thresh
self.jitter_chnl = hist
self.jitter_ext_chnl = hist_synth
self.jitter_bins = bin_centers
self.jitter_spectrum_chnl = jitter_spectrum
self.jitter_ind_spectrum_chnl = jitter_ind_spectrum
self.f_MHz = array(spectrum_freqs) * 1.e-6
# - Tx output
actual_xings = find_crossings(t, rx_in, decision_scaler, mod_type=mod_type)
(jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \
thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \
hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh)
self.isi_tx = isi
self.dcd_tx = dcd
self.pj_tx = pj
self.rj_tx = rj
self.thresh_tx = thresh
self.jitter_tx = hist
self.jitter_ext_tx = hist_synth
self.jitter_spectrum_tx = jitter_spectrum
self.jitter_ind_spectrum_tx = jitter_ind_spectrum
# - CTLE output
actual_xings = find_crossings(t, ctle_out, decision_scaler, mod_type=mod_type)
(jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \
thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \
hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh)
self.isi_ctle = isi
self.dcd_ctle = dcd
self.pj_ctle = pj
self.rj_ctle = rj
self.thresh_ctle = thresh
self.jitter_ctle = hist
self.jitter_ext_ctle = hist_synth
self.jitter_spectrum_ctle = jitter_spectrum
self.jitter_ind_spectrum_ctle = jitter_ind_spectrum
# - DFE output
ignore_until = (nui - eye_uis) * ui + 0.75 * ui # 0.5 was causing an occasional misalignment.
ideal_xings = array(filter(lambda x: x > ignore_until, list(ideal_xings)))
min_delay = ignore_until + conv_dly
actual_xings = find_crossings(t, dfe_out, decision_scaler, min_delay=min_delay,
mod_type=mod_type,
rising_first=False,
)
(jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \
thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \
hist, hist_synth, bin_centers) = calc_jitter(ui, eye_uis, pattern_len, ideal_xings, actual_xings, rel_thresh)
self.isi_dfe = isi
self.dcd_dfe = dcd
self.pj_dfe = pj
self.rj_dfe = rj
self.thresh_dfe = thresh
self.jitter_dfe = hist
self.jitter_ext_dfe = hist_synth
self.jitter_spectrum_dfe = jitter_spectrum
self.jitter_ind_spectrum_dfe = jitter_ind_spectrum
self.f_MHz_dfe = array(spectrum_freqs) * 1.e-6
skip_factor = nbits / eye_bits
ctle_spec = self.jitter_spectrum_ctle
dfe_spec = self.jitter_spectrum_dfe
self.jitter_rejection_ratio = zeros(len(dfe_spec))
self.jitter_perf = nbits * nspb / (clock() - split_time)
self.total_perf = nbits * nspb / (clock() - start_time)
split_time = clock()
self.status = 'Updating plots...(sweep %d of %d)' % (sweep_num, num_sweeps)
except Exception:
self.status = 'Exception: jitter'
# raise
# Update plots.
try:
if(update_plots):
update_results(self)
if(not initial_run):
update_eyes(self)
self.plotting_perf = nbits * nspb / (clock() - split_time)
self.status = 'Ready.'
except Exception:
self.status = 'Exception: plotting'
raise
def a_in_scan_read_numpy(self, samples_per_channel, timeout):
# pylint: disable=too-many-locals
"""
Read scan status and data (as a NumPy array).
This function is similar to :py:func:`a_in_scan_read` except that the
*data* key in the returned namedtuple is a NumPy array of float64 values
and may be used directly with NumPy functions.
Args:
samples_per_channel (int): The number of samples per channel to read
from the scan buffer. Specify a negative number to read all
available samples or 0 to only read the scan status and return
no data.
timeout (float): The amount of time in seconds to wait for the
samples to be read. Specify a negative number to wait
indefinitely, or 0 to return immediately with the samples that
are already in the scan buffer. If the timeout is met and the
specified number of samples have not been read, then the
function will return with the amount that has been read and the
timeout status set.
Returns:
namedtuple: a namedtuple containing the following field names:
* **running** (bool): True if the scan is running, False if it has
stopped or completed.
* **hardware_overrun** (bool): True if the hardware could not
acquire and unload samples fast enough and data was lost.
* **buffer_overrun** (bool): True if the background scan buffer was
not read fast enough and data was lost.
* **triggered** (bool): True if the trigger conditions have been met
and data acquisition started.
* **timeout** (bool): True if the timeout time expired before the
specified number of samples were read.
* **data** (NumPy array of float64): The data that was read from the
scan buffer.
Raises:
HatError: A scan is not active, the board is not initialized, does
not respond, or responds incorrectly.
ValueError: Incorrect argument.
"""
try:
import numpy
from numpy.ctypeslib import ndpointer
except ImportError:
raise
if not self._initialized:
raise HatError(self._address, "Not initialized.")
self._lib.mcc118_a_in_scan_read.argtypes = [
c_ubyte,
POINTER(c_ushort), c_long, c_double,
ndpointer(c_double, flags="C_CONTIGUOUS"), c_ulong,
POINTER(c_ulong)
]
num_channels = self._lib.mcc118_a_in_scan_channel_count(self._address)
samples_read_per_channel = c_ulong()
status = c_ushort()
timed_out = False
samples_to_read = 0
if samples_per_channel < 0:
# read all available data
# first, get the number of available samples
samples_available = c_ulong(0)
result = self._lib.mcc118_a_in_scan_status(
self._address, byref(status), byref(samples_available))
if result != self._RESULT_SUCCESS:
raise HatError(self._address,
"Incorrect response {}.".format(result))
# allocate a buffer large enough for all the data
samples_to_read = samples_available.value
buffer_size = samples_available.value * num_channels
data_buffer = numpy.empty(buffer_size, dtype=numpy.float64)
elif samples_per_channel == 0:
# only read the status
samples_to_read = 0
buffer_size = 0
data_buffer = None
elif samples_per_channel > 0:
# read the specified number of samples
samples_to_read = samples_per_channel
# create a C buffer for the read
buffer_size = samples_per_channel * num_channels
data_buffer = numpy.empty(buffer_size, dtype=numpy.float64)
else:
# invalid samples_per_channel
raise ValueError(
"Invalid samples_per_channel {}.".format(samples_per_channel))
result = self._lib.mcc118_a_in_scan_read(
self._address, byref(status), samples_to_read, timeout,
data_buffer, buffer_size, byref(samples_read_per_channel))
if result == self._RESULT_BAD_PARAMETER:
raise ValueError("Invalid parameter.")
elif result == self._RESULT_RESOURCE_UNAVAIL:
raise HatError(self._address, "Scan not active.")
elif result == self._RESULT_TIMEOUT:
timed_out = True
elif result != self._RESULT_SUCCESS:
raise HatError(self._address,
"Incorrect response {}.".format(result))
total_read = samples_read_per_channel.value * num_channels
if total_read < buffer_size:
data_buffer = numpy.resize(data_buffer, (total_read, ))
scan_status = namedtuple('MCC118ScanRead', [
'running', 'hardware_overrun', 'buffer_overrun', 'triggered',
'timeout', 'data'
])
return scan_status(
running=(status.value & self._STATUS_RUNNING) != 0,
hardware_overrun=(status.value & self._STATUS_HW_OVERRUN) != 0,
buffer_overrun=(status.value & self._STATUS_BUFFER_OVERRUN) != 0,
triggered=(status.value & self._STATUS_TRIGGERED) != 0,
timeout=timed_out,
data=data_buffer)
def intcode_program(state, max_size, n_read):
output = []
# Get state
intcode_aux = state[0][:]
inputs = state[1]
pointer = state[2]
relative_base = state[3]
# Resize array (get more memory)
intcode_aux = np.resize(intcode_aux, max_size)
# Browse array by steps
while pointer < len(intcode):
# Get opcode and mode of parameters
opcode_instruction = str(intcode_aux[pointer])
while len(opcode_instruction) < 5:
opcode_instruction = '0' + opcode_instruction
opcode = int(opcode_instruction[-2:])
mode_params = [
int(opcode_instruction[-3]),
int(opcode_instruction[-4]),
int(opcode_instruction[-5])
]
# Get index of each parameter
indexes = [-1, -1, -1]
for i in range(len(indexes)):
pos = pointer + 1 + i
if pos < len(intcode_aux):
if mode_params[i] == 0:
indexes[i] = intcode_aux[pos]
elif mode_params[i] == 1:
indexes[i] = pos
elif mode_params[i] == 2:
indexes[i] = relative_base + intcode_aux[pos]
# Apply opcode operation
# STOP
if opcode == 99:
break
# ADDITION
elif opcode == 1:
op1 = intcode_aux[indexes[0]]
op2 = intcode_aux[indexes[1]]
intcode_aux[indexes[2]] = op1 + op2
pointer += 4
# MULTIPLICATION
elif opcode == 2:
op1 = intcode_aux[indexes[0]]
op2 = intcode_aux[indexes[1]]
intcode_aux[indexes[2]] = op1 * op2
pointer += 4
# INPUT
elif opcode == 3:
intcode_aux[indexes[0]] = inputs.pop(0)
pointer += 2
# OUTPUT
elif opcode == 4:
#print('Output: '+intcode_aux[indexes[0]].__str__())
output.append(intcode_aux[indexes[0]])
pointer += 2
if len(output) == n_read:
break
# JUMP-IF-TRUE
elif opcode == 5:
if intcode_aux[indexes[0]] != 0:
pointer = intcode_aux[indexes[1]]
else:
pointer += 3
# JUMP-IF-FALSE
elif opcode == 6:
if intcode_aux[indexes[0]] == 0:
pointer = intcode_aux[indexes[1]]
else:
pointer += 3
# LESS-THAN
elif opcode == 7:
if intcode_aux[indexes[0]] < intcode_aux[indexes[1]]:
intcode_aux[indexes[2]] = 1
else:
intcode_aux[indexes[2]] = 0
pointer += 4
# EQUAL-TO
elif opcode == 8:
if intcode_aux[indexes[0]] == intcode_aux[indexes[1]]:
intcode_aux[indexes[2]] = 1
else:
intcode_aux[indexes[2]] = 0
pointer += 4
# ADJUST RELATIVE BASE
elif opcode == 9:
relative_base += intcode_aux[indexes[0]]
pointer += 2
return [output, intcode_aux, pointer, relative_base]
#
# Call Option Pricing with Discrete Fourier Transforms (DFT/FFT)
import math
import numpy as np
from numpy.fft import fft, ifft
from convolution import revnp
from parameters import *
# Parameter Adjustments
M = 3 #numberoftimesteps
dt, df, u, d, q = get_binomial_parameters(M)
# Array Generation for Stock Prices
mu = np.arange(M + 1)
mu = np.resize(mu, (M + 1, M + 1))
md = np.transpose(mu)
mu = u**(mu - md)
md = d**md
S = S0 * mu * md
# Valuation by fft
CT = np.maximum(S[:, -1] - K, 0)
qv = np.zeros(M + 1, dtype=np.float)
qv[0] = q
qv[1] = 1 - q
C0_a = fft(math.exp(-r * T) * ifft(CT) * ((M + 1) * ifft(revnp(qv)))**M)
C0_b = fft(math.exp(-r * T) * ifft(CT) * fft(qv)**M)
C0_c = ifft(math.exp(-r * T) * fft(CT) * fft(revnp(qv))**M)
print("Value of European option is %8.3f" % np.real(C0_a[0]))
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import Base_Utils as bu
number_iteractions =10
img = Image.open('/Users/Ashley/Desktop/test.tif')
img.seek(5000)
out=np.array(img.copy())
[X,Y]=np.shape(out)
out_line=np.resize(out,[X*Y,1])
N=np.size(out_line)
mean_val=np.mean(out_line) #Select the mean value of the data and use it
selected_points = np.where(out_line > mean_val) #to select the points above that value
selected_points_signal=np.zeros((N,1)) #Build empty array
selected_points_signal[selected_points]=True #True means there is signal there
selected_points_noise = np.logical_not(selected_points_signal) #True means there is noise there
pms=np.zeros((2,1))
pmm=np.zeros((2,1))
#pms =1 x 2 double [signal max and signal widrh
pms[0]=np.max(out_line)
pms[1]=10
#pmn =1 x 2 double [signal max and noise widrh
pmm[0]=np.mean(out_line)
pmm[1]=10
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
"""Returns TEC screen parmdb parameters
H - H5parm object
solset - solution set with TEC screen parameters
TECsolTab = solution table with tecscreen values
timewidths - time widths of output parmdb
freq - frequency of output parmdb
freqwidth - frequency width of output parmdb
"""
global ipbar, pbar
solset = H.getSolset(solset)
station_dict = solset.getAnt()
station_names = station_dict.keys()
station_positions = station_dict.values()
source_dict = solset.getSou()
source_names = source_dict.keys()
source_positions = source_dict.values()
tec_sf = solset.getSoltab(TECsolTab)
tec_screen, axis_vals = tec_sf.getValues()
times = axis_vals['time']
beta = TECsolTab._v_attrs['beta']
r_0 = TECsolTab._v_attrs['r_0']
height = TECsolTab._v_attrs['height']
order = TECsolTab._v_attrs['order']
pp = tec_sf.t.piercepoint
N_sources = len(source_names)
N_times = len(times)
N_freqs = 1
N_stations = len(station_names)
N_piercepoints = N_sources * N_stations
freqs = freq
freqwidths = freqwidth
parms = {}
v = {}
v['times'] = times
v['timewidths'] = timewidths
v['freqs'] = freqs
v['freqwidths'] = freqwidths
for station_name in station_names:
for source_name in source_names:
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname] = v.copy()
for k in range(N_times):
D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
D = np.transpose(D, (1, 0, 2)) - D
D2 = np.sum(D**2, axis=2)
C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
tec_fit_white = np.dot(np.linalg.inv(C),
tec_screen[:, k, :].reshape(N_piercepoints))
pp_idx = 0
for src, source_name in enumerate(source_names):
for sta, station_name in enumerate(station_names):
parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]
parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]
parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]
parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]
pp_idx += 1
pbar.update(ipbar)
ipbar += 1
time_start = times[0] - timewidths[0] / 2
time_end = times[-1] + timewidths[-1] / 2
v['times'] = np.array([(time_start + time_end) / 2])
v['timewidths'] = np.array([time_end - time_start])
v_r0 = v.copy()
v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
parms['r_0'] = v_r0
v_beta = v.copy()
v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
parms['beta'] = v_beta
v_height = v.copy()
v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
parms['height'] = v_height
return parms
def process_chunk(start_id, end_id, raw_data):
"""This function extracts features and assigns labels from a good chunk of raw data
Following step is taken:
+ determine aircon status from `power`
+ identify user actions (classification labels) from the aircon status
+ remove spurious actions (TURN ON, TURN OFF that are too close to each other)
+ fix aircon status to match with the user actions (after removing spurious ones)
+ sub-sample and extract data features (outlier samples will be removed)
@:param raw_data the whole raw data array
@:param start_id where the chunk starts
@:param end_id where the chunk ends
@:return a list of feature array, label vector and aircon status vector
"""
nrows = end_id - start_id
saving_time = raw_data[start_id:end_id, 0]
power = raw_data[start_id:end_id, 1]
#prepare vectors to hold `user action` and `aircon status`
action = np.empty(nrows, int)
status = np.empty(nrows, int)
status.fill(STATUS_OFF)
action.fill(ACTION_NOTHING)
#detect power ON status
status[np.where(power > POWER_CUT)] = STATUS_ON
#detect TURN ON, TURN OFF action
action_id = []
for i in range(1, nrows):
status_diff = status[i] - status[i - 1]
if status_diff == 1:
action[i] = ACTION_TURN_ON
action_id.append(i)
elif status_diff == -1:
action[i] = ACTION_TURN_OFF
action_id.append(i)
num_actions = len(action_id) #total number of actions
action_id = np.array(action_id) #row index of actions
#remove any (TURN_OFF, TURN_ON) tuple whose time interval is too short (less than 5, for example)
min_diff = OFF_MIN_DURATION * 60 # define the minimum time interval, for example 5 minutes time 60 secs
i = num_actions - 1
while i > 0:
if action[action_id[i]] == ACTION_TURN_ON:
time_diff = saving_time[action_id[i]] - saving_time[action_id[i -
1]]
if time_diff < min_diff:
action[action_id[i]] = ACTION_NOTHING # remove TURN ON
action[action_id[i - 1]] = ACTION_NOTHING # remove TURN OFF
i -= 2
else:
i -= 1
#remove any (TURN_ON, TURN_OFF) tuple whose time interval is too short (less than 5, for example)
min_diff = ON_MIN_DURATION * 60 # define the minimum time interval, for example 5 minutes time 60 secs
i = 0
while i < num_actions - 1:
if action[action_id[i]] == ACTION_TURN_ON:
#find the next TURN OFF action
j = i + 1
while (j < num_actions
and action[action_id[j]] != ACTION_TURN_OFF):
j += 1
if j < num_actions: # found TURN OFF action
time_diff = saving_time[action_id[j]] - saving_time[
action_id[i]]
if time_diff < min_diff:
action[action_id[i]] = ACTION_NOTHING # remove TURN ON
action[action_id[j]] = ACTION_NOTHING # remove TURN OFF
i = j + 1
else:
i += 1
#once actions are correctly recognized, now it's time to fix the aircon status
prev_status = status[0]
for i in range(1, nrows):
if action[i] == ACTION_NOTHING:
status[i] = prev_status
elif action[i] == ACTION_TURN_ON:
status[i] = STATUS_ON
else:
status[i] = STATUS_OFF
prev_status = status[i]
# sub-sampling (reduce the data size), but retain all data points having an action
n_obs = SAMPLING_SIZE #sampling every 10 timepoints (i.e. 5 minutes)
nfrows = int(nrows / n_obs) #number of data rows after sampling
if nfrows <= 0:
return None
#allocate data arrays
x = np.empty([nfrows, NUM_FEATURES], float)
y = np.empty(nfrows, int)
y.fill(ACTION_NOTHING)
sampled_status = np.empty(nfrows, int)
sampled_id = n_obs
i = 0
feat_period = PAST_DURATION * 60 #extract feature from previous 5 minutes
#sub-sampling is happening in this loop
while sampled_id < nrows:
#find any near by actions. actions is precious, we must retain all actions
for j in range(n_obs):
new_id = sampled_id + j
if new_id < nrows and action[new_id] != ACTION_NOTHING:
sampled_id = new_id
break
#collect samples which are obs_period seconds from sampled_id sample
feature_ids = []
cur_id = start_id + sampled_id - 1
while (cur_id >= 0) and ((raw_data[start_id + sampled_id, 0] -
raw_data[cur_id, 0]) < feat_period):
#check if the sample at cur_id is an outlier
bOutlier = False
for j in range(2, NUM_CSV_COLS):
if (raw_data[cur_id, j] < SENSOR_RANGES[j - 2][0]) or (
raw_data[cur_id, j] > SENSOR_RANGES[j - 2][1]):
bOutlier = True
break
if not bOutlier:
feature_ids.append(cur_id)
cur_id -= 1
if len(feature_ids) > 0:
#extract feature from these samples
for j in range(2, NUM_CSV_COLS):
x[i, j - 2] = np.mean(raw_data[feature_ids, j])
#x[i, 2*j] = np.mean(raw_data[feature_ids, j])
#x[i, 2*j+1] = np.std(raw_data[feature_ids, j])
x[i, NUM_FEATURES - 2] = raw_data[sampled_id,
NUM_CSV_COLS] #week day
x[i, NUM_FEATURES - 1] = raw_data[sampled_id,
NUM_CSV_COLS + 1] #hour
sampled_status[i] = status[sampled_id - 1]
y[i] = action[sampled_id]
i += 1
#move on to the next id for sampling
sampled_id += n_obs
nrows = i #correct number of data rows
if nrows <= 0:
return None
#resize arrays to their correct sizes
x = np.resize(x, [nrows, NUM_FEATURES])
y = np.resize(y, nrows)
sampled_status = np.resize(sampled_status, nrows)
return [x, y, sampled_status]
def KM2():
matrix = numpy.loadtxt('D:/EE569_Assignment/4/C++/Kmeans2/x64/Debug/Weight_W.txt');
temp = numpy.resize(matrix, (1, 5 * 5 * 6 * 16))
temp = tf.convert_to_tensor(temp, tf.float32)
initial = tflearn.layers.core.reshape(temp, [5, 5, 6, 16])
return initial
def get_rgb(imgs,
bands,
mnmx=None,
arcsinh=None,
scales=None,
imgname='test',
desaturate=False):
'''
Given a list of images in the given bands, returns a scaled RGB
image.
*imgs* a list of numpy arrays, all the same size, in nanomaggies
*bands* a list of strings, eg, ['g','r','z']
*mnmx* = (min,max), values that will become black/white *after* scaling.
Default is (-3,10)
*arcsinh* use nonlinear scaling as in SDSS
*scales*
Returns a (H,W,3) numpy array with values between 0 and 1.
'''
bands = ''.join(bands)
grzscales = dict(
g=(2, 0.0066),
r=(1, 0.01385),
z=(0, 0.025),
)
if scales is None:
if bands == 'grz':
scales = grzscales
elif bands == 'urz':
scales = dict(
u=(2, 0.0066),
r=(1, 0.01),
z=(0, 0.025),
)
elif bands == 'gri':
# scales = dict(g = (2, 0.004),
# r = (1, 0.0066),
# i = (0, 0.01),
# )
scales = dict(
g=(2, 0.002),
r=(1, 0.004),
i=(0, 0.005),
)
else:
scales = grzscales
h, w = imgs[0].shape
rgb = np.zeros((h, w, 3), np.float32)
# Convert to ~ sigmas
for im, band in zip(imgs, bands):
plane, scale = scales[band]
rgb[:, :, plane] = (im / scale).astype(np.float32)
#print 'rgb: plane', plane, 'range', rgb[:,:,plane].min(), rgb[:,:,plane].max()
if mnmx is None:
mn, mx = -3, 10
else:
mn, mx = mnmx
if arcsinh is not None:
def nlmap(x):
return np.arcsinh(x * arcsinh) / np.sqrt(arcsinh)
rgb = nlmap(rgb)
mn = nlmap(mn)
mx = nlmap(mx)
rgb = (rgb - mn) / (mx - mn)
if desaturate:
# optionally desaturate pixels that are dominated by a single
# colour to avoid colourful speckled sky
RGBim = np.array([rgb[:, :, 0], rgb[:, :, 1], rgb[:, :, 2]])
a = RGBim.mean(axis=0)
np.putmask(a, a == 0.0, 1.0)
acube = np.resize(a, (3, h, w))
bcube = (RGBim / acube) / 2.5
mask = np.array(bcube)
wt = np.max(mask, axis=0)
np.putmask(wt, wt > 1.0, 1.0)
wt = 1 - wt
wt = np.sin(wt * np.pi / 2.0)
temp = RGBim * wt + a * (1 - wt) + a * (1 - wt)**2 * RGBim
rgb = np.zeros((h, w, 3), np.float32)
for idx, im in enumerate(
(temp[0, :, :], temp[1, :, :], temp[2, :, :])):
rgb[:, :, idx] = im
clipped = np.clip(rgb, 0., 1.)
# Save hardcopy as JPG
#out_jpg = '%s/decals/imagetests/dstn/test.jpeg' % gzpath
out_jpg = '%s/decals/imagetests/sugata/%s.jpeg' % (gzpath, imgname)
plt.imsave(out_jpg, clipped, origin='lower')
return clipped
def body1(self, num, object_num, loss, predict, labels, nilboy):
"""
calculate loss
Args:
predict: 3-D tensor [cell_size, cell_size, 5 * boxes_per_cell]
labels : [max_objects, 5] (x_center, y_center, w, h, class)
"""
label = labels[num:num + 1, :]
label = tf.reshape(label, [-1])
#calculate objects tensor [CELL_SIZE, CELL_SIZE]
min_x = (label[0] - label[2] / 2) / (self.image_size / self.cell_size)
max_x = (label[0] + label[2] / 2) / (self.image_size / self.cell_size)
min_y = (label[1] - label[3] / 2) / (self.image_size / self.cell_size)
max_y = (label[1] + label[3] / 2) / (self.image_size / self.cell_size)
min_x = tf.floor(min_x)
min_y = tf.floor(min_y)
max_x = tf.ceil(max_x)
max_y = tf.ceil(max_y)
temp = tf.cast(tf.stack([max_y - min_y, max_x - min_x]),
dtype=tf.int32)
objects = tf.ones(temp, tf.float32)
temp = tf.cast(
tf.stack(
[min_y, self.cell_size - max_y, min_x,
self.cell_size - max_x]), tf.int32)
temp = tf.reshape(temp, (2, 2))
objects = tf.pad(objects, temp, "CONSTANT")
#calculate objects tensor [CELL_SIZE, CELL_SIZE]
#calculate responsible tensor [CELL_SIZE, CELL_SIZE]
center_x = label[0] / (self.image_size / self.cell_size)
center_x = tf.floor(center_x)
center_y = label[1] / (self.image_size / self.cell_size)
center_y = tf.floor(center_y)
response = tf.ones([1, 1], tf.float32)
temp = tf.cast(
tf.stack([
center_y, self.cell_size - center_y - 1, center_x,
self.cell_size - center_x - 1
]), tf.int32)
temp = tf.reshape(temp, (2, 2))
response = tf.pad(response, temp, "CONSTANT")
#objects = response
#calculate iou_predict_truth [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
predict_boxes = predict[:, :, self.num_classes + self.boxes_per_cell:]
predict_boxes = tf.reshape(
predict_boxes,
[self.cell_size, self.cell_size, self.boxes_per_cell, 4])
predict_boxes = predict_boxes * [
self.image_size / self.cell_size, self.image_size / self.cell_size,
self.image_size, self.image_size
]
base_boxes = np.zeros([self.cell_size, self.cell_size, 4])
for y in range(self.cell_size):
for x in range(self.cell_size):
#nilboy
base_boxes[y, x, :] = [
self.image_size / self.cell_size * x,
self.image_size / self.cell_size * y, 0, 0
]
base_boxes = np.tile(
np.resize(base_boxes, [self.cell_size, self.cell_size, 1, 4]),
[1, 1, self.boxes_per_cell, 1])
predict_boxes = base_boxes + predict_boxes
iou_predict_truth = self.iou(predict_boxes, label[0:4])
#calculate C [cell_size, cell_size, boxes_per_cell]
C = iou_predict_truth * tf.reshape(response,
[self.cell_size, self.cell_size, 1])
#calculate I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
I = iou_predict_truth * tf.reshape(response,
(self.cell_size, self.cell_size, 1))
max_I = tf.reduce_max(I, 2, keep_dims=True)
I = tf.cast((I >= max_I), tf.float32) * tf.reshape(
response, (self.cell_size, self.cell_size, 1))
#calculate no_I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
no_I = tf.ones_like(I, dtype=tf.float32) - I
p_C = predict[:, :,
self.num_classes:self.num_classes + self.boxes_per_cell]
#calculate truth x,y,sqrt_w,sqrt_h 0-D
x = label[0]
y = label[1]
sqrt_w = tf.sqrt(tf.abs(label[2]))
sqrt_h = tf.sqrt(tf.abs(label[3]))
#sqrt_w = tf.abs(label[2])
#sqrt_h = tf.abs(label[3])
#calculate predict p_x, p_y, p_sqrt_w, p_sqrt_h 3-D [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
p_x = predict_boxes[:, :, :, 0]
p_y = predict_boxes[:, :, :, 1]
#p_sqrt_w = tf.sqrt(tf.abs(predict_boxes[:, :, :, 2])) * ((tf.cast(predict_boxes[:, :, :, 2] > 0, tf.float32) * 2) - 1)
#p_sqrt_h = tf.sqrt(tf.abs(predict_boxes[:, :, :, 3])) * ((tf.cast(predict_boxes[:, :, :, 3] > 0, tf.float32) * 2) - 1)
#p_sqrt_w = tf.sqrt(tf.maximum(0.0, predict_boxes[:, :, :, 2]))
#p_sqrt_h = tf.sqrt(tf.maximum(0.0, predict_boxes[:, :, :, 3]))
#p_sqrt_w = predict_boxes[:, :, :, 2]
#p_sqrt_h = predict_boxes[:, :, :, 3]
p_sqrt_w = tf.sqrt(
tf.minimum(self.image_size * 1.0,
tf.maximum(0.0, predict_boxes[:, :, :, 2])))
p_sqrt_h = tf.sqrt(
tf.minimum(self.image_size * 1.0,
tf.maximum(0.0, predict_boxes[:, :, :, 3])))
#calculate truth p 1-D tensor [NUM_CLASSES]
P = tf.one_hot(tf.cast(label[4], tf.int32),
self.num_classes,
dtype=tf.float32)
#calculate predict p_P 3-D tensor [CELL_SIZE, CELL_SIZE, NUM_CLASSES]
p_P = predict[:, :, 0:self.num_classes]
#class_loss
class_loss = tf.nn.l2_loss(
tf.reshape(objects, (self.cell_size, self.cell_size, 1)) *
(p_P - P)) * self.class_scale
#class_loss = tf.nn.l2_loss(tf.reshape(response, (self.cell_size, self.cell_size, 1)) * (p_P - P)) * self.class_scale
#object_loss
object_loss = tf.nn.l2_loss(I * (p_C - C)) * self.object_scale
#object_loss = tf.nn.l2_loss(I * (p_C - (C + 1.0)/2.0)) * self.object_scale
#noobject_loss
#noobject_loss = tf.nn.l2_loss(no_I * (p_C - C)) * self.noobject_scale
noobject_loss = tf.nn.l2_loss(no_I * (p_C)) * self.noobject_scale
#coord_loss
coord_loss = (tf.nn.l2_loss(I * (p_x - x) /
(self.image_size / self.cell_size)) +
tf.nn.l2_loss(I * (p_y - y) /
(self.image_size / self.cell_size)) +
tf.nn.l2_loss(I *
(p_sqrt_w - sqrt_w)) / self.image_size +
tf.nn.l2_loss(I * (p_sqrt_h - sqrt_h)) /
self.image_size) * self.coord_scale
nilboy = I
return num + 1, object_num, [
loss[0] + class_loss, loss[1] + object_loss,
loss[2] + noobject_loss, loss[3] + coord_loss
], predict, labels, nilboy
def KM4():
matrix = numpy.loadtxt('D:/EE569_Assignment/4/C++/Kmeans4/x64/Debug/Weight_W.txt');
temp = numpy.resize(matrix, (1, 120 * 84))
temp = tf.convert_to_tensor(temp, tf.float32)
initial = tflearn.layers.core.reshape(temp, [120, 84])
return initial
def log_encoding_ACESproxy(lin_AP1,
bit_depth=10,
out_int=False,
constants=ACES_PROXY_CONSTANTS):
"""
Defines the *ACESproxy* colourspace log encoding curve / opto-electronic
transfer function.
Parameters
----------
lin_AP1 : numeric or array_like
*lin_AP1* value.
bit_depth : int, optional
**{10, 12}**,
*ACESproxy* bit depth.
out_int : bool, optional
Whether to return value as integer code value or float equivalent of a
code value at a given bit depth.
constants : Structure, optional
*ACESproxy* constants.
Returns
-------
numeric or ndarray
*ACESproxy* non-linear value.
Notes
-----
+---------------+-----------------------+---------------+
| **Domain \\*** | **Scale - Reference** | **Scale - 1** |
+===============+=======================+===============+
| ``lin_AP1`` | [0, 1] | [0, 1] |
+---------------+-----------------------+---------------+
+---------------+-----------------------+---------------+
| **Range \\*** | **Scale - Reference** | **Scale - 1** |
+===============+=======================+===============+
| ``ACESproxy`` | [0, 1] | [0, 1] |
+---------------+-----------------------+---------------+
- \\* This definition has an output integer switch, thus the domain-range
scale information is only given for the floating point mode.
References
----------
:cite:`TheAcademyofMotionPictureArtsandSciences2014q`,
:cite:`TheAcademyofMotionPictureArtsandSciences2014r`,
:cite:`TheAcademyofMotionPictureArtsandSciences2014s`,
:cite:`TheAcademyofMotionPictureArtsandSciencese`
Examples
--------
>>> log_encoding_ACESproxy(0.18) # doctest: +ELLIPSIS
0.4164222...
>>> log_encoding_ACESproxy(0.18, out_int=True)
426
"""
lin_AP1 = to_domain_1(lin_AP1)
constants = constants[bit_depth]
CV_min = np.resize(constants.CV_min, lin_AP1.shape)
CV_max = np.resize(constants.CV_max, lin_AP1.shape)
def float_2_cv(x):
"""
Converts given numeric to code value.
"""
return np.maximum(CV_min, np.minimum(CV_max, np.round(x)))
ACESproxy = np.where(
lin_AP1 > 2**-9.72,
float_2_cv((np.log2(lin_AP1) + constants.mid_log_offset) *
constants.steps_per_stop + constants.mid_CV_offset),
np.resize(CV_min, lin_AP1.shape),
)
if out_int:
return as_int(np.round(ACESproxy))
else:
return as_float(from_range_1(ACESproxy / (2**bit_depth - 1)))
def _spectral_helper(x,
y=None,
NFFT=None,
Fs=None,
detrend_func=None,
window=None,
noverlap=None,
pad_to=None,
sides=None,
scale_by_freq=None,
mode=None):
"""
Private helper implementing the common parts between the psd, csd,
spectrogram and complex, magnitude, angle, and phase spectrums.
"""
if y is None:
# if y is None use x for y
same_data = True
else:
# The checks for if y is x are so that we can use the same function to
# implement the core of psd(), csd(), and spectrogram() without doing
# extra calculations. We return the unaveraged Pxy, freqs, and t.
same_data = y is x
if Fs is None:
Fs = 2
if noverlap is None:
noverlap = 0
if detrend_func is None:
detrend_func = detrend_none
if window is None:
window = window_hanning
# if NFFT is set to None use the whole signal
if NFFT is None:
NFFT = 256
if mode is None or mode == 'default':
mode = 'psd'
_api.check_in_list(
['default', 'psd', 'complex', 'magnitude', 'angle', 'phase'],
mode=mode)
if not same_data and mode != 'psd':
raise ValueError("x and y must be equal if mode is not 'psd'")
# Make sure we're dealing with a numpy array. If y and x were the same
# object to start with, keep them that way
x = np.asarray(x)
if not same_data:
y = np.asarray(y)
if sides is None or sides == 'default':
if np.iscomplexobj(x):
sides = 'twosided'
else:
sides = 'onesided'
_api.check_in_list(['default', 'onesided', 'twosided'], sides=sides)
# zero pad x and y up to NFFT if they are shorter than NFFT
if len(x) < NFFT:
n = len(x)
x = np.resize(x, NFFT)
x[n:] = 0
if not same_data and len(y) < NFFT:
n = len(y)
y = np.resize(y, NFFT)
y[n:] = 0
if pad_to is None:
pad_to = NFFT
if mode != 'psd':
scale_by_freq = False
elif scale_by_freq is None:
scale_by_freq = True
# For real x, ignore the negative frequencies unless told otherwise
if sides == 'twosided':
numFreqs = pad_to
if pad_to % 2:
freqcenter = (pad_to - 1) // 2 + 1
else:
freqcenter = pad_to // 2
scaling_factor = 1.
elif sides == 'onesided':
if pad_to % 2:
numFreqs = (pad_to + 1) // 2
else:
numFreqs = pad_to // 2 + 1
scaling_factor = 2.
if not np.iterable(window):
window = window(np.ones(NFFT, x.dtype))
if len(window) != NFFT:
raise ValueError(
"The window length must match the data's first dimension")
result = stride_windows(x, NFFT, noverlap, axis=0)
result = detrend(result, detrend_func, axis=0)
result = result * window.reshape((-1, 1))
result = np.fft.fft(result, n=pad_to, axis=0)[:numFreqs, :]
freqs = np.fft.fftfreq(pad_to, 1 / Fs)[:numFreqs]
if not same_data:
# if same_data is False, mode must be 'psd'
resultY = stride_windows(y, NFFT, noverlap)
resultY = detrend(resultY, detrend_func, axis=0)
resultY = resultY * window.reshape((-1, 1))
resultY = np.fft.fft(resultY, n=pad_to, axis=0)[:numFreqs, :]
result = np.conj(result) * resultY
elif mode == 'psd':
result = np.conj(result) * result
elif mode == 'magnitude':
result = np.abs(result) / np.abs(window).sum()
elif mode == 'angle' or mode == 'phase':
# we unwrap the phase later to handle the onesided vs. twosided case
result = np.angle(result)
elif mode == 'complex':
result /= np.abs(window).sum()
if mode == 'psd':
# Also include scaling factors for one-sided densities and dividing by
# the sampling frequency, if desired. Scale everything, except the DC
# component and the NFFT/2 component:
# if we have a even number of frequencies, don't scale NFFT/2
if not NFFT % 2:
slc = slice(1, -1, None)
# if we have an odd number, just don't scale DC
else:
slc = slice(1, None, None)
result[slc] *= scaling_factor
# MATLAB divides by the sampling frequency so that density function
# has units of dB/Hz and can be integrated by the plotted frequency
# values. Perform the same scaling here.
if scale_by_freq:
result /= Fs
# Scale the spectrum by the norm of the window to compensate for
# windowing loss; see Bendat & Piersol Sec 11.5.2.
result /= (np.abs(window)**2).sum()
else:
# In this case, preserve power in the segment, not amplitude
result /= np.abs(window).sum()**2
t = np.arange(NFFT / 2, len(x) - NFFT / 2 + 1, NFFT - noverlap) / Fs
if sides == 'twosided':
# center the frequency range at zero
freqs = np.roll(freqs, -freqcenter, axis=0)
result = np.roll(result, -freqcenter, axis=0)
elif not pad_to % 2:
# get the last value correctly, it is negative otherwise
freqs[-1] *= -1
# we unwrap the phase here to handle the onesided vs. twosided case
if mode == 'phase':
result = np.unwrap(result, axis=0)
return result, freqs, t
output = 'datatype:{}, mean:{}, sd:{}, train time:{}s, explain time:{}s \n'.format(
"mnist",
0, #np.mean(median_ranks),
0, #np.std(median_ranks),
train_time,
exp_time)
print(scores.shape)
#exit(0)
print(scores_grp, "smaple_grp")
print(scores, "samples")
group_dir_name = 'img_groups_new'
if not os.path.exists(group_dir_name):
os.mkdir(group_dir_name)
num_dir_name = 'img_numbers_new'
print(x_val, "x_val")
print(new_model_input4, "new 4")
if not os.path.exists(num_dir_name):
os.mkdir(num_dir_name)
for num in tqdm(range(100)):
visualize_group(scores[num], group_dir_name + '/group_%03d.png' % num,
scores_grp[num])
img = np.resize(
np.squeeze(x_val[num] * 255.0).astype(np.uint8), (14, 14))
imsave(num_dir_name + '/num_%03d.png' % num, img)
pickle.dump(scores, open("./score.pkl", "wb"))
pickle.dump(scores_grp, open("./score_grp.pkl", "wb"))
pickle.dump(x_val, open("./x_val.pkl", "wb"))
pickle.dump(y_val, open("./y_val.pkl", "wb"))
pickle.dump(preds, open("./preds.pkl", "wb"))
def getitem(self, global_index, crop=None):
# select frame from sequence
index = global_index
# get data ids
audio_id = self.audio_ids[index]
image_id = self.image_ids[index]
uv_id = self.uvs_ids[index]
#print('GET ITEM: ', index)
img_fname = os.path.join(self.image_dir,
str(image_id).zfill(5) + '.png')
img_numpy = np.asarray(Image.open(img_fname))
TARGET = 2.0 * transforms.ToTensor()(img_numpy.astype(
np.float32)) / 255.0 - 1.0
uv_fname = os.path.join(self.uvs_dir, str(uv_id).zfill(5) + '.exr')
uv_numpy = util.load_exr(uv_fname)
UV = transforms.ToTensor()(uv_numpy.astype(np.float32))
UV = torch.where(UV > 1.0, torch.zeros_like(UV), UV)
UV = torch.where(UV < 0.0, torch.zeros_like(UV), UV)
UV = 2.0 * UV - 1.0
# intrinsics and extrinsics
intrinsics = self.intrinsics
extrinsics = self.extrinsics[uv_id]
# expressions
expressions = np.asarray(self.expressions[audio_id], dtype=np.float32)
#print('expressions:', expressions.shape)
expressions[32] *= 0.0 # remove eye brow movements
expressions[41] *= 0.0 # remove eye brow movements
expressions[71:75] *= 0.0 # remove eye brow movements
expressions = torch.tensor(expressions)
# identity
identity = torch.tensor(self.identities[audio_id])
# load deepspeech feature
dsf_fname = os.path.join(self.audio_feature_dir,
str(audio_id) + '.deepspeech.npy')
feature_array = np.load(dsf_fname)
dsf_np = np.resize(feature_array, (16, 29, 1))
dsf = transforms.ToTensor()(dsf_np.astype(np.float32))
# load sequence data if necessary
if self.opt.look_ahead: # use prev and following frame infos
r = self.opt.seq_len // 2
last_valid_idx = audio_id
for i in range(1, r): # prev frames
index_seq = index - i
if index_seq < 0: index_seq = 0
audio_id_seq = self.audio_ids[index_seq]
if audio_id_seq == audio_id - i: last_valid_idx = audio_id_seq
else: audio_id_seq = last_valid_idx
dsf_fname = os.path.join(self.audio_feature_dir,
str(audio_id_seq) + '.deepspeech.npy')
feature_array = np.load(dsf_fname)
dsf_np = np.resize(feature_array, (16, 29, 1))
dsf_seq = transforms.ToTensor()(dsf_np.astype(
np.float32)) # 1 x 16 x 29
dsf = torch.cat([dsf_seq, dsf], 0) # seq_len x 16 x 29
# note the ordering [old ... current]
last_valid_idx = audio_id
for i in range(1, self.opt.seq_len - r + 1): # following frames
index_seq = index + i
max_idx = len(self.audio_ids) - 1
if index_seq > max_idx: index_seq = max_idx
audio_id_seq = self.audio_ids[index_seq]
if audio_id_seq == audio_id + i: last_valid_idx = audio_id_seq
else: audio_id_seq = last_valid_idx
dsf_fname = os.path.join(self.audio_feature_dir,
str(audio_id_seq) + '.deepspeech.npy')
feature_array = np.load(dsf_fname)
dsf_np = np.resize(feature_array, (16, 29, 1))
dsf_seq = transforms.ToTensor()(dsf_np.astype(
np.float32)) # 1 x 16 x 29
dsf = torch.cat([dsf, dsf_seq], 0) # seq_len x 16 x 29
# note the ordering [old ... current ... future]
else:
last_valid_idx = audio_id
for i in range(1, self.opt.seq_len):
index_seq = index - i
if index_seq < 0: index_seq = 0
audio_id_seq = self.audio_ids[index_seq]
if audio_id_seq == audio_id - i: last_valid_idx = audio_id_seq
else: audio_id_seq = last_valid_idx
dsf_fname = os.path.join(self.audio_feature_dir,
str(audio_id_seq) + '.deepspeech.npy')
feature_array = np.load(dsf_fname)
dsf_np = np.resize(feature_array, (16, 29, 1))
dsf_seq = transforms.ToTensor()(dsf_np.astype(
np.float32)) # 1 x 16 x 29
dsf = torch.cat([dsf_seq, dsf], 0) # seq_len x 16 x 29
# note the ordering [old ... current]
#################################
####### apply augmentation ######
#################################
if not self.opt.no_augmentation:
if type(crop) == type(None):
INVALID_UV = -1
mask = ((UV[0:1, :, :] != INVALID_UV) |
(UV[1:2, :, :] != INVALID_UV))
crop = self.computeCrop(
mask, MULTIPLE_OF=64
) # << dependent on the network structure !! 64 => 6 layers
offset_x, offset_y, new_dim_x, new_dim_y = crop
# select subwindow
TARGET = TARGET[:, offset_y:offset_y + new_dim_y,
offset_x:offset_x + new_dim_x]
UV = UV[:, offset_y:offset_y + new_dim_y,
offset_x:offset_x + new_dim_x]
# compute new intrinsics
# TODO: atm not needed but maybe later
#################################
return {
'TARGET': TARGET,
'UV': UV,
'paths': dsf_fname, #img_path,
'intrinsics': np.array(intrinsics),
'extrinsics': np.array(extrinsics),
'expressions': expressions,
'identity': identity,
'audio_deepspeech': dsf, # deepspeech feature
#'target_id':target_id,
'crop': crop,
'internal_id': 0
}
