def _aces_proxy_OECF(value, bit_depth='10 Bit'): """ Defines the *ACESproxy* colourspace opto-electronic conversion function. Parameters ---------- value : numeric or array_like Value. bit_depth : unicode, optional **{'10 Bit', '12 Bit'}**, *ACESproxy* bit depth. Returns ------- numeric or ndarray Companded value. """ value = np.asarray(value) constants = ACES_PROXY_CONSTANTS.get(bit_depth) CV_min = np.resize(constants.CV_min, value.shape) CV_max = np.resize(constants.CV_max, value.shape) float_2_cv = lambda x: np.maximum(CV_min, np.minimum(CV_max, np.round(x))) output = np.where(value > 2 ** -9.72, float_2_cv((np.log2(value) + constants.mid_log_offset) * constants.steps_per_stop + constants.mid_CV_offset), np.resize(CV_min, value.shape)) return output
def feature_normalization(train, test): """Rescale the data so that each feature in the training set is in the interval [0,1], and apply the same transformations to the test set, using the statistics computed on the training set. Args: train - training set, a 2D numpy array of size (num_instances, num_features) test - test set, a 2D numpy array of size (num_instances, num_features) Returns: train_normalized - training set after normalization test_normalized - test set after normalization """ ## TODONE trainMin = np.resize(train.min(axis=0), (100, 48)) trainMax = np.resize(train.max(axis=0), (100, 48)) topTrain = train - trainMin bottomTrain = trainMax - trainMin outputTrain = topTrain / bottomTrain topTest = test - trainMin bottomTest = bottomTrain outputTest = topTest / bottomTest return outputTrain, outputTest
def get_spectral_magnitude(y_data,time_data, fs): tStep = np.max(time_data)/len(time_data) timeV = np.arange(0, np.max(time_data), tStep) numsamp = 512 #timeDomainVectorLength(timeV) if (len(y_data) < numsamp): y_data = np.resize(y_data, (numsamp,)) window = hann(numsamp) ## setup the fft spectrum arrays mag_spectrum = np.zeros([numsamp,int(np.ceil(float(len(timeV))/numsamp))]) #print 'time.len= %d, numsamp=%d, loop:%d' % (len(timeV), numsamp, int(np.ceil(float(len(timeV))/numsamp))) for k in range(0,int(np.ceil(float(len(timeV))/numsamp))): slice_dat = y_data[k*numsamp:numsamp*(k+1)] if (len(slice_dat) < numsamp): if (len(slice_dat) < numsamp/2): # WE DISCARDS LAST SLICE POINTS IF < NUMSAMP/2 break; slice_dat = np.resize(slice_dat,(numsamp,)) #multiply it with the window and transform it into frequency domain spectrum_dat = fft(slice_dat*window); #get the spectrum mag @ each of the 256 frequency points and store it #print 'k:',k,' spectrum_dat.len:',len(spectrum_dat) mag_spectrum[:,k]= 20 * np.log10(abs(spectrum_dat)) mag_spectrum[:,k]= abs(spectrum_dat) #print "fs= %.4g, NFFT= %ld, y_data.shape= %d, mag_spectrum= %dx%d" % (fs, numsamp,np.shape(y_data)[0], np.shape(mag_spectrum)[0],np.shape(mag_spectrum)[1]) ## DOUBLE CHECK THE SIZE OF THE MATRIX avg_fft_foreach = np.mean(mag_spectrum, axis=1) # print "np.shape(avg_fft_foreach):", np.shape(avg_fft_foreach) return avg_fft_foreach
def genImg(self,xml,compressionLevel): xml = re.split('\n',xml) img_data = re.split('"',xml[1]) type = img_data[1] size = [int(x) for x in img_data[3].split(',')] compressed = img_data[5] pixels = xml[2] pixels = base64.b64decode(pixels) # decode base 64 encoding if compressed == 'True': pixels = zlib.decompress(pixels) # if data was compressed, decompress it pixels = list(pixels) # converting byte data into a list, which will give us the actual numbers of ndarray(i.e. the image) if type == 'Gray': # Based on image type, reconstruct numpy.ndarray return numpy.resize(pixels,tuple(size)) else: r = pixels[:size[0]*size[1]] g = pixels[size[0]*size[1]:2*size[0]*size[1]] b = pixels[2*size[0]*size[1]:3*size[0]*size[1]] image = [] for i in range(size[0] * size[1]): image.append(r[i]) image.append(g[i]) image.append(b[i]) size.append(3) return numpy.resize(image,tuple(size))
def aabut (source, *args): """ Like the |Stat abut command. It concatenates two arrays column-wise and returns the result. CAUTION: If one array is shorter, it will be repeated until it is as long as the other. Usage: aabut (source, args) where args=any # of arrays Returns: an array as long as the LONGEST array past, source appearing on the 'left', arrays in <args> attached on the 'right'. """ if len(source.shape)==1: width = 1 source = N.resize(source,[source.shape[0],width]) else: width = source.shape[1] for addon in args: if len(addon.shape)==1: width = 1 addon = N.resize(addon,[source.shape[0],width]) else: width = source.shape[1] if len(addon) < len(source): addon = N.resize(addon,[source.shape[0],addon.shape[1]]) elif len(source) < len(addon): source = N.resize(source,[addon.shape[0],source.shape[1]]) source = N.concatenate((source,addon),1) return source
def twopoint_spidx_bootstrap(freq, flux, flux_err, niter=10000): """ Quick bootstrap for spectral index calulcation freq: 2 array flux: 2 or 2xN array flux_err: 2 or 2xN array N is the number of sources """ # calculate spidx assuming [iter,source,freq_point] shapes def spidx(freq, flux): return np.log10(flux[:,:,0]/flux[:,:,1])/np.log10(freq[:,:,0]/freq[:,:,1]) freq = np.array(freq).astype(float) flux = np.array(flux).astype(float) flux_err = np.array(flux_err).astype(float) # if only 1 source, add degenerate axis if flux.shape == (2,): flux = np.expand_dims(flux, axis=1) if flux_err.shape == (2,): flux_err = np.expand_dims(flux_err, axis=1) flux = flux.T flux_err = flux_err.T nsource = flux.shape[0] results = np.zeros(shape=(niter,nsource)) random_flux = np.resize(flux, (niter, nsource, 2)) + np.resize(flux_err, (niter, nsource, 2)) * np.random.randn(niter, nsource, 2) random_flux[random_flux <= 0] = np.nan # remove negative, this create a bias freq = np.resize(freq, (niter, nsource, 2)) results = spidx(freq, random_flux) mean = np.nanmean(results,axis=0) err = np.nanstd(results,axis=0) return mean, err
def batchsd(trace, batches=5): """ Calculates the simulation standard error, accounting for non-independent samples. The trace is divided into batches, and the standard deviation of the batch means is calculated. """ if len(np.shape(trace)) > 1: dims = np.shape(trace) # ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:])))) ttrace = np.transpose([t.ravel() for t in trace]) return np.reshape([batchsd(t, batches) for t in ttrace], dims[1:]) else: if batches == 1: return np.std(trace) / np.sqrt(len(trace)) try: batched_traces = np.resize(trace, (batches, len(trace) / batches)) except ValueError: # If batches do not divide evenly, trim excess samples resid = len(trace) % batches batched_traces = np.resize(trace[:-resid], (batches, len(trace[:-resid]) / batches)) means = np.mean(batched_traces, 1) return np.std(means) / np.sqrt(batches)
def draw_imf_samples(**kwargs): ''' Draw samples for power-law model Parameters ---------- **kwargs: string Keyword arguments as model parameters and number of samples Returns ------- array The first mass array The second mass ''' alpha_salpeter = kwargs.get('alpha', -2.35) nsamples = kwargs.get('nsamples', 1) min_mass = kwargs.get('min_mass', 5.) max_mass = kwargs.get('max_mass', 95.) max_mtotal = min_mass + max_mass a = (max_mass/min_mass)**(alpha_salpeter + 1.0) - 1.0 beta = 1.0 / (alpha_salpeter + 1.0) k = nsamples * int(1.5 + log(1 + 100./nsamples)) aa = min_mass * (1.0 + a * np.random.random(k))**beta bb = np.random.uniform(min_mass, aa, k) idx = np.where(aa + bb < max_mtotal) m1, m2 = (np.maximum(aa, bb))[idx], (np.minimum(aa, bb))[idx] return np.resize(m1, nsamples), np.resize(m2, nsamples)
def draw_lnm_samples(**kwargs): ''' Draw samples for uniform-in-log model Parameters ---------- **kwargs: string Keyword arguments as model parameters and number of samples Returns ------- array The first mass array The second mass ''' #PDF doesnt match with sampler nsamples = kwargs.get('nsamples', 1) min_mass = kwargs.get('min_mass', 5.) max_mass = kwargs.get('max_mass', 95.) max_mtotal = min_mass + max_mass lnmmin = log(min_mass) lnmmax = log(max_mass) k = nsamples * int(1.5 + log(1 + 100./nsamples)) aa = np.exp(np.random.uniform(lnmmin, lnmmax, k)) bb = np.exp(np.random.uniform(lnmmin, lnmmax, k)) idx = np.where(aa + bb < max_mtotal) m1, m2 = (np.maximum(aa, bb))[idx], (np.minimum(aa, bb))[idx] return np.resize(m1, nsamples), np.resize(m2, nsamples)
def JacobiMatrix(F, x, h = 1e-05): J = np.matrix([[],[]]) np.resize((len(F), len(F))) for i in range(0, len(F)): for j in range(0, len(F)): J[i][j] = derivative(F[i], x, j, h) return J
def mc_error(x, batches=5): """ Calculates the simulation standard error, accounting for non-independent samples. The trace is divided into batches, and the standard deviation of the batch means is calculated. :Arguments: x : Numpy array An array containing MCMC samples batches : integer Number of batchas """ if x.ndim > 1: dims = np.shape(x) #ttrace = np.transpose(np.reshape(trace, (dims[0], sum(dims[1:])))) trace = np.transpose([t.ravel() for t in x]) return np.reshape([mc_error(t, batches) for t in trace], dims[1:]) else: if batches == 1: return np.std(x)/np.sqrt(len(x)) try: batched_traces = np.resize(x, (batches, len(x)/batches)) except ValueError: # If batches do not divide evenly, trim excess samples resid = len(x) % batches batched_traces = np.resize(x[:-resid], (batches, len(x)/batches)) means = np.mean(batched_traces, 1) return np.std(means)/np.sqrt(batches)
def _evaluatePart(self, expr, part): """Evaluate expression expr for part part. Returns True if succeeded """ # replace dataset names with calls newexpr = substituteDatasets(self.document.data, expr, part)[0] comp = self.document.evaluate.compileCheckedExpression( newexpr, origexpr=expr) if comp is None: return False # set up environment to evaluate expressions in environment = self.document.evaluate.context.copy() # create dataset using parametric expression if self.parametric: p = self.parametric if p[2] >= 2: deltat = (p[1]-p[0]) / (p[2]-1) t = N.arange(p[2])*deltat + p[0] else: t = N.array([p[0]]) environment['t'] = t # this fn gets called to return the value of a dataset environment['_DS_'] = self.evaluateDataset # actually evaluate the expression try: result = eval(comp, environment) evalout = N.array(result, N.float64) if len(evalout.shape) > 1: raise RuntimeError("Number of dimensions is not 1") except Exception as ex: self.document.log( _("Error evaluating expression: %s\n" "Error: %s") % (self.expr[part], cstr(ex)) ) return False # make evaluated error expression have same shape as data if part != 'data': data = self.evaluated['data'] if evalout.shape == (): # zero dimensional - expand to data shape evalout = N.resize(evalout, data.shape) else: # 1-dimensional - make it right size and trim oldsize = evalout.shape[0] evalout = N.resize(evalout, data.shape) evalout[oldsize:] = N.nan else: if evalout.shape == (): # zero dimensional - make a single point evalout = N.resize(evalout, 1) self.evaluated[part] = evalout return True
def __init__(self, name, geometry, order, init_context=True): LagrangeSimplexPolySpace.__init__(self, name, geometry, order, init_context=False) nodes, nts, node_coors = self.nodes, self.nts, self.node_coors shape = [nts.shape[0] + 1, 2] nts = nm.resize(nts, shape) nts[-1,:] = [3, 0] shape = [nodes.shape[0] + 1, nodes.shape[1]] nodes = nm.resize(nodes, shape) # Make a 'hypercubic' (cubic in 2D) node. nodes[-1,:] = 1 n_v = self.geometry.n_vertex tmp = nm.ones((n_v,), nm.int32) node_coors = nm.vstack((node_coors, nm.dot(tmp, self.geometry.coors) / n_v)) self.nodes, self.nts = nodes, nts self.node_coors = nm.ascontiguousarray(node_coors) self.bnode = nodes[-1:,:] self.n_nod = self.nodes.shape[0] if init_context: self.eval_ctx = self.create_context(None, 0, 1e-15, 100, 1e-8, tdim=n_v - 1) else: self.eval_ctx = None
def build_data_dict(data1, data2, match): """Build a dictionary of zeros like the union of the two data dictionaries, but with 'length' equal to the shorter dict. """ data = {} keys1 = [key for key in data1.keys() if key != match] keys2 = [key for key in data2.keys() if key != match] nfinal = min(np.shape(data1[match])[0], np.shape(data2[match])[0]) if nfinal == np.shape(data1[match])[0]: data[match] = np.zeros_like(data1[match]) else: data[match] = np.zeros_like(data2[match]) for k in keys1: data[k] = np.zeros_like(data1[k]) shp = list(np.shape(data1[k])) shp[0] = nfinal data[k] = np.resize(data[k], tuple(shp)) for k in keys2: data[k] = np.zeros_like(data2[k]) shp = list(np.shape(data2[k])) shp[0] = nfinal data[k] = np.resize(data[k], tuple(shp)) return data, keys1, keys2
def test(npoints): xx = numpy.arange(npoints) xx=numpy.resize(xx,(npoints,1)) #yy = 1000.0 * exp (- 0.5 * (xx * xx) /15)+ 2.0 * xx + 10.5 yy = gauss([10.5,2,1000.0,20.,15],xx) yy=numpy.resize(yy,(npoints,1)) sy = numpy.sqrt(abs(yy)) sy=numpy.resize(sy,(npoints,1)) data = numpy.concatenate((xx, yy, sy),1) parameters = [0.0,1.0,900.0, 25., 10] stime = time.time() if 0: #old fashion fittedpar, chisq, sigmapar = LeastSquaresFit(gauss,parameters,data) else: #easier to handle fittedpar, chisq, sigmapar = LeastSquaresFit(gauss,parameters, xdata=xx.reshape((-1,)), ydata=yy.reshape((-1,)), sigmadata=sy.reshape((-1,))) etime = time.time() print("Took ",etime - stime, "seconds") print("chi square = ",chisq) print("Fitted pars = ",fittedpar) print("Sigma pars = ",sigmapar)
def abut(source, *args): # comment: except for the repetition, this is equivalent to hstack. """\nLike the |Stat abut command. It concatenates two arrays column-wise and returns the result. CAUTION: If one array is shorter, it will be repeated until it is as long as the other. Format: abut (source, args) where args=any # of arrays Returns: an array as long as the LONGEST array past, source appearing on the 'left', arrays in <args> attached on the 'right'.\n""" source = asarray(source) if len(source.shape) == 1: width = 1 source = np.resize(source,[source.shape[0],width]) else: width = source.shape[1] for addon in args: if len(addon.shape) == 1: width = 1 addon = np.resize(addon,[source.shape[0],width]) else: width = source.shape[1] if len(addon) < len(source): addon = np.resize(addon,[source.shape[0],addon.shape[1]]) elif len(source) < len(addon): source = np.resize(source,[addon.shape[0],source.shape[1]]) source = np.concatenate((source,addon),1) return source
def _assignInitialPoints(cvmat,S): h,w,c = cvmat.shape # Compute the max grid assignment nx = w/S ny = h/S # Compute the super pixel x,y grid xgrid = np.arange(nx).reshape(1,nx)*np.ones(ny,dtype=np.int).reshape(ny,1) ygrid = np.arange(ny).reshape(ny,1)*np.ones(nx,dtype=np.int).reshape(1,nx) # compute an x,y lookup to a label look up label_map = nx*ygrid + xgrid # Compute the x groups in pixel space tmp = np.arange(nx) tmp = np.resize(tmp,(w,)) xgroups = tmp[tmp.argsort()] # Compute the y groups in pixel space tmp = np.arange(ny) tmp = np.resize(tmp,(h,)) ygroups = tmp[tmp.argsort()] labels = np.zeros((h,w),dtype=np.int) for x in range(w): for y in range(h): labels[y,x] = label_map[ygroups[y],xgroups[x]] return label_map,xgroups,ygroups,labels
def preprocessForTwoPhenotypeAsso(self, snpData, which_phenotype_ls, data_matrix_phen): """ 2009-3-20 refactored out of run(), easy for MpiAssociation.py to call """ if len(which_phenotype_ls) < 2: sys.stderr.write("Error: Require to specify 2 phenotypes in order to carry out this test type.\n") sys.exit(3) snpData.data_matrix = numpy.vstack((snpData.data_matrix, snpData.data_matrix)) # stack the two phenotypes, fake a data_matrix_phen, which_phenotype_ls no_of_strains = len(strain_acc_list) which_phenotype1, which_phenotype2 = which_phenotype_ls[:2] phenotype1 = data_matrix_phen[:, which_phenotype1] phenotype1 = numpy.resize( phenotype1, [no_of_strains, 1] ) # phenotype1.resize([10,1]) doesn't work. ValueError: 'resize only works on single-segment arrays' phenotype2 = data_matrix_phen[:, which_phenotype2] phenotype2 = numpy.resize(phenotype2, [no_of_strains, 1]) data_matrix_phen = numpy.vstack((phenotype1, phenotype2)) which_phenotype_index_ls = [0] # stack the PC_matrix as well if PC_matrix is not None: PC_matrix = numpy.vstack((PC_matrix, PC_matrix)) # create an environment variable a = numpy.zeros(no_of_strains) a.resize([no_of_strains, 1]) b = numpy.ones(no_of_strains) b.resize([no_of_strains, 1]) environment_matrix = numpy.vstack((a, b)) return which_phenotype_index_ls, environment_matrix, data_matrix_phen
def convert_to_batched_episodes(self, episodes, max_length=None): """Convert batch-major list of episodes to time-major batch of episodes.""" lengths = [len(ep[-2]) for ep in episodes] max_length = max_length or max(lengths) new_episodes = [] for ep, length in zip(episodes, lengths): initial, observations, actions, rewards, terminated = ep observations = [np.resize(obs, [max_length + 1] + list(obs.shape)[1:]) for obs in observations] actions = [np.resize(act, [max_length + 1] + list(act.shape)[1:]) for act in actions] pads = np.array([0] * length + [1] * (max_length - length)) rewards = np.resize(rewards, [max_length]) * (1 - pads) new_episodes.append([initial, observations, actions, rewards, terminated, pads]) (initial, observations, actions, rewards, terminated, pads) = zip(*new_episodes) observations = [np.swapaxes(obs, 0, 1) for obs in zip(*observations)] actions = [np.swapaxes(act, 0, 1) for act in zip(*actions)] rewards = np.transpose(rewards) pads = np.transpose(pads) return (initial, observations, actions, rewards, terminated, pads)
def __init__(self, name, geometry, order): LagrangeSimplexPolySpace.__init__(self, name, geometry, order) nodes, nts, node_coors = self.nodes, self.nts, self.node_coors shape = [nts.shape[0] + 1, 2] nts = nm.resize(nts, shape) nts[-1,:] = [3, 0] shape = [nodes.shape[0] + 1, nodes.shape[1]] nodes = nm.resize(nodes, shape) # Make a 'hypercubic' (cubic in 2D) node. nodes[-1,:] = 1 n_v = self.geometry.n_vertex tmp = nm.ones((n_v,), nm.int32) node_coors = nm.vstack((node_coors, nm.dot(tmp, self.geometry.coors) / n_v)) self.nodes, self.nts = nodes, nts self.node_coors = nm.ascontiguousarray(node_coors) self.bnode = nodes[-1:,:] self.n_nod = self.nodes.shape[0]
def partial_autocorr(series, lag): #------------------------------------------------------------------------------- """Partial autocorrelation function, using Durbin (1960) recursive algorithm """ # Initialize matrices of phi and rho phi = resize(0.0, (lag, lag)) rho = resize(0.0, lag) if isinstance(series, list): series = numpy.array(series) # \phi_{1,1} = \rho_1 phi[0, 0] = rho[0] = autocorr(series, 1) for k in range(1, lag): # Calculate autocorrelation for current lag rho[k] = autocorr(series, k + 1) for j in range(k - 1): # \phi_{k+1,j} = \phi_{k,j} - \phi_{k+1,k+1} * \phi_{k,k+1-j} phi[k - 1, j] = phi[k - 2, j] - phi[k - 1, k - 1] * phi[k - 2, k - 2 - j] # Numerator: \rho_{k+1} - \sum_{j=1}^k \phi_{k,j}\rho_j phi[k, k] = rho[k] - sum([phi[k - 1, j] * rho[k - 1 - j] for j in range(k)]) # Denominator: 1 - \sum_{j=1}^k \phi_{k,j}\rho_j phi[k, k] /= 1 - sum([phi[k - 1, j] * rho[j] for j in range(k)]) # Return partial autocorrelation value return phi[lag - 1, lag - 1]
def eulertrap(ode, vardict, soln, h, relerr): """ Implementation of the Euler-Trapezoidal method. """ eqnum = len(ode) dim = [eqnum, 3] dim.extend(soln[0][0].shape) dim = tuple(dim) if numpy.iscomplexobj(soln[0]): aux = numpy.resize([0. + 0j], dim) else: aux = numpy.resize([0.], dim) dim = soln[0][0].shape for vari in range(eqnum): vardict.update({'y_{}'.format(vari): soln[vari][-1]}) for vari in range(eqnum): aux[vari][0] = numpy.resize(seval(ode[vari], **vardict), dim) for vari in range(eqnum): aux[vari][1] = numpy.resize(seval(ode[vari], **vardict) * h[0] + soln[vari][-1], dim) for vari in range(eqnum): vardict.update({'y_{}'.format(vari): aux[vari][1]}) vardict.update({'t': vardict['t'] + h[0]}) for vari in range(eqnum): aux[vari][2] = numpy.resize(seval(ode[vari], **vardict), dim) for vari in range(eqnum): vardict.update({"y_{}".format(vari): soln[vari][-1] + h[0] * (aux[vari][0] + aux[vari][2])}) pt = soln[vari] kt = numpy.array([vardict['y_{}'.format(vari)]]) soln[vari] = numpy.concatenate((pt, kt))
def test_QuaternionClass(self): v1 = np.array([0.2, 0.2, 0.4]) v2 = np.array([1, 0, 0]) q1 = Quaternion.q_exp(v1) q2 = Quaternion.q_exp(v2) v=np.array([1, 2, 3]) # Testing Mult and rotate np.testing.assert_almost_equal(Quaternion.q_rotate(Quaternion.q_mult(q1,q2),v), Quaternion.q_rotate(q1,Quaternion.q_rotate(q2,v)), decimal=7) np.testing.assert_almost_equal(Quaternion.q_rotate(q1,v2), np.resize(Quaternion.q_toRotMat(q1),(3,3)).dot(v2), decimal=7) # Testing Boxplus, Boxminus, Log and Exp np.testing.assert_almost_equal(Quaternion.q_boxPlus(q1,Quaternion.q_boxMinus(q2,q1)), q2, decimal=7) np.testing.assert_almost_equal(Quaternion.q_log(q1), v1, decimal=7) # Testing Lmat and Rmat np.testing.assert_almost_equal(Quaternion.q_mult(q1,q2), Quaternion.q_Lmat(q1).dot(q2), decimal=7) np.testing.assert_almost_equal(Quaternion.q_mult(q1,q2), Quaternion.q_Rmat(q2).dot(q1), decimal=7) # Testing ypr and quat roll = 0.2 pitch = -0.5 yaw = 2.5 q_test = Quaternion.q_mult(np.array([np.cos(0.5*pitch), 0, np.sin(0.5*pitch), 0]),np.array([np.cos(0.5*yaw), 0, 0, np.sin(0.5*yaw)])) q_test = Quaternion.q_mult(np.array([np.cos(0.5*roll), np.sin(0.5*roll), 0, 0]),q_test) np.testing.assert_almost_equal(Quaternion.q_toYpr(q_test), np.array([roll, pitch, yaw]), decimal=7) # Testing Jacobian of Ypr for i in np.arange(0,3): dv1 = np.array([0.0, 0.0, 0.0]) dv1[i] = 1.0 epsilon = 1e-6 ypr1 = Quaternion.q_toYpr(q1) ypr1_dist = Quaternion.q_toYpr(Quaternion.q_boxPlus(q1,dv1*epsilon)) dypr1_1 = (ypr1_dist-ypr1)/epsilon J = np.resize(Quaternion.q_toYprJac(q1),(3,3)) dypr1_2 = J.dot(dv1) np.testing.assert_almost_equal(dypr1_1,dypr1_2, decimal=5)
def append(self, value): # convert the appended object to an array if it starts as something else if type(value) is not np.ndarray: value = np.array(value) # add the data if value.ndim == 1: # adding a single row of data n = self.__n if n + 1 > self.__N: # need to allocate more memory self.__N += self.__n_grow self.__data = np.resize(self.__data, (self.__N, self.__cols)) self.__data[n] = value self.__n = n + 1 elif value.ndim == 2: # adding multiple rows of data # avoid loops for appending large arrays n = self.__n L = value.shape[0] N_needed = n + L - self.__N if N_needed > 0: # need to allocate more memory self.__N += (N_needed / self.__n_grow + 1) * self.__n_grow self.__data = np.resize(self.__data, (self.__N, self.__cols)) self.__data[n:n+L] = value self.__n += L
def initialize(dx, x_shore, eta_shore, eta_toe, S_d, S_b, S, Q_w, B0): """Initialize the variables for the simulation. Args: Returns: eta_b : array of z-coordinate of the basement at every node. Has the same size as dx. [ L ] Comments: """ # First thing we should do is specify our computational domain. N, dx_shore = nodes_in_domain(x_shore, dx) N_old = N.copy() dx = init_domain(N, dx_shore, dx) x_shore = x(dx)[-1] # Then we compute the basement elevation and the location of the delta toe. eta_b = np.resize(eta_toe, N) eta_b, x_toe = init_basement(dx, eta_shore, eta_toe, eta_b, S_d, S_b) # Next, we compute the initial fluvial profile eta, S = init_flumen(dx, eta_shore, S) # Instantiate a water-depth array for the domain. H = np.zeros_like(dx) # Compute unit flow rate qw = unit_flowrate(Q_w, B0) # [ L**2 / T ] # Compute critical depth for the section. Hc = critical_flow(qw) # Redefine B0 as a vector, to have that information on every node. B0 = np.resize(B0, N) # Instantiate a sediment transport capacity array for the domain qt = np.zeros_like(dx) return (N, N_old, dx_shore, dx, x_shore, eta_b, x_toe, eta, S, H, Hc, qw, B0, qt)
def cube(): """ Build vertices for a colored cube. V is the vertices I1 is the indices for a filled cube (use with GL_TRIANGLES) I2 is the indices for an outline cube (use with GL_LINES) """ vtype = [('a_position', np.float32, 3), ('a_normal' , np.float32, 3), ('a_color', np.float32, 4)] # Vertices positions v = [ [ 1, 1, 1], [-1, 1, 1], [-1,-1, 1], [ 1,-1, 1], [ 1,-1,-1], [ 1, 1,-1], [-1, 1,-1], [-1,-1,-1] ] # Face Normals n = [ [ 0, 0, 1], [ 1, 0, 0], [ 0, 1, 0] , [-1, 0, 1], [ 0,-1, 0], [ 0, 0,-1] ] # Vertice colors c = [ [0, 1, 1, 1], [0, 0, 1, 1], [0, 0, 0, 1], [0, 1, 0, 1], [1, 1, 0, 1], [1, 1, 1, 1], [1, 0, 1, 1], [1, 0, 0, 1] ]; V = np.array([(v[0],n[0],c[0]), (v[1],n[0],c[1]), (v[2],n[0],c[2]), (v[3],n[0],c[3]), (v[0],n[1],c[0]), (v[3],n[1],c[3]), (v[4],n[1],c[4]), (v[5],n[1],c[5]), (v[0],n[2],c[0]), (v[5],n[2],c[5]), (v[6],n[2],c[6]), (v[1],n[2],c[1]), (v[1],n[3],c[1]), (v[6],n[3],c[6]), (v[7],n[3],c[7]), (v[2],n[3],c[2]), (v[7],n[4],c[7]), (v[4],n[4],c[4]), (v[3],n[4],c[3]), (v[2],n[4],c[2]), (v[4],n[5],c[4]), (v[7],n[5],c[7]), (v[6],n[5],c[6]), (v[5],n[5],c[5]) ], dtype = vtype) I1 = np.resize( np.array([0,1,2,0,2,3], dtype=np.uint32), 6*(2*3)) I1 += np.repeat( 4*np.arange(2*3), 6) I2 = np.resize( np.array([0,1,1,2,2,3,3,0], dtype=np.uint32), 6*(2*4)) I2 += np.repeat( 4*np.arange(6), 8) return V, I1, I2
def _wrap(vector, pad_tuple, iaxis, kwargs): ''' Private function to calculate the before/after vectors for pad_wrap. Parameters ---------- vector : ndarray Input vector that already includes empty padded values. pad_tuple : tuple This tuple represents the (before, after) width of the padding along this particular iaxis. iaxis : int The axis currently being looped across. Not used in _wrap. kwargs : keyword arguments Keyword arguments. Not used in _wrap. Return ------ _wrap : ndarray Padded vector ''' if pad_tuple[1] == 0: after_vector = vector[pad_tuple[0]:None] else: after_vector = vector[pad_tuple[0]:-pad_tuple[1]] before_vector = np.resize(after_vector[::-1], pad_tuple[0])[::-1] after_vector = np.resize(after_vector, pad_tuple[1]) return _create_vector(vector, pad_tuple, before_vector, after_vector)
def explicitmidpoint(ode, vardict, soln, h, relerr): """ Implementation of the Explicit Midpoint method. """ eqnum = len(ode) dim = [eqnum, 2] dim.extend(soln[0][0].shape) dim = tuple(dim) if numpy.iscomplexobj(soln[0]): aux = numpy.resize([0. + 0j], dim) else: aux = numpy.resize([0.], dim) dim = soln[0][0].shape for vari in range(eqnum): vardict.update({'y_{}'.format(vari): soln[vari][-1]}) for vari in range(eqnum): aux[vari][0] = numpy.resize([seval(ode[vari], **vardict) * h[0] + soln[vari][-1]], dim) for vari in range(eqnum): vardict.update({"y_{}".format(vari): aux[vari][0]}) vardict.update({'t': vardict['t'] + 0.5 * h[0]}) for vari in range(eqnum): aux[vari][0] = numpy.resize([seval(ode[vari], **vardict)], dim) for vari in range(eqnum): vardict.update({"y_{}".format(vari): numpy.array(soln[vari][-1] + h[0] * aux[vari][0])}) pt = soln[vari] kt = numpy.array([vardict['y_{}'.format(vari)]]) soln[vari] = numpy.concatenate((pt, kt)) vardict.update({'t': vardict['t'] + 0.5 * h[0]})
def plot_image(self, image, nb_repeat=40, show_plot=True): """Plot augmented variations of an image. This method takes an image and plots it by default in 40 differently augmented versions. This method is intended to visualize the strength of your chosen augmentations (so for debugging). Args: image: The image to plot. nb_repeat: How often to plot the image. Each time it is plotted, the chosen augmentation will be different. (Default: 40). show_plot: Whether to show the plot. False makes sense if you don't have a graphical user interface on the machine. (Default: True) Returns: The figure of the plot. Use figure.savefig() to save the image. """ if len(image.shape) == 2: images = np.resize(image, (nb_repeat, image.shape[0], image.shape[1])) else: images = np.resize(image, (nb_repeat, image.shape[0], image.shape[1], image.shape[2])) return self.plot_images(images, True, show_plot=show_plot)
def sympforeuler(ode, vardict, soln, h, relerr): """ Implementation of the Symplectic Euler method. """ eqnum = len(ode) dim = [eqnum, 1] dim.extend(soln[0][0].shape) dim = tuple(dim) if numpy.iscomplexobj(soln[0]): aux = numpy.resize([0. + 0j], dim) else: aux = numpy.resize([0.], dim) dim = soln[0][0].shape for vari in range(eqnum): vardict.update({'y_{}'.format(vari): soln[vari][-1]}) for vari in range(eqnum): aux[vari][0] = numpy.resize((seval(ode[vari], **vardict) * h[0] + soln[vari][-1]), dim) if vari % 2 == 0: vardict.update({"y_{}".format(vari): aux[vari][0]}) for vari in range(eqnum): vardict.update({"y_{}".format(vari): aux[vari][0]}) pt = soln[vari] kt = numpy.array([vardict['y_{}'.format(vari)]]) soln[vari] = numpy.concatenate((pt, kt)) vardict.update({'t': vardict['t'] + h[0]})
def main(argv): # --------------------- Initializations ----------------------- # now = datetime.datetime.now() date = str('{:04d}'.format(now.year)) + str('{:02d}'.format( now.month)) + str('{:02d}'.format(now.day)) # -------- Flags ---------- # chunk_time = FLAGS.chunk_time db = FLAGS.use_database expID = FLAGS.expID patience = FLAGS.patience percentage_test = FLAGS.percentage_test percentage_validation = FLAGS.percentage_validation use_class_weights = FLAGS.use_class_weights target_id = FLAGS.ini_spk # ---------- NN ------------ # restart_ID_model = FLAGS.restart_ID_model filters = eval(FLAGS.num_filters) filter_sizes = eval(FLAGS.filter_sizes) # ---- Training options ---- # batch_size = FLAGS.batch_size earlystop = FLAGS.earlystop epochs = FLAGS.num_epochs ini_spk = FLAGS.ini_spk nusr_train = FLAGS.nusr_train n_spks = FLAGS.n_spks pool_factor = FLAGS.pool_factor random_shuffles = FLAGS.random_shuffles if not os.path.exists('../models'): os.makedirs('../models') # -------------------------- Main ----------------------------- # print('=-=-=-=-=-==-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-') print(bcolors.OKBLUE + 'Chunk_Time:' + bcolors.ENDC, chunk_time) if db == 'PulseID': fs = 200 path_df = './DataFrame_PulseID.pkl' signal = 'PPG_GDF' df, num_subjects, num_segments = split_db_pulseID( path_df=path_df, fs=fs, chunk_time=chunk_time, percentage_validation=percentage_validation, percentage_test=percentage_test, nusr_train=nusr_train, target=target_id) elif db == 'TROIKA': fs = 125 signal = 'PPG' path_df = './DataFrame_TROIKA.pkl' df, num_subjects, num_segments = split_db_TROIKA( path_df=path_df, fs=fs, chunk_time=chunk_time, percentage_validation=percentage_validation, percentage_test=percentage_test, nusr_train=nusr_train, target=target_id) #print(df.shape) data_train, labels_train, data_validation, labels_validation, data_test, labels_test, data_testIDs, labels_testIDs, data_zscore = atomic_test( df=df, signal=signal, num_subjects=num_subjects, num_segments=num_segments, target=target_id, percentage_test=percentage_test ) # data_train & labels_train are BALANCED #print(data_train.shape[1]) data_train = data_train.reshape(data_train.shape[0], 1) # Reshaping necessary to train if restart_ID_model != True: model = get_model(filters=filters, kernel_size=filter_sizes, x_train=data_train, cnn_stride=1, pool_factor=pool_factor) print(bcolors.WARNING + 'Keeping previous trained parameters from previous models' + bcolors.ENDC) for i in range(ini_spk, n_spks + 1): # round(num_subjects * percentage_test) + 1): # Different model for each subject ID if restart_ID_model == True: model = get_model(filters=filters, kernel_size=filter_sizes, x_train=data_train, cnn_stride=1, pool_factor=pool_factor) print( bcolors.WARNING + 'Reseting NN model per userID ' + bcolors.ENDC, i) sgd = SGD(lr=0.0001, momentum=0.0, decay=0.0, nesterov=True) model.compile(optimizer=sgd, loss='binary_crossentropy', metrics=['accuracy']) model.summary() print('*****************************************') print(bcolors.OKGREEN + 'ID:' + bcolors.ENDC, i) target_id = i labels_validation = [] labels_test = [] score_validation = [] score_test = [] # For the Circular Queue: buffer_auc = collections.deque(maxlen=patience + 1) #circular queue declaration buffer_models = collections.deque(maxlen=patience + 1) buffer_score_validation = collections.deque(maxlen=patience + 1) buffer_score_test = collections.deque(maxlen=patience + 1) buffer_score_testIDs = collections.deque(maxlen=patience + 1) buffer_labels_validation = collections.deque(maxlen=patience + 1) buffer_labels_test = collections.deque(maxlen=patience + 1) buffer_labels_testIDs = collections.deque(maxlen=patience + 1) for i in range(patience + 1): buffer_auc.append(0.0) print(bcolors.WARNING + 'Entering at Random_Shuffle for' + bcolors.ENDC) for r in range(1, random_shuffles + 1): print('------------------------------------------') data_train, labels_train, data_validation, labels_validation, data_test, labels_test, data_testIDs, labels_testIDs, data_zscore = atomic_test( df=df, signal=signal, num_subjects=num_subjects, num_segments=num_segments, target=target_id, percentage_test=percentage_test) data_train = data_train.reshape(data_train.shape[0], 1, 1) data_test = data_test.reshape(data_test.shape[0], 1, 1) data_testIDs = data_testIDs.reshape(data_testIDs.shape[0], 1, 1) data_validation = data_validation.reshape(data_validation.shape[0], 1, 1) #up=np.array(labels_validation,dtype='int') develop_weights = class_weight.compute_class_weight( "balanced", (np.unique(labels_validation)), (labels_validation)) develop_weights_vec = [] #develop_weights=develop_weights.reshape(2,1,1) #print((develop_weights[0])) for j in range(0, len(labels_validation)): if labels_validation[j] == 0: develop_weights_vec.append(develop_weights[0]) elif labels_validation[j] == 1: develop_weights_vec.append(develop_weights[1]) develop_weights_vec = np.array(develop_weights_vec) class_weights = { 0: 1., 1: 150. } # Weights in case of unbalanced validation data # 1 appearence of class 0 equals 150 appearences of class 1 #data_validation=[0]*19 data_train = np.resize(data_train, (19, 19, 1)) labels_train = labels_train[19:] #print(batch_size) data_validation = np.resize(data_validation, (2, 19, 1)) #labels_train=np.reshape(1,19) labels_train = np.reshape(labels_train, (19, 1)) if use_class_weights == True: history = model.fit(x=data_train, y=labels_train, batch_size=batch_size, epochs=epochs, verbose=0, callbacks=None, validation_split=0.0, validation_data=(data_validation, labels_validation, develop_weights_vec), shuffle=True, class_weight=class_weights, sample_weight=None, initial_epoch=0) # default values else: """history = model.fit(x=data_train, y=labels_train, batch_size=batch_size, epochs=epochs, verbose=0, validation_split=0.0, validation_data=(data_validation, labels_validation, develop_weights_vec), shuffle=True, class_weight=None, sample_weight=None, initial_epoch=0)""" # default values #print(data_test.shape) data_test = np.resize(data_test, (7, 19, 1)) data_testIDs = np.resize(data_testIDs, (7, 19, 1)) score_validation = model.predict(x=data_validation, batch_size=1, verbose=0) score_test = model.predict(x=data_test, batch_size=1, verbose=0) score_testIDs = model.predict(x=data_testIDs, batch_size=1, verbose=0) score_validation = score_validation.reshape( score_validation.shape[0]) score_test = score_test.reshape(score_test.shape[0]) score_testIDs = score_testIDs.reshape(score_testIDs.shape[0]) labels_validation = labels_validation.reshape( labels_validation.shape[0]) labels_test = labels_test.reshape(labels_test.shape[0]) labels_testIDs = labels_testIDs.reshape(labels_testIDs.shape[0]) roc_auc = roc_auc_score(labels_validation, score_validation) roc_auc_test = roc_auc_score(labels_test, score_test) print('random shuffle iteration: ', r, 'Validation AUC: ', roc_auc, 'Test AUC: ', roc_auc_test) model_copy = clone_model( model ) # necesary to clone to create a copy of the whole model. If we do not do this, # the model is passed as pointer by default model_copy.compile(optimizer=sgd, loss='binary_crossentropy', metrics=['accuracy']) print( "\n---------------------------------------\nmodel_copy summary: \n" ) model_copy.summary() print("\n---------------------------------------\n") # Circular queue # If the AUC decreases 'patience' consecutive stop training and save the model & scores for the last 'best' model # (actual model -patience steps) buffer_auc.append(roc_auc) #buffer_auc[0 --> patience] buffer_models.append(model_copy) buffer_score_validation.append(score_validation) buffer_score_test.append(score_test) buffer_score_testIDs.append(score_testIDs) buffer_labels_validation.append(labels_validation) buffer_labels_test.append(labels_test) buffer_labels_testIDs.append(labels_testIDs) # Boundaries to set the maximum FPR value fprbound1 = 0.2 # more restrictive fprbound2 = 0.3 # "checkpoint" value if (buffer_auc.index(max(buffer_auc)) == 0 or r == random_shuffles) and earlystop == True: if buffer_auc.index(max(buffer_auc)) == 0: index_modelok = 0 elif r == random_shuffles: index_modelok = -1 if not os.path.exists('../models/' + db + '/' + date + '/'): os.makedirs('../models/' + db + '/' + date + '/') buffer_models[index_modelok].save('../models/' + db + '/' + date + '/model_' + db + '_ES' + str(patience) + '_' + expID + '.h5') if not os.path.exists('../results/' + db + '/' + 'scores_ES' + str(patience) + '_' + date + '/'): os.makedirs('../results/' + db + '/' + 'scores_ES' + str(patience) + '_' + date + '/') filename = '../results/' + db + '/' + 'scores_ES' + str( patience) + '_' + date + '/' + expID + '_scores' # Computing the optimal threshold for a given scores validation_metrics, test_metrics, testIDs_metrics, threshold = optimal_threshold( buffer_score_validation[index_modelok], buffer_score_test[index_modelok], buffer_score_testIDs[index_modelok], buffer_labels_validation[index_modelok], buffer_labels_test[index_modelok], buffer_labels_testIDs[index_modelok], fprbound1, fprbound2) # Saving scores in a .json file scores = { 'score_validation': buffer_score_validation[index_modelok].tolist(), 'score_test': buffer_score_test[index_modelok].tolist(), 'score_testIDs': buffer_score_testIDs[index_modelok].tolist(), 'labels_validation': buffer_labels_validation[index_modelok].tolist(), 'labels_test': buffer_labels_test[index_modelok].tolist(), 'labels_testIDs': buffer_labels_testIDs[index_modelok].tolist(), 'threshold': threshold.tolist() } with open(filename + '.json', 'w') as fp: json.dump(scores, fp) # pickle.dump((buffer_score_validation[index_modelok], buffer_score_test[index_modelok], buffer_labels_validation[index_modelok], # buffer_predict_labels_validation[index_modelok], buffer_labels_test[index_modelok]),open(filename,"wb")) break if earlystop == False: if not os.path.exists('../models/' + db + '/' + date + '/'): os.makedirs('../models/' + db + '/' + date + '/') model.save('../models/' + db + '/' + date + '/model_' + db + '_' + expID + '.h5') if not os.path.exists('../results/' + db + '/' + 'scores_' + date + '/'): os.makedirs('../results/' + db + '/' + 'scores_' + date + '/') filename = '../results/' + db + '/' + 'scores_' + date + '/' + expID + '_scores' # Computing the optimal threshold for a given scores validation_metrics, test_metrics, testIDs_metrics, threshold = optimal_threshold( score_validation, score_test, score_testIDs, labels_validation, labels_test, labels_testIDs, fprbound1, fprbound2) # Saving the scores in a .json file scores = { 'score_validation': score_validation.tolist(), 'score_test': score_test.tolist(), 'score_testIDs': score_testIDs.tolist(), 'labels_validation': labels_validation.tolist(), 'labels_test': labels_test.tolist(), 'labels_testIDs': labels_testIDs.tolist(), 'threshold': threshold.tolist() } with open(filename + '.json', 'w') as fp: json.dump(scores, fp) # Unpacking variables fpr_validation = validation_metrics[0] tpr_validation = validation_metrics[1] roc_auc = validation_metrics[2] report_validation = validation_metrics[3] cnf_matrix_validation = validation_metrics[4] fpr_test = test_metrics[0] tpr_test = test_metrics[1] roc_auc_test = test_metrics[2] report_test = test_metrics[3] cnf_matrix_test = test_metrics[4] fpr_testIDs = testIDs_metrics[0] tpr_testIDs = testIDs_metrics[1] roc_auc_testIDs = testIDs_metrics[2] report_testIDs = testIDs_metrics[3] cnf_matrix_testIDs = testIDs_metrics[4] print('FINAL ITERATION -> Validation AUC: ', roc_auc, 'Test AUC: ', roc_auc_test, 'TestIDs AUC: ', roc_auc_testIDs) plot_ROC_and_Report(db, expID, date, 'validation', False, filter_sizes, filters, fpr_validation, tpr_validation, roc_auc, report_validation, cnf_matrix_validation, threshold) plot_ROC_and_Report(db, expID, date, 'test', False, filter_sizes, filters, fpr_test, tpr_test, roc_auc_test, report_test, cnf_matrix_test, threshold) plot_ROC_and_Report(db, expID, date, 'testIDs', False, filter_sizes, filters, fpr_testIDs, tpr_testIDs, roc_auc_testIDs, report_testIDs, cnf_matrix_testIDs, threshold)
def saveToCenter(i, rList, jList, bufferArray, summaryLength, h_size, sess, mainQN, time_per_step): with open('./Center/log.csv', 'a') as myfile: state_display = (np.zeros([1, h_size]), np.zeros([1, h_size])) imagesS = [] for idx, z in enumerate(np.vstack(bufferArray[:, 0])): img, state_display = sess.run( [mainQN.salience, mainQN.rnn_state], feed_dict={ mainQN.scalarInput: np.reshape(bufferArray[idx, 0], [1, 21168]) / 255.0, mainQN.trainLength: 1, mainQN.state_in: state_display, mainQN.batch_size: 1 }) imagesS.append(img) imagesS = (imagesS - np.main(imagesS)) / (np.max(imagesS) - np.min(imagesS)) imagesS = np.vstack(imagesS) imagesS = np.resize(imagesS, [len(imagesS), 84, 84, 3]) luminance = np.max(imagesS, 3) imagesS = np.multiply(np.ones([len(imagesS), 84, 84, 3]), np.reshape(luminance, [len(imagesS), 84, 84, 1])) make_gif(np.ones([len(imagesS), 84, 84, 3]), './Center/frames/sal' + str(i) + '.gif', duration=len(imagesS) * time_per_step, true_image=False, salience=True, salIMGS=luminance) images = list(zip(bufferArray[:, 0])) images.append(bufferArray[-1, 3]) images = np.vstack(images) images = np.resize(images, [len(images), 84, 84, 3]) make_gif(images, './Center/frames/image' + str(i) + '.gif', duration=len(images) * time_per_step, true_image=True, salience=False) wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) wr.writerow([ i, np, mean(jList[-100:0]), np.mean(rList[-summaryLength:]), './frames/image' + str(i) + '.gif', './frames/log' + str(i) + '.csv', './frames/sal' + str(i) + '.gif' ]) myfile.close() with open('./Center/frames/log' + str(i) + '.csv', 'w') as myfile: state_train = (np.zeros([1, h_size]), np.zeros([1, h_size])) wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) wr.writerow(["ACTION", "REWARD", "A0", "A1", 'A2', 'A3', 'V']) a, v = sess.run( [mainQN.Advantage, mainQN.Value], feed_dict={ mainQN.scalarInput: np.vstack(bufferArray[:, 0]) / 255.0, mainQN.trainLength: len(bufferArray), mainQN.state_in: state_train, mainQN.batch_size: 1 }) wr.writerows( zip(bufferArray[:, 1], bufferArray[:, 2], a[:, 0], a[:, 1], a[:, 2], a[:, 3], v[:, 0]))
def train_neural_network(x): prediction = neural_network_model(x) cost = tf.reduce_sum(tf.square(y - prediction)) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) #wr.writerow([ "Run Number", "Training Accuracy", "Testing Accuracy" ]) with tf.Session() as sess: ########################## # TRAINING FUNCTIONALITY # ########################## sess.run(tf.global_variables_initializer()) # This loop will run of the number of total run required. for i in range(numOfRuns): accuracy = 0 for epoch in range(hm_epochs): epoch_loss = 0 accuracy = 0 np.random.shuffle(xy_dataTraining) data_x = xy_dataTraining[0:trainingDataSize, 0:36] data_y = xy_dataTraining[0:trainingDataSize, 36:38] data_y = np.resize(data_y, (328, 2)) for piece in range(len(data_x)): input_x = [data_x[piece]] expected_y = [data_y[piece]] _, c, predict = sess.run( [optimizer, cost, prediction], feed_dict={ x: input_x, y: expected_y, keep_probL1: dropOutRateL1, keep_probL2: dropOutRateL2 }) epoch_loss += c if np.argmax(predict[0]) == np.argmax(expected_y[0]): accuracy += 1 accuracyPercentage = (accuracy / len(data_x)) print("Epoch: ", epoch, ", accuracy_standing: ", accuracyPercentage) runLabel = "Accuracy for run " + str(i) + ":" ###################### # IMAGE TESTING BULK # ###################### accuracy = 0 accuracyPercentage2 = 0 data_x = xy_dataTesting[0:testingDataSize, 0:36] data_y = xy_dataTesting[0:testingDataSize, 36:38] for piece in range(len(data_x)): input_x = [data_x[piece]] expected_y = [data_y[piece]] predict = sess.run([prediction], feed_dict={ x: input_x, y: expected_y, keep_probL1: 1.0, keep_probL2: 1.0 }) if np.argmax(predict[0]) == np.argmax(expected_y[0]): accuracy += 1 accuracyPercentage2 = (accuracy / len(data_x)) trialLabel = "Testing Accuracy on Entire Set: " print(trialLabel, accuracyPercentage2) # wr.writerow([i, accuracyPercentage, accuracyPercentage2]) # Play a sound to signal the numOfRuns complete winsound.PlaySound("SystemExit", winsound.SND_ALIAS) ######################## # AD HOC IMAGE TESTING # ######################## # Preload cmu dG = dataGenerator.dataScriptGenerator() continueProcessing = "Y" while (continueProcessing == "Y" or continueProcessing == "y"): # Call data script generator on new image and generate csv dG.adHocData() # Pass new csv data into network and print out prediction data_x = np.genfromtxt(sys.path[0] + '\\aClean.csv', delimiter=',') if data_x.ndim == 1: data_x = np.resize(data_x, (1, 36)) data_x[data_x < 0] = 0 if (len(data_x)) > 0: for piece in range(len(data_x)): input_x = [data_x[piece]] predict = sess.run([prediction], feed_dict={ x: input_x, keep_probL1: 1.0, keep_probL2: 1.0 }) print("Network Output: ", predict[0]) if (np.argmax(predict[0]) == 0): print("SQUATTING") elif (np.argmax(predict[0]) == 1): print("STANDING") continueProcessing = input("Continue processing? ('Y' or 'N'): ")
cfg=cfg) if args.cuda: newmodel.cuda() start_mask = torch.ones(3) layer_id_in_cfg = 0 linear_number = 1 end_mask = cfg_mask[layer_id_in_cfg] for [m0, m1] in zip(model.modules(), newmodel.modules()): if isinstance(m0, nn.Conv2d): idx0 = np.squeeze(np.argwhere(np.asarray(start_mask.cpu().numpy()))) idx1 = np.squeeze(np.argwhere(np.asarray(end_mask.cpu().numpy()))) print('In shape: {:d}, Out shape {:d}.'.format(idx0.size, idx1.size)) if idx0.size == 1: idx0 = np.resize(idx0, (1, )) if idx1.size == 1: idx1 = np.resize(idx1, (1, )) w1 = m0.weight.data[:, idx0.tolist(), :, :].clone() w1 = w1[idx1.tolist(), :, :, :].clone() m1.weight.data = w1.clone() elif isinstance(m0, nn.Linear): if layer_id_in_cfg == len(cfg_mask): idx0 = np.squeeze( np.argwhere(np.asarray(cfg_mask[-1].cpu().numpy()))) if idx0.size == 1: idx0 = np.resize(idx0, (1, )) m1.weight.data = m0.weight.data[:, idx0].clone() m1.bias.data = m0.bias.data.clone() layer_id_in_cfg += 1 continue
cv2.imshow("Legend", legend) #----------------------循环主体------------------------------------------- time1 = time() while True: ret, frame = cap.read() #ret是否有读取到----------frame帧图像-------- #input_image = cap.read(args.input_vedio, 1).astype(np.float32) input_image = frame input_image = cv2.resize(input_image, (input_shape[3], input_shape[2])) input_image = input_image.transpose((2, 0, 1)) input_image = np.asarray([input_image]) out = net.forward_all(**{net.inputs[0]: input_image}) prediction = net.blobs['deconv6_0_0'].data[0].argmax(axis=0) prediction = np.squeeze(prediction) prediction = np.resize(prediction, (3, input_shape[2], input_shape[3])) prediction = prediction.transpose(1, 2, 0).astype(np.uint8) prediction_rgb = np.zeros(prediction.shape, dtype=np.uint8) label_colours_bgr = label_colours[..., ::-1] cv2.LUT(prediction, label_colours_bgr, prediction_rgb) frame = cv2.resize(frame, (1280, 720), interpolation=cv2.INTER_CUBIC) #修改大小 prediction_rgb = cv2.resize(prediction_rgb, (1280, 720), interpolation=cv2.INTER_CUBIC) output = ((0.3 * frame) + (0.7 * prediction_rgb)).astype("uint8") cv2.imshow("input", frame) #cv2.imshow("ENet", prediction_rgb) cv2.imshow("ENet", output) # if writer is None: # initialize our video writer #------- fourcc = cv2.VideoWriter_fourcc(*"MPEG")
# <b>resize() only works if no otehr references to array, else error</b> # In[158]: ar2 = ar # In[159]: ar.resize((8, )) # <b>Workaround is to use numpy.ndarray.resize() instead</b> # In[160]: np.resize(ar, (8, )) # In[161]: ar # <h2>Adding a dimension</h2> # In[162]: ar = np.array([14, 15, 16]) ar.shape # In[163]: ar
def test_check_different_dimensions(): """ Ensure an error is raised if the dimensions are different. """ XA = np.resize(np.arange(45), (5, 9)) XB = np.resize(np.arange(32), (4, 8)) assert_raises(ValueError, check_pairwise_arrays, XA, XB)
img = Image.open(os.path.join(PATH_TO_TEST_IMAGES_DIR, PATH_TO_TEST_IMAGE)) image.imsave(os.path.join(PATH_TO_INPUT_IMAGE, '{}.png'.format(0)), img) for i in range(conv1.shape[3]): # Get the 9x9x1 filter: extracted_filter = conv1[:, :, :, i] # Get rid of the last dimension (hence get 9x9): extracted_filter = np.squeeze(extracted_filter) image.imsave(os.path.join(PATH_TO_CONV1_KERNEL, '{}.png'.format(i)), extracted_filter, vmin=-0.6064218, vmax=0.24946211) img = Image.open(os.path.join(PATH_TO_TEST_IMAGES_DIR, PATH_TO_TEST_IMAGE)) arr2 = np.array(img.getdata(), dtype=np.uint8) arr2 = np.resize(arr2, (img.size[1], img.size[0], 4)) arr2 = arr2[:, :, 1] conv = signal.correlate2d(arr2, extracted_filter, 'valid') conv = conv + conv1_bias[i] conv = np.maximum(0, conv) output[:, :, i] = conv image.imsave(os.path.join(PATH_TO_RELU, '{}.png'.format(i)), conv, cmap=CMAP, vmin=0, vmax=255) for i in range(primary_caps.shape[3]): # 9,9,256,256 extracted_filter = primary_caps[:, :, :, i]
def _fit_phase_screen(station_names, source_names, pp, airmass, rr, weights, times, height, order, r_0, beta, outQueue): """ Fits a screen to given phase values using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of phase values to fit screen to weights: array Array of weights times: array Array of times height: float Height of screen (m) order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of phase fluctuations (m) beta: float Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye logging.info('Fitting screens...') # Initialize arrays N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) N_piercepoints = N_sources * N_stations real_fit_white_all = np.zeros((N_times, N_sources, N_stations)) imag_fit_white_all = np.zeros((N_times, N_sources, N_stations)) phase_fit_white_all = np.zeros((N_times, N_sources, N_stations)) real_residual_all = np.zeros((N_times, N_sources, N_stations)) imag_residual_all = np.zeros((N_times, N_sources, N_stations)) phase_residual_all = np.zeros((N_times, N_sources, N_stations)) # Change phase to real/imag rr_real = np.cos(rr) rr_imag = np.sin(rr) for k in range(N_times): try: D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3) # Calculate real screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_real[:, k])) real_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) real_fit_white_all[k, :, :] = real_fit.reshape( (N_sources, N_stations)) residual = rr_real - np.dot(C, real_fit)[:, newaxis] real_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) # Calculate imag screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr_imag[:, k])) imag_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) imag_fit_white_all[k, :, :] = imag_fit.reshape( (N_sources, N_stations)) residual = rr_imag - np.dot(C, imag_fit)[:, newaxis] imag_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) # Calculate phase screen phase_fit = np.dot( pinvC, np.arctan2(np.dot(C, imag_fit), np.dot(C, real_fit))) phase_fit_white_all[k, :, :] = phase_fit.reshape( (N_sources, N_stations)) residual = rr - np.dot(C, phase_fit)[:, newaxis] phase_residual_all[k, :, :] = residual.reshape( (N_sources, N_stations)) except: # Set screen to zero if fit did not work logging.debug('Screen fit failed for timeslot {}'.format(k)) real_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) real_residual_all[k, :, :] = np.ones((N_sources, N_stations)) imag_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) imag_residual_all[k, :, :] = np.ones((N_sources, N_stations)) phase_fit_white_all[k, :, :] = np.zeros((N_sources, N_stations)) phase_residual_all[k, :, :] = np.ones((N_sources, N_stations)) outQueue.put([ real_fit_white_all, real_residual_all, imag_fit_white_all, imag_residual_all, phase_fit_white_all, phase_residual_all, times ])
u_exact = x_grid**2 u_pred_err = u_exact - u_pred #print X_star #np.savetxt('inp.out', X_star, delimiter=',') #print u_pred np.savetxt('uPred.out', u_pred, delimiter=',') np.savetxt('uPredErr.out', u_pred_err, delimiter=',') np.savetxt('uExact.out', u_pred_err, delimiter=',') np.savetxt('f_uPred.out', f_uPred, delimiter=',') error_u = (np.linalg.norm(u_exact - u_pred, 2) / np.linalg.norm(u_exact, 2)) print('Relative error u: %e' % (error_u)) # #plot the solution u_comp u_pred = np.resize(u_pred, [nPred, nPred]) CS = plt.contour(xPred, yPred, u_pred, 20, linewidths=0.5, colors='k') CS = plt.contourf(xPred, yPred, u_pred, 20, cmap=plt.cm.jet) plt.colorbar() # draw colorbar plt.title('$u_{comp}$') plt.show() #plot the error u_ex - u_comp u_pred_err = np.resize(u_pred_err, [nPred, nPred]) CS2 = plt.contour(xPred, yPred, u_pred_err, 20, linewidths=0.5, colors='k') CS2 = plt.contourf(xPred, yPred, u_pred_err, 20, cmap=plt.cm.jet) plt.colorbar() # draw colorbar plt.title('$u_{ex}-u_{comp}$') plt.show() #
import numpy as np from sys import argv from keras.preprocessing import image from keras.models import load_model img = argv[1] # Loads the image test_image = image.load_img(img, target_size=(28, 28, 1)) # Transform the img into a array test_image = image.img_to_array(test_image) test_image = np.resize(test_image, (28, 28, 1)) test_image = np.expand_dims(test_image, axis=0) test_image = test_image.astype('float32') test_image /= 255 # Loads the classifier model classifier = load_model('results/models/cnn_30e.h5') # Predicts the label res = classifier.predict_classes(test_image) print(res) print(classifier.predict(test_image))
def _fit_tec_screen(station_names, source_names, pp, airmass, rr, weights, times, height, order, r_0, beta, outQueue): """ Fits a screen to given TEC values using Karhunen-Lo`eve base vectors Parameters ---------- station_names: array Array of station names source_names: array Array of source names pp: array Array of piercepoint locations airmass: array Array of airmass values (note: not currently used) rr: array Array of TEC values to fit screen to weights: array Array of weights times: array Array of times height: float Height of screen (m) order: int Order of screen (i.e., number of KL base vectors to keep) r_0: float Scale size of phase fluctuations (m) beta: float Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) """ import numpy as np from pylab import kron, concatenate, pinv, norm, newaxis, find, amin, svd, eye logging.info('Fitting screens...') # Initialize arrays N_stations = len(station_names) N_sources = len(source_names) N_times = len(times) N_piercepoints = N_sources * N_stations tec_fit_white_all = np.zeros((N_times, N_sources, N_stations)) tec_residual_all = np.zeros((N_times, N_sources, N_stations)) for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / r_0**2)**(beta / 2.0) / 2.0 pinvC = pinv(C, rcond=1e-3) U, S, V = svd(C) invU = pinv(np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], U[:, :order])), rcond=1e-3) # Calculate screen rr1 = np.dot(np.transpose(U[:, :order]), np.dot(weights[:, :, k], rr[:, k])) tec_fit = np.dot(pinvC, np.dot(U[:, :order], np.dot(invU, rr1))) tec_fit_white_all[k, :, :] = tec_fit.reshape((N_sources, N_stations)) residual = rr - np.dot(C, tec_fit)[:, newaxis] tec_residual_all[k, :, :] = residual.reshape((N_sources, N_stations)) outQueue.put([tec_fit_white_all, tec_residual_all, times])
labels=one_hot(org_y)) # Run optimization op (backprop) and cost op (to get loss value) _, c = sess.run([train_op, loss_op], {X: batch_x, Y: batch_y}) # Compute average loss avg_cost += c / batch_size # Display logs per epoch s1tep if epoch % display_step == 0: print("Epoch:", '%04d' % (epoch + 1), "cost={:.9f}".format(avg_cost)) saver.save(sess, './model/TensorFlow/model.ckpt') print("Optimization Finished!") print("Predicting sample from train set!") random_index = np.random.randint(0, 5000) sample_x = org_X[random_index] logits = np.argmax(predict(np.resize(sample_x, (1, 400))).eval()[0]) + 1 print("Predicted X:", logits) print("True X:", org_y[random_index][0]) print("Prediction finished!") p = [] logits = predict(org_X) for i in logits.eval(): p.append(np.argmax(i) + 1) p = np.asarray(p) c = 0 for i in range(org_y.shape[0]): if p[i] == org_y[i][0]: c += 1 print("Accuracy: %.6f" % (c / org_y.shape[0]))
"""""" """""" """""" """""" """ INTEGRATOR """ """""" """""" """""" """""" # Variable extraction xAcc = DataSet["Denoised + Offset Removed X Accel. (g)"] yAcc = DataSet["Denoised + Offset Removed Y Accel. (g)"] zAcc = DataSet["Denoised + Offset Removed Z Accel. (g)"] time = DataSet["Time (all)"] # Integrating acceleration vectors, producing velocities xVel = it.cumtrapz(xAcc, time) yVel = it.cumtrapz(yAcc, time) zVel = it.cumtrapz(zAcc, time) # Plotting Velocities in X, Y, and Z time = np.resize(time, time.size - 1) plt.figure(figsize=(10, 6)) plt.plot(time, xVel, linewidth=2.25, alpha=0.6, label='x', color='r') plt.plot(time, yVel, linewidth=2.25, alpha=0.6, label='y', color='g') plt.plot(time, zVel, linewidth=2.25, alpha=0.6, label='z', color='b') plt.title("Velocities in X, Y, and Z") plt.xlabel("Time (s)") plt.ylabel("Velocity (m/s)") plt.grid() plt.xlim() plt.ylim() plt.legend(loc='upper left') plt.show() # Adding new dictionary entries DataSet["X Velocity"] = xVel
def calculate_pattern(length, repeats): return np.resize(np.repeat(np.array((0, 1, 0, -1)), repeats), length + 1)[1:length + 1]
def my_run_simulation(self, initial_run=False, update_plots=True): """ Runs the simulation. Args: self(PyBERT): Reference to an instance of the *PyBERT* class. initial_run(Bool): If True, don't update the eye diagrams, since they haven't been created, yet. (Optional; default = False.) update_plots(Bool): If True, update the plots, after simulation completes. This option can be used by larger scripts, which import *pybert*, in order to avoid graphical back-end conflicts and speed up this function's execution time. (Optional; default = True.) """ num_sweeps = self.num_sweeps sweep_num = self.sweep_num start_time = clock() self.status = 'Running channel...(sweep %d of %d)' % (sweep_num, num_sweeps) self.run_count += 1 # Force regeneration of bit stream. # Pull class variables into local storage, performing unit conversion where necessary. t = self.t t_ns = self.t_ns f = self.f w = self.w bits = self.bits symbols = self.symbols ffe = self.ffe nbits = self.nbits nui = self.nui bit_rate = self.bit_rate * 1.e9 eye_bits = self.eye_bits eye_uis = self.eye_uis nspb = self.nspb nspui = self.nspui rn = self.rn pn_mag = self.pn_mag pn_freq = self.pn_freq * 1.e6 Vod = self.vod Rs = self.rs Cs = self.cout * 1.e-12 RL = self.rin CL = self.cac * 1.e-6 Cp = self.cin * 1.e-12 R0 = self.R0 w0 = self.w0 Rdc = self.Rdc Z0 = self.Z0 v0 = self.v0 * 3.e8 Theta0 = self.Theta0 l_ch = self.l_ch pattern_len = self.pattern_len rx_bw = self.rx_bw * 1.e9 peak_freq = self.peak_freq * 1.e9 peak_mag = self.peak_mag ctle_offset = self.ctle_offset ctle_mode = self.ctle_mode delta_t = self.delta_t * 1.e-12 alpha = self.alpha ui = self.ui n_taps = self.n_taps gain = self.gain n_ave = self.n_ave decision_scaler = self.decision_scaler n_lock_ave = self.n_lock_ave rel_lock_tol = self.rel_lock_tol lock_sustain = self.lock_sustain bandwidth = self.sum_bw * 1.e9 rel_thresh = self.thresh mod_type = self.mod_type[0] try: # Calculate misc. values. fs = bit_rate * nspb Ts = t[1] ts = Ts # Generate the ideal over-sampled signal. # # Duo-binary is problematic, in that it requires convolution with the ideal duobinary # impulse response, in order to produce the proper ideal signal. x = repeat(symbols, nspui) self.x = x if(mod_type == 1): # Handle duo-binary case. duob_h = array(([0.5] + [0.] * (nspui - 1)) * 2) x = convolve(x, duob_h)[:len(t)] self.ideal_signal = x # Find the ideal crossing times, for subsequent jitter analysis of transmitted signal. ideal_xings = find_crossings(t, x, decision_scaler, min_delay=(ui / 2.), mod_type=mod_type) self.ideal_xings = ideal_xings # Calculate the channel output. # # Note: We're not using 'self.ideal_signal', because we rely on the system response to # create the duobinary waveform. We only create it explicitly, above, # so that we'll have an ideal reference for comparison. chnl_h = self.calc_chnl_h() chnl_out = convolve(self.x, chnl_h)[:len(x)] self.channel_perf = nbits * nspb / (clock() - start_time) split_time = clock() self.status = 'Running Tx...(sweep %d of %d)' % (sweep_num, num_sweeps) except Exception: self.status = 'Exception: channel' raise self.chnl_out = chnl_out self.chnl_out_H = fft(chnl_out) # Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the Tx. try: if(self.tx_use_ami): # Note: Within the PyBERT computational environment, we use normalized impulse responses, # which have units of (V/ts), where 'ts' is the sample interval. However, IBIS-AMI models expect # units of (V/s). So, we have to scale accordingly, as we transit the boundary between these two worlds. tx_cfg = self._tx_cfg # Grab the 'AMIParamConfigurator' instance for this model. # Get the model invoked and initialized, except for 'channel_response', which # we need to do several different ways, in order to gather all the data we need. tx_param_dict = tx_cfg.input_ami_params tx_model_init = AMIModelInitializer(tx_param_dict) tx_model_init.sample_interval = ts # Must be set, before 'channel_response'! tx_model_init.channel_response = [1. / ts] + [0.] * (len(chnl_h) - 1) # Start with a delta function, to capture the model's impulse response. tx_model_init.bit_time = ui tx_model = AMIModel(self.tx_dll_file) tx_model.initialize(tx_model_init) self.log("Tx IBIS-AMI model initialization results:\nInput parameters: {}\nOutput parameters: {}\nMessage: {}".format( tx_model.ami_params_in, tx_model.ami_params_out, tx_model.msg)) if(tx_cfg.fetch_param_val(['Reserved_Parameters', 'Init_Returns_Impulse'])): tx_h = array(tx_model.initOut) * ts elif(not tx_cfg.fetch_param_val(['Reserved_Parameters', 'GetWave_Exists'])): error("ERROR: Both 'Init_Returns_Impulse' and 'GetWave_Exists' are False!\n \ I cannot continue.\nThis condition is supposed to be caught sooner in the flow.") self.status = "Simulation Error." return elif(not self.tx_use_getwave): error("ERROR: You have elected not to use GetWave for a model, which does not return an impulse response!\n \ I cannot continue.\nPlease, select 'Use GetWave' and try again.", 'PyBERT Alert') self.status = "Simulation Error." return if(self.tx_use_getwave): # For GetWave, use a step to extract the model's native properties. # Position the input edge at the center of the vector, in # order to minimize high frequency artifactual energy # introduced by frequency domain processing in some models. half_len = len(chnl_h) // 2 tx_s = tx_model.getWave(array([0.] * half_len + [1.] * half_len)) # Shift the result back to the correct location, extending the last sample. tx_s = pad(tx_s[half_len:], (0, half_len), 'edge') tx_h = diff(concatenate((array([0.0]), tx_s))) # Without the leading 0, we miss the pre-tap. tx_out = tx_model.getWave(self.x) else: # Init()-only. tx_s = tx_h.cumsum() tx_out = convolve(tx_h, self.x)[:len(self.x)] else: # - Generate the ideal, post-preemphasis signal. # To consider: use 'scipy.interp()'. This is what Mark does, in order to induce jitter in the Tx output. ffe_out = convolve(symbols, ffe)[:len(symbols)] self.rel_power = mean(ffe_out ** 2) # Store the relative average power dissipated in the Tx. tx_out = repeat(ffe_out, nspui) # oversampled output # - Calculate the responses. # - (The Tx is unique in that the calculated responses aren't used to form the output. # This is partly due to the out of order nature in which we combine the Tx and channel, # and partly due to the fact that we're adding noise to the Tx output.) tx_h = array(sum([[x] + list(zeros(nspui - 1)) for x in ffe], [])) # Using sum to concatenate. tx_h.resize(len(chnl_h)) tx_s = tx_h.cumsum() temp = tx_h.copy() temp.resize(len(w)) tx_H = fft(temp) tx_H *= tx_s[-1] / abs(tx_H[0]) # - Generate the uncorrelated periodic noise. (Assume capacitive coupling.) # - Generate the ideal rectangular aggressor waveform. pn_period = 1. / pn_freq pn_samps = int(pn_period / Ts + 0.5) pn = zeros(pn_samps) pn[pn_samps // 2:] = pn_mag pn = resize(pn, len(tx_out)) # - High pass filter it. (Simulating capacitive coupling.) (b, a) = iirfilter(2, gFc/(fs/2), btype='highpass') pn = lfilter(b, a, pn)[:len(pn)] # - Add the uncorrelated periodic and random noise to the Tx output. tx_out += pn tx_out += normal(scale=rn, size=(len(tx_out),)) # - Convolve w/ channel. tx_out_h = convolve(tx_h, chnl_h)[:len(chnl_h)] temp = tx_out_h.copy() temp.resize(len(w)) tx_out_H = fft(temp) rx_in = convolve(tx_out, chnl_h)[:len(tx_out)] self.tx_s = tx_s self.tx_out = tx_out self.rx_in = rx_in self.tx_out_s = tx_out_h.cumsum() self.tx_out_p = self.tx_out_s[nspui:] - self.tx_out_s[:-nspui] self.tx_H = tx_H self.tx_h = tx_h self.tx_out_H = tx_out_H self.tx_out_h = tx_out_h self.tx_perf = nbits * nspb / (clock() - split_time) split_time = clock() self.status = 'Running CTLE...(sweep %d of %d)' % (sweep_num, num_sweeps) except Exception: self.status = 'Exception: Tx' raise # Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the CTLE. try: if(self.rx_use_ami): rx_cfg = self._rx_cfg # Grab the 'AMIParamConfigurator' instance for this model. # Get the model invoked and initialized, except for 'channel_response', which # we need to do several different ways, in order to gather all the data we need. rx_param_dict = rx_cfg.input_ami_params rx_model_init = AMIModelInitializer(rx_param_dict) rx_model_init.sample_interval = ts # Must be set, before 'channel_response'! rx_model_init.channel_response = tx_out_h / ts rx_model_init.bit_time = ui rx_model = AMIModel(self.rx_dll_file) rx_model.initialize(rx_model_init) self.log("Rx IBIS-AMI model initialization results:\nInput parameters: {}\nMessage: {}\nOutput parameters: {}".format( rx_model.ami_params_in, rx_model.msg, rx_model.ami_params_out)) if(rx_cfg.fetch_param_val(['Reserved_Parameters', 'Init_Returns_Impulse'])): ctle_out_h = array(rx_model.initOut) * ts elif(not rx_cfg.fetch_param_val(['Reserved_Parameters', 'GetWave_Exists'])): error("ERROR: Both 'Init_Returns_Impulse' and 'GetWave_Exists' are False!\n \ I cannot continue.\nThis condition is supposed to be caught sooner in the flow.") self.status = "Simulation Error." return elif(not self.rx_use_getwave): error("ERROR: You have elected not to use GetWave for a model, which does not return an impulse response!\n \ I cannot continue.\nPlease, select 'Use GetWave' and try again.", 'PyBERT Alert') self.status = "Simulation Error." return if(self.rx_use_getwave): if(False): ctle_out, clock_times = rx_model.getWave(rx_in, 32) else: ctle_out, clock_times = rx_model.getWave(rx_in, len(rx_in)) self.log(rx_model.ami_params_out) ctle_H = fft(ctle_out * signal.hann(len(ctle_out))) / fft(rx_in * signal.hann(len(rx_in))) ctle_h = real(ifft(ctle_H)[:len(chnl_h)]) ctle_out_h = convolve(ctle_h, tx_out_h)[:len(chnl_h)] else: # Init() only. ctle_out_h_padded = pad(ctle_out_h, (nspb, len(rx_in) - nspb - len(ctle_out_h)), 'linear_ramp', end_values=(0., 0.)) tx_out_h_padded = pad(tx_out_h, (nspb, len(rx_in) - nspb - len(tx_out_h)), 'linear_ramp', end_values=(0., 0.)) ctle_H = fft(ctle_out_h_padded) / fft(tx_out_h_padded) ctle_h = real(ifft(ctle_H)[:len(chnl_h)]) ctle_out = convolve(rx_in, ctle_h)[:len(tx_out)] ctle_s = ctle_h.cumsum() else: if(self.use_ctle_file): ctle_h = import_channel(self.ctle_file, ts) if(max(abs(ctle_h)) < 100.): # step response? ctle_h = diff(ctle_h) # impulse response is derivative of step response. else: ctle_h *= ts # Normalize to (V/sample) ctle_h.resize(len(t)) ctle_H = fft(ctle_h) ctle_H *= sum(ctle_h) / ctle_H[0] else: w_dummy, ctle_H = make_ctle(rx_bw, peak_freq, peak_mag, w, ctle_mode, ctle_offset) ctle_h = real(ifft(ctle_H))[:len(chnl_h)] ctle_h *= abs(ctle_H[0]) / sum(ctle_h) ctle_out = convolve(rx_in, ctle_h)[:len(tx_out)] ctle_out -= mean(ctle_out) # Force zero mean. if(self.ctle_mode == 'AGC'): # Automatic gain control engaged? ctle_out *= 2. * decision_scaler / ctle_out.ptp() ctle_s = ctle_h.cumsum() ctle_out_h = convolve(tx_out_h, ctle_h)[:len(tx_out_h)] self.ctle_s = ctle_s ctle_out_h_main_lobe = where(ctle_out_h >= max(ctle_out_h) / 2.)[0] if(len(ctle_out_h_main_lobe)): conv_dly_ix = ctle_out_h_main_lobe[0] else: conv_dly_ix = self.chnl_dly / Ts conv_dly = t[conv_dly_ix] ctle_out_s = ctle_out_h.cumsum() temp = ctle_out_h.copy() temp.resize(len(w)) ctle_out_H = fft(temp) # - Store local variables to class instance. self.ctle_out_s = ctle_out_s # Consider changing this; it could be sensitive to insufficient "front porch" in the CTLE output step response. self.ctle_out_p = self.ctle_out_s[nspui:] - self.ctle_out_s[:-nspui] self.ctle_H = ctle_H self.ctle_h = ctle_h self.ctle_out_H = ctle_out_H self.ctle_out_h = ctle_out_h self.ctle_out = ctle_out self.conv_dly = conv_dly self.conv_dly_ix = conv_dly_ix self.ctle_perf = nbits * nspb / (clock() - split_time) split_time = clock() self.status = 'Running DFE/CDR...(sweep %d of %d)' % (sweep_num, num_sweeps) except Exception: self.status = 'Exception: Rx' raise # Generate the output from, and the incremental/cumulative impulse/step/frequency responses of, the DFE. try: if(self.use_dfe): dfe = DFE(n_taps, gain, delta_t, alpha, ui, nspui, decision_scaler, mod_type, n_ave=n_ave, n_lock_ave=n_lock_ave, rel_lock_tol=rel_lock_tol, lock_sustain=lock_sustain, bandwidth=bandwidth, ideal=self.sum_ideal) else: dfe = DFE(n_taps, 0., delta_t, alpha, ui, nspui, decision_scaler, mod_type, n_ave=n_ave, n_lock_ave=n_lock_ave, rel_lock_tol=rel_lock_tol, lock_sustain=lock_sustain, bandwidth=bandwidth, ideal=True) (dfe_out, tap_weights, ui_ests, clocks, lockeds, clock_times, bits_out) = dfe.run(t, ctle_out) bits_out = array(bits_out) auto_corr = 1. * correlate(bits_out[(nbits - eye_bits):], bits[(nbits - eye_bits):], mode='same') \ / sum(bits[(nbits - eye_bits):]) auto_corr = auto_corr[len(auto_corr) // 2 :] self.auto_corr = auto_corr bit_dly = where(auto_corr == max(auto_corr))[0][0] bits_ref = bits[(nbits - eye_bits) :] bits_tst = bits_out[(nbits + bit_dly - eye_bits) :] if(len(bits_ref) > len(bits_tst)): bits_ref = bits_ref[: len(bits_tst)] elif(len(bits_tst) > len(bits_ref)): bits_tst = bits_tst[: len(bits_ref)] bit_errs = where(bits_tst ^ bits_ref)[0] self.bit_errs = len(bit_errs) dfe_h = array([1.] + list(zeros(nspb - 1)) + sum([[-x] + list(zeros(nspb - 1)) for x in tap_weights[-1]], [])) dfe_h.resize(len(ctle_out_h)) temp = dfe_h.copy() temp.resize(len(w)) dfe_H = fft(temp) self.dfe_s = dfe_h.cumsum() dfe_out_H = ctle_out_H * dfe_H dfe_out_h = convolve(ctle_out_h, dfe_h)[:len(ctle_out_h)] dfe_out_s = dfe_out_h.cumsum() self.dfe_out_p = dfe_out_s - pad(dfe_out_s[:-nspui], (nspui,0), 'constant', constant_values=(0,0)) self.dfe_H = dfe_H self.dfe_h = dfe_h self.dfe_out_H = dfe_out_H self.dfe_out_h = dfe_out_h self.dfe_out_s = dfe_out_s self.dfe_out = dfe_out self.dfe_perf = nbits * nspb / (clock() - split_time) split_time = clock() self.status = 'Analyzing jitter...(sweep %d of %d)' % (sweep_num, num_sweeps) except Exception: self.status = 'Exception: DFE' raise # Save local variables to class instance for state preservation, performing unit conversion where necessary. self.adaptation = tap_weights self.ui_ests = array(ui_ests) * 1.e12 # (ps) self.clocks = clocks self.lockeds = lockeds self.clock_times = clock_times # Analyze the jitter. self.thresh_tx = array([]) self.jitter_ext_tx = array([]) self.jitter_tx = array([]) self.jitter_spectrum_tx = array([]) self.jitter_ind_spectrum_tx = array([]) self.thresh_ctle = array([]) self.jitter_ext_ctle = array([]) self.jitter_ctle = array([]) self.jitter_spectrum_ctle = array([]) self.jitter_ind_spectrum_ctle = array([]) self.thresh_dfe = array([]) self.jitter_ext_dfe = array([]) self.jitter_dfe = array([]) self.jitter_spectrum_dfe = array([]) self.jitter_ind_spectrum_dfe = array([]) self.f_MHz_dfe = array([]) self.jitter_rejection_ratio = array([]) try: if(mod_type == 1): # Handle duo-binary case. pattern_len *= 2 # Because, the XOR pre-coding can invert every other pattern rep. if(mod_type == 2): # Handle PAM-4 case. if(pattern_len % 2): pattern_len *= 2 # Because, the bits are taken in pairs, to form the symbols. # - channel output actual_xings = find_crossings(t, chnl_out, decision_scaler, mod_type=mod_type) (jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \ thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \ hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh) self.t_jitter = t_jitter self.isi_chnl = isi self.dcd_chnl = dcd self.pj_chnl = pj self.rj_chnl = rj self.thresh_chnl = thresh self.jitter_chnl = hist self.jitter_ext_chnl = hist_synth self.jitter_bins = bin_centers self.jitter_spectrum_chnl = jitter_spectrum self.jitter_ind_spectrum_chnl = jitter_ind_spectrum self.f_MHz = array(spectrum_freqs) * 1.e-6 # - Tx output actual_xings = find_crossings(t, rx_in, decision_scaler, mod_type=mod_type) (jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \ thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \ hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh) self.isi_tx = isi self.dcd_tx = dcd self.pj_tx = pj self.rj_tx = rj self.thresh_tx = thresh self.jitter_tx = hist self.jitter_ext_tx = hist_synth self.jitter_spectrum_tx = jitter_spectrum self.jitter_ind_spectrum_tx = jitter_ind_spectrum # - CTLE output actual_xings = find_crossings(t, ctle_out, decision_scaler, mod_type=mod_type) (jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \ thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \ hist, hist_synth, bin_centers) = calc_jitter(ui, nui, pattern_len, ideal_xings, actual_xings, rel_thresh) self.isi_ctle = isi self.dcd_ctle = dcd self.pj_ctle = pj self.rj_ctle = rj self.thresh_ctle = thresh self.jitter_ctle = hist self.jitter_ext_ctle = hist_synth self.jitter_spectrum_ctle = jitter_spectrum self.jitter_ind_spectrum_ctle = jitter_ind_spectrum # - DFE output ignore_until = (nui - eye_uis) * ui + 0.75 * ui # 0.5 was causing an occasional misalignment. ideal_xings = array(filter(lambda x: x > ignore_until, list(ideal_xings))) min_delay = ignore_until + conv_dly actual_xings = find_crossings(t, dfe_out, decision_scaler, min_delay=min_delay, mod_type=mod_type, rising_first=False, ) (jitter, t_jitter, isi, dcd, pj, rj, jitter_ext, \ thresh, jitter_spectrum, jitter_ind_spectrum, spectrum_freqs, \ hist, hist_synth, bin_centers) = calc_jitter(ui, eye_uis, pattern_len, ideal_xings, actual_xings, rel_thresh) self.isi_dfe = isi self.dcd_dfe = dcd self.pj_dfe = pj self.rj_dfe = rj self.thresh_dfe = thresh self.jitter_dfe = hist self.jitter_ext_dfe = hist_synth self.jitter_spectrum_dfe = jitter_spectrum self.jitter_ind_spectrum_dfe = jitter_ind_spectrum self.f_MHz_dfe = array(spectrum_freqs) * 1.e-6 skip_factor = nbits / eye_bits ctle_spec = self.jitter_spectrum_ctle dfe_spec = self.jitter_spectrum_dfe self.jitter_rejection_ratio = zeros(len(dfe_spec)) self.jitter_perf = nbits * nspb / (clock() - split_time) self.total_perf = nbits * nspb / (clock() - start_time) split_time = clock() self.status = 'Updating plots...(sweep %d of %d)' % (sweep_num, num_sweeps) except Exception: self.status = 'Exception: jitter' # raise # Update plots. try: if(update_plots): update_results(self) if(not initial_run): update_eyes(self) self.plotting_perf = nbits * nspb / (clock() - split_time) self.status = 'Ready.' except Exception: self.status = 'Exception: plotting' raise
def a_in_scan_read_numpy(self, samples_per_channel, timeout): # pylint: disable=too-many-locals """ Read scan status and data (as a NumPy array). This function is similar to :py:func:`a_in_scan_read` except that the *data* key in the returned namedtuple is a NumPy array of float64 values and may be used directly with NumPy functions. Args: samples_per_channel (int): The number of samples per channel to read from the scan buffer. Specify a negative number to read all available samples or 0 to only read the scan status and return no data. timeout (float): The amount of time in seconds to wait for the samples to be read. Specify a negative number to wait indefinitely, or 0 to return immediately with the samples that are already in the scan buffer. If the timeout is met and the specified number of samples have not been read, then the function will return with the amount that has been read and the timeout status set. Returns: namedtuple: a namedtuple containing the following field names: * **running** (bool): True if the scan is running, False if it has stopped or completed. * **hardware_overrun** (bool): True if the hardware could not acquire and unload samples fast enough and data was lost. * **buffer_overrun** (bool): True if the background scan buffer was not read fast enough and data was lost. * **triggered** (bool): True if the trigger conditions have been met and data acquisition started. * **timeout** (bool): True if the timeout time expired before the specified number of samples were read. * **data** (NumPy array of float64): The data that was read from the scan buffer. Raises: HatError: A scan is not active, the board is not initialized, does not respond, or responds incorrectly. ValueError: Incorrect argument. """ try: import numpy from numpy.ctypeslib import ndpointer except ImportError: raise if not self._initialized: raise HatError(self._address, "Not initialized.") self._lib.mcc118_a_in_scan_read.argtypes = [ c_ubyte, POINTER(c_ushort), c_long, c_double, ndpointer(c_double, flags="C_CONTIGUOUS"), c_ulong, POINTER(c_ulong) ] num_channels = self._lib.mcc118_a_in_scan_channel_count(self._address) samples_read_per_channel = c_ulong() status = c_ushort() timed_out = False samples_to_read = 0 if samples_per_channel < 0: # read all available data # first, get the number of available samples samples_available = c_ulong(0) result = self._lib.mcc118_a_in_scan_status( self._address, byref(status), byref(samples_available)) if result != self._RESULT_SUCCESS: raise HatError(self._address, "Incorrect response {}.".format(result)) # allocate a buffer large enough for all the data samples_to_read = samples_available.value buffer_size = samples_available.value * num_channels data_buffer = numpy.empty(buffer_size, dtype=numpy.float64) elif samples_per_channel == 0: # only read the status samples_to_read = 0 buffer_size = 0 data_buffer = None elif samples_per_channel > 0: # read the specified number of samples samples_to_read = samples_per_channel # create a C buffer for the read buffer_size = samples_per_channel * num_channels data_buffer = numpy.empty(buffer_size, dtype=numpy.float64) else: # invalid samples_per_channel raise ValueError( "Invalid samples_per_channel {}.".format(samples_per_channel)) result = self._lib.mcc118_a_in_scan_read( self._address, byref(status), samples_to_read, timeout, data_buffer, buffer_size, byref(samples_read_per_channel)) if result == self._RESULT_BAD_PARAMETER: raise ValueError("Invalid parameter.") elif result == self._RESULT_RESOURCE_UNAVAIL: raise HatError(self._address, "Scan not active.") elif result == self._RESULT_TIMEOUT: timed_out = True elif result != self._RESULT_SUCCESS: raise HatError(self._address, "Incorrect response {}.".format(result)) total_read = samples_read_per_channel.value * num_channels if total_read < buffer_size: data_buffer = numpy.resize(data_buffer, (total_read, )) scan_status = namedtuple('MCC118ScanRead', [ 'running', 'hardware_overrun', 'buffer_overrun', 'triggered', 'timeout', 'data' ]) return scan_status( running=(status.value & self._STATUS_RUNNING) != 0, hardware_overrun=(status.value & self._STATUS_HW_OVERRUN) != 0, buffer_overrun=(status.value & self._STATUS_BUFFER_OVERRUN) != 0, triggered=(status.value & self._STATUS_TRIGGERED) != 0, timeout=timed_out, data=data_buffer)
def intcode_program(state, max_size, n_read): output = [] # Get state intcode_aux = state[0][:] inputs = state[1] pointer = state[2] relative_base = state[3] # Resize array (get more memory) intcode_aux = np.resize(intcode_aux, max_size) # Browse array by steps while pointer < len(intcode): # Get opcode and mode of parameters opcode_instruction = str(intcode_aux[pointer]) while len(opcode_instruction) < 5: opcode_instruction = '0' + opcode_instruction opcode = int(opcode_instruction[-2:]) mode_params = [ int(opcode_instruction[-3]), int(opcode_instruction[-4]), int(opcode_instruction[-5]) ] # Get index of each parameter indexes = [-1, -1, -1] for i in range(len(indexes)): pos = pointer + 1 + i if pos < len(intcode_aux): if mode_params[i] == 0: indexes[i] = intcode_aux[pos] elif mode_params[i] == 1: indexes[i] = pos elif mode_params[i] == 2: indexes[i] = relative_base + intcode_aux[pos] # Apply opcode operation # STOP if opcode == 99: break # ADDITION elif opcode == 1: op1 = intcode_aux[indexes[0]] op2 = intcode_aux[indexes[1]] intcode_aux[indexes[2]] = op1 + op2 pointer += 4 # MULTIPLICATION elif opcode == 2: op1 = intcode_aux[indexes[0]] op2 = intcode_aux[indexes[1]] intcode_aux[indexes[2]] = op1 * op2 pointer += 4 # INPUT elif opcode == 3: intcode_aux[indexes[0]] = inputs.pop(0) pointer += 2 # OUTPUT elif opcode == 4: #print('Output: '+intcode_aux[indexes[0]].__str__()) output.append(intcode_aux[indexes[0]]) pointer += 2 if len(output) == n_read: break # JUMP-IF-TRUE elif opcode == 5: if intcode_aux[indexes[0]] != 0: pointer = intcode_aux[indexes[1]] else: pointer += 3 # JUMP-IF-FALSE elif opcode == 6: if intcode_aux[indexes[0]] == 0: pointer = intcode_aux[indexes[1]] else: pointer += 3 # LESS-THAN elif opcode == 7: if intcode_aux[indexes[0]] < intcode_aux[indexes[1]]: intcode_aux[indexes[2]] = 1 else: intcode_aux[indexes[2]] = 0 pointer += 4 # EQUAL-TO elif opcode == 8: if intcode_aux[indexes[0]] == intcode_aux[indexes[1]]: intcode_aux[indexes[2]] = 1 else: intcode_aux[indexes[2]] = 0 pointer += 4 # ADJUST RELATIVE BASE elif opcode == 9: relative_base += intcode_aux[indexes[0]] pointer += 2 return [output, intcode_aux, pointer, relative_base]
# # Call Option Pricing with Discrete Fourier Transforms (DFT/FFT) import math import numpy as np from numpy.fft import fft, ifft from convolution import revnp from parameters import * # Parameter Adjustments M = 3 #numberoftimesteps dt, df, u, d, q = get_binomial_parameters(M) # Array Generation for Stock Prices mu = np.arange(M + 1) mu = np.resize(mu, (M + 1, M + 1)) md = np.transpose(mu) mu = u**(mu - md) md = d**md S = S0 * mu * md # Valuation by fft CT = np.maximum(S[:, -1] - K, 0) qv = np.zeros(M + 1, dtype=np.float) qv[0] = q qv[1] = 1 - q C0_a = fft(math.exp(-r * T) * ifft(CT) * ((M + 1) * ifft(revnp(qv)))**M) C0_b = fft(math.exp(-r * T) * ifft(CT) * fft(qv)**M) C0_c = ifft(math.exp(-r * T) * fft(CT) * fft(revnp(qv))**M) print("Value of European option is %8.3f" % np.real(C0_a[0]))
from PIL import Image import matplotlib.pyplot as plt import numpy as np import Base_Utils as bu number_iteractions =10 img = Image.open('/Users/Ashley/Desktop/test.tif') img.seek(5000) out=np.array(img.copy()) [X,Y]=np.shape(out) out_line=np.resize(out,[X*Y,1]) N=np.size(out_line) mean_val=np.mean(out_line) #Select the mean value of the data and use it selected_points = np.where(out_line > mean_val) #to select the points above that value selected_points_signal=np.zeros((N,1)) #Build empty array selected_points_signal[selected_points]=True #True means there is signal there selected_points_noise = np.logical_not(selected_points_signal) #True means there is noise there pms=np.zeros((2,1)) pmm=np.zeros((2,1)) #pms =1 x 2 double [signal max and signal widrh pms[0]=np.max(out_line) pms[1]=10 #pmn =1 x 2 double [signal max and noise widrh pmm[0]=np.mean(out_line) pmm[1]=10
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth): """Returns TEC screen parmdb parameters H - H5parm object solset - solution set with TEC screen parameters TECsolTab = solution table with tecscreen values timewidths - time widths of output parmdb freq - frequency of output parmdb freqwidth - frequency width of output parmdb """ global ipbar, pbar solset = H.getSolset(solset) station_dict = solset.getAnt() station_names = station_dict.keys() station_positions = station_dict.values() source_dict = solset.getSou() source_names = source_dict.keys() source_positions = source_dict.values() tec_sf = solset.getSoltab(TECsolTab) tec_screen, axis_vals = tec_sf.getValues() times = axis_vals['time'] beta = TECsolTab._v_attrs['beta'] r_0 = TECsolTab._v_attrs['r_0'] height = TECsolTab._v_attrs['height'] order = TECsolTab._v_attrs['order'] pp = tec_sf.t.piercepoint N_sources = len(source_names) N_times = len(times) N_freqs = 1 N_stations = len(station_names) N_piercepoints = N_sources * N_stations freqs = freq freqwidths = freqwidth parms = {} v = {} v['times'] = times v['timewidths'] = timewidths v['freqs'] = freqs v['freqwidths'] = freqwidths for station_name in station_names: for source_name in source_names: v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() v['values'] = np.zeros((N_times, N_freqs), dtype=np.double) parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname] = v.copy() for k in range(N_times): D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3)) D = np.transpose(D, (1, 0, 2)) - D D2 = np.sum(D**2, axis=2) C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0 tec_fit_white = np.dot(np.linalg.inv(C), tec_screen[:, k, :].reshape(N_piercepoints)) pp_idx = 0 for src, source_name in enumerate(source_names): for sta, station_name in enumerate(station_names): parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0] parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1] parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2] parmname = 'TECfit_white:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name) parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx] pp_idx += 1 pbar.update(ipbar) ipbar += 1 time_start = times[0] - timewidths[0] / 2 time_end = times[-1] + timewidths[-1] / 2 v['times'] = np.array([(time_start + time_end) / 2]) v['timewidths'] = np.array([time_end - time_start]) v_r0 = v.copy() v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2) parms['r_0'] = v_r0 v_beta = v.copy() v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2) parms['beta'] = v_beta v_height = v.copy() v_height['values'] = np.array(height, dtype=np.double, ndmin=2) parms['height'] = v_height return parms
def process_chunk(start_id, end_id, raw_data): """This function extracts features and assigns labels from a good chunk of raw data Following step is taken: + determine aircon status from `power` + identify user actions (classification labels) from the aircon status + remove spurious actions (TURN ON, TURN OFF that are too close to each other) + fix aircon status to match with the user actions (after removing spurious ones) + sub-sample and extract data features (outlier samples will be removed) @:param raw_data the whole raw data array @:param start_id where the chunk starts @:param end_id where the chunk ends @:return a list of feature array, label vector and aircon status vector """ nrows = end_id - start_id saving_time = raw_data[start_id:end_id, 0] power = raw_data[start_id:end_id, 1] #prepare vectors to hold `user action` and `aircon status` action = np.empty(nrows, int) status = np.empty(nrows, int) status.fill(STATUS_OFF) action.fill(ACTION_NOTHING) #detect power ON status status[np.where(power > POWER_CUT)] = STATUS_ON #detect TURN ON, TURN OFF action action_id = [] for i in range(1, nrows): status_diff = status[i] - status[i - 1] if status_diff == 1: action[i] = ACTION_TURN_ON action_id.append(i) elif status_diff == -1: action[i] = ACTION_TURN_OFF action_id.append(i) num_actions = len(action_id) #total number of actions action_id = np.array(action_id) #row index of actions #remove any (TURN_OFF, TURN_ON) tuple whose time interval is too short (less than 5, for example) min_diff = OFF_MIN_DURATION * 60 # define the minimum time interval, for example 5 minutes time 60 secs i = num_actions - 1 while i > 0: if action[action_id[i]] == ACTION_TURN_ON: time_diff = saving_time[action_id[i]] - saving_time[action_id[i - 1]] if time_diff < min_diff: action[action_id[i]] = ACTION_NOTHING # remove TURN ON action[action_id[i - 1]] = ACTION_NOTHING # remove TURN OFF i -= 2 else: i -= 1 #remove any (TURN_ON, TURN_OFF) tuple whose time interval is too short (less than 5, for example) min_diff = ON_MIN_DURATION * 60 # define the minimum time interval, for example 5 minutes time 60 secs i = 0 while i < num_actions - 1: if action[action_id[i]] == ACTION_TURN_ON: #find the next TURN OFF action j = i + 1 while (j < num_actions and action[action_id[j]] != ACTION_TURN_OFF): j += 1 if j < num_actions: # found TURN OFF action time_diff = saving_time[action_id[j]] - saving_time[ action_id[i]] if time_diff < min_diff: action[action_id[i]] = ACTION_NOTHING # remove TURN ON action[action_id[j]] = ACTION_NOTHING # remove TURN OFF i = j + 1 else: i += 1 #once actions are correctly recognized, now it's time to fix the aircon status prev_status = status[0] for i in range(1, nrows): if action[i] == ACTION_NOTHING: status[i] = prev_status elif action[i] == ACTION_TURN_ON: status[i] = STATUS_ON else: status[i] = STATUS_OFF prev_status = status[i] # sub-sampling (reduce the data size), but retain all data points having an action n_obs = SAMPLING_SIZE #sampling every 10 timepoints (i.e. 5 minutes) nfrows = int(nrows / n_obs) #number of data rows after sampling if nfrows <= 0: return None #allocate data arrays x = np.empty([nfrows, NUM_FEATURES], float) y = np.empty(nfrows, int) y.fill(ACTION_NOTHING) sampled_status = np.empty(nfrows, int) sampled_id = n_obs i = 0 feat_period = PAST_DURATION * 60 #extract feature from previous 5 minutes #sub-sampling is happening in this loop while sampled_id < nrows: #find any near by actions. actions is precious, we must retain all actions for j in range(n_obs): new_id = sampled_id + j if new_id < nrows and action[new_id] != ACTION_NOTHING: sampled_id = new_id break #collect samples which are obs_period seconds from sampled_id sample feature_ids = [] cur_id = start_id + sampled_id - 1 while (cur_id >= 0) and ((raw_data[start_id + sampled_id, 0] - raw_data[cur_id, 0]) < feat_period): #check if the sample at cur_id is an outlier bOutlier = False for j in range(2, NUM_CSV_COLS): if (raw_data[cur_id, j] < SENSOR_RANGES[j - 2][0]) or ( raw_data[cur_id, j] > SENSOR_RANGES[j - 2][1]): bOutlier = True break if not bOutlier: feature_ids.append(cur_id) cur_id -= 1 if len(feature_ids) > 0: #extract feature from these samples for j in range(2, NUM_CSV_COLS): x[i, j - 2] = np.mean(raw_data[feature_ids, j]) #x[i, 2*j] = np.mean(raw_data[feature_ids, j]) #x[i, 2*j+1] = np.std(raw_data[feature_ids, j]) x[i, NUM_FEATURES - 2] = raw_data[sampled_id, NUM_CSV_COLS] #week day x[i, NUM_FEATURES - 1] = raw_data[sampled_id, NUM_CSV_COLS + 1] #hour sampled_status[i] = status[sampled_id - 1] y[i] = action[sampled_id] i += 1 #move on to the next id for sampling sampled_id += n_obs nrows = i #correct number of data rows if nrows <= 0: return None #resize arrays to their correct sizes x = np.resize(x, [nrows, NUM_FEATURES]) y = np.resize(y, nrows) sampled_status = np.resize(sampled_status, nrows) return [x, y, sampled_status]
def KM2(): matrix = numpy.loadtxt('D:/EE569_Assignment/4/C++/Kmeans2/x64/Debug/Weight_W.txt'); temp = numpy.resize(matrix, (1, 5 * 5 * 6 * 16)) temp = tf.convert_to_tensor(temp, tf.float32) initial = tflearn.layers.core.reshape(temp, [5, 5, 6, 16]) return initial
def get_rgb(imgs, bands, mnmx=None, arcsinh=None, scales=None, imgname='test', desaturate=False): ''' Given a list of images in the given bands, returns a scaled RGB image. *imgs* a list of numpy arrays, all the same size, in nanomaggies *bands* a list of strings, eg, ['g','r','z'] *mnmx* = (min,max), values that will become black/white *after* scaling. Default is (-3,10) *arcsinh* use nonlinear scaling as in SDSS *scales* Returns a (H,W,3) numpy array with values between 0 and 1. ''' bands = ''.join(bands) grzscales = dict( g=(2, 0.0066), r=(1, 0.01385), z=(0, 0.025), ) if scales is None: if bands == 'grz': scales = grzscales elif bands == 'urz': scales = dict( u=(2, 0.0066), r=(1, 0.01), z=(0, 0.025), ) elif bands == 'gri': # scales = dict(g = (2, 0.004), # r = (1, 0.0066), # i = (0, 0.01), # ) scales = dict( g=(2, 0.002), r=(1, 0.004), i=(0, 0.005), ) else: scales = grzscales h, w = imgs[0].shape rgb = np.zeros((h, w, 3), np.float32) # Convert to ~ sigmas for im, band in zip(imgs, bands): plane, scale = scales[band] rgb[:, :, plane] = (im / scale).astype(np.float32) #print 'rgb: plane', plane, 'range', rgb[:,:,plane].min(), rgb[:,:,plane].max() if mnmx is None: mn, mx = -3, 10 else: mn, mx = mnmx if arcsinh is not None: def nlmap(x): return np.arcsinh(x * arcsinh) / np.sqrt(arcsinh) rgb = nlmap(rgb) mn = nlmap(mn) mx = nlmap(mx) rgb = (rgb - mn) / (mx - mn) if desaturate: # optionally desaturate pixels that are dominated by a single # colour to avoid colourful speckled sky RGBim = np.array([rgb[:, :, 0], rgb[:, :, 1], rgb[:, :, 2]]) a = RGBim.mean(axis=0) np.putmask(a, a == 0.0, 1.0) acube = np.resize(a, (3, h, w)) bcube = (RGBim / acube) / 2.5 mask = np.array(bcube) wt = np.max(mask, axis=0) np.putmask(wt, wt > 1.0, 1.0) wt = 1 - wt wt = np.sin(wt * np.pi / 2.0) temp = RGBim * wt + a * (1 - wt) + a * (1 - wt)**2 * RGBim rgb = np.zeros((h, w, 3), np.float32) for idx, im in enumerate( (temp[0, :, :], temp[1, :, :], temp[2, :, :])): rgb[:, :, idx] = im clipped = np.clip(rgb, 0., 1.) # Save hardcopy as JPG #out_jpg = '%s/decals/imagetests/dstn/test.jpeg' % gzpath out_jpg = '%s/decals/imagetests/sugata/%s.jpeg' % (gzpath, imgname) plt.imsave(out_jpg, clipped, origin='lower') return clipped
def body1(self, num, object_num, loss, predict, labels, nilboy): """ calculate loss Args: predict: 3-D tensor [cell_size, cell_size, 5 * boxes_per_cell] labels : [max_objects, 5] (x_center, y_center, w, h, class) """ label = labels[num:num + 1, :] label = tf.reshape(label, [-1]) #calculate objects tensor [CELL_SIZE, CELL_SIZE] min_x = (label[0] - label[2] / 2) / (self.image_size / self.cell_size) max_x = (label[0] + label[2] / 2) / (self.image_size / self.cell_size) min_y = (label[1] - label[3] / 2) / (self.image_size / self.cell_size) max_y = (label[1] + label[3] / 2) / (self.image_size / self.cell_size) min_x = tf.floor(min_x) min_y = tf.floor(min_y) max_x = tf.ceil(max_x) max_y = tf.ceil(max_y) temp = tf.cast(tf.stack([max_y - min_y, max_x - min_x]), dtype=tf.int32) objects = tf.ones(temp, tf.float32) temp = tf.cast( tf.stack( [min_y, self.cell_size - max_y, min_x, self.cell_size - max_x]), tf.int32) temp = tf.reshape(temp, (2, 2)) objects = tf.pad(objects, temp, "CONSTANT") #calculate objects tensor [CELL_SIZE, CELL_SIZE] #calculate responsible tensor [CELL_SIZE, CELL_SIZE] center_x = label[0] / (self.image_size / self.cell_size) center_x = tf.floor(center_x) center_y = label[1] / (self.image_size / self.cell_size) center_y = tf.floor(center_y) response = tf.ones([1, 1], tf.float32) temp = tf.cast( tf.stack([ center_y, self.cell_size - center_y - 1, center_x, self.cell_size - center_x - 1 ]), tf.int32) temp = tf.reshape(temp, (2, 2)) response = tf.pad(response, temp, "CONSTANT") #objects = response #calculate iou_predict_truth [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] predict_boxes = predict[:, :, self.num_classes + self.boxes_per_cell:] predict_boxes = tf.reshape( predict_boxes, [self.cell_size, self.cell_size, self.boxes_per_cell, 4]) predict_boxes = predict_boxes * [ self.image_size / self.cell_size, self.image_size / self.cell_size, self.image_size, self.image_size ] base_boxes = np.zeros([self.cell_size, self.cell_size, 4]) for y in range(self.cell_size): for x in range(self.cell_size): #nilboy base_boxes[y, x, :] = [ self.image_size / self.cell_size * x, self.image_size / self.cell_size * y, 0, 0 ] base_boxes = np.tile( np.resize(base_boxes, [self.cell_size, self.cell_size, 1, 4]), [1, 1, self.boxes_per_cell, 1]) predict_boxes = base_boxes + predict_boxes iou_predict_truth = self.iou(predict_boxes, label[0:4]) #calculate C [cell_size, cell_size, boxes_per_cell] C = iou_predict_truth * tf.reshape(response, [self.cell_size, self.cell_size, 1]) #calculate I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] I = iou_predict_truth * tf.reshape(response, (self.cell_size, self.cell_size, 1)) max_I = tf.reduce_max(I, 2, keep_dims=True) I = tf.cast((I >= max_I), tf.float32) * tf.reshape( response, (self.cell_size, self.cell_size, 1)) #calculate no_I tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] no_I = tf.ones_like(I, dtype=tf.float32) - I p_C = predict[:, :, self.num_classes:self.num_classes + self.boxes_per_cell] #calculate truth x,y,sqrt_w,sqrt_h 0-D x = label[0] y = label[1] sqrt_w = tf.sqrt(tf.abs(label[2])) sqrt_h = tf.sqrt(tf.abs(label[3])) #sqrt_w = tf.abs(label[2]) #sqrt_h = tf.abs(label[3]) #calculate predict p_x, p_y, p_sqrt_w, p_sqrt_h 3-D [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL] p_x = predict_boxes[:, :, :, 0] p_y = predict_boxes[:, :, :, 1] #p_sqrt_w = tf.sqrt(tf.abs(predict_boxes[:, :, :, 2])) * ((tf.cast(predict_boxes[:, :, :, 2] > 0, tf.float32) * 2) - 1) #p_sqrt_h = tf.sqrt(tf.abs(predict_boxes[:, :, :, 3])) * ((tf.cast(predict_boxes[:, :, :, 3] > 0, tf.float32) * 2) - 1) #p_sqrt_w = tf.sqrt(tf.maximum(0.0, predict_boxes[:, :, :, 2])) #p_sqrt_h = tf.sqrt(tf.maximum(0.0, predict_boxes[:, :, :, 3])) #p_sqrt_w = predict_boxes[:, :, :, 2] #p_sqrt_h = predict_boxes[:, :, :, 3] p_sqrt_w = tf.sqrt( tf.minimum(self.image_size * 1.0, tf.maximum(0.0, predict_boxes[:, :, :, 2]))) p_sqrt_h = tf.sqrt( tf.minimum(self.image_size * 1.0, tf.maximum(0.0, predict_boxes[:, :, :, 3]))) #calculate truth p 1-D tensor [NUM_CLASSES] P = tf.one_hot(tf.cast(label[4], tf.int32), self.num_classes, dtype=tf.float32) #calculate predict p_P 3-D tensor [CELL_SIZE, CELL_SIZE, NUM_CLASSES] p_P = predict[:, :, 0:self.num_classes] #class_loss class_loss = tf.nn.l2_loss( tf.reshape(objects, (self.cell_size, self.cell_size, 1)) * (p_P - P)) * self.class_scale #class_loss = tf.nn.l2_loss(tf.reshape(response, (self.cell_size, self.cell_size, 1)) * (p_P - P)) * self.class_scale #object_loss object_loss = tf.nn.l2_loss(I * (p_C - C)) * self.object_scale #object_loss = tf.nn.l2_loss(I * (p_C - (C + 1.0)/2.0)) * self.object_scale #noobject_loss #noobject_loss = tf.nn.l2_loss(no_I * (p_C - C)) * self.noobject_scale noobject_loss = tf.nn.l2_loss(no_I * (p_C)) * self.noobject_scale #coord_loss coord_loss = (tf.nn.l2_loss(I * (p_x - x) / (self.image_size / self.cell_size)) + tf.nn.l2_loss(I * (p_y - y) / (self.image_size / self.cell_size)) + tf.nn.l2_loss(I * (p_sqrt_w - sqrt_w)) / self.image_size + tf.nn.l2_loss(I * (p_sqrt_h - sqrt_h)) / self.image_size) * self.coord_scale nilboy = I return num + 1, object_num, [ loss[0] + class_loss, loss[1] + object_loss, loss[2] + noobject_loss, loss[3] + coord_loss ], predict, labels, nilboy
def KM4(): matrix = numpy.loadtxt('D:/EE569_Assignment/4/C++/Kmeans4/x64/Debug/Weight_W.txt'); temp = numpy.resize(matrix, (1, 120 * 84)) temp = tf.convert_to_tensor(temp, tf.float32) initial = tflearn.layers.core.reshape(temp, [120, 84]) return initial
def log_encoding_ACESproxy(lin_AP1, bit_depth=10, out_int=False, constants=ACES_PROXY_CONSTANTS): """ Defines the *ACESproxy* colourspace log encoding curve / opto-electronic transfer function. Parameters ---------- lin_AP1 : numeric or array_like *lin_AP1* value. bit_depth : int, optional **{10, 12}**, *ACESproxy* bit depth. out_int : bool, optional Whether to return value as integer code value or float equivalent of a code value at a given bit depth. constants : Structure, optional *ACESproxy* constants. Returns ------- numeric or ndarray *ACESproxy* non-linear value. Notes ----- +---------------+-----------------------+---------------+ | **Domain \\*** | **Scale - Reference** | **Scale - 1** | +===============+=======================+===============+ | ``lin_AP1`` | [0, 1] | [0, 1] | +---------------+-----------------------+---------------+ +---------------+-----------------------+---------------+ | **Range \\*** | **Scale - Reference** | **Scale - 1** | +===============+=======================+===============+ | ``ACESproxy`` | [0, 1] | [0, 1] | +---------------+-----------------------+---------------+ - \\* This definition has an output integer switch, thus the domain-range scale information is only given for the floating point mode. References ---------- :cite:`TheAcademyofMotionPictureArtsandSciences2014q`, :cite:`TheAcademyofMotionPictureArtsandSciences2014r`, :cite:`TheAcademyofMotionPictureArtsandSciences2014s`, :cite:`TheAcademyofMotionPictureArtsandSciencese` Examples -------- >>> log_encoding_ACESproxy(0.18) # doctest: +ELLIPSIS 0.4164222... >>> log_encoding_ACESproxy(0.18, out_int=True) 426 """ lin_AP1 = to_domain_1(lin_AP1) constants = constants[bit_depth] CV_min = np.resize(constants.CV_min, lin_AP1.shape) CV_max = np.resize(constants.CV_max, lin_AP1.shape) def float_2_cv(x): """ Converts given numeric to code value. """ return np.maximum(CV_min, np.minimum(CV_max, np.round(x))) ACESproxy = np.where( lin_AP1 > 2**-9.72, float_2_cv((np.log2(lin_AP1) + constants.mid_log_offset) * constants.steps_per_stop + constants.mid_CV_offset), np.resize(CV_min, lin_AP1.shape), ) if out_int: return as_int(np.round(ACESproxy)) else: return as_float(from_range_1(ACESproxy / (2**bit_depth - 1)))
def _spectral_helper(x, y=None, NFFT=None, Fs=None, detrend_func=None, window=None, noverlap=None, pad_to=None, sides=None, scale_by_freq=None, mode=None): """ Private helper implementing the common parts between the psd, csd, spectrogram and complex, magnitude, angle, and phase spectrums. """ if y is None: # if y is None use x for y same_data = True else: # The checks for if y is x are so that we can use the same function to # implement the core of psd(), csd(), and spectrogram() without doing # extra calculations. We return the unaveraged Pxy, freqs, and t. same_data = y is x if Fs is None: Fs = 2 if noverlap is None: noverlap = 0 if detrend_func is None: detrend_func = detrend_none if window is None: window = window_hanning # if NFFT is set to None use the whole signal if NFFT is None: NFFT = 256 if mode is None or mode == 'default': mode = 'psd' _api.check_in_list( ['default', 'psd', 'complex', 'magnitude', 'angle', 'phase'], mode=mode) if not same_data and mode != 'psd': raise ValueError("x and y must be equal if mode is not 'psd'") # Make sure we're dealing with a numpy array. If y and x were the same # object to start with, keep them that way x = np.asarray(x) if not same_data: y = np.asarray(y) if sides is None or sides == 'default': if np.iscomplexobj(x): sides = 'twosided' else: sides = 'onesided' _api.check_in_list(['default', 'onesided', 'twosided'], sides=sides) # zero pad x and y up to NFFT if they are shorter than NFFT if len(x) < NFFT: n = len(x) x = np.resize(x, NFFT) x[n:] = 0 if not same_data and len(y) < NFFT: n = len(y) y = np.resize(y, NFFT) y[n:] = 0 if pad_to is None: pad_to = NFFT if mode != 'psd': scale_by_freq = False elif scale_by_freq is None: scale_by_freq = True # For real x, ignore the negative frequencies unless told otherwise if sides == 'twosided': numFreqs = pad_to if pad_to % 2: freqcenter = (pad_to - 1) // 2 + 1 else: freqcenter = pad_to // 2 scaling_factor = 1. elif sides == 'onesided': if pad_to % 2: numFreqs = (pad_to + 1) // 2 else: numFreqs = pad_to // 2 + 1 scaling_factor = 2. if not np.iterable(window): window = window(np.ones(NFFT, x.dtype)) if len(window) != NFFT: raise ValueError( "The window length must match the data's first dimension") result = stride_windows(x, NFFT, noverlap, axis=0) result = detrend(result, detrend_func, axis=0) result = result * window.reshape((-1, 1)) result = np.fft.fft(result, n=pad_to, axis=0)[:numFreqs, :] freqs = np.fft.fftfreq(pad_to, 1 / Fs)[:numFreqs] if not same_data: # if same_data is False, mode must be 'psd' resultY = stride_windows(y, NFFT, noverlap) resultY = detrend(resultY, detrend_func, axis=0) resultY = resultY * window.reshape((-1, 1)) resultY = np.fft.fft(resultY, n=pad_to, axis=0)[:numFreqs, :] result = np.conj(result) * resultY elif mode == 'psd': result = np.conj(result) * result elif mode == 'magnitude': result = np.abs(result) / np.abs(window).sum() elif mode == 'angle' or mode == 'phase': # we unwrap the phase later to handle the onesided vs. twosided case result = np.angle(result) elif mode == 'complex': result /= np.abs(window).sum() if mode == 'psd': # Also include scaling factors for one-sided densities and dividing by # the sampling frequency, if desired. Scale everything, except the DC # component and the NFFT/2 component: # if we have a even number of frequencies, don't scale NFFT/2 if not NFFT % 2: slc = slice(1, -1, None) # if we have an odd number, just don't scale DC else: slc = slice(1, None, None) result[slc] *= scaling_factor # MATLAB divides by the sampling frequency so that density function # has units of dB/Hz and can be integrated by the plotted frequency # values. Perform the same scaling here. if scale_by_freq: result /= Fs # Scale the spectrum by the norm of the window to compensate for # windowing loss; see Bendat & Piersol Sec 11.5.2. result /= (np.abs(window)**2).sum() else: # In this case, preserve power in the segment, not amplitude result /= np.abs(window).sum()**2 t = np.arange(NFFT / 2, len(x) - NFFT / 2 + 1, NFFT - noverlap) / Fs if sides == 'twosided': # center the frequency range at zero freqs = np.roll(freqs, -freqcenter, axis=0) result = np.roll(result, -freqcenter, axis=0) elif not pad_to % 2: # get the last value correctly, it is negative otherwise freqs[-1] *= -1 # we unwrap the phase here to handle the onesided vs. twosided case if mode == 'phase': result = np.unwrap(result, axis=0) return result, freqs, t
output = 'datatype:{}, mean:{}, sd:{}, train time:{}s, explain time:{}s \n'.format( "mnist", 0, #np.mean(median_ranks), 0, #np.std(median_ranks), train_time, exp_time) print(scores.shape) #exit(0) print(scores_grp, "smaple_grp") print(scores, "samples") group_dir_name = 'img_groups_new' if not os.path.exists(group_dir_name): os.mkdir(group_dir_name) num_dir_name = 'img_numbers_new' print(x_val, "x_val") print(new_model_input4, "new 4") if not os.path.exists(num_dir_name): os.mkdir(num_dir_name) for num in tqdm(range(100)): visualize_group(scores[num], group_dir_name + '/group_%03d.png' % num, scores_grp[num]) img = np.resize( np.squeeze(x_val[num] * 255.0).astype(np.uint8), (14, 14)) imsave(num_dir_name + '/num_%03d.png' % num, img) pickle.dump(scores, open("./score.pkl", "wb")) pickle.dump(scores_grp, open("./score_grp.pkl", "wb")) pickle.dump(x_val, open("./x_val.pkl", "wb")) pickle.dump(y_val, open("./y_val.pkl", "wb")) pickle.dump(preds, open("./preds.pkl", "wb"))
def getitem(self, global_index, crop=None): # select frame from sequence index = global_index # get data ids audio_id = self.audio_ids[index] image_id = self.image_ids[index] uv_id = self.uvs_ids[index] #print('GET ITEM: ', index) img_fname = os.path.join(self.image_dir, str(image_id).zfill(5) + '.png') img_numpy = np.asarray(Image.open(img_fname)) TARGET = 2.0 * transforms.ToTensor()(img_numpy.astype( np.float32)) / 255.0 - 1.0 uv_fname = os.path.join(self.uvs_dir, str(uv_id).zfill(5) + '.exr') uv_numpy = util.load_exr(uv_fname) UV = transforms.ToTensor()(uv_numpy.astype(np.float32)) UV = torch.where(UV > 1.0, torch.zeros_like(UV), UV) UV = torch.where(UV < 0.0, torch.zeros_like(UV), UV) UV = 2.0 * UV - 1.0 # intrinsics and extrinsics intrinsics = self.intrinsics extrinsics = self.extrinsics[uv_id] # expressions expressions = np.asarray(self.expressions[audio_id], dtype=np.float32) #print('expressions:', expressions.shape) expressions[32] *= 0.0 # remove eye brow movements expressions[41] *= 0.0 # remove eye brow movements expressions[71:75] *= 0.0 # remove eye brow movements expressions = torch.tensor(expressions) # identity identity = torch.tensor(self.identities[audio_id]) # load deepspeech feature dsf_fname = os.path.join(self.audio_feature_dir, str(audio_id) + '.deepspeech.npy') feature_array = np.load(dsf_fname) dsf_np = np.resize(feature_array, (16, 29, 1)) dsf = transforms.ToTensor()(dsf_np.astype(np.float32)) # load sequence data if necessary if self.opt.look_ahead: # use prev and following frame infos r = self.opt.seq_len // 2 last_valid_idx = audio_id for i in range(1, r): # prev frames index_seq = index - i if index_seq < 0: index_seq = 0 audio_id_seq = self.audio_ids[index_seq] if audio_id_seq == audio_id - i: last_valid_idx = audio_id_seq else: audio_id_seq = last_valid_idx dsf_fname = os.path.join(self.audio_feature_dir, str(audio_id_seq) + '.deepspeech.npy') feature_array = np.load(dsf_fname) dsf_np = np.resize(feature_array, (16, 29, 1)) dsf_seq = transforms.ToTensor()(dsf_np.astype( np.float32)) # 1 x 16 x 29 dsf = torch.cat([dsf_seq, dsf], 0) # seq_len x 16 x 29 # note the ordering [old ... current] last_valid_idx = audio_id for i in range(1, self.opt.seq_len - r + 1): # following frames index_seq = index + i max_idx = len(self.audio_ids) - 1 if index_seq > max_idx: index_seq = max_idx audio_id_seq = self.audio_ids[index_seq] if audio_id_seq == audio_id + i: last_valid_idx = audio_id_seq else: audio_id_seq = last_valid_idx dsf_fname = os.path.join(self.audio_feature_dir, str(audio_id_seq) + '.deepspeech.npy') feature_array = np.load(dsf_fname) dsf_np = np.resize(feature_array, (16, 29, 1)) dsf_seq = transforms.ToTensor()(dsf_np.astype( np.float32)) # 1 x 16 x 29 dsf = torch.cat([dsf, dsf_seq], 0) # seq_len x 16 x 29 # note the ordering [old ... current ... future] else: last_valid_idx = audio_id for i in range(1, self.opt.seq_len): index_seq = index - i if index_seq < 0: index_seq = 0 audio_id_seq = self.audio_ids[index_seq] if audio_id_seq == audio_id - i: last_valid_idx = audio_id_seq else: audio_id_seq = last_valid_idx dsf_fname = os.path.join(self.audio_feature_dir, str(audio_id_seq) + '.deepspeech.npy') feature_array = np.load(dsf_fname) dsf_np = np.resize(feature_array, (16, 29, 1)) dsf_seq = transforms.ToTensor()(dsf_np.astype( np.float32)) # 1 x 16 x 29 dsf = torch.cat([dsf_seq, dsf], 0) # seq_len x 16 x 29 # note the ordering [old ... current] ################################# ####### apply augmentation ###### ################################# if not self.opt.no_augmentation: if type(crop) == type(None): INVALID_UV = -1 mask = ((UV[0:1, :, :] != INVALID_UV) | (UV[1:2, :, :] != INVALID_UV)) crop = self.computeCrop( mask, MULTIPLE_OF=64 ) # << dependent on the network structure !! 64 => 6 layers offset_x, offset_y, new_dim_x, new_dim_y = crop # select subwindow TARGET = TARGET[:, offset_y:offset_y + new_dim_y, offset_x:offset_x + new_dim_x] UV = UV[:, offset_y:offset_y + new_dim_y, offset_x:offset_x + new_dim_x] # compute new intrinsics # TODO: atm not needed but maybe later ################################# return { 'TARGET': TARGET, 'UV': UV, 'paths': dsf_fname, #img_path, 'intrinsics': np.array(intrinsics), 'extrinsics': np.array(extrinsics), 'expressions': expressions, 'identity': identity, 'audio_deepspeech': dsf, # deepspeech feature #'target_id':target_id, 'crop': crop, 'internal_id': 0 }