def objective(pars,Z,ycovar): """The objective function""" bcost= pars[0] t= pars[1] Pb= pars[2] Xb= pars[3] Yb= pars[4] Zb= sc.array([Xb,Yb]) Vb1= sc.exp(pars[5]) Vb2= sc.exp(pars[6]) corr= pars[7] V= sc.array([[Vb1,sc.sqrt(Vb1*Vb2)*corr],[sc.sqrt(Vb1*Vb2)*corr,Vb2]]) v= sc.array([-sc.sin(t),sc.cos(t)]) if Pb < 0. or Pb > 1.: return -sc.finfo(sc.dtype(sc.float64)).max if corr < -1. or corr > 1.: return -sc.finfo(sc.dtype(sc.float64)).max delta= sc.dot(v,Z.T)-bcost sigma2= sc.dot(v,sc.dot(ycovar,v)) ndata= Z.shape[0] detVycovar= sc.zeros(ndata) deltaOUT= sc.zeros(ndata) for ii in range(ndata): detVycovar[ii]= m.sqrt(linalg.det(V+ycovar[:,ii,:])) deltaOUT[ii]= sc.dot(Z[ii,:]-Zb,sc.dot(linalg.inv(V+ycovar[:,ii,:]),Z[ii,:]-Zb)) return sc.sum(sc.log((1.-Pb)/sc.sqrt(2.*m.pi*sigma2)* sc.exp(-0.5*delta**2./sigma2) +Pb/2./m.pi/detVycovar *sc.exp(-0.5*deltaOUT)))
def objective(pars, Z, ycovar): """The objective function""" bcost = pars[0] t = pars[1] Pb = pars[2] Xb = pars[3] Yb = pars[4] Zb = sc.array([Xb, Yb]) Vb1 = sc.exp(pars[5]) Vb2 = sc.exp(pars[6]) corr = pars[7] V = sc.array([[Vb1, sc.sqrt(Vb1 * Vb2) * corr], [sc.sqrt(Vb1 * Vb2) * corr, Vb2]]) v = sc.array([-sc.sin(t), sc.cos(t)]) if Pb < 0. or Pb > 1.: return -sc.finfo(sc.dtype(sc.float64)).max if corr < -1. or corr > 1.: return -sc.finfo(sc.dtype(sc.float64)).max delta = sc.dot(v, Z.T) - bcost sigma2 = sc.dot(v, sc.dot(ycovar, v)) ndata = Z.shape[0] detVycovar = sc.zeros(ndata) deltaOUT = sc.zeros(ndata) for ii in range(ndata): detVycovar[ii] = m.sqrt(linalg.det(V + ycovar[:, ii, :])) deltaOUT[ii] = sc.dot( Z[ii, :] - Zb, sc.dot(linalg.inv(V + ycovar[:, ii, :]), Z[ii, :] - Zb)) return sc.sum( sc.log((1. - Pb) / sc.sqrt(2. * m.pi * sigma2 / sc.cos(t)**2.) * sc.exp(-0.5 * delta**2. / sigma2) + Pb / 2. / m.pi / detVycovar * sc.exp(-0.5 * deltaOUT)))
def doTestMapElements(self, use1to1cache): dds = mango.data.gaussian_noise(self.shape, mean=1000, stdd=2, dtype="int32") mapDds = mango.map_element_values(dds, lambda x: x * x, use1to1cache=use1to1cache) self.assertTrue(sp.all(dds.asarray()**2 == mapDds.asarray())) mapDds = mango.map_element_values(dds, lambda x: sp.sqrt(x), dtype=sp.dtype("float32"), use1to1cache=use1to1cache) self.assertEqual(sp.dtype("float32"), mapDds.dtype) self.assertTrue(not hasattr(mapDds, "mtype")) diff = sp.sqrt(dds.asarray()) - mapDds.asarray() rootLogger.info("min, max diff = %s,%s" % (np.min(diff), np.max(diff))) self.assertTrue(sp.all(sp.absolute(diff) <= 1.0e-6)) mapDds = mango.map_element_values(dds, lambda x: sp.sqrt(x), mtype="tomo_float", use1to1cache=use1to1cache) self.assertEqual(sp.dtype("float32"), mapDds.dtype) self.assertTrue(hasattr(mapDds, "mtype")) self.assertEqual(mapDds.mtype, mango.mtype("tomo_float")) diff = sp.sqrt(dds.asarray()) - mapDds.asarray() rootLogger.info("min, max diff = %s,%s" % (np.min(diff), np.max(diff))) self.assertTrue(sp.all(sp.absolute(diff) <= 1.0e-6))
def get_motor_cal(motor_maps, table_size=50): """ Returns list of motors calibrations sorted by motor number. """ cal_list = [] # Sort motor calibrations by motor number num_key_pairs = [(v['number'], k) for k, v in motor_maps.iteritems()] num_key_pairs.sort() # Loop over motors and get motor calibration data for n, k in num_key_pairs: cal = {} if motor_maps[k]['type'] == 'RC': cal['type'] = 'table' # Get bounds and produce lookup table max_pos = motor_maps[k]['caldata'][:, 1].max() min_pos = motor_maps[k]['caldata'][:, 1].min() unit_data = scipy.linspace(min_pos, max_pos, table_size) ind_data = _convert_unit2ind(unit_data, motor_maps[k]) unit_data = unit_data.reshape((table_size, 1)) ind_data = ind_data.reshape((table_size, 1)) cal['unit_data'] = unit_data.astype(scipy.dtype('float32')) cal['ind_data'] = ind_data.astype(scipy.dtype('float32')) else: cal['type'] = 'mult' cal['unit_per_ind'] = motor_maps[k]['unit_per_ind'] cal_list.append(cal) return cal_list
def densidad(qe): """El metodo de ponderacion implementado es el CIC""" global x, rhoe, rhoi, dx, nparticulas, npuntos_malla, pared_izquierda, pared_derecha j1 = sp.dtype(sp.int32) # Asegura que la variable permanezca entera j2 = sp.dtype(sp.int32) # Factor de ponderacion de carga re = qe / dx # Densidad electronica rhoe = sp.zeros(npuntos_malla + 1) # Mapa de cargas sobre la malla for i in range(nparticulas): xa = x[i] / dx # xparticula/dx j1 = int(xa) # indices de la malla fija xmalla/dx j2 = j1 + 1 # Siguiente punto en la malla f2 = xa - j1 # |xmalla - xparticula|/dx f1 = 1.0 - f2 rhoe[j1] = rhoe[j1] + re * f1 rhoe[j2] = rhoe[j2] + re * f2 # Condiciones de frontera periodica rhoe[0] += rhoe[npuntos_malla] rhoe[npuntos_malla] = rhoe[0] # Se agrega una densidad de iones neutral rhoi = rho0 return True
def _leastSquaresXY(self, tilts, positions, tilt): m = len(tilts) n = 3 a = scipy.zeros((m, n), scipy.dtype('d')) b = scipy.zeros((m, 1), scipy.dtype('d')) for i in range(m): v = tilts[i] for j in range(n): a[i, j] = v**j b[i] = positions[i] x, resids, rank, s = lstsq(a, b) position = 0 for j in range(n): position += x[j] * tilt**j return position
def _leastSquaresXY(self, tilts, positions, tilt): m = len(tilts) n = 3 a = scipy.zeros((m, n), scipy.dtype('d')) b = scipy.zeros((m, 1), scipy.dtype('d')) for i in range(m): v = tilts[i] for j in range(n): a[i, j] = v**j b[i] = positions[i] x, resids, rank, s = lstsq(a, b) position = 0 for j in range(n): position += x[j]*tilt**j return position
def unique_rows(x): """This function takes a 2D scipy array x and makes it unique by rows.""" y = sp.ascontiguousarray(x).view(sp.dtype((sp.void, x.dtype.itemsize * x.shape[1]))) _, idx = sp.unique(y, return_index=True) return x[idx]
def __init__(self, system): self.system = system self.screen = array([800,800]) #Initialize pygame pygame.init() pygame.display.set_caption('Dynamics Visualizer') pygame.display.set_mode(self.screen,OPENGL|DOUBLEBUF) glViewport(0, 0, self.screen[0], self.screen[1]) glPixelStorei(GL_PACK_ALIGNMENT,1) glPixelStorei(GL_UNPACK_ALIGNMENT,1) #HACK: PyOpenGL is stupid, sometimes it returns an array other times it doesn't. WTF? shape_list = system.shape_db.shape_list num_shapes = len(shape_list) if(num_shapes == 0): self.textures = [] elif(num_shapes == 1): self.textures = [glGenTextures(num_shapes)] else: self.textures = glGenTextures(num_shapes) #Cache all of the textures for s in shape_list: glBindTexture(GL_TEXTURE_RECTANGLE_ARB, self.textures[s.shape_num]) glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_S, GL_CLAMP); glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_T, GL_CLAMP); #Need to flatten array and threshold colors properly s_flat = array(255. * s.indicator / max(s.indicator.flatten()), dtype('uint8')) glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, GL_LUMINANCE, s_flat.shape[1], s_flat.shape[0], 0, GL_RED, GL_UNSIGNED_BYTE, s_flat.tostring('C'))
def unique_rows(arr): """Returns a copy of arr with duplicate rows removed. From Stackoverflow "Find unique rows in numpy.array." Parameters ---------- arr : :py:class:`Array`, (`m`, `n`). The array to find the unique rows of. Returns ------- unique : :py:class:`Array`, (`p`, `n`) where `p` <= `m` The array `arr` with duplicate rows removed. """ b = scipy.ascontiguousarray(arr).view( scipy.dtype((scipy.void, arr.dtype.itemsize * arr.shape[1]))) try: dum, idx = scipy.unique(b, return_index=True) except TypeError: # Handle bug in numpy 1.6.2: rows = [_Row(row) for row in b] srt_idx = sorted(range(len(rows)), key=rows.__getitem__) rows = scipy.asarray(rows)[srt_idx] row_cmp = [-1] for k in xrange(1, len(srt_idx)): row_cmp.append(rows[k - 1].__cmp__(rows[k])) row_cmp = scipy.asarray(row_cmp) transition_idxs = scipy.where(row_cmp != 0)[0] idx = scipy.asarray(srt_idx)[transition_idxs] return arr[idx]
def unique_rows(arr): """Returns a copy of arr with duplicate rows removed. From Stackoverflow "Find unique rows in numpy.array." Parameters ---------- arr : :py:class:`Array`, (`m`, `n`). The array to find the unique rows of. Returns ------- unique : :py:class:`Array`, (`p`, `n`) where `p` <= `m` The array `arr` with duplicate rows removed. """ b = scipy.ascontiguousarray(arr).view( scipy.dtype((scipy.void, arr.dtype.itemsize * arr.shape[1])) ) try: dum, idx = scipy.unique(b, return_index=True) except TypeError: # Handle bug in numpy 1.6.2: rows = [_Row(row) for row in b] srt_idx = sorted(range(len(rows)), key=rows.__getitem__) rows = scipy.asarray(rows)[srt_idx] row_cmp = [-1] for k in xrange(1, len(srt_idx)): row_cmp.append(rows[k-1].__cmp__(rows[k])) row_cmp = scipy.asarray(row_cmp) transition_idxs = scipy.where(row_cmp != 0)[0] idx = scipy.asarray(srt_idx)[transition_idxs] return arr[idx]
def residuals(self, parameters, args_list): # residual_list is the values which sum of squares is to be minimized residuals_list = [] position_groups = self.model(parameters, args_list) for i, positions in enumerate(position_groups): n = positions.shape[0] cos_tilts, sin_tilts, x0, y0, x, y = args_list[i] residuals = scipy.zeros((n, 2), scipy.dtype('d')) residuals[:, 0] = x residuals[:, 1] = y residuals -= positions[:, :2] residuals_list.extend(residuals[:, 0]) residuals_list.extend(residuals[:, 1]) residuals_list = scipy.array(residuals_list, scipy.dtype('d')) residuals_list.shape = (residuals_list.size,) return residuals_list
def _getCorrelationCoefficient(self,xs,ys): if len(xs) != len(ys): return 0 m = len(xs) xa = scipy.zeros((m, 1), scipy.dtype('d')) ya = scipy.zeros((m, 1), scipy.dtype('d')) for i in range(m): xa[i] = xs[i] ya[i] = ys[i] xmean = sum(xa)/m ssxx = sum(xa*xa) - m*xmean*xmean ymean = sum(ya)/m ssyy = sum(ya*ya) - m*ymean*ymean ssxy = sum(xa*ya) - m*xmean*ymean r2 = ssxy * ssxy / (ssxx * ssyy) return r2
def loadTextPDB(filename): with open(filename) as f: data = f.readlines() pdbtype = [('atomId', int), ('atom', ('S4', [('element', 'S2'), ('distance', 'c'), ('branch', 'c')])), ('altLoc', 'S1'), ('resName', 'S4'), ('chain', 'S1'), ('resID', 'S5'), ('resNum', int), ('iCode', 'S1'), ('coords', float, 3), ('occupancy', float), ('beta', float), ('segment', 'S4'), ('element', 'S2'), ('charge', 'S2'), ('weight', float)] atomLines = [line for line in data if line[0:6] == 'ATOM '] pdb = empty(len(atomLines), dtype(pdbtype)) for n, line in enumerate(atomLines): pdb[n] = (int(line[6:11]), line[12:16], line[16], line[17:21].strip(), line[21], line[22:27], int(line[22:26]), line[26], (float(line[30:38]), float(line[38:46]), float(line[46:54])), float(line[54:60]), float(line[60:67]), line[72:76].strip(), line[76:78].strip(), line[78:80].strip(), 0) pdb = rec.array(pdb) weights = [(' C', 12.01), (' H', 1.01), (' O', 16.00), (' P', 30.97), (' N', 14.01), ('Na', 22.99), ('Cl', 35.45), (' S', 28.09)] for at, w in weights: pdb.weight[pdb.element == at] = w return pdb
def leastSquaresXY(self, tilts, xs, ys, tilt, n=5): n = self.fitdata + 1 position = scipy.zeros(2, scipy.dtype('d')) for i, positions in enumerate((xs, ys)): position[i] = self._leastSquaresXY(tilts[-n:], positions[-n:], tilt) return position
def __init__(self, tdvp, n, KLnm1, tau=1, sanity_checks=False): """ """ self.D = tdvp.D self.q = tdvp.q self.tdvp = tdvp self.n = n self.KLnm1 = KLnm1 self.sanity_checks = sanity_checks self.sanity_tol = 1E-12 d = self.D[n - 1] * self.D[n] * self.q[n] self.shape = (d, d) self.dtype = sp.dtype(tdvp.typ) self.calls = 0 self.tau = tau if n > 1: try: self.lm1_i = tdvp.l[n - 1].inv() except AttributeError: self.lm1_i = m.invmh(tdvp.l[n - 1]) else: self.lm1_i = tdvp.l[n - 1]
def _getCorrelationCoefficient(self, xs, ys): if len(xs) != len(ys): return 0 m = len(xs) xa = scipy.zeros((m, 1), scipy.dtype('d')) ya = scipy.zeros((m, 1), scipy.dtype('d')) for i in range(m): xa[i] = xs[i] ya[i] = ys[i] xmean = sum(xa) / m ssxx = sum(xa * xa) - m * xmean * xmean ymean = sum(ya) / m ssyy = sum(ya * ya) - m * ymean * ymean ssxy = sum(xa * ya) - m * xmean * ymean r2 = ssxy * ssxy / (ssxx * ssyy) return r2
def residuals(self, parameters, args_list): # residual_list is the values which sum of squares is to be minimized residuals_list = [] position_groups = self.model(parameters, args_list) for i, positions in enumerate(position_groups): n = positions.shape[0] cos_tilts, sin_tilts, x0, y0, x, y = args_list[i] residuals = scipy.zeros((n, 2), scipy.dtype('d')) residuals[:, 0] = x residuals[:, 1] = y residuals -= positions[:, :2] residuals_list.extend(residuals[:, 0]) residuals_list.extend(residuals[:, 1]) residuals_list = scipy.array(residuals_list, scipy.dtype('d')) residuals_list.shape = (residuals_list.size, ) return residuals_list
def sixaxff_c_wrapper(kine, config): """ Python wrapper for sixaxwff function. Performs force-feedback task. Spawns real-time thread to handle kinematics outscan, data acquisition, and yaw dynamics. During outscaning displays information regarding ongoing real-time task. Inputs: kine = Nx7 array of wing kinematics in indices, one column is for yaw. config = system configuration dictionary """ # Convert kinematics to integers kine = convert2int(kine) config_struct = create_config_struct(config) # Create c kinematics array structure kine_int = kine.astype(scipy.dtype('int')) kine_struct = get_c_array_struct(kine_int) # Time, position, velocity, and torque arrays for return data n = kine.shape[0] t = scipy.zeros((n, 1), dtype=scipy.dtype('float64')) pos = scipy.zeros((n, 2), dtype=scipy.dtype('float32')) vel = scipy.zeros((n, 2), dtype=scipy.dtype('float32')) ft = scipy.zeros((n, 6), dtype=scipy.dtype('float32')) # Create c data structure data_struct = data_t() data_struct.t = get_c_array_struct(t) data_struct.pos = get_c_array_struct(pos) data_struct.vel = get_c_array_struct(vel) data_struct.ft = get_c_array_struct(ft) # Create array for ending positions end_pos = (ctypes.c_int * config['num_motor'])() # Call C library sixaxff function ret_val = lib.sixaxff(kine_struct, config_struct, data_struct, end_pos) if ret_val == FAIL: raise RuntimeError, "lib.sizaxff call failed" end_pos = scipy.array(end_pos) return t, pos, vel, ft, end_pos
def __init__(self, A1, A2, left, use_batch=False): """Creates a new LinearOperator interface to the superoperator E. This is a wrapper to be used with SciPy's sparse linear algebra routines. Parameters ---------- A1 : ndarray Ket parameter tensor. A2 : ndarray Bra parameter tensor. left : bool Whether to multiply with a vector to the left (or to the right). """ self.A1G = [list(map(garr.to_gpu, A1k)) for A1k in A1] self.A2G = [list(map(garr.to_gpu, A2k)) for A2k in A2] self.tmp = list(map(garr.empty_like, self.A1G[0])) self.tmp2 = list(map(garr.empty_like, self.A1G[0])) self.use_batch = use_batch self.left = left self.D = A1[0].shape[1] self.shape = (self.D**2, self.D**2) self.dtype = sp.dtype(A1[0][0].dtype) self.calls = 0 self.out = garr.empty((self.D, self.D), dtype=self.dtype) self.xG = garr.empty((self.D, self.D), dtype=self.dtype) if use_batch: self.A1G_p = list(map(get_batch_ptrs, self.A1G)) self.A2G_p = list(map(get_batch_ptrs, self.A2G)) self.tmp_p = get_batch_ptrs(self.tmp) self.tmp2_p = get_batch_ptrs(self.tmp2) self.xG_p = get_batch_ptrs([self.xG] * len(A1[0])) self.out_p = get_batch_ptrs([self.out] * len(A1[0])) else: self.A1G_p = None self.A2G_p = None self.tmp_p = None self.tmp2_p = None self.xG_p = None self.out_p = None self.ones = [ garr.zeros((1), dtype=sp.complex128) for s in range(len(A1[0])) ] self.ones = [one.fill(1) for one in self.ones] self.zeros = [ garr.zeros((1), dtype=sp.complex128) for s in range(len(A1[0])) ] self.streams = [] for s in range(A1[0].shape[0]): self.streams.append(cd.Stream()) self.hdl = cb.cublasCreate()
def calc(self,T,concentrations,params): """Converts the arguments to their appropriate ctypes, passes to the DLL, and returns the calculated enthalpies. Arguments --------- T : float The experimental temperature (in Kelvin) concentrations : list of dicts Concentrations of components from the experiment at each injection point. params : list of floats Model parameter values Returns ------- list of doubles The integrated enthalpies at each injection point. Raises ------ Exception If DLL returns a non-zero error code. Notes ----- Named components are "TRAP" (lattice) and "Trp" (ligand). """ n = len(concentrations) Q = scipy.zeros(n,scipy.dtype('d')) # patch for compatibility with general model nomenclature if 'TRAP' not in concentrations[0]: concentrations = [{'TRAP':concentrations[i]['Macromolecule'],'Trp':concentrations[i]['Ligand']} for i in range(n)] status = self._lib.calc( ctypes.c_int( n ), ctypes.c_double( T ), scipy.array([c['TRAP'] for c in concentrations],scipy.dtype('d')).ctypes, scipy.array([c['Trp'] for c in concentrations],scipy.dtype('d')).ctypes, Q.ctypes, scipy.array(params,scipy.dtype('d')).ctypes ) if status != 0: raise Exception("DLL returned a non-zero error code: %i"%(status)) return Q
def unique_rows(x): """This function takes a 2D scipy array x and makes it unique by rows.""" y = sp.ascontiguousarray(x).view( sp.dtype((sp.void, x.dtype.itemsize * x.shape[1]))) _, idx = sp.unique(y, return_index=True) return x[idx]
def getParameters(self, parameters): phi = parameters[0] optical_axis = parameters[1] if self.fixed_model == True: phi = self.phi0 optical_axis = self.offset0 zs = scipy.array(parameters[2:], scipy.dtype('d')) return phi, optical_axis, zs
def __init__(self, iqfile, datatype, block_length, block_offset, sample_rate): self.iqfile = iqfile self.datatype = datatype self.sizeof_data = self.datatype.nbytes # number of bytes per sample in file self.block_length = block_length self.sample_rate = sample_rate self.block_offset = block_offset self.binary_offset = self.block_offset * scipy.dtype( self.datatype).itemsize
def eps(value): """ Give the smallest epsilon that value can accept as a change. It depends both on value and type. """ tv = type(value) t = scipy.dtype(type(value)) tv = t.type return value * eps.values[tv]
def get_c_array_struct(x): """ Get the C array structure associated with the scipy or numpy array. """ x_struct = array_t() x_struct.data = x.ctypes.data_as(ctypes.c_void_p) x_struct.nrow = x.ctypes.shape[0] x_struct.ncol = x.ctypes.shape[1] x_struct.s0 = x.ctypes.strides[0] x_struct.s1 = x.ctypes.strides[1] if x.dtype == scipy.dtype('int'): x_struct.type = INT_ARRAY elif x.dtype == scipy.dtype('float32'): x_struct.type = FLT_ARRAY elif x.dtype == scipy.dtype('float64'): x_struct.type = DBL_ARRAY else: raise ValueError, "array must be of type INT_ARRAY, FLT_ARRAY or DBL_ARRAY" return x_struct
def objective(pars,X,Y,yerr): """The objective function""" b= pars[0] s= pars[1] Pb= pars[2] Yb= pars[3] Vb= m.exp(pars[4]) if Pb < 0. or Pb > 1.: return -sc.finfo(sc.dtype(sc.float64)).max return sc.sum(sc.log((1.-Pb)/sc.sqrt(2.*m.pi)/yerr*sc.exp(-0.5*(Y-s*X-b)**2./yerr**2.)+Pb/sc.sqrt(2.*m.pi*(Vb+yerr**2.))*sc.exp(-0.5*(Y-Yb)**2./(Vb+yerr**2.))))#+pars[4]
def eps(value): """ Give the smallest epsilon that value can accept as a change. It depends both on value and type. """ tv = type(value) t = scipy.dtype(type(value)) tv = t.type return value*eps.values[tv]
def __init__(self, A1, A2, left, use_batch=False): """Creates a new LinearOperator interface to the superoperator E. This is a wrapper to be used with SciPy's sparse linear algebra routines. Parameters ---------- A1 : ndarray Ket parameter tensor. A2 : ndarray Bra parameter tensor. left : bool Whether to multiply with a vector to the left (or to the right). """ self.A1G = [list(map(garr.to_gpu, A1k)) for A1k in A1] self.A2G = [list(map(garr.to_gpu, A2k)) for A2k in A2] self.tmp = list(map(garr.empty_like, self.A1G[0])) self.tmp2 = list(map(garr.empty_like, self.A1G[0])) self.use_batch = use_batch self.left = left self.D = A1[0].shape[1] self.shape = (self.D**2, self.D**2) self.dtype = sp.dtype(A1[0][0].dtype) self.calls = 0 self.out = garr.empty((self.D, self.D), dtype=self.dtype) self.xG = garr.empty((self.D, self.D), dtype=self.dtype) if use_batch: self.A1G_p = list(map(get_batch_ptrs, self.A1G)) self.A2G_p = list(map(get_batch_ptrs, self.A2G)) self.tmp_p = get_batch_ptrs(self.tmp) self.tmp2_p = get_batch_ptrs(self.tmp2) self.xG_p = get_batch_ptrs([self.xG] * len(A1[0])) self.out_p = get_batch_ptrs([self.out] * len(A1[0])) else: self.A1G_p = None self.A2G_p = None self.tmp_p = None self.tmp2_p = None self.xG_p = None self.out_p = None self.ones = [garr.zeros((1), dtype=sp.complex128) for s in range(len(A1[0]))] self.ones = [one.fill(1) for one in self.ones] self.zeros = [garr.zeros((1), dtype=sp.complex128) for s in range(len(A1[0]))] self.streams = [] for s in range(A1[0].shape[0]): self.streams.append(cd.Stream()) self.hdl = cb.cublasCreate()
def calc(self,T,concentrations,params): n = len(concentrations) Q = scipy.zeros(n,scipy.dtype('d')) # patch for compatibility with general model nomenclature if 'TRAP' not in concentrations[0]: concentrations = [{'TRAP':concentrations[i]['Macromolecule'],'Trp':concentrations[i]['Ligand']} for i in xrange(n)] status = self.lib.calc( ctypes.c_int( n ), ctypes.c_double( T ), scipy.array([c['TRAP'] for c in concentrations],scipy.dtype('d')).ctypes, scipy.array([c['Trp'] for c in concentrations],scipy.dtype('d')).ctypes, Q.ctypes, scipy.array(params,scipy.dtype('d')).ctypes ) if status != 0: raise Exception("DLL returned a non-zero error code: %i"%(status)) return Q
def map_element_values(input, ufunc, mtype=None, dtype=None, use1to1cache=True): """ Maps each element value if :samp:`input` array to new value calculated by :samp:`ufunc`. :type input: :obj:`mango.Dds` :param input: Elements of this array are mapped to new values via the :samp:`ufunc` argument. :type ufunc: callable :obj:`object` :param ufunc: Unary function which accepts a :samp:`input.dtype` argument and returns a :samp:`dtype` (or :samp:`mtype.dtype`) value. :type mtype: :obj:`mango.mtype` or :obj:`str` :param mtype: The :obj:`mango.mtype` of the returned :obj:`mango.Dds` object. :type dtype: :obj:`numpy.dtype` or :obj:`str` :param dtype: The :obj:`numpy.dtype` of the returned :obj:`mango.Dds` array. :type use1to1cache: :obj:`bool` :param use1to1cache: If :samp:`True`, an internal lookup table is kept which saves multiple calls of the :samp:`ufunc` function for the same argument. Can excessively use memory (for float data types, say) and is not correct if the :samp:`ufunc` function is not a 1-to-1 mapping. :rtype: :obj:`mango.Dds` :return: Array with elements mapped as per :samp:`ufunc` mapping. Example:: import mango import mango.data import math def cubed_root_func(x): return math.pow(x,1.0/3.0) srcDds = mango.data.gaussian_noise(shape=(64,128,128), mean=2000, stdd=10, dtype="int32") sqrdDds = mango.map_element_values(srcDds, lambda x: x*x) sqrtDds = mango.map_element_values(srcDds, lambda x: math.sqrt(x), dtype="float32") cubdDds = mango.map_element_values(srcDds, cubed_root_func, mtype="tomo_float") """ if (mtype != None): # Convert to object in case mtype argument is a string mtype = mango.mtype(mtype) if (dtype != None): # Convert to object in case dtype argument is a string dtype = sp.dtype(dtype) return \ _mango_open_core_so._map_element_values( input=input, ufunc=ufunc, mtype=mtype, dtype=dtype, use1to1cache=use1to1cache )
def loadPDBsingle(atomLines): pdb = empty(len(atomLines), dtype(pdbtype)) for n, line in enumerate(atomLines): pdb[n] = (int(line[6:11]), line[12:16], line[17:21], line[21], int(line[22:26]), (float(line[30:38]), float(line[38:46]), float(line[46:54])), float(line[54:60]), float(line[60:67]), line[72:76].strip(), line[76:78].strip(), line[78:80].strip(), 0) pdb = pdb.view(recarray) weights = [('C', 12.01), ('H', 1.01), ('O', 16.00), ('P', 30.97), ('N', 14.01), ('Na', 22.99), ('Cl', 35.45), ('S', 28.09)] # for at,w in weights: # pdb.weight[pdb.element == at] = w return pdb
def evaluate_hull(x, hull): """evaluate_hull: evaluate h_u(x) and (optional) h_l(x) Input: x - abcissa hull - the hull (see setup_hull for a definition) Output: hu(x) (optional), hl(x) History: 2009-05-21 - Written - Bovy (NYU) """ #Find in which [z_{i-1},z_i] interval x lies if x < hull[4][0]: #x lies in the first interval hux = hull[3][0] * (x - hull[1][0]) + hull[2][0] indx = 0 else: if len(hull[5]) == 1: #There are only two intervals indx = 1 else: indx = 1 while indx < len(hull[4]) and hull[4][indx] < x: indx = indx + 1 indx = indx - 1 hux = hull[3][indx] * (x - hull[1][indx]) + hull[2][indx] #Now evaluate hlx neginf = sc.finfo(sc.dtype(sc.float64)).min if x < hull[1][0] or x > hull[1][-1]: hlx = neginf else: if indx == 0: hlx = ((hull[1][1] - x) * hull[2][0] + (x - hull[1][0]) * hull[2][1]) / (hull[1][1] - hull[1][0]) elif indx == len(hull[4]): hlx = ( (hull[1][-1] - x) * hull[2][-2] + (x - hull[1][-2]) * hull[2][-1]) / (hull[1][-1] - hull[1][-2]) elif x < hull[1][indx + 1]: hlx = ((hull[1][indx + 1] - x) * hull[2][indx] + (x - hull[1][indx]) * hull[2][indx + 1]) / ( hull[1][indx + 1] - hull[1][indx]) else: hlx = ((hull[1][indx + 2] - x) * hull[2][indx + 1] + (x - hull[1][indx + 1]) * hull[2][indx + 2]) / ( hull[1][indx + 2] - hull[1][indx + 1]) return hux, hlx
def __init__(self, filename=None, dtype=None, **kwargs): """ :type filename: str :param filename: Path on the filesystem to the physical file. :type dtype: dtype resolveable :param dtype: A numpy dtype Default=float32 """ # members self.dtype = sp.dtype(dtype or sp.float32) self.fp = None # initialize self._initialize_file(filename, **kwargs)
def get_start_pos(setpt, config): """ Python wrapper for get_start_pos function. Returns starting position of kinematics. """ config_struct = create_config_struct(config) kine = scipy.zeros((1, config['num_motor']), dtype=scipy.dtype('float32')) kine_c_array = get_c_array_struct(kine) setpt_c_float = ctypes.c_float(setpt) ret_val = lib.get_start_pos(setpt_c_float, kine_c_array, config_struct) if ret_val == FAIL: raise RuntimeError, "lib.get_start_pos call failed" return kine
def __addClassAttributes(): """ Adds some attributes to the _mango_open_core_so._Dds_(.*) classes, in particular __getitem__ method, __setitem__ method, and the dtype attribute. """ thingStrList = dir(_mango_open_core_so) ddsClsRegEx = re.compile("_Dds_(.*)") for thingStr in thingStrList: mtch = ddsClsRegEx.match(thingStr) if (mtch != None): thing = getattr(_mango_open_core_so, thingStr) thing.dtype = sp.dtype(mtch.group(1)) _ddsDtypeList.append(thing.dtype) thing.__getitem__ = _ddsGetItem thing.__setitem__ = _ddsSetItem thing.copy = _ddsCopy _ddsTypeList.append(thing)
def yawff_ctlr_c_wrapper(setpt, config): """ Python wrapper for yawff_w_ctlr function. Perform yaw force-feedback task with yaw controller. Spawns real-time thread to handle controller, data acquisition, and yaw dynamics. During outscan displays information regarding ongoing real-time task. Inputs: setpt = Nx1 array of setpt values at time steps 0, dt, ... (N-1)*dt config = system configuration dictionary """ config_struct = create_config_struct(config) # Create c structure for setpt data setpt_float32 = setpt.astype(scipy.dtype('float32')) setpt_struct = get_c_array_struct(setpt_float32) # Create time, position, velocity, torque, and kinematics arrarys for # return data n = setpt.shape[0] t = scipy.zeros((n, 1), dtype=scipy.dtype('float64')) pos = scipy.zeros((n, 1), dtype=scipy.dtype('float32')) vel = scipy.zeros((n, 1), dtype=scipy.dtype('float32')) torq = scipy.zeros((n, 2), dtype=scipy.dtype('float32')) kine = scipy.zeros((n, config['num_motor']), scipy.dtype('float32')) u = scipy.zeros((n, 1), dtype=scipy.dtype('float32')) # Create c data structure data_struct = data_t() data_struct.t = get_c_array_struct(t) data_struct.pos = get_c_array_struct(pos) data_struct.vel = get_c_array_struct(vel) data_struct.torq = get_c_array_struct(torq) kine_struct = get_c_array_struct(kine) u_struct = get_c_array_struct(u) # Create array for ending positions end_pos = (ctypes.c_int * config['num_motor'])() # Call C library yawff function ret_val = lib.yawff_w_ctlr(setpt_struct, config_struct, kine_struct, u_struct, data_struct, end_pos) if ret_val == FAIL: raise RuntimeError, "lib.yawff_w_ctlr call failed" end_pos = scipy.array(end_pos) return t, pos, vel, torq, kine, u, end_pos
def from_data(data): """produce a SimPkg from bytedata :Parameters: data : str The data to produce the package from. """ # length check if len(data) < SimPkg.HLEN: raise ValueError('length < SimPkg.HLEN') # read header idx = SimPkg.HLEN tid, ident, frame, nitems = unpack(SimPkg.HDEF, data[:idx]) cont = [] # content loop while idx < len(data): # read contents header dim0, dim1, nbytes, dtype_str = unpack( ContentItem.HDEF, data[idx:idx + ContentItem.HLEN]) idx += ContentItem.HLEN # read content data cont_item = N.fromstring(data[idx:idx + nbytes], dtype=N.dtype(dtype_str)) if dim0 >= 0: if dim1 >= 0: dim = [dim0, dim1] else: dim = [dim0] cont_item.shape = dim cont.append(cont_item) idx += nbytes # return assert len( cont ) == nitems, 'cont list length (%s) does not match nitems (%s)!' % ( len(cont), nitems) return SimPkg(tid, ident, frame, tuple(cont))
def g(N): ab_type = sp.dtype([('val',sp.float64),('a',sp.int16),('b',sp.int16)]) k_max = int(N*sp.log(sp.pi+1)) + 1 ff = [f(N,k) for k in range(0,k_max+1)] ab_dat = sp.zeros((k_max*(k_max+1))/2, dtype = ab_type) ix = 0 for a in xrange(1,k_max+1): for b in xrange(1,a+1): ab_dat[ix]['val'] = ff[a] + ff[b] ab_dat[ix]['a'] = a ab_dat[ix]['b'] = b ix += 1 if ix%1000000 == 0: print 'Building %d/%d'%(ix,len(ab_dat)) ab_dat.sort() gmin = sp.pi amin,bmin,cmin,dmin = 0,0,0,0 for ix in xrange(len(ab_dat)): if ix%1000000 == 0: print 'Searching %d/%d'%(ix,len(ab_dat)) jx = bisect(ab_dat, sp.pi-ab_dat[ix]['val']) if jx > 0: v = ab_dat[ix]['val'] + ab_dat[jx-1]['val'] if abs(v-sp.pi) < gmin: gmin = abs(v - sp.pi) listmin = [ab_dat[ix]['a'],ab_dat[ix]['b'], ab_dat[jx-1]['a'],ab_dat[jx-1]['b']] listmin.sort() amin,bmin,cmin,dmin = tuple(listmin) print v,amin,bmin,cmin,dmin,amin**2+bmin**2+cmin**2+dmin**2 if jx < len(ab_dat): v = ab_dat[ix]['val'] + ab_dat[jx]['val'] if abs(v-sp.pi) < gmin: gmin = abs(v - sp.pi) listmin = [ab_dat[ix]['a'],ab_dat[ix]['b'], ab_dat[jx]['a'],ab_dat[jx]['b']] listmin.sort() amin,bmin,cmin,dmin = tuple(listmin) print v,amin,bmin,cmin,dmin,amin**2+bmin**2+cmin**2+dmin**2 return amin,bmin,cmin,dmin,ff[amin]+ff[bmin]+ff[cmin]+ff[dmin],gmin
def _loadEGM96(): """load the EGM96 geoid model into a spline object""" # load the data resource file into a string flc = resource_string(__name__, "data/egm96.dac") # setup basic coordinates lon = sp.linspace(0, 2 * sp.pi, 1440, False) lat = sp.linspace(0, sp.pi, 721) # parse the raw data string data = sp.fromstring(flc, sp.dtype(sp.int16).newbyteorder("B"), 1038240).reshape((lat.size, lon.size)) / 100.0 # interpolate data lut = RectSphereBivariateSpline(lat[1: -1], lon, data[1: -1], pole_values=(sp.mean(data[1]), sp.mean(data[-1]))) return lut
def from_data(data): """produce a SimPkg from bytedata :Parameters: data : str The data to produce the package from. """ # length check if len(data) < SimPkg.HLEN: raise ValueError('length < SimPkg.HLEN') # read header idx = SimPkg.HLEN tid, ident, frame, nitems = unpack(SimPkg.HDEF, data[:idx]) cont = [] # content loop while idx < len(data): # read contents header dim0, dim1, nbytes, dtype_str = unpack(ContentItem.HDEF, data[idx:idx + ContentItem.HLEN]) idx += ContentItem.HLEN # read content data cont_item = N.fromstring( data[idx:idx + nbytes], dtype=N.dtype(dtype_str) ) if dim0 >= 0: if dim1 >= 0: dim = [dim0, dim1] else: dim = [dim0] cont_item.shape = dim cont.append(cont_item) idx += nbytes # return assert len(cont) == nitems, 'cont list length (%s) does not match nitems (%s)!' % (len(cont), nitems) return SimPkg(tid, ident, frame, tuple(cont))
def evaluate_hull(x,hull): """evaluate_hull: evaluate h_u(x) and (optional) h_l(x) Input: x - abcissa hull - the hull (see setup_hull for a definition) Output: hu(x) (optional), hl(x) History: 2009-05-21 - Written - Bovy (NYU) """ #Find in which [z_{i-1},z_i] interval x lies if x < hull[4][0]: #x lies in the first interval hux= hull[3][0]*(x-hull[1][0])+hull[2][0] indx= 0 else: if len(hull[5]) == 1: #There are only two intervals indx= 1 else: indx= 1 while indx < len(hull[4]) and hull[4][indx] < x: indx= indx+1 indx= indx-1 hux= hull[3][indx]*(x-hull[1][indx])+hull[2][indx] #Now evaluate hlx neginf= sc.finfo(sc.dtype(sc.float64)).min if x < hull[1][0] or x > hull[1][-1]: hlx= neginf else: if indx == 0: hlx= ((hull[1][1]-x)*hull[2][0]+(x-hull[1][0])*hull[2][1])/(hull[1][1]-hull[1][0]) elif indx == len(hull[4]): hlx= ((hull[1][-1]-x)*hull[2][-2]+(x-hull[1][-2])*hull[2][-1])/(hull[1][-1]-hull[1][-2]) elif x < hull[1][indx+1]: hlx= ((hull[1][indx+1]-x)*hull[2][indx]+(x-hull[1][indx])*hull[2][indx+1])/(hull[1][indx+1]-hull[1][indx]) else: hlx= ((hull[1][indx+2]-x)*hull[2][indx+1]+(x-hull[1][indx+1])*hull[2][indx+2])/(hull[1][indx+2]-hull[1][indx+1]) return hux, hlx
def __init__(self, tdvp, n, KLn, tau=1, sanity_checks=False): """ """ self.D = tdvp.D self.q = tdvp.q self.tdvp = tdvp self.n = n self.KLn = KLn self.sanity_checks = sanity_checks self.sanity_tol = 1E-12 d = self.D[n] * self.D[n] self.shape = (d, d) self.dtype = sp.dtype(tdvp.typ) self.calls = 0 self.tau = tau
def read_csv(self, path): self.__data_h5 = h5.File('memfile.h5', 'w', driver='core', backing_store=False) self.__data_h5.create_group('/pvalues') self.__data_h5.create_group('/quantiles') csvdata = pd.read_csv(path, header=0) #convert chromosome numbers to strings if not already done. if csvdata['chromosomes'].dtype == sp.dtype('int'): csvdata['chromosomes'] = [ "chr{:d}".format(x) for x in csvdata['chromosomes'] ] chr_names = sorted(set(csvdata['chromosomes'].values)) csvdata['scores'] = -sp.log10(csvdata['pvals'].values) for chr_name in chr_names: subdata = csvdata[csvdata['chromosomes'] == chr_name] h5_chr_group = self.__data_h5.create_group( '/pvalues/{}'.format(chr_name)) for col in subdata.columns: if col == 'chromosomes' or col == 'pvals': continue h5_chr_group.create_dataset(col, data=subdata[col].values) # create attributes h5_pvalues_group = self.__data_h5['/pvalues'] h5_pvalues_group.attrs['analysis_method'] = 'N/A' h5_pvalues_group.attrs['bh_thres'] = -sp.log10( mtcorr.get_bhy_thres(csvdata['pvals'].values)['thes_pval']) h5_pvalues_group.attrs['bonferroni_threshold'] = -sp.log10( 0.05 / csvdata.shape[0]) ks_stats = stats.calc_ks_stats(csvdata['pvals'].values) h5_pvalues_group.attrs['ks_pval'] = ks_stats['p_val'] h5_pvalues_group.attrs['ks_stat'] = ks_stats['D'] h5_pvalues_group.attrs['max_score'] = csvdata['scores'].values.max() h5_pvalues_group.attrs['med_pval'] = sp.median(csvdata['pvals'].values) h5_pvalues_group.attrs['numberOfSNPs'] = csvdata.shape[0] h5_pvalues_group.attrs['transformation'] = 'raw'
def __init__(self, capacity=64, dimension=1, dtype=None): """ :type capacity: int :param capacity: capacity of the ringbuffer (rows) Default=64 :type dimension: tuple or int :param dimension: dimensionality of the items to store in the ringbuffer. If int, this will be converted internally to (int,) Default=1 :type dtype: dtype resolvable :param dtype: dtype of single entries Default=float32 """ # checks if capacity < 1: raise ValueError('capacity < 1') if isinstance(dimension, int): dimension = (dimension, ) elif isinstance(dimension, tuple): pass else: raise ValueError('dimension has to be tuple or int') # members self._capacity = int(capacity) self._dimension = dimension self._dtype = sp.dtype(dtype or sp.float32) self._data = sp.empty((self._capacity, ) + self._dimension, dtype=self._dtype) self._next = 0 self._full = False # mapping prototypes self._idx_belowcap_proto = lambda: range(self._next) self._idx_fullcap_proto =\ lambda:range(self._next, self._capacity) + range(self._next) # mappings self._idx_append = self._idx_fullcap_proto self._idx_retrieve = self._idx_belowcap_proto
def __new__(cls, lon=sp.zeros((1, 1)), lat=sp.zeros((1, 1)), lev=sp.zeros(1)): if (lon.ndim == 1) and (lat.ndim == 1): lon, lat = sp.meshgrid(lon, lat) im = lon.shape jm = lat.shape km = lev.shape dtype=sp.dtype([('lev',sp.float32,km),\ ('lat',sp.float32,jm),\ ('lon',sp.float32,im)]) obj = sp.ndarray.__new__(cls, (), dtype=dtype) obj['lon'] = lon obj['lat'] = lat obj['lev'] = lev return obj
def __init__(self, capacity=64, dimension=1, dtype=None): """ :type capacity: int :param capacity: capacity of the ringbuffer (rows) Default=64 :type dimension: tuple or int :param dimension: dimensionality of the items to store in the ringbuffer. If int, this will be converted internally to (int,) Default=1 :type dtype: dtype resolvable :param dtype: dtype of single entries Default=float32 """ # checks if capacity < 1: raise ValueError("capacity < 1") if isinstance(dimension, int): dimension = (dimension,) elif isinstance(dimension, tuple): pass else: raise ValueError("dimension has to be tuple or int") # members self._capacity = int(capacity) self._dimension = dimension self._dtype = sp.dtype(dtype or sp.float32) self._data = sp.empty((self._capacity,) + self._dimension, dtype=self._dtype) self._next = 0 self._full = False # mapping prototypes self._idx_belowcap_proto = lambda: range(self._next) self._idx_fullcap_proto = lambda: range(self._next, self._capacity) + range(self._next) # mappings self._idx_append = self._idx_fullcap_proto self._idx_retrieve = self._idx_belowcap_proto
def _loadEGM96(): """load the EGM96 geoid model into a spline object""" #load the data resource file into a string flc = resource_string(__name__, "data/egm96.dac") #setup basic coordinates lon = sp.linspace(0, 2 * sp.pi, 1440, False) lat = sp.linspace(0, sp.pi, 721) #parse the raw data string data = sp.fromstring(flc, sp.dtype(sp.int16).newbyteorder("B"), 1038240).reshape((lat.size, lon.size)) / 100.0 #interpolate the bad boy lut = RectSphereBivariateSpline(lat[1:-1], lon, data[1:-1], pole_values=(sp.mean(data[1]), sp.mean(data[-1]))) return lut
def setUp(self): self.rows = 100 self.intSample = FieldContainer( scipy.arange(0, self.rows), Quantity("1m"), # dimensions=INDEX, longname=u"Integer sample", shortname=u"i", ) self.intSample2 = FieldContainer( 2 * scipy.arange(0, self.rows), Quantity("1m"), longname=u"Integer sample No2", shortname=u"i2" ) self.floatSample = FieldContainer( scipy.arange(self.rows / 2, self.rows, 0.5), Quantity("1s"), longname=u"Float sample", shortname=u"t" ) self.shortFloatSample1 = FieldContainer( scipy.array([1.0, 2.0, 10.0]), Quantity("1.0 s**2"), longname=u"Short Float sample 1", shortname=u"short1" ) self.shortFloatSample2 = FieldContainer( scipy.array([5.0, 10.0, 1.0]), Quantity("1.0 kg * m"), longname=u"Short Float sample 2", shortname=u"short2" ) self.desc = scipy.dtype( { "names": [u"i", u"t"], "formats": [self.intSample.data.dtype, self.floatSample.data.dtype], "titles": [self.intSample.longname, self.floatSample.longname], } ) self.data = scipy.rec.fromarrays([self.intSample.data, self.floatSample.data], dtype=self.desc) self.longname = u"Toller Sample" self.shortname = u"phi" self.sampleContainer = SampleContainer([self.intSample, self.floatSample], self.longname, self.shortname) self.sampleContainerNeu = SampleContainer( [self.intSample, self.intSample2, self.floatSample, self.shortFloatSample1, self.shortFloatSample2], "New Sample", "NewSC", )
def __init__(self, weight=0.05, cond=50, dtype=None): """ :type weight: float :param weight: from [0.0, 1.0]. new observations will be weighted and contribute to the update with the factor weight. (exp model) Default=0.05 :type cond: float :param cond: condition number to assert if the loaded matrix is requested. Default=50 :type dtype: dtype resolvable :param dtype: anything that can be used as a numpy.dtype object Default=float32 """ # members self.dtype = sp.dtype(dtype or sp.float32) # privates self._weight = weight self._cond = float(cond) self._is_initialised = False self._n_upd = 0 self._n_upd_smpl = 0
def itemfreq(a): items,ind, inv = sp.unique(a, return_inverse=True,return_index=True) freq = sp.bincount(inv) return sp.array([ind, items, freq]).T if __name__ in "__main__": f = h5py.File(sys.argv[1]) raw = f['Genotype']['raw'][:] sample_ids = f['Genotype']['sample_ids'][:] chr_index = f['Genotype']['chr_index'][:] position_index = f['Genotype']['position_index'][:] rawT = raw.T snp_strings = sp.ascontiguousarray(rawT).view(sp.dtype((sp.void,rawT.dtype.itemsize * rawT.shape[1]))) frequencies = itemfreq(snp_strings) ind = sp.where(frequencies[:,2]<=int(sys.argv[3]))[0] indices = sp.array(frequencies[ind,0],dtype="int") print "Number of Samples:\t\t\t", raw.shape[0] print "Number of SNPs before Filtering:\t", raw.shape[1] print "Number of truly unique SNPs:\t\t", sp.where(frequencies[ind,2]==1)[0].shape[0] print sp.histogram(frequencies[:,2],bins=[1,2,5,10,50,100,500,1000,2000]) out = h5py.File(sys.argv[2],'w') g = out.create_group("Genotype") g.create_dataset("raw",data=raw[:,indices],chunks=True) g.create_dataset("sample_ids",data=sample_ids,chunks=True) g.create_dataset("chr_index",data=chr_index[indices],chunks=True)
def loadData(self,settings=None): ''' Load Phenotype Data ''' try: if settings.phenotype_file==None: self.__y = self.__dbfile['Phenotypes/' + str(settings.phenotype_id) + '/y'][:] self.__sample_ids = self.__dbfile['Phenotypes/' + str(settings.phenotype_id) + '/sample_ids'][:] self.__phenotype_name = self.__dbfile['Phenotypes/' + str(settings.phenotype_id) + "/name"].value else: f = h5py.File(settings.phenotype_file,'r') self.__y = f['Phenotypes/' + str(settings.phenotype_id) + '/y'][:] self.__sample_ids = f['Phenotypes/' + str(settings.phenotype_id) + '/sample_ids'][:] self.__phenotype_name = f['Phenotypes/' + str(settings.phenotype_id) + "/name"].value f.close() except: print "[ERROR] Loading Phenotype went wrong" quit() #remove missing values ind = sp.where(~sp.isnan(self.__y))[0] self.__y = self.__y[ind] self.__sample_ids = self.__sample_ids[ind] #transform phenotypes self.__y = self.transformData(self.__y,settings.phenotype_transformation) ''' Load Covariate Data and restrict samples ''' self.__cov = None covariates = settings.covariates for covariate in covariates: try: cov = self.__dbfile['Covariates/' + str(covariate) + '/y'][:] sample_ids = self.__dbfile['Covariates/' + str(covariate) + '/sample_ids'][:] except: print "[ERROR] Loading Covariate went wrong" quit() #transform covariates cov = self.transformData(cov,settings.covariate_transformation) #match samples sample_indices = (sp.reshape(sample_ids,(sample_ids.shape[0],1))==self.__sample_ids).nonzero() sample_ids = sample_ids[sample_indices[0]] if self.__cov==None: self.__cov = cov[sample_indices[0]] else: self.__cov = sp.column_stack([self.__cov,cov]) self.__y = self.__y[sample_indices[1]] self.__sample_ids = self.__sample_ids[sample_indices[1]] ''' Load Genotype Data and restrict samples ''' sample_ids_file = self.__dbfile['Genotype/sample_ids'][:] raw_data = self.__dbfile['Genotype/raw'][:] self.__chr_index = self.__dbfile['Genotype/chr_index'][:] self.__pos_index = self.__dbfile['Genotype/position_index'][:] sample_indices = (sp.reshape(self.__sample_ids,(self.__sample_ids.shape[0],1))==sample_ids_file).nonzero() self.__sample_ids = self.__sample_ids[sample_indices[0]] self.__y = self.__y[sample_indices[0]] if not self.__cov is None: self.__cov = self.__cov[sample_indices[0]] raw_data = raw_data[sample_indices[1],:] self.__raw = raw_data if settings.homozygous==True: [self.__x, self.__maf_data] = self.encodeHomozygousData(raw_data) else: [self.__x, self.__maf_data] = self.encodeHeterozygousData(raw_data,settings.snp_encoding) #if settings.snp_encoding!="additive": #[self.__x_additive, self.__maf_data] = self.encodeHeterozygousData(raw_data) #This was experimental to use an additve Kinship in case a other encoding was selected, now we only use the MAF filtering based #on an additve model! Comment the next line out if you wish to use the additve kinship matrix again # self.__x_additive = None if settings.maf > 0.0: self.filter_mAF(settings.maf) self.filterNonInformativeSNPs() if settings.principle_components > 0: if not self.__x_additive is None: cov = sp.real(self.computePCA(X=self.__x_additive,number_pcs=settings.principle_components)) else: cov = sp.real(self.computePCA(X=self.__x,number_pcs=settings.principle_components)) if self.__cov == None: self.__cov = cov else: self.__cov = sp.column_stack([self.__cov,cov]) if not self.__x_additive is None: tmpX = self.__x_additive.T else: tmpX = self.__x.T usnps,self.__snp_hash=sp.unique(sp.ascontiguousarray(tmpX).view(sp.dtype((sp.void,tmpX.dtype.itemsize*tmpX.shape[1]))),return_inverse=True) #compute kinship kernel if necessary if self.__algorithm == "FaSTLMM" or self.__algorithm=="EMMAX" or self.__algorithm=="EMMAXperm": if settings.unique_snps_only: tmp,uindex = sp.unique(self.__snp_hash,return_index=True) if not self.__x_additive is None: self.__ass.setK(gwas_core.CKernels.realizedRelationshipKernel(self.__x_additive[:,uindex])) #self.__ass.setK(self.computeRealizedRelationshipKernel(genotype=self.__x_additive[:,uindex])) else: self.__ass.setK(gwas_core.CKernels.realizedRelationshipKernel(self.__x[:,uindex])) #self.__ass.setK(self.computeRealizedRelationshipKernel(genotype=self.__x[:,uindex])) else: if not self.__x_additive is None: self.__ass.setK(gwas_core.CKernels.realizedRelationshipKernel(self.__x_additive)) #self.__ass.setK(self.computeRealizedRelationshipKernel(genotype=self.__x_additive)) else: self.__ass.setK(gwas_core.CKernels.realizedRelationshipKernel(self.__x))
'UI16MAX', 'UI32MAX', 'UI64MAX', 'VERBOSE', 'log'] ##---IMPORTS import logging import scipy as sp ##---PACKAGE-LOGGING logging.basicConfig(level=logging.DEBUG, format='') log = logging.getLogger('BOTMpy') ##---CONSTANTS ## index type INDEX_DTYPE = sp.dtype(sp.int64) ## integer max values SI8MAX = sp.iinfo(sp.int8).max SI16MAX = sp.iinfo(sp.int16).max SI32MAX = sp.iinfo(sp.int32).max SI64MAX = sp.iinfo(sp.int64).max UI8MAX = sp.iinfo(sp.uint8).max UI16MAX = sp.iinfo(sp.uint16).max UI32MAX = sp.iinfo(sp.uint32).max UI64MAX = sp.iinfo(sp.uint64).max ## CLASSES class VERBOSE(object):
def to_ind(f, alpha=0.): return array(f > alpha, dtype('float'))
return v**(-i+1) return 0 for v in scipy.floating.__subclasses__(): eps.values[v] = find_eps(v) def centered(arr, newsize): # Return the center newsize portion of the array. newsize = asarray(newsize) currsize = array(arr.shape) startind = (currsize - newsize) / 2 endind = startind + newsize myslice = [slice(startind[k], endind[k]) for k in range(len(endind))] return arr[tuple(myslice)] __bigendian = dtype("=i") == dtype(">i") def bigendian(): return __bigendian def compare_versions(v1, v2): n1 = [ int(i) for i in v1.split('.') ] n2 = [ int(i) for i in v2.split('.') ] for i1,i2 in zip(n1, n2): if i1 < i2: return -1 elif i1 > i2: return 1 if len(n1) < len(n2): return -1 elif len(n1) > len(n2): return 1
def partial_dot(left, right): """Perform matrix multiplication on some subset of the axes. This is similar to a numpy `tensordot` but it is aware of the matrix and vector nature of the inputs and returns appropriate objects. It decides which axes to 'dot' based on the axis names. If a `vect` is passed, it is treated as `mat` with one row if it's the first arguments and a matrix with one column if it's the second. If the output matrix has either only a single row or a single column, it is cast as a `vect`. This function can properly deal with block diagonal structure and axes sorted in any order. The axes in the output array as sorted such in order of block diagonal axes then row-only axes then col-only axes. Parameters ---------- left: mat or vect right: mat or vect Returns ------- out: mat or vect Tensor product of `left` and `right`, with any named axes appearing in both `left`'s columns and `right`'s rows contracted. """ # Figure out what kind of object the inputs are. msg = "Inputs must be either mat or vect objects." if isinstance(left, matrix.MatrixObject): left_rows = list(left.rows) left_cols = list(left.cols) elif isinstance(left, vector.VectorObject): left_rows = [] left_cols = range(left.ndim) else: raise TypeError(msg) if isinstance(right, matrix.MatrixObject): right_rows = list(right.rows) right_cols = list(right.cols) elif isinstance(right, vector.VectorObject): right_rows = range(right.ndim) right_cols = [] else: raise TypeError(msg) # Find axes that are block diagonal make copies of the rows and cols that # ommits the block diagonal ones. left_cols_only = list(left_cols) left_rows_only = list(left_rows) right_cols_only = list(right_cols) right_rows_only = list(right_rows) left_diag = [] left_diag_names = [] left_col_only_names = [] for axis in left_cols: if axis in left_rows: left_diag.append(axis) left_diag_names.append(left.axes[axis]) left_rows_only.remove(axis) left_cols_only.remove(axis) else: left_col_only_names.append(left.axes[axis]) right_diag = [] right_diag_names = [] right_row_only_names = [] for axis in list(right_rows): if axis in right_cols: right_diag.append(axis) right_diag_names.append(right.axes[axis]) right_rows_only.remove(axis) right_cols_only.remove(axis) else: right_row_only_names.append(right.axes[axis]) # Divide all axes into groups based on what we are going to do with them. # To not be dotted. left_notdot = [] # Block diagonal axis to not dot. left_notdot_diag = [] # To be dotted with a normal axis. left_todot = [] right_todot = [] # To be dotted with a block diagonal axis. left_todot_with_diag = [] right_todot_with_diag = [] # Block diagonal axes to be dotted with a normal axis. left_todot_diag = [] right_todot_diag = [] # Block diagonal axes to be dotted with a block diagonal axis. left_todot_diag_diag = [] right_todot_diag_diag = [] for axis in left_cols: axis_name = left.axes[axis] if axis_name in left_col_only_names and \ axis_name not in right_row_only_names and \ axis_name not in right_diag_names: left_notdot.append(axis) elif axis_name in left_diag_names and \ axis_name not in right_row_only_names and \ axis_name not in right_diag_names: left_notdot_diag.append(axis) elif axis_name in left_col_only_names and \ axis_name in right_row_only_names: left_todot.append(axis) right_todot.append(right.axes.index(axis_name)) elif axis_name in left_diag_names and \ axis_name in right_row_only_names: left_todot_diag.append(axis) right_todot_with_diag.append(right.axes.index(axis_name)) elif axis_name in left_col_only_names and \ axis_name in right_diag_names: left_todot_with_diag.append(axis) right_todot_diag.append(right.axes.index(axis_name)) elif axis_name in left_diag_names and \ axis_name in right_diag_names: left_todot_diag_diag.append(axis) right_todot_diag_diag.append(right.axes.index(axis_name)) right_notdot = list(set(right_rows_only) - set(right_todot) - set(right_todot_with_diag)) right_notdot.sort() right_notdot_diag = list(set(right_diag) - set(right_todot_diag) - set(right_todot_diag_diag)) right_notdot_diag.sort() # Need shapes and names for all of these. left_notdot_shape = [left.shape[axis] for axis in left_notdot] left_notdot_names = [left.axes[axis] for axis in left_notdot] left_notdot_diag_shape = [left.shape[axis] for axis in left_notdot_diag] left_notdot_diag_names = [left.axes[axis] for axis in left_notdot_diag] left_todot_shape = [left.shape[axis] for axis in left_todot] left_todot_names = [left.axes[axis] for axis in left_todot] left_todot_with_diag_shape = [left.shape[axis] for axis in left_todot_with_diag] left_todot_with_diag_names = [left.axes[axis] for axis in left_todot_with_diag] left_todot_diag_shape = [left.shape[axis] for axis in left_todot_diag] left_todot_diag_names = [left.axes[axis] for axis in left_todot_diag] left_todot_diag_diag_shape = [left.shape[axis] for axis in left_todot_diag_diag] left_todot_diag_diag_names = [left.axes[axis] for axis in left_todot_diag_diag] left_rows_only_shape = [left.shape[axis] for axis in left_rows_only] left_rows_only_names = [left.axes[axis] for axis in left_rows_only] right_notdot_shape = [right.shape[axis] for axis in right_notdot] right_notdot_names = [right.axes[axis] for axis in right_notdot] right_notdot_diag_shape = [right.shape[axis] for axis in right_notdot_diag] right_notdot_diag_names = [right.axes[axis] for axis in right_notdot_diag] right_todot_shape = [right.shape[axis] for axis in right_todot] right_todot_names = [right.axes[axis] for axis in right_todot] right_todot_with_diag_shape = [right.shape[axis] for axis in right_todot_with_diag] right_todot_with_diag_names = [right.axes[axis] for axis in right_todot_with_diag] right_todot_diag_shape = [right.shape[axis] for axis in right_todot_diag] right_todot_diag_names = [right.axes[axis] for axis in right_todot_diag] right_todot_diag_diag_shape = [right.shape[axis] for axis in right_todot_diag_diag] right_todot_diag_diag_names = [right.axes[axis] for axis in right_todot_diag_diag] right_cols_only_shape = [right.shape[axis] for axis in right_cols_only] right_cols_only_names = [right.axes[axis] for axis in right_cols_only] # Figure out the shape, names, rows and cols of the output. out_shape = (left_notdot_diag_shape + left_todot_diag_diag_shape + right_notdot_diag_shape + left_todot_diag_shape + left_rows_only_shape + right_notdot_shape + left_notdot_shape + right_todot_diag_shape + right_cols_only_shape) out_names = (left_notdot_diag_names + left_todot_diag_diag_names + right_notdot_diag_names + left_todot_diag_names + left_rows_only_names + right_notdot_names + left_notdot_names + right_todot_diag_names + right_cols_only_names) # First add the block diagonal axes as both rows and columns. out_rows = range(len(left_notdot_diag) + len(left_todot_diag_diag) + len(right_notdot_diag)) out_cols = list(out_rows) # Now add the others. out_rows += range(len(out_rows), len(out_rows) + len(left_todot_diag) + len(left_rows_only) + len(right_notdot)) out_cols += range(len(out_rows), len(out_shape)) # Output data type. # This is no good because it crashes for length 0 arrays. #out_dtype = (left.flat[[0]] * right.flat[[0]]).dtype # There are functions that do this in higher versions of numpy. out_dtype = sp.dtype(float) # Allowcate memory. out = sp.empty(out_shape, dtype=out_dtype) # All the block diagonal axes will be treated together. Get the global # shape of them. all_diag_shape = (left_notdot_diag_shape + left_todot_diag_diag_shape + right_notdot_diag_shape + left_todot_diag_shape + right_todot_diag_shape) n_diag_axes = len(all_diag_shape) all_diag_size = 1 for s in all_diag_shape: all_diag_size *= s # Each of these block diagonal axes are associated with different axes in # the input and output arrays. Figure out the associations for each of # them. These arrays are the length of the number of diagonal axes and # each entry refers to an axis in that array. If the axis does not apply # to that array, put None. out_diag_inds = [] left_diag_inds = [] right_diag_inds = [] tmp_n_out_axes_passed = 0 for ii in range(len(left_notdot_diag)): left_diag_inds.append(left_notdot_diag[ii]) right_diag_inds.append(None) out_diag_inds.append(ii) tmp_n_out_axes_passed += len(left_notdot_diag) for ii in range(len(left_todot_diag_diag)): left_diag_inds.append(left_todot_diag_diag[ii]) right_diag_inds.append(right_todot_diag_diag[ii]) out_diag_inds.append(ii + tmp_n_out_axes_passed) tmp_n_out_axes_passed += len(left_todot_diag_diag) for ii in range(len(right_notdot_diag)): left_diag_inds.append(None) right_diag_inds.append(right_notdot_diag[ii]) out_diag_inds.append(ii + tmp_n_out_axes_passed) tmp_n_out_axes_passed += len(right_notdot_diag) for ii in range(len(left_todot_diag)): left_diag_inds.append(left_todot_diag[ii]) right_diag_inds.append(right_todot_with_diag[ii]) out_diag_inds.append(ii + tmp_n_out_axes_passed) tmp_n_out_axes_passed += (len(left_todot_diag) + len(left_rows_only) + len(right_notdot) + len(left_notdot)) for ii in range(len(right_todot_diag)): left_diag_inds.append(left_todot_with_diag[ii]) right_diag_inds.append(right_todot_diag[ii]) out_diag_inds.append(ii + tmp_n_out_axes_passed) # Once we index all the diagonal axes, the ones we want to dot will be in # the wrong place. Find the location of the axes we'll need in the new # array. left_sliced_rows_only = list(left_rows_only) left_sliced_notdot = list(left_notdot) left_sliced_todot = list(left_todot) for diag_axis in left_diag_inds: if diag_axis is not None: for ii in range(len(left_rows_only)): if diag_axis < left_rows_only[ii]: left_sliced_rows_only[ii] -= 1 for ii in range(len(left_notdot)): if diag_axis < left_notdot[ii]: left_sliced_notdot[ii] -= 1 for ii in range(len(left_todot)): if diag_axis < left_todot[ii]: left_sliced_todot[ii] -= 1 right_sliced_cols_only = list(right_cols_only) right_sliced_notdot = list(right_notdot) right_sliced_todot = list(right_todot) for diag_axis in right_diag_inds: if diag_axis is not None: for ii in range(len(right_cols_only)): if diag_axis < right_cols_only[ii]: right_sliced_cols_only[ii] -= 1 for ii in range(len(right_notdot)): if diag_axis < right_notdot[ii]: right_sliced_notdot[ii] -= 1 for ii in range(len(right_todot)): if diag_axis < right_todot[ii]: right_sliced_todot[ii] -= 1 # Once we slice the arrays to get one block, we will permute the axes to be # in the proper order for dotting. left_sliced_permute = (left_sliced_rows_only + left_sliced_notdot + left_sliced_todot) right_sliced_permute = (right_sliced_todot + right_sliced_notdot + right_sliced_cols_only) # Then we'll reshape both into 2D arrays (matricies). left_sliced_reshape = (sp.prod(left_rows_only_shape + left_notdot_shape), sp.prod(left_todot_shape)) right_sliced_reshape = (sp.prod(right_todot_shape), sp.prod(right_notdot_shape + right_cols_only_shape)) # After the dot, we will neet to reshape back. out_sliced_reshape = tuple(left_rows_only_shape + left_notdot_shape + right_notdot_shape + right_cols_only_shape) # And finally we will need to permute the out axes. out_sliced_permute = range(len(left_rows_only)) out_sliced_permute += range(len(left_rows_only) + len(left_notdot_shape), len(left_rows_only) + len(left_notdot_shape) + len(right_notdot)) out_sliced_permute += range(len(left_rows_only), len(left_rows_only) + len(left_notdot)) out_sliced_permute += range(len(out_sliced_reshape) - len(right_cols_only), len(out_sliced_reshape)) # Create an index for each of left, right and out. left_slice = [slice(None)] * left.ndim right_slice = [slice(None)] * right.ndim out_slice = [slice(None)] * out.ndim # Flags for corner cases of 0D arrays. left_scalar_flag = False right_scalar_flag = False # Now we loop over all the block diagonal axes. for ii in xrange(all_diag_size): # Figure out exactly which blocks we are dealing with. tmp_ii = ii for kk in xrange(n_diag_axes - 1, -1, -1): this_index = tmp_ii % all_diag_shape[kk] out_slice[out_diag_inds[kk]] = this_index if left_diag_inds[kk] is not None: left_slice[left_diag_inds[kk]] = this_index if right_diag_inds[kk] is not None: right_slice[right_diag_inds[kk]] = this_index tmp_ii = tmp_ii // all_diag_shape[kk] this_left = left[tuple(left_slice)] this_right = right[tuple(right_slice)] # Permute and reshape the axes so the fast dot function can be used. # Corner case, if this_left or this_right are scalars (0D arrays). # TODO: Really these flags can be set outside the loop with a few more # lines of code. if ii == 0: if this_left.ndim == 0: left_scalar_flag = True if this_left.ndim == 0 and this_right.ndim == 0: right_scalar_flag = True if not left_scalar_flag: this_left = this_left.transpose(left_sliced_permute) this_left = sp.reshape(this_left, left_sliced_reshape) if not right_scalar_flag: this_right = this_right.transpose(right_sliced_permute) this_right = sp.reshape(this_right, right_sliced_reshape) # Dot them. if left_scalar_flag or right_scalar_flag: this_out = this_left * this_right else: this_out = sp.dot(this_left, this_right) # Reshape, permute and copy to the output. if not (left_scalar_flag and right_scalar_flag): this_out.shape = out_sliced_reshape this_out = this_out.transpose(out_sliced_permute) out[tuple(out_slice)] = this_out # XXX There is a bug where this crashes for certain cases if these lines # are before the loop. if not out_rows or not out_cols: out = vector.make_vect(out, out_names) else: out = matrix.make_mat(out, axis_names=out_names, row_axes=out_rows, col_axes=out_cols) return out
if socket.gethostname() == "Christopher-Kleins-MacBook-Pro.local" or \ socket.gethostname()[:8] == "airbears": coadd_directory = "/Volumes/Extra_HDD/Research/DECam_Data/coadds/chip21_coadds_and_data" else: coadd_directory = "/global/scratch2/sd/cenko/DES/Jan2014/coadds/" """ End of local laptop testing and development modification. """ chip_num_list = range(1, 61) chip_num_list.append(62) rrl_datatype = dtype([('ra', 'float'), ('dec', 'float'), ('ogle_id', '|S20'), ('type', '|S4'), ('period', 'float'), ('V_mag', 'float'), ('I_mag', 'float'), ('decam_z_mag', 'float'), ('decam_z_magerr', 'float'), ('sdss_z_mag', 'float'), ('sdss_z_magerr', 'float')]) rrl_data_flag = False for chip_num in chip_num_list: chip_str = "C%02d" % (chip_num,) for field in range(30): field_str = "1" + "%02d" % (field,) rrl_datafile = coadd_directory + "/" + field_str + "-" + chip_str \ + "_psf_coadd_rrl_data.txt" if not rrl_data_flag: try: rrl_data = loadtxt(rrl_datafile, dtype=rrl_datatype)