def propagate_distribution(pomdp, D_ux, u_dim=None, x_dim=None): '''evolve input/state distribution D_ux into output distribution D_xz D_xz(x', z) = \sum_{x', u) P(X+ = x, Z = z | U = u X = x' ) D_ux(u, x') ''' if u_dim is None: u_dim = tuple(range(len(pomdp.M))) if x_dim is None: x_dim = (len(pomdp.M), ) if len(u_dim) != len(pomdp.M) or len(x_dim) != 1: raise Exception('dimension problem') if len(D_ux.shape) <= max(u_dim + x_dim) or len( set(u_dim + x_dim)) < len(u_dim + x_dim) or sum(D_ux.data) != 1: raise Exception('D_ux not a valid distribution') T_uxXz = sparse.stack([sparse.stack([sparse.COO(pomdp.Tuz(m_tuple, z)) for z in range(pomdp.O)], axis=-1) for m_tuple in pomdp.m_tuple_iter()]) \ .reshape(pomdp.M + (pomdp.N, pomdp.N, pomdp.O)) T_zx = sparse.tensordot(D_ux, T_uxXz, axes=(u_dim + x_dim, range(len(pomdp.M) + 1))) return sparse.COO(T_zx)
def __init__(self, dim, basis=None): if basis is None: self.dim = dim self.basis = sparse.stack(gm.get_basis(dim, sparse=True)) else: self.dim = basis[0].shape[0] self.basis = COO.from_numpy(np.array(basis)) # Diagonal metric (since we're working with what are assumed to be # orthogonal but not necessarily normalized basis vectors) self.sq_norms = COO.from_numpy( sparse.tensordot( self.basis, self.basis, ([1, 2], [2, 1])).to_scipy_sparse().diagonal()) # Diagonal inverse metric sq_norms_inv = COO.from_numpy(1 / self.sq_norms.todense()) # Dual basis obtained from the original basis by the inverse metric self.dual = self.basis * sq_norms_inv[:,None,None] # Structure coefficients for the Lie algebra showing how to represent a # product of two basis elements as a complex-linear combination of basis # elements self.struct = sparse.tensordot(sparse.tensordot(self.basis, self.basis, ([2], [1])), self.dual, ([1, 3], [2, 1])) if isinstance(self.struct, np.ndarray): # Sometimes sparse.tensordot returns numpy arrays. We want to force # it to be sparse, since sparse.tensordot fails when passed two # numpy arrays. self.struct = COO.from_numpy(self.struct)
def test_stack(sd): shape, axis, xx, yy, zz = sd x = xx.todense() y = yy.todense() z = zz.todense() assert_eq(np.stack([x, y, z], axis=axis), sparse.stack([xx, yy, zz], axis=axis))
def fit(self, x, y_multiclass, kernel=poly(1), C=0.001): y_multiclass=y_multiclass.reshape(-1).astype(np.float64) self.x = sparse.COO(x.astype(np.float64)) self.m = self.x.shape[0] self.y_multiclass = y_multiclass self.kernel = kernel self.C = C ys = [sparse.COO(self.cast(y_multiclass, k)) for k in range(self.n_svm)] self.y_matrix = sparse.stack(ys,0) del ys for k in range(self.n_svm): print("training ",k,"th SVM in ",self.n_svm) y = self.y_matrix[k, :].reshape((-1,1)) yx = y * self.x G = kernel(yx, yx) # Gram matrix compensate = (sparse.eye(self.m)*1e-7).astype(np.float64) G = (G + compensate) objective = cp.Maximize(cp.sum(self.a[k])-(1/2)*cp.quad_form(self.a[k], G.tocsr())) if not objective.is_dcp(): print("Not solvable!") assert objective.is_dcp() constraints = [self.a[k] <= C, cp.sum(cp.multiply(self.a[k],y.todense())) == 0] # box constraint prob = cp.Problem(objective, constraints) result = prob.solve() x_pos = x[y.todense()[:,0]==1,:] x_neg = x[y.todense()[:,0]==-1,:] b_min = -np.min(self.wTx(k,x_pos)) if x_pos.shape[0]!=0 else 0 b_max = -np.max(self.wTx(k,x_neg)) if x_neg.shape[0]!=0 else 0 self.b[k,0] = (1/2)*(b_min + b_max) self.a_matrix = np.stack([i.value.reshape(-1) for i in self.a],0) self.a_matrix = sparse.COO(self.a_matrix)
def test_stack(shape, axis): xx = sparse.random(shape, density=0.5, format="gcxs") x = xx.todense() yy = sparse.random(shape, density=0.5, format="gcxs") y = yy.todense() zz = sparse.random(shape, density=0.5, format="gcxs") z = zz.todense() assert_eq(np.stack([x, y, z], axis=axis), sparse.stack([xx, yy, zz], axis=axis))
def radial_bins(centerX, centerY, imageSizeX, imageSizeY, radius=None, radius_inner=0, n_bins=None, normalize=False, use_sparse=None, dtype=None): ''' Generate antialiased rings ''' if radius is None: radius = bounding_radius(centerX, centerY, imageSizeX, imageSizeY) if n_bins is None: n_bins = int(np.round(radius - radius_inner)) r, phi = polar_map(centerX, centerY, imageSizeX, imageSizeY) r = r.flatten() width = (radius - radius_inner) / n_bins bin_area = np.pi * (radius**2 - (radius - width)**2) if use_sparse is None: use_sparse = bin_area / (imageSizeX * imageSizeY) < 0.1 if use_sparse: jjs = np.arange(len(r), dtype=np.int64) slices = [] for r0 in np.linspace(radius_inner, radius - width, n_bins) + width/2: diff = np.abs(r - r0) # The "0.5" ensures that the bins overlap and sum up to exactly 1 vals = np.maximum(0, np.minimum(1, width/2 + 0.5 - diff)) if use_sparse: select = vals != 0 vals = vals[select] if normalize: # Make sure each bin has a sum of 1 s = vals.sum() if not np.isclose(s, 0): vals /= s slices.append(sparse.COO(shape=len(r), data=vals.astype(dtype), coords=(jjs[select],))) else: if normalize: # Make sure each bin has a sum of 1 s = vals.sum() if not np.isclose(s, 0): vals /= s slices.append(vals.reshape((imageSizeY, imageSizeX)).astype(dtype)) # Patch a singularity at the center if radius_inner < 0.5: yy = int(np.round(centerY)) xx = int(np.round(centerX)) if use_sparse: index = yy * imageSizeX + xx diff = 1 - slices[0][index] - radius_inner patch = sparse.COO(shape=len(r), data=[diff], coords=[index]) slices[0] += patch else: slices[0][yy, xx] = 1 - radius_inner if use_sparse: return sparse.stack(slices).reshape((-1, imageSizeY, imageSizeX)) else: return np.stack(slices)
def convert_ndarray(value): if isinstance(value, sparse.SparseArray): return value if isinstance(value, np.ndarray): return sparse.COO(value) try: return sparse.COO(np.asarray(value)) except RuntimeError: return sparse.stack([convert_ndarray(v) for v in value])
def make_D(alphas, thick, thetas): """ Makes the bulk absorption vector for the bulk material :param alphas: absorption coefficient (m^{-1}) :param thick: thickness of the slab in m :param thetas: incident thetas in angle_vector (second column) :return: """ #print(alphas, abs(np.cos(thetas[None, :]))) diag = np.exp(-alphas[:, None] * thick / abs(np.cos(thetas[None, :]))) #print(diag) D_1 = stack([COO(np.diag(x)) for x in diag]) return D_1
def test_upcast(): a = sparse.random((50, 50, 50), density=0.1, format="coo", idx_dtype=np.uint8) b = a.asformat("gcxs") assert b.indices.dtype == np.uint16 a = sparse.random((8, 7, 6), density=0.5, format="gcxs", idx_dtype=np.uint8) b = sparse.random((6, 6, 6), density=0.8, format="gcxs", idx_dtype=np.uint8) assert sparse.concatenate((a, a)).indptr.dtype == np.uint16 assert sparse.stack((b, b)).indptr.dtype == np.uint16
def __init__(self, langs, order=1, prediction_cutoff=.9, base_path='./models/{}_o{}.pkl', class_dict_path='./unique_bytes.npy'): self.langs = langs self.order = order self.prediction_cutoff = prediction_cutoff #Encode unique bytes unique_bytes = np.load(class_dict_path) byte_classes = dict() for i, byt in enumerate(unique_bytes): #correct off-by-one issue byt = int(byt) byt -= 1 byt = str(byt) byte_classes[byt] = i #Add 'other' class in case unexpected byte encountered. byte_classes['other'] = i + 1 self.class_dict = byte_classes print('loading models...') #order of models matches order of languages self.models = list() for lang in langs: path = base_path.format(lang, self.order) try: with open(path, 'rb') as f: model = pickle.load(f) except FileNotFoundError: raise FileNotFoundError('No model found for language: ', lang) self.models.append(model) self.models = sparse.stack(self.models, axis=0) print('Done!')
def build(self, cohort, cache_file='/tmp/store.csv', nontemporal_cache_file='/tmp/store_ntmp.csv', from_cached=False): joined_sql = "{} order by {} asc".format( " union all ".join( f._sql_raw.format(cdm_schema=config.OMOP_CDM_SCHEMA, cohort_table='{}.{}'.format( cohort._schema_name, cohort._cohort_table_name)) for f in self._temporal_features), ",".join([ self.unique_id_col, ## Order by unique_id self.time_col, self.feature_col ])) if not from_cached: copy_sql = """ copy ({query}) to stdout with csv {head} """.format(query=joined_sql, head="HEADER") t = time.time() conn = self._db.engine.raw_connection() cur = conn.cursor() store = open(cache_file, 'wb') cur.copy_expert(copy_sql, store) store.seek(0) print( 'Data loaded to buffer in {0:.2f} seconds'.format(time.time() - t)) t = time.time() store = open(cache_file, 'rb') self.concepts = set() self.times = set() self.seen_ids = set() chunksize = int(2e6) for chunk in pd.read_csv(store, chunksize=chunksize): self.concepts = self.concepts.union( set(chunk[self.feature_col].unique())) self.times = self.times.union(set(chunk[self.time_col].unique())) self.seen_ids = self.seen_ids.union( set(chunk[self.unique_id_col].unique())) self.times = sorted(list(self.times)) self.concepts = sorted(list(self.concepts)) self.seen_ids = sorted(list(self.seen_ids)) print('Got Unique Concepts and Timestamps in {0:.2f} seconds'.format( time.time() - t)) t = time.time() store.seek(0) self.ids = cohort._cohort[self.unique_id_col].unique() self.id_map = { i: example_id for i, example_id in enumerate(self.seen_ids) } self.id_map_rev = { example_id: i for i, example_id in enumerate(self.seen_ids) } self.concept_map = { i: concept_name for i, concept_name in enumerate(self.concepts) } self.concept_map_rev = { concept_name: i for i, concept_name in enumerate(self.concepts) } self.time_map = {i: t for i, t in enumerate(self.times)} self.time_map_rev = {t: i for i, t in enumerate(self.times)} print('Created Index Mappings in {0:.2f} seconds'.format(time.time() - t)) t = time.time() last = None spm_stored = None spm_arr = [] self.recorded_ids = set() for chunk_num, chunk in enumerate( pd.read_csv(store, chunksize=chunksize)): first = chunk.iloc[0][self.unique_id_col] vals = chunk[self.unique_id_col].unique() indices = np.searchsorted(chunk[self.unique_id_col], vals) self.recorded_ids = self.recorded_ids.union(set(vals)) chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply( self.concept_map_rev.get) chunk.loc[:, self.time_col] = chunk[self.time_col].apply( self.time_map_rev.get) df_split = [ chunk.iloc[indices[i]:indices[i + 1]] for i in range(len(indices) - 1) ] + [chunk.iloc[indices[-1]:]] def gen_sparr(sub_df): sparr = coo_matrix( (np.ones(len(sub_df[self.feature_col])), (sub_df[self.feature_col], sub_df[self.time_col])), shape=(len(self.concepts), len(self.times))) return sparr spm_local = [gen_sparr(s) for s in df_split] if first == last: spm_local[0] += spm_stored else: if spm_stored is not None: spm_arr.append(spm_stored) spm_arr += spm_local[:-1] spm_stored = spm_local[-1] # last = chunk.iloc[-1][sep_col] last = chunk.iloc[-1][self.unique_id_col] spm_arr.append(spm_stored) print(len(spm_arr)) self._spm_arr = sparse.stack( [sparse.COO.from_scipy_sparse(m) for m in spm_arr], 2) print('Generated Sparse Representation of Data in {0:.2f} seconds'. format(time.time() - t)) # Build nontemporal feature matrix if len(self._nontemporal_features) > 0: joined_sql = "{} order by {} asc".format( " union all ".join( f._sql_raw.format(cdm_schema=config.OMOP_CDM_SCHEMA, cohort_table='{}.{}'.format( cohort._schema_name, cohort._cohort_table_name)) for f in self._nontemporal_features), ",".join([ self.unique_id_col, ## Order by unique_id self.feature_col ])) if not from_cached: copy_sql = """ copy ({query}) to stdout with csv {head} """.format(query=joined_sql, head="HEADER") t = time.time() conn = self._db.engine.raw_connection() cur = conn.cursor() store = open(nontemporal_cache_file, 'wb') cur.copy_expert(copy_sql, store) store.seek(0) print('Nontemporal data loaded to buffer in {0:.2f} seconds'. format(time.time() - t)) t = time.time() store = open(nontemporal_cache_file, 'rb') self.ntmp_concepts = set() self.ntmp_seen_ids = set() chunksize = int(2e6) for chunk in pd.read_csv(store, chunksize=chunksize): self.ntmp_seen_ids = self.ntmp_seen_ids.union( set(chunk[self.unique_id_col].unique())) self.ntmp_concepts = self.ntmp_concepts.union( set(chunk[self.feature_col].unique())) self.ntmp_concepts = sorted(list(self.ntmp_concepts)) print('Got Unique Nontemporal Concepts in {0:.2f} seconds'.format( time.time() - t)) t = time.time() store.seek(0) self.ntmp_id_map = { i: example_id for i, example_id in enumerate(self.ntmp_seen_ids) } self.ntmp_id_map_rev = { example_id: i for i, example_id in enumerate(self.ntmp_seen_ids) } self.ntmp_concept_map = { i: concept_name for i, concept_name in enumerate(self.ntmp_concepts) } self.ntmp_concept_map_rev = { concept_name: i for i, concept_name in enumerate(self.ntmp_concepts) } print( 'Created Nontemporal Index Mappings in {0:.2f} seconds'.format( time.time() - t)) t = time.time() ntmp_data = [] ntmp_unique_id = [] ntmp_feature_id = [] for chunk_num, chunk in enumerate( pd.read_csv(store, chunksize=chunksize)): chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply( self.ntmp_concept_map_rev.get) ntmp_data.append(chunk[self.ntmp_val_col]) ntmp_unique_id.append(chunk[self.unique_id_col]) ntmp_feature_id.append(chunk[self.feature_col]) ntmp_data = np.concatenate(ntmp_data) ntmp_unique_id = np.concatenate(ntmp_unique_id) ntmp_feature_id = np.concatenate(ntmp_feature_id) sparr = coo_matrix((ntmp_data, (ntmp_unique_id, ntmp_feature_id)), shape=(len(self.ids), len(self.ntmp_concepts))) self._ntmp_spm = sparr print( 'Generated Sparse Representation of Nontemporal Data in {0:.2f} seconds' .format(time.time() - t))
def st(*XS): return sp.stack(XS, axis=axis) if iscoo(XS[0]) else np.stack(XS, axis=axis)
def test_upcast_2(a, b): assert sparse.concatenate((a, a)).indptr.dtype == np.uint16 assert sparse.stack((b, b)).indptr.dtype == np.uint16
def RCWA(structure, size, orders, options, incidence, transmission, only_incidence_angle=False, front_or_rear='front', surf_name='', detail_layer=False, save=True): """ Calculates the reflected, absorbed and transmitted intensity of the structure for the wavelengths and angles defined using an RCWA method implemented using the S4 package. :param structure: A solcore Structure object with layers and materials or a OptiStack object. :param size: list with 2 entries, size of the unit cell (right now, can only be rectangular :param orders: number of orders to retain in the RCWA calculations. :param wavelength: Wavelengths (in nm) in which calculate the data. :param theta: polar incidence angle (in degrees) of the incident light. Default: 0 (normal incidence) :param phi: azimuthal incidence angle in degrees. Default: 0 :param pol: Polarisation of the light: 's', 'p' or 'u'. Default: 'u' (unpolarised). :param transmission: semi-infinite transmission medium :return: A dictionary with the R, A and T at the specified wavelengths and angle. """ # TODO: when non-zero incidence angle, not binned correctly in matrix (just goes in theta = 0) # TODO: when doing unpolarized, why not just set s=0.5 p=0.5 in S4? (Maybe needs to be normalised differently). Also don't know if this is faster, # or if internally it will still do s & p separately # TODO: if incidence angle is zero, s and p polarization are the same so no need to do both structpath = os.path.join(results_path, options['project_name']) if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') full_mat = load_npz(savepath_RT) A_mat = load_npz(savepath_A) else: wavelengths = options['wavelengths'] if front_or_rear == 'front': layers = structure trns = transmission inc = incidence else: layers = structure[::-1] trns = incidence inc = transmission # write a separate function that makes the OptiStack structure into an S4 object, defined materials etc. geom_list = [layer.geometry for layer in structure] geom_list.insert(0, {}) # incidence medium geom_list.append({}) # transmission medium ## Materials for the shapes need to be defined before you can do .SetRegion shape_mats, geom_list_str = necessary_materials(geom_list) shapes_oc = np.zeros((len(wavelengths), len(shape_mats)), dtype=complex) for i1, x in enumerate(shape_mats): shapes_oc[:, i1] = (x.n(wavelengths) + 1j * x.k(wavelengths))**2 stack_OS = OptiStack(layers, no_back_reflection=False) widths = stack_OS.get_widths() layers_oc = np.zeros((len(wavelengths), len(structure) + 2), dtype=complex) layers_oc[:, 0] = (inc.n(wavelengths))**2 #+ 1j*inc.k(wavelengths))**2 layers_oc[:, -1] = (trns.n(wavelengths) + 1j * trns.k(wavelengths))**2 for i1, x in enumerate(layers): layers_oc[:, i1 + 1] = (x.material.n(wavelengths) + 1j * x.material.k(wavelengths))**2 shapes_names = [str(x) for x in shape_mats] #nm_spacing = options['nm_spacing'] phi_sym = options['phi_symmetry'] n_theta_bins = options['n_theta_bins'] c_az = options['c_azimuth'] pol = options['pol'] # RCWA options rcwa_options = dict(LatticeTruncation='Circular', DiscretizedEpsilon=False, DiscretizationResolution=8, PolarizationDecomposition=False, PolarizationBasis='Default', LanczosSmoothing=False, SubpixelSmoothing=False, ConserveMemory=False, WeismannFormulation=False) user_options = options[ 'rcwa_options'] if 'rcwa_options' in options.keys() else {} rcwa_options.update(user_options) print(rcwa_options) theta_intv, phi_intv, angle_vector = make_angle_vector( n_theta_bins, phi_sym, c_az) if only_incidence_angle: thetas_in = np.array([options['theta_in']]) phis_in = np.array([options['phi_in']]) else: angles_in = angle_vector[:int(len(angle_vector) / 2), :] thetas_in = angles_in[:, 1] phis_in = angles_in[:, 2] # angle in degrees thetas_in = thetas_in * 180 / np.pi phis_in = phis_in * 180 / np.pi # initialise_S has to happen inside parallel job (get Pickle errors otherwise); # just pass relevant optical constants for each wavelength, like for RT angle_vector_0 = angle_vector[:, 0] if front_or_rear == "front": side = 1 else: side = -1 if options['parallel']: allres = Parallel(n_jobs=options['n_jobs'])(delayed(RCWA_wl)( wavelengths[i1] * 1e9, geom_list, layers_oc[i1], shapes_oc[i1], shapes_names, pol, thetas_in, phis_in, widths, size, orders, phi_sym, theta_intv, phi_intv, angle_vector_0, rcwa_options, detail_layer, side) for i1 in range(len(wavelengths))) else: allres = [ RCWA_wl(wavelengths[i1] * 1e9, geom_list, layers_oc[i1], shapes_oc[i1], shapes_names, pol, thetas_in, phis_in, widths, size, orders, phi_sym, theta_intv, phi_intv, angle_vector_0, rcwa_options, detail_layer, side) for i1 in range(len(wavelengths)) ] R = np.stack([item[0] for item in allres]) T = np.stack([item[1] for item in allres]) A_mat = np.stack([item[2] for item in allres]) full_mat = stack([item[3] for item in allres]) int_mat = stack([item[4] for item in allres]) #T_mat = np.stack([item[4] for item in allres]) #full_mat = np.hstack((R_mat, T_mat)) #full_mat = COO(full_mat) A_mat = COO(A_mat) if save: save_npz(savepath_RT, full_mat) save_npz(savepath_A, A_mat) #R_pfbo = np.stack([item[3] for item in allres]) #T_pfbo = np.stack([item[4] for item in allres]) #phi_rt = np.stack([item[5] for item in allres]) #theta_r = np.stack([item[6] for item in allres]) #theta_t = np.stack([item[7] for item in allres]) #R_pfbo_2 = np.stack([item[8] for item in allres]) #return {'R': R, 'T':T, 'A_layer': A_mat, 'full_mat': full_mat, 'int_mat': int_mat}#'R_pfbo': R_pfbo, 'T_pfbo': T_pfbo, 'phi_rt': phi_rt, 'theta_r': theta_r, 'theta_t': theta_t}#, 'R_pfbo_2': R_pfbo_2} return full_mat, A_mat # , R, T
def build(self, omop_cdm_schema=None, cohort_table_name=None, cohort_generation_script=None, cohort_generation_kwargs=None, first=None, verbose=True, outcome_col_name='y', cache_file='/tmp/store.csv', from_cached=False): # Generate cohort here with open(cohort_generation_script, 'r') as f: cohort_generation_sql_raw = f.read() # sep_col = self.id_col joined_sql = """with {} as ({}) select example_id, person_id, concept_name, feature_start_date, person_start_date, person_end_date from ({}) as t order by {} asc""".format( cohort_table_name, cohort_generation_sql_raw.format(**cohort_generation_kwargs), " union all ".join( f._sql_raw.format(cdm_schema=omop_cdm_schema, cohort_table=cohort_table_name) for f in self._temporal_features), ",".join([ self.unique_id_col, ## Order by unique_id # sep_col, self.time_col, self.feature_col ])) # print(joined_sql) if not from_cached: # copy_sql = """ # copy # ({query}) # to # stdout # with # csv {head} # """.format( # query=joined_sql, # head="HEADER" # ) t = time.time() conn = self._db.engine.raw_connection() # cur = conn.cursor() # store = open(cache_file,'wb') # cur.copy_expert(copy_sql, store) result = pd.read_sql(joined_sql, conn) print(result.shape) # print(result.columns) result.to_csv(cache_file) # store.seek(0) print( 'Data loaded to buffer in {0:.2f} seconds'.format(time.time() - t)) # Get outcomes separately cohort = pd.read_sql( cohort_generation_sql_raw.format(**cohort_generation_kwargs), conn) t = time.time() store = open(cache_file, 'rb') self.concepts = set() self.times = set() self.seen_ids = set() chunksize = int(2e6) for chunk in pd.read_csv(store, chunksize=chunksize): self.concepts = self.concepts.union( set(chunk[self.feature_col].unique())) self.times = self.times.union(set(chunk[self.time_col].unique())) # self.seen_ids = self.seen_ids.union(set(chunk[self.id_col].unique())) self.seen_ids = self.seen_ids.union( set(chunk[self.unique_id_col].unique())) self.times = sorted(list(self.times)) self.concepts = sorted(list(self.concepts)) self.seen_ids = sorted(list(self.seen_ids)) print('Got Unique Concepts and Timestamps in {0:.2f} seconds'.format( time.time() - t)) t = time.time() store.seek(0) # self.ids = cohort._cohort[self.id_col].unique() # self.ids = cohort._cohort[self.unique_id_col].unique() # self.id_map = {i:person_id for i,person_id in enumerate(self.seen_ids)} # self.id_map_rev = {person_id:i for i,person_id in enumerate(self.seen_ids)} self.id_map = { i: example_id for i, example_id in enumerate(self.seen_ids) } self.id_map_rev = { example_id: i for i, example_id in enumerate(self.seen_ids) } self.concept_map = { i: concept_name for i, concept_name in enumerate(self.concepts) } self.concept_map_rev = { concept_name: i for i, concept_name in enumerate(self.concepts) } self.time_map = {i: t for i, t in enumerate(self.times)} self.time_map_rev = {t: i for i, t in enumerate(self.times)} print('Created Index Mappings in {0:.2f} seconds'.format(time.time() - t)) t = time.time() last = None spm_stored = None spm_arr = [] self.recorded_ids = set() for chunk_num, chunk in enumerate( pd.read_csv(store, chunksize=chunksize)): # first = chunk.iloc[0][sep_col] first = chunk.iloc[0][self.unique_id_col] # vals = chunk[sep_col].unique() vals = chunk[self.unique_id_col].unique() # indices = np.searchsorted(chunk[sep_col], vals) indices = np.searchsorted(chunk[self.unique_id_col], vals) self.recorded_ids = self.recorded_ids.union(set(vals)) chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply( self.concept_map_rev.get) chunk.loc[:, self.time_col] = chunk[self.time_col].apply( self.time_map_rev.get) df_split = [ chunk.iloc[indices[i]:indices[i + 1]] for i in range(len(indices) - 1) ] + [chunk.iloc[indices[-1]:]] def gen_sparr(sub_df): sparr = coo_matrix( (np.ones(len(sub_df[self.feature_col])), (sub_df[self.feature_col], sub_df[self.time_col])), shape=(len(self.concepts), len(self.times))) return sparr spm_local = [gen_sparr(s) for s in df_split] if first == last: spm_local[0] += spm_stored else: if spm_stored is not None: spm_arr.append(spm_stored) spm_arr += spm_local[:-1] spm_stored = spm_local[-1] # last = chunk.iloc[-1][sep_col] last = chunk.iloc[-1][self.unique_id_col] spm_arr.append(spm_stored) print(len(spm_arr)) self._spm_arr = sparse.stack( [sparse.COO.from_scipy_sparse(m) for m in spm_arr], 2) print('Generated Sparse Representation of Data in {0:.2f} seconds'. format(time.time() - t)) return cohort
def generate_masks(reconstruct_shape, mask_shape, dtype, lamb, dpix, semiconv, semiconv_pix, transformation=None, center=None, cutoff=1, cutoff_freq=np.float32('inf'), method='subpix'): reconstruct_shape = np.array(reconstruct_shape) dpix = np.array(dpix) d_Kf = np.sin(semiconv) / lamb / semiconv_pix d_Qp = 1 / dpix / reconstruct_shape if center is None: center = np.array(mask_shape) / 2 if transformation is None: transformation = identity() cy, cx = center filter_center = circular(centerX=cx, centerY=cy, imageSizeX=mask_shape[1], imageSizeY=mask_shape[0], radius=semiconv_pix, antialiased=True) half_reconstruct = (reconstruct_shape[0] // 2 + 1, reconstruct_shape[1]) masks = [] for row in range(half_reconstruct[0]): for column in range(half_reconstruct[1]): # Do an fftshift of q and p qp = np.array((row, column)) flip = qp > (reconstruct_shape / 2) real_qp = qp.copy() real_qp[flip] = qp[flip] - reconstruct_shape[flip] if np.sum(real_qp**2) > cutoff_freq**2: masks.append(empty_mask(mask_shape, dtype=dtype)) continue # Shift of diffraction order relative to zero order # without rotation in physical coordinates real_sy_phys, real_sx_phys = real_qp * d_Qp # We apply the transformation backwards to go # from physical orientation to detector orientation, # while the forward direction in center of mass analysis # goes from detector coordinates to physical coordinates # Afterwards, we transform from physical detector coordinates # to pixel coordinates sy, sx = ((real_sy_phys, real_sx_phys) @ transformation) / d_Kf masks.append( generate_mask( cy=cy, cx=cx, sy=sy, sx=sx, filter_center=filter_center, semiconv_pix=semiconv_pix, cutoff=cutoff, mask_shape=mask_shape, dtype=dtype, method=method, )) # Since we go through masks in order, this gives a mask stack with # flattened (q, p) dimension to work with dot product and mask container masks = sparse.stack(masks) return masks
def create_interaction_list(interaction_df, num_individuals, fps=3, ringbuffer_size=5): ts = interaction_df.timestamp.min() rbs = ringbuffer_size rbs_hp = (rbs // 2) + 1 # cumulative interactions over whole period previous_interactions = sparse.COO([], shape=(num_individuals, num_individuals)) # frame sliding window interaction_ringbuffer = [ sparse.COO([], shape=(num_individuals, num_individuals)) for i in range(rbs) ] # cumulative interactions for all cameras within a 1/fps frame period # == 1 frame combined for all cameras current_interactions = sparse.COO([], shape=(num_individuals, num_individuals)) interval_counter = 0 events = [] print("Number of events {}".format(len(interaction_df)), flush=True) print("Number of timestamps {}".format( len(interaction_df.timestamp.unique())), flush=True) for timestamp, group in list( interaction_df.sort_values("timestamp").groupby("timestamp")): # still within current time interval if (timestamp - ts) < datetime.timedelta(milliseconds=int(900 / fps)): pass # end of current time interval else: # count as interaction if more than half of rbs consecutive frames had interactions # == median filter over temporal dimension with kernel size rbs if interval_counter >= rbs: new_interactions = sparse.stack(interaction_ringbuffer).sum( axis=0) > rbs_hp stopped_interactions = (previous_interactions.astype(np.int) - new_interactions.astype(np.int)) == 1 if stopped_interactions.sum() > 0: for bee_id_a, bee_id_b in np.argwhere( stopped_interactions): events.append((timestamp, bee_id_a, bee_id_b)) previous_interactions = new_interactions interaction_ringbuffer[interval_counter % rbs] = current_interactions # new time interval => reset adjacency matrix and timestamp current_interactions = sparse.COO([], shape=(num_individuals, num_individuals)) ts = group.timestamp.min() interval_counter += 1 # interaction adjacency matrix adj_data = {(min(k), max(k)): 1 for k in tuple(group["bee_id"].values)} adj = sparse.DOK(shape=(num_individuals, num_individuals), data=adj_data) # logical or => accumulate interactions from different cameras # for current time interval (~1/fps of a second) current_interactions += adj current_interactions.clip(0, 1, current_interactions) return events
def generate_masks(reconstruct_shape, mask_shape, dtype, lamb, dpix, semiconv, semiconv_pix, transformation=None, cy=None, cx=None, cutoff=1, cutoff_freq=np.float32('inf'), method='subpix'): ''' Generate the trotter mask stack. The y dimension is trimmed to size(y)//2 + 1 to exploit the inherent symmetry of the mask stack. Parameters ---------- reconstruct_shape : tuple(int) Shape of the reconstructed area mask_shape : tuple(int) Shape of the detector dtype : numpy dtype dtype to use for the mask stack lamb : float Wavelength of the illuminating radiation in m dpix : float or (float, float) Scan step in m. Tuple (y, x) in case scan step is different in x and y direction. semiconv : float Semiconvergence angle of the illumination in radians semiconv_pix : float Semiconvergence angle in measured in detector pixel, i.e. radius of the zero order disk. transformation : numpy.ndarray, optional Matrix for affine transformation from the scan coordinate directions to the detector coordinate directions. This does not include the scale, which is handled by dpix, lamb, semiconv and semiconv_pix. It should only be used to rotate and flip the coordinate system as necessary. See also https://github.com/LiberTEM/LiberTEM/blob/master/src/libertem/corrections/coordinates.py cy, cx : float, optional Position of the optical axis on the detector in px, center of illumination. Default: Center of the detector cutoff : int, optional Minimum number of pixels in the positive and negative trotter. This can be used to purge very small trotters to reduce noise. Default is 1, i.e. no cutoff unless one trotter is empty. cutoff_freq: float Trotters belonging to a spatial frequency higher than this value in reciprocal pixel coordinates will be cut off. method : str, optional Can be :code:`'subpix'`(default) or :code:`'shift'` to switch between :meth:`mask_pair_subpix` and :meth:`mask_pair_shift` to generate a trotter pair. Returns ------- masks : sparse.COO Masks in sparse.pydata.org COO format. y and x frequency index are FFT shifted, i.e. the zero frequency is at (0,0) and negative frequencies are in the far quadrant and reversed. The y frequency index is cut in half with size(y)//2 + 1 to exploit the inherent symmetry of a real-valued Fourier transform. The y and x index are then flattened to make it suitable for using it with MaskContainer. ''' reconstruct_shape = np.array(reconstruct_shape) dpix = np.array(dpix) d_Kf = np.sin(semiconv) / lamb / semiconv_pix d_Qp = 1 / dpix / reconstruct_shape if cy is None: cy = mask_shape[0] / 2 if cx is None: cx = mask_shape[1] / 2 if transformation is None: transformation = identity() filter_center = circular(centerX=cx, centerY=cy, imageSizeX=mask_shape[1], imageSizeY=mask_shape[0], radius=semiconv_pix, antialiased=True) half_reconstruct = (reconstruct_shape[0] // 2 + 1, reconstruct_shape[1]) masks = [] for row in range(half_reconstruct[0]): for column in range(half_reconstruct[1]): # Do an fftshift of q and p qp = np.array((row, column)) flip = qp > (reconstruct_shape / 2) real_qp = qp.copy() real_qp[flip] = qp[flip] - reconstruct_shape[flip] if np.sum(real_qp**2) > cutoff_freq**2: masks.append(empty_mask(mask_shape, dtype=dtype)) continue # Shift of diffraction order relative to zero order # without rotation in physical coordinates real_sy_phys, real_sx_phys = real_qp * d_Qp # We apply the transformation backwards to go # from physical orientation to detector orientation, # while the forward direction in center of mass analysis # goes from detector coordinates to physical coordinates # Afterwards, we transform from physical detector coordinates # to pixel coordinates sy, sx = ((real_sy_phys, real_sx_phys) @ transformation) / d_Kf masks.append( generate_mask( cy=cy, cx=cx, sy=sy, sx=sx, filter_center=filter_center, semiconv_pix=semiconv_pix, cutoff=cutoff, dtype=dtype, method=method, )) # Since we go through masks in order, this gives a mask stack with # flattened (q, p) dimension to work with dot product and mask container masks = sparse.stack(masks) return masks
def RT(group, incidence, transmission, surf_name, options, Fr_or_TMM = 0, front_or_rear = 'front', n_absorbing_layers=0, calc_profile=[], only_incidence_angle=False, widths=[], save=True): """Calculates the reflection/transmission and absorption redistribution matrices for an interface using either a previously calculated TMM lookup table or the Fresnel equations. :param group: an RTgroup object containing the surface textures :param incidence: incidence medium :param: transmission: transmission medium :param surf_name: name of the surface (to save matrices) :param options: dictionary of options :param Fr_or_TMM: whether to use the Fresnel equations (0) or a TMM lookup table (1) :param front_or_rear: whether light is incident from the front or rear :param for a structure with multiple interface layers, where a TMM lookuptable is used, the number of layers in the interface :param calc_profile: whether to save the relevant information to calculate the depth-dependent absorption profile :param only_incidence_angle: if True, the ray-tracing will only be performed for the incidence theta and phi specified in the options. :return out_mat: the R/T redistribution matrix at each wavelength, indexed as (wavelength, angle_bin_out, angle_bin_in) :return A_mat: the absorption redistribution matrix (total absorption per layer), indexed as (wavelength, layer_out, angle_bin_in) :return local_angle_mat: only if calc_profile = True. A matrix storing the local incidence angles for rays which were absorbed. This is used to calculate absorption profiles using TMM. """ if Fr_or_TMM > 0 or save: structpath = os.path.join(results_path, options['project_name']) # also need this to get lookup table if save: if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if Fr_or_TMM > 0: savepath_prof = os.path.join(structpath, surf_name + front_or_rear + 'Aprof.npz') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') allArrays = load_npz(savepath_RT) absArrays = load_npz(savepath_A) if Fr_or_TMM > 0 and os.path.isfile(savepath_prof): local_angles = load_npz(savepath_prof) if os.path.isfile(prof_mat_path): prof_int = xr.load_dataset(prof_mat_path) profile = prof_int['profile'] intgr = prof_int['intgr'] return allArrays, absArrays, local_angles, profile, intgr else: return allArrays, absArrays, local_angles else: return allArrays, absArrays else: wavelengths = options['wavelengths'] n_rays = options['n_rays'] nx = options['nx'] ny = options['ny'] n_angles = int(np.ceil(n_rays/(nx*ny))) phi_sym = options['phi_symmetry'] n_theta_bins = options['n_theta_bins'] c_az = options['c_azimuth'] pol = options['pol'] nm_spacing = options['nm_spacing'] if front_or_rear == 'front': side = 1 else: side = -1 if Fr_or_TMM == 1: lookuptable = xr.open_dataset(os.path.join(structpath, surf_name + '.nc')) if front_or_rear == 'rear': lookuptable = lookuptable.assign_coords(side=np.flip(lookuptable.side)) else: lookuptable = None theta_intv, phi_intv, angle_vector = make_angle_vector(n_theta_bins, phi_sym, c_az) if only_incidence_angle: print('Calculating matrix only for incidence theta/phi') if options['theta_in'] == 0: th_in = 0.0001 else: th_in = options['theta_in'] angles_in = angle_vector[:int(len(angle_vector) / 2), :] n_reps = int(np.ceil(n_angles / len(angles_in))) thetas_in = np.tile(th_in, n_reps) print('only inc angle' , thetas_in) n_angles = n_reps if options['phi_in'] == 'all': # get relevant phis phis_in = np.tile(options['phi_in'], n_reps) else: if options['phi_in'] == 0: phis_in = np.tile(0.0001, n_reps) else: phis_in = np.tile(options['phi_in'], n_reps) else: if options['random_angles']: thetas_in = np.random.random(n_angles)*np.pi/2 phis_in = np.random.random(n_angles)*2*np.pi else: angles_in = angle_vector[:int(len(angle_vector)/2),:] if n_angles/len(angles_in) < 1: warn('The number of rays is not sufficient to populate the redistribution matrix!') n_reps = int(np.ceil(n_angles/len(angles_in))) thetas_in = np.tile(angles_in[:,1], n_reps)[:n_angles] phis_in = np.tile(angles_in[:,2], n_reps)[:n_angles] if front_or_rear == 'front': mats = [incidence] else: mats = [transmission] for i1 in range(len(group.materials)): mats.append(group.materials[i1]) if front_or_rear == 'front': mats.append(transmission) else: mats.append(incidence) # list of lists: first in tuple is front incidence if front_or_rear == 'front': surfaces = [x[0] for x in group.textures] else: surfaces = [x[1] for x in group.textures] nks = np.empty((len(mats), len(wavelengths)), dtype=complex) for i1, mat in enumerate(mats): nks[i1] = mat.n(wavelengths) + 1j*mat.k(wavelengths) h = max(surfaces[0].Points[:, 2]) x_lim = surfaces[0].Lx y_lim = surfaces[0].Ly if options['random_ray_position']: xs = np.random.uniform(0, x_lim, nx) ys = np.random.uniform(0, y_lim, ny) else: if options['avoid_edges']: xs = np.linspace(x_lim / 4, x_lim - (x_lim / 4), nx) ys = np.linspace(y_lim / 4, y_lim - (y_lim / 4), ny) else: xs = np.linspace(x_lim/101, x_lim-(x_lim/99), nx) ys = np.linspace(y_lim/100, y_lim-(y_lim/102), ny) print('n_th_in', len(thetas_in), len(xs)) if options['parallel']: allres = Parallel(n_jobs=options['n_jobs'])(delayed(RT_wl) (i1, wavelengths[i1], n_angles, nx, ny, widths, thetas_in, phis_in, h, xs, ys, nks, surfaces, pol, phi_sym, theta_intv, phi_intv, angle_vector, Fr_or_TMM, n_absorbing_layers, lookuptable, calc_profile, nm_spacing, side) for i1 in range(len(wavelengths))) else: allres = [RT_wl(i1, wavelengths[i1], n_angles, nx, ny, widths, thetas_in, phis_in, h, xs, ys, nks, surfaces, pol, phi_sym, theta_intv, phi_intv, angle_vector, Fr_or_TMM, n_absorbing_layers, lookuptable, calc_profile, nm_spacing, side) for i1 in range(len(wavelengths))] allArrays = stack([item[0] for item in allres]) absArrays = stack([item[1] for item in allres]) if save: save_npz(savepath_RT, allArrays) save_npz(savepath_A, absArrays) if Fr_or_TMM > 0: local_angles = stack([item[2] for item in allres]) if save: save_npz(savepath_prof, local_angles) #make_profile_data(options, np.unique(angle_vector[:,1]), int(len(angle_vector) / 2), # front_or_rear, surf_name, n_absorbing_layers, widths) if len(calc_profile) > 0: profile = xr.concat([item[3] for item in allres], 'wl') intgr = xr.concat([item[4] for item in allres], 'wl') intgr.name = 'intgr' profile.name = 'profile' allres = xr.merge([intgr, profile]) if save: allres.to_netcdf(prof_mat_path) save_npz(savepath_prof, local_angles) return allArrays, absArrays, local_angles, profile, intgr else: return allArrays, absArrays, local_angles else: return allArrays, absArrays
def TMM(layers, incidence, transmission, surf_name, options, coherent=True, coherency_list=None, prof_layers=[], front_or_rear='front', save=True): """Function which takes a layer stack and creates an angular redistribution matrix. :param layers: A list with one or more layers. :param transmission: transmission medium :param incidence: incidence medium :param surf_name: name of the surface (to save the matrices generated. :param options: a list of options :param coherent: whether or not the layer stack is coherent. If None, it is assumed to be fully coherent :param coherency: a list with the same number of entries as the layers, either 'c' for a coherent layer or 'i' for an incoherent layer :param prof_layers: layers for which the absorption profile should be calculated (if None, do not calculate absorption profile at all) :param front_or_rear: a string, either 'front' or 'rear'; front incidence on the stack, from the incidence medium, or rear incidence on the stack, from the transmission medium. :return full_mat: R and T redistribution matrix :return A_mat: matrix describing absorption per layer """ def make_matrix_wl(wl): # binning into matrix, including phi RT_mat = np.zeros((len(theta_bins_in)*2, len(theta_bins_in))) A_mat = np.zeros((n_layers, len(theta_bins_in))) for i1 in range(len(theta_bins_in)): theta = theta_lookup[i1]#angle_vector[i1, 1] data = allres.loc[dict(angle=theta, wl=wl)] R_prob = np.real(data['R'].data.item(0)) T_prob = np.real(data['T'].data.item(0)) Alayer_prob = np.real(data['Alayer'].data) phi_out = phis_out[i1] #print(R_prob, T_prob) # reflection phi_int = phi_intv[theta_bins_in[i1]] phi_ind = np.digitize(phi_out, phi_int, right=True) - 1 bin_out_r = np.argmin(abs(angle_vector[:, 0] - theta_bins_in[i1])) + phi_ind #print(bin_out_r, i1+offset) RT_mat[bin_out_r, i1] = R_prob #print(R_prob) # transmission theta_t = np.abs(-np.arcsin((inc.n(wl * 1e-9) / trns.n(wl * 1e-9)) * np.sin(theta_lookup[i1])) + quadrant) #print('angle in, transmitted', angle_vector_th[i1], theta_t) # theta switches half-plane (th < 90 -> th >90 if ~np.isnan(theta_t): theta_out_bin = np.digitize(theta_t, theta_intv, right=True) - 1 phi_int = phi_intv[theta_out_bin] phi_ind = np.digitize(phi_out, phi_int, right=True) - 1 bin_out_t = np.argmin(abs(angle_vector[:, 0] - theta_out_bin)) + phi_ind RT_mat[bin_out_t, i1] = T_prob #print(bin_out_t, i1+offset) # absorption A_mat[:, i1] = Alayer_prob fullmat = COO(RT_mat) A_mat = COO(A_mat) return fullmat, A_mat structpath = os.path.join(results_path, options['project_name']) if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') fullmat = load_npz(savepath_RT) A_mat = load_npz(savepath_A) if len(prof_layers) > 0: profile = xr.load_dataarray(prof_mat_path) return fullmat, A_mat, profile else: wavelengths = options['wavelengths']*1e9 # convert to nm theta_intv, phi_intv, angle_vector = make_angle_vector(options['n_theta_bins'], options['phi_symmetry'], options['c_azimuth']) angles_in = angle_vector[:int(len(angle_vector) / 2), :] thetas = np.unique(angles_in[:, 1]) n_angles = len(thetas) n_layers = len(layers) if front_or_rear == 'front': optlayers = OptiStack(layers, substrate=transmission, incidence=incidence) trns = transmission inc = incidence else: optlayers = OptiStack(layers[::-1], substrate=incidence, incidence=transmission) trns = incidence inc = transmission if len(prof_layers) > 0: profile = True z_limit = np.sum(np.array(optlayers.widths)) full_dist = np.arange(0, z_limit, options['nm_spacing']) layer_start = np.insert(np.cumsum(np.insert(optlayers.widths, 0, 0)), 0, 0) layer_end = np.cumsum(np.insert(optlayers.widths, 0, 0)) dist = [] for l in prof_layers: dist = np.hstack((dist, full_dist[np.all((full_dist >= layer_start[l], full_dist < layer_end[l]), 0)])) else: profile = False if options['pol'] == 'u': pols = ['s', 'p'] else: pols = [options['pol']] R = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='R') T = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='T') Alayer = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), n_layers)), dims=['pol', 'angle', 'wl', 'layer'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas, 'layer': range(1, n_layers + 1)}, name='Alayer') theta_t = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='theta_t') if profile: Aprof = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), len(dist))), dims=['pol', 'angle', 'wl', 'z'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas, 'z': dist}, name='Aprof') R_loop = np.empty((len(wavelengths), n_angles)) T_loop = np.empty((len(wavelengths), n_angles)) Alayer_loop = np.empty((n_angles, len(wavelengths), n_layers), dtype=np.complex_) th_t_loop = np.empty((len(wavelengths), n_angles)) if profile: Aprof_loop = np.empty((n_angles, len(wavelengths), len(dist))) tmm_struct = tmm_structure(optlayers, coherent=coherent, coherency_list=coherency_list, no_back_reflection=False) for i2, pol in enumerate(pols): for i3, theta in enumerate(thetas): res = tmm_struct.calculate(wavelengths, angle=theta, pol=pol, profile=profile, layers=prof_layers, nm_spacing = options['nm_spacing']) R_loop[:, i3] = np.real(res['R']) T_loop[:, i3] = np.real(res['T']) Alayer_loop[i3, :, :] = np.real(res['A_per_layer'].T) if profile: Aprof_loop[i3, :, :] = res['profile'] # sometimes get very small negative values (like -1e-20) R_loop[R_loop < 0] = 0 T_loop[T_loop < 0] = 0 Alayer_loop[Alayer_loop < 0] = 0 if front_or_rear == 'rear': Alayer_loop = np.flip(Alayer_loop, axis=2) print('flipping') R.loc[dict(pol=pol)] = R_loop T.loc[dict(pol=pol)] = T_loop Alayer.loc[dict(pol=pol)] = Alayer_loop theta_t.loc[dict(pol=pol)] = th_t_loop if profile: Aprof.loc[dict(pol=pol)] = Aprof_loop Aprof.transpose('pol', 'wl', 'angle', 'z') Alayer = Alayer.transpose('pol', 'wl', 'angle', 'layer') if profile: allres = xr.merge([R, T, Alayer, Aprof]) else: allres = xr.merge([R, T, Alayer]) if options['pol'] == 'u': allres = allres.reduce(np.mean, 'pol').assign_coords(pol='u').expand_dims('pol') # populate matrices if front_or_rear == "front": angle_vector_th = angle_vector[:int(len(angle_vector)/2),1] angle_vector_phi = angle_vector[:int(len(angle_vector)/2),2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) theta_lookup = angles_in[:,1] quadrant = np.pi else: angle_vector_th = angle_vector[int(len(angle_vector) / 2):, 1] angle_vector_phi = angle_vector[int(len(angle_vector) / 2):, 2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) theta_lookup = angles_in[:,1][::-1] quadrant = 0 phis_out[phis_out == 0] = 1e-10 theta_bins_in = np.digitize(angle_vector_th, theta_intv, right=True) -1 print(theta_bins_in) mats = [make_matrix_wl(wl) for wl in wavelengths] fullmat = stack([item[0] for item in mats]) A_mat = stack([item[1] for item in mats]) if save: save_npz(savepath_RT, fullmat) save_npz(savepath_A, A_mat) return fullmat, A_mat #, allres