def cache_tm(nspins): """ Parameters ---------- nspins Returns ------- """ """spin11 test indicates this leads to faster overall simsignals(). 11 spin x 6: 29.6 vs. 35.1 s 8 spin x 60: 2.2 vs 3.0 s""" filename = f'T{nspins}.npz' bin_dir = os.path.join(os.path.dirname(__file__), 'bin') path = os.path.join(bin_dir, filename) try: T = sparse.load_npz(path) return T except FileNotFoundError: print(f'creating {filename}') T = transition_matrix_dense(nspins) T_sparse = sparse.COO(T) sparse.save_npz(path, T_sparse) return T_sparse
def test_save_load_npz_file(tmp_path, compression, format): x = sparse.random((2, 3, 4, 5), density=0.25, format=format) y = x.todense() filename = tmp_path / "mat.npz" save_npz(filename, x, compressed=compression) z = load_npz(filename) assert_eq(x, z) assert_eq(y, z.todense())
def save_file(array, fname, path=PATH): """ Save a numpy matrix/array. """ if not os.path.exists(path): os.mkdir(path) print("Saving " + fname) point = sparse.COO(array) sparse.save_npz(path + "/" + fname, point)
def writeMatrix(arr, filename): try: sparse.save_npz(filename.replace(".txt", ""), arr) except: with open(filename, 'a') as outfile: outfile.truncate(0) outfile.write('# Array shape: {0}\n'.format(arr.shape)) for slice_2d in arr: np.savetxt(outfile, slice_2d) outfile.write("# New slice\n")
def test_save_load_npz_file(compression, format): with tempfile.TemporaryDirectory() as tmp_path_str: tmp_path = pathlib.Path(tmp_path_str) x = sparse.random((2, 3, 4, 5), density=0.25, format=format) y = x.todense() filename = tmp_path / "mat.npz" save_npz(filename, x, compressed=compression) z = load_npz(filename) assert_eq(x, z) assert_eq(y, z.todense())
def _so_sparse(nspins): """ Either load a presaved set of spin operators as numpy arrays, or calculate them and save them if a presaved set wasn't found. Parameters ---------- nspins : int the number of spins in the spin system Returns ------- (Lz, Lproduct) : a tuple of: Lz : 3d sparse.COO array of shape (n, 2^n, 2^n) representing [Lz1, Lz2, ...Lzn] Lproduct : 4d sparse.COO array of shape (n, n, 2^n, 2^n), representing an n x n array (cartesian product) for all combinations of Lxa*Lxb + Lya*Lyb + Lza*Lzb, where 1 <= a, b <= n. Side Effect ----------- Saves the results as .npz files to the bin directory if they were not found there. """ # TODO: once nmrsim demonstrates installing via the PyPI *test* server, # need to determine how the saved solutions will be handled. For example, # part of the final build may be generating these files then testing. # Also, need to consider different users with different system capabilities # (e.g. at extreme, Raspberry Pi). Some way to let user select, or select # for user? filename_Lz = f'Lz{nspins}.npz' filename_Lproduct = f'Lproduct{nspins}.npz' bin_path = _bin_path() path_Lz = bin_path.joinpath(filename_Lz) path_Lproduct = bin_path.joinpath(filename_Lproduct) # with path_context_Lz as p: # path_Lz = p # with path_context_Lproduct as p: # path_Lproduct = p try: Lz = sparse.load_npz(path_Lz) Lproduct = sparse.load_npz(path_Lproduct) return Lz, Lproduct except FileNotFoundError: print('no SO file ', path_Lz, ' found.') print(f'creating {filename_Lz} and {filename_Lproduct}') Lz, Lproduct = _so_dense(nspins) Lz_sparse = sparse.COO(Lz) Lproduct_sparse = sparse.COO(Lproduct) sparse.save_npz(path_Lz, Lz_sparse) sparse.save_npz(path_Lproduct, Lproduct_sparse) return Lz_sparse, Lproduct_sparse
def test_save_load_npz_file(compression): x = sparse.random((2, 3, 4, 5), density=.25) y = x.todense() dir_name = tempfile.mkdtemp() filename = os.path.join(dir_name, 'mat.npz') save_npz(filename, x, compressed=compression) z = load_npz(filename) assert_eq(x, z) assert_eq(y, z.todense()) shutil.rmtree(dir_name)
def so_sparse(nspins): filename_Lz = f'sparse_Lz{nspins}.npz' filename_Lproduct = f'sparse_Lproduct{nspins}.npz' try: Lz = sparse.load_npz(filename_Lz) Lproduct = sparse.load_npz(filename_Lproduct) return Lz, Lproduct except FileNotFoundError: print(f'creating {filename_Lz} and {filename_Lproduct}') sigma_x = np.array([[0, 1 / 2], [1 / 2, 0]]) sigma_y = np.array([[0, -1j / 2], [1j / 2, 0]]) sigma_z = np.array([[1 / 2, 0], [0, -1 / 2]]) unit = np.array([[1, 0], [0, 1]]) L = np.empty((3, nspins, 2 ** nspins, 2 ** nspins), dtype=np.complex128) # consider other dtype? for n in range(nspins): Lx_current = 1 Ly_current = 1 Lz_current = 1 for k in range(nspins): if k == n: Lx_current = np.kron(Lx_current, sigma_x) Ly_current = np.kron(Ly_current, sigma_y) Lz_current = np.kron(Lz_current, sigma_z) else: Lx_current = np.kron(Lx_current, unit) Ly_current = np.kron(Ly_current, unit) Lz_current = np.kron(Lz_current, unit) L[0][n] = Lx_current L[1][n] = Ly_current L[2][n] = Lz_current L_T = L.transpose(1, 0, 2, 3) Lproduct = np.tensordot(L_T, L, axes=((1, 3), (0, 2))).swapaxes(1, 2) Lz_sparse = sparse.COO(L[2]) # for i in range(nspins): # for j in range(nspins): # Lproduct[i, j] = csr_matrix(Lproduct[i, j]) Lproduct_sparse = sparse.COO(Lproduct) sparse.save_npz(filename_Lz, Lz_sparse) sparse.save_npz(filename_Lproduct, Lproduct_sparse) return Lz_sparse, Lproduct_sparse
def so_sparse(nspins): """Either load a presaved set of spin operators as numpy arrays, or calculate them and save them if a presaved set wasn't found. Parameters ---------- nspins: int the number of spins in the spin system Returns ------- (Lz, Lproduct): a tuple of: Lz: 3d sparse.COO array of shape (n, 2^n, 2^n) representing [Lz1, Lz2, ...Lzn] Lproduct: 4d sparse.COO array of shape (n, n, 2^n, 2^n), representing an n x n array (cartesian product) for all combinations of Lxa*Lxb + Lya*Lyb + Lza*Lzb, where 1 <= a, b <= n. Side Effect ----------- Saves the results as .npz files to the bin directory if they were not found there. """ filename_Lz = f'Lz{nspins}.npz' filename_Lproduct = f'Lproduct{nspins}.npz' bin_dir = os.path.join(os.path.dirname(__file__), 'bin') path_Lz = os.path.join(bin_dir, filename_Lz) path_Lproduct = os.path.join(bin_dir, filename_Lproduct) try: Lz = sparse.load_npz(path_Lz) Lproduct = sparse.load_npz(path_Lproduct) return Lz, Lproduct except FileNotFoundError: print('no SO file ', filename_Lz, ' found in: ', bin_dir) print(f'creating {filename_Lz} and {filename_Lproduct}') Lz, Lproduct = so_dense(nspins) Lz_sparse = sparse.COO(Lz) Lproduct_sparse = sparse.COO(Lproduct) sparse.save_npz(path_Lz, Lz_sparse) sparse.save_npz(path_Lproduct, Lproduct_sparse) return Lz_sparse, Lproduct_sparse
def resolve_relations(db_file, rel_file, meta_file, id_file): """ """ conn = open_db_connection(db_file) c = conn.cursor() # load or compute unique IDs if os.path.isfile(meta_file): meta = np.load(meta_file) off = meta[0] num_unique = meta[1] unique_ids = np.load(id_file) else: off = 0 c.execute("SELECT DISTINCT event1_id FROM Relations;") event_ids = set(c.fetchall()) for id2 in c.execute("SELECT event2_id FROM Relations;"): if not id2 in event_ids: event_ids.add(id2) unique_ids = np.char.array(list(event_ids)) num_unique = len(event_ids) np.save(id_file, unique_ids) np.save(meta_file, np.array([off, num_unique])) id_lookup = dict() for i, id_entr in enumerate(unique_ids): id_lookup[id_entr[0]] = i # load or compute (compressed) relations if os.path.isfile(rel_file): relations = sparse.load_npz(rel_file) else: relations = sparse.DOK((num_unique, num_unique, RELATION_COUNT), dtype=np.float32) for row in c.execute("SELECT * FROM Relations;"): id_out = row[1] id_in = row[2] relations[id_lookup[id_out], id_lookup[id_in], :] = row[3:] relations = sparse.COO(relations) sparse.save_npz(rel_file, relations) conn.close()
def cache_tm(n): """spin11 test indicates this leads to faster overall simsignals(). 11 spin x 6: 29.6 vs. 35.1 s 8 spin x 60: 2.2 vs 3.0 s""" filename = f'transitions{n}.npz' try: T = sparse.load_npz(filename) return T except FileNotFoundError: print(f'creating {filename}') T = np.zeros((n, n)) for i in range(n - 1): for j in range(i + 1, n): if bin(i ^ j).count('1') == 1: T[i, j] = 1 # T = T + T.T T += T.T T_sparse = sparse.COO(T) sparse.save_npz(filename, T_sparse) return T_sparse
def _tm_cache(nspins): """ Loads a saved sparse transition matrix if it exists, or creates and saves one if it is not. Parameters ---------- nspins : int The number of spins in the spin system. Returns ------- T_sparse : sparse.COO The sparse transition matrix. Side Effects ------------ Saves a sparse array to the bin folder if the required array was not found there. """ # Speed tests indicated that using sparse-array transition matrices # provides a modest speed improvement on larger spin systems. filename = f'T{nspins}.npz' # init_path_context = resources.path(nmrsim.bin, '__init__.py') # with init_path_context as p: # init_path = p # print('path to init: ', init_path) # bin_path = init_path.parent bin_path = _bin_path() path = bin_path.joinpath(filename) try: T_sparse = sparse.load_npz(path) return T_sparse except FileNotFoundError: print(f'creating {filename}') T_sparse = _transition_matrix_dense(nspins) T_sparse = sparse.COO(T_sparse) print('_tm_cache will save on path: ', path) sparse.save_npz(path, T_sparse) return T_sparse
def mirror_matrix(angle_vector, theta_intv, phi_intv, surf_name, options, front_or_rear='front', save=True): if save: structpath = os.path.join( results_path, options['project_name']) # also need this to get lookup table if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') allArray = load_npz(savepath_RT) else: if front_or_rear == "front": angle_vector_th = angle_vector[:int(len(angle_vector) / 2), 1] angle_vector_phi = angle_vector[:int(len(angle_vector) / 2), 2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) else: angle_vector_th = angle_vector[int(len(angle_vector) / 2):, 1] angle_vector_phi = angle_vector[int(len(angle_vector) / 2):, 2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) # matrix will be all zeros with just one '1' in each column/row. Just need to determine where it goes binned_theta = np.digitize(angle_vector_th, theta_intv, right=True) - 1 # print(binned_theta_out, theta_out, theta_intv) # print(binned_theta_in) # print(binned_theta_out) # -1 to give the correct index for the bins in phi_intv bin_in = np.arange(len(angle_vector_phi)) phi_ind = [ np.digitize(phi, phi_intv[binned_theta[i1]], right=True) - 1 for i1, phi in enumerate(phis_out) ] overall_bin = [ np.argmin(abs(angle_vector[:, 0] - binned_theta[i1])) + phi_i for i1, phi_i in enumerate(phi_ind) ] whole_matrix = np.zeros((len(overall_bin) * 2, len(overall_bin))) whole_matrix[overall_bin, bin_in] = 1 A_matrix = np.zeros((1, len(overall_bin))) allArray = COO(whole_matrix) absArray = COO(A_matrix) if save: save_npz(savepath_RT, allArray) save_npz(savepath_A, absArray) return allArray
def lambertian_matrix(angle_vector, theta_intv, surf_name, options, front_or_rear='front', save=True): if save: structpath = os.path.join( results_path, options['project_name']) # also need this to get lookup table if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') allArray = load_npz(savepath_RT) else: theta_values = np.unique(angle_vector[angle_vector[:, 1] < np.pi / 2, 1]) dtheta = np.diff(theta_intv[theta_intv <= np.pi / 2]) dP = np.cos(theta_values) * dtheta # matrix has indexing (out, in): row picks out 'out' entry, column picks out which v0 element # since it doesn't matter what the incidence angle is for Lambertian scattering, all the columns rows be identical! # how many phi entries are there for each theta? n_phis = [ np.sum(angle_vector[:, 1] == theta) for theta in theta_values ] column = [ x for sublist in [[dP[i1] / n] * n for i1, n in enumerate(n_phis)] for x in sublist ] whole_matrix = np.vstack([column] * int(len(angle_vector) / 2)).T # renormalize (rounding errors) whole_matrix_R = whole_matrix / np.sum(whole_matrix, 0) whole_matrix_T = np.zeros_like(whole_matrix_R) whole_matrix = np.vstack([whole_matrix_R, whole_matrix_T]) print(whole_matrix.shape) A_matrix = np.zeros((1, int(len(angle_vector) / 2))) allArray = COO(whole_matrix) absArray = COO(A_matrix) if save: save_npz(savepath_RT, allArray) save_npz(savepath_A, absArray) return allArray
def RT(group, incidence, transmission, surf_name, options, Fr_or_TMM = 0, front_or_rear = 'front', n_absorbing_layers=0, calc_profile=[], only_incidence_angle=False, widths=[], save=True): """Calculates the reflection/transmission and absorption redistribution matrices for an interface using either a previously calculated TMM lookup table or the Fresnel equations. :param group: an RTgroup object containing the surface textures :param incidence: incidence medium :param: transmission: transmission medium :param surf_name: name of the surface (to save matrices) :param options: dictionary of options :param Fr_or_TMM: whether to use the Fresnel equations (0) or a TMM lookup table (1) :param front_or_rear: whether light is incident from the front or rear :param for a structure with multiple interface layers, where a TMM lookuptable is used, the number of layers in the interface :param calc_profile: whether to save the relevant information to calculate the depth-dependent absorption profile :param only_incidence_angle: if True, the ray-tracing will only be performed for the incidence theta and phi specified in the options. :return out_mat: the R/T redistribution matrix at each wavelength, indexed as (wavelength, angle_bin_out, angle_bin_in) :return A_mat: the absorption redistribution matrix (total absorption per layer), indexed as (wavelength, layer_out, angle_bin_in) :return local_angle_mat: only if calc_profile = True. A matrix storing the local incidence angles for rays which were absorbed. This is used to calculate absorption profiles using TMM. """ if Fr_or_TMM > 0 or save: structpath = os.path.join(results_path, options['project_name']) # also need this to get lookup table if save: if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if Fr_or_TMM > 0: savepath_prof = os.path.join(structpath, surf_name + front_or_rear + 'Aprof.npz') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') allArrays = load_npz(savepath_RT) absArrays = load_npz(savepath_A) if Fr_or_TMM > 0 and os.path.isfile(savepath_prof): local_angles = load_npz(savepath_prof) if os.path.isfile(prof_mat_path): prof_int = xr.load_dataset(prof_mat_path) profile = prof_int['profile'] intgr = prof_int['intgr'] return allArrays, absArrays, local_angles, profile, intgr else: return allArrays, absArrays, local_angles else: return allArrays, absArrays else: wavelengths = options['wavelengths'] n_rays = options['n_rays'] nx = options['nx'] ny = options['ny'] n_angles = int(np.ceil(n_rays/(nx*ny))) phi_sym = options['phi_symmetry'] n_theta_bins = options['n_theta_bins'] c_az = options['c_azimuth'] pol = options['pol'] nm_spacing = options['nm_spacing'] if front_or_rear == 'front': side = 1 else: side = -1 if Fr_or_TMM == 1: lookuptable = xr.open_dataset(os.path.join(structpath, surf_name + '.nc')) if front_or_rear == 'rear': lookuptable = lookuptable.assign_coords(side=np.flip(lookuptable.side)) else: lookuptable = None theta_intv, phi_intv, angle_vector = make_angle_vector(n_theta_bins, phi_sym, c_az) if only_incidence_angle: print('Calculating matrix only for incidence theta/phi') if options['theta_in'] == 0: th_in = 0.0001 else: th_in = options['theta_in'] angles_in = angle_vector[:int(len(angle_vector) / 2), :] n_reps = int(np.ceil(n_angles / len(angles_in))) thetas_in = np.tile(th_in, n_reps) print('only inc angle' , thetas_in) n_angles = n_reps if options['phi_in'] == 'all': # get relevant phis phis_in = np.tile(options['phi_in'], n_reps) else: if options['phi_in'] == 0: phis_in = np.tile(0.0001, n_reps) else: phis_in = np.tile(options['phi_in'], n_reps) else: if options['random_angles']: thetas_in = np.random.random(n_angles)*np.pi/2 phis_in = np.random.random(n_angles)*2*np.pi else: angles_in = angle_vector[:int(len(angle_vector)/2),:] if n_angles/len(angles_in) < 1: warn('The number of rays is not sufficient to populate the redistribution matrix!') n_reps = int(np.ceil(n_angles/len(angles_in))) thetas_in = np.tile(angles_in[:,1], n_reps)[:n_angles] phis_in = np.tile(angles_in[:,2], n_reps)[:n_angles] if front_or_rear == 'front': mats = [incidence] else: mats = [transmission] for i1 in range(len(group.materials)): mats.append(group.materials[i1]) if front_or_rear == 'front': mats.append(transmission) else: mats.append(incidence) # list of lists: first in tuple is front incidence if front_or_rear == 'front': surfaces = [x[0] for x in group.textures] else: surfaces = [x[1] for x in group.textures] nks = np.empty((len(mats), len(wavelengths)), dtype=complex) for i1, mat in enumerate(mats): nks[i1] = mat.n(wavelengths) + 1j*mat.k(wavelengths) h = max(surfaces[0].Points[:, 2]) x_lim = surfaces[0].Lx y_lim = surfaces[0].Ly if options['random_ray_position']: xs = np.random.uniform(0, x_lim, nx) ys = np.random.uniform(0, y_lim, ny) else: if options['avoid_edges']: xs = np.linspace(x_lim / 4, x_lim - (x_lim / 4), nx) ys = np.linspace(y_lim / 4, y_lim - (y_lim / 4), ny) else: xs = np.linspace(x_lim/101, x_lim-(x_lim/99), nx) ys = np.linspace(y_lim/100, y_lim-(y_lim/102), ny) print('n_th_in', len(thetas_in), len(xs)) if options['parallel']: allres = Parallel(n_jobs=options['n_jobs'])(delayed(RT_wl) (i1, wavelengths[i1], n_angles, nx, ny, widths, thetas_in, phis_in, h, xs, ys, nks, surfaces, pol, phi_sym, theta_intv, phi_intv, angle_vector, Fr_or_TMM, n_absorbing_layers, lookuptable, calc_profile, nm_spacing, side) for i1 in range(len(wavelengths))) else: allres = [RT_wl(i1, wavelengths[i1], n_angles, nx, ny, widths, thetas_in, phis_in, h, xs, ys, nks, surfaces, pol, phi_sym, theta_intv, phi_intv, angle_vector, Fr_or_TMM, n_absorbing_layers, lookuptable, calc_profile, nm_spacing, side) for i1 in range(len(wavelengths))] allArrays = stack([item[0] for item in allres]) absArrays = stack([item[1] for item in allres]) if save: save_npz(savepath_RT, allArrays) save_npz(savepath_A, absArrays) if Fr_or_TMM > 0: local_angles = stack([item[2] for item in allres]) if save: save_npz(savepath_prof, local_angles) #make_profile_data(options, np.unique(angle_vector[:,1]), int(len(angle_vector) / 2), # front_or_rear, surf_name, n_absorbing_layers, widths) if len(calc_profile) > 0: profile = xr.concat([item[3] for item in allres], 'wl') intgr = xr.concat([item[4] for item in allres], 'wl') intgr.name = 'intgr' profile.name = 'profile' allres = xr.merge([intgr, profile]) if save: allres.to_netcdf(prof_mat_path) save_npz(savepath_prof, local_angles) return allArrays, absArrays, local_angles, profile, intgr else: return allArrays, absArrays, local_angles else: return allArrays, absArrays
import sparse from ase.build import molecule from dscribe.descriptors import SOAP # Let's create SOAP feature vectors for two structures and all positions. If # the output sizes are the same for each structure, a single 3D array is # created. soap = SOAP( species=["C", "H", "O"], periodic=False, rcut=5, nmax=8, lmax=8, average="off", sparse=True ) soap_features = soap.create([molecule("H2O"), molecule("CO2")]) # Save the output to disk and load it back. sparse.save_npz("soap.npz", soap_features) soap_features = sparse.load_npz("soap.npz") # Convert to numpy/scipy formats: dense = soap_features.todense() csr = soap_features[0, :, :].tocsr() csc = soap_features[0, :, :].tocsc()
def matrix(self, lastfm, tags=None, dim=3, save_to=None): ''' Computes a n-dimensional matrix where the (i_1, ... ,i_n)-th entry contains the number of tracks having all the i_1-th, ..., i_n-th tags (where the i's are the indexes in self.m_tags). Notes ----- To optimize performance, values are computed only with indexes in increasing order (which means, we only compute the number of tracks having tag-0 and tag-1, not vice-versa). This is something to keep in mind when indexing the matrix. To optimize memory, the matrix is saved in sparse format. DOK is the preferred sparse format for building and indexing, while COO is the preferred sparse format to perform mathematical operations). The dimension of the matrix captures the kind of queries which you will be able to perform. A matrix of dim=2 on tracks=['rock', 'pop', 'hip-hop'] will capture how many tracks have tags rock and pop, or pop and hip-hop, but not rock, pop and hip-hop at the same time. A matrix of dim=len(tags) will fully describe the database (or the subset of the database having the given tags). A matrix of dim>len(tags) will be rather pointless (but we won't prevent you from doing it). Parameters ---------- lastfm: LastFm, LastFm2Pandas Instance of tags database. Using LastFm2Pandas is strongly recommended here. tags: list List of tags to use. If None, all the tags will be used. dim: int The dimension of the matrix. save_to: str Filename or full path of the .npz file to save matrix and matrix tags. Use to load_from in the future. ''' # initialize matrix tags if tags is None: tags = lastfm.get_tags() else: tags = [tag for tag in tags if tag in lastfm.get_tags() ] # possibly purge inexistent tags # initialize matrix matrix = sparse.DOK( (len(tags), ) * dim, dtype=np.int32 ) # sparse dict-of-keys matrix (for easy creation, awful for calculations) # compute total number of steps to comatplotlibetion (see http://www.iosrjournals.org/iosr-jm/papers/Vol8-issue3/A0830110.pdf) n_steps = crazysum(n=len(tags), s=3, k=dim - 1) # check whether a progress bar is needed verbose = n_steps > 100 if verbose: progbar = Progbar(n_steps) # instantiate progress bar def count_intersect_tags(tags): tids_list = [lastfm.with_tag(tag) for tag in tags] tids_list.sort(key=len, reverse=True) tids = set( tids_list.pop() ) # start with shortest list of tids to improve performance; convert to set to be able to intersect for _ in range(len(tids_list)): tids = tids.intersection(tids_list.pop( )) # intersections performed from shortest list to longest return len(tids) # how many tids have all tags def count_intersect_tags_recursive( tags_idxs, dim ): # recursively iterate count_intersect_tags dim times; avoid repetitions such as 'rock AND pop AND folk' vs. 'rock AND folk AND pop' vs. 'folk AND pop AND rock' if dim >= 1: for i in range(tags_idxs[-1] + 1): count_intersect_tags_recursive(tags_idxs + (i, ), dim - 1) else: matrix[tags_idxs] = count_intersect_tags( np.take(tags, tags_idxs)) # add count to sparse matrix if verbose: progbar.add(1) # instantiate recursive loop for i in range(len(tags)): count_intersect_tags_recursive((i, ), dim - 1) matrix = matrix.to_coo() # convert to coordinate matrix if save_to is not None: # save matrix sparse.save_npz(save_to, matrix.to_coo( )) # default to compressed format (i.e. sparse format) # save matrix tags in serialized format with open(os.path.splitext(save_to)[0] + '.nfo', 'wb') as f: pickle.dump(tags, f) return matrix, tags
def RCWA(structure, size, orders, options, incidence, transmission, only_incidence_angle=False, front_or_rear='front', surf_name='', detail_layer=False, save=True): """ Calculates the reflected, absorbed and transmitted intensity of the structure for the wavelengths and angles defined using an RCWA method implemented using the S4 package. :param structure: A solcore Structure object with layers and materials or a OptiStack object. :param size: list with 2 entries, size of the unit cell (right now, can only be rectangular :param orders: number of orders to retain in the RCWA calculations. :param wavelength: Wavelengths (in nm) in which calculate the data. :param theta: polar incidence angle (in degrees) of the incident light. Default: 0 (normal incidence) :param phi: azimuthal incidence angle in degrees. Default: 0 :param pol: Polarisation of the light: 's', 'p' or 'u'. Default: 'u' (unpolarised). :param transmission: semi-infinite transmission medium :return: A dictionary with the R, A and T at the specified wavelengths and angle. """ # TODO: when non-zero incidence angle, not binned correctly in matrix (just goes in theta = 0) # TODO: when doing unpolarized, why not just set s=0.5 p=0.5 in S4? (Maybe needs to be normalised differently). Also don't know if this is faster, # or if internally it will still do s & p separately # TODO: if incidence angle is zero, s and p polarization are the same so no need to do both structpath = os.path.join(results_path, options['project_name']) if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') full_mat = load_npz(savepath_RT) A_mat = load_npz(savepath_A) else: wavelengths = options['wavelengths'] if front_or_rear == 'front': layers = structure trns = transmission inc = incidence else: layers = structure[::-1] trns = incidence inc = transmission # write a separate function that makes the OptiStack structure into an S4 object, defined materials etc. geom_list = [layer.geometry for layer in structure] geom_list.insert(0, {}) # incidence medium geom_list.append({}) # transmission medium ## Materials for the shapes need to be defined before you can do .SetRegion shape_mats, geom_list_str = necessary_materials(geom_list) shapes_oc = np.zeros((len(wavelengths), len(shape_mats)), dtype=complex) for i1, x in enumerate(shape_mats): shapes_oc[:, i1] = (x.n(wavelengths) + 1j * x.k(wavelengths))**2 stack_OS = OptiStack(layers, no_back_reflection=False) widths = stack_OS.get_widths() layers_oc = np.zeros((len(wavelengths), len(structure) + 2), dtype=complex) layers_oc[:, 0] = (inc.n(wavelengths))**2 #+ 1j*inc.k(wavelengths))**2 layers_oc[:, -1] = (trns.n(wavelengths) + 1j * trns.k(wavelengths))**2 for i1, x in enumerate(layers): layers_oc[:, i1 + 1] = (x.material.n(wavelengths) + 1j * x.material.k(wavelengths))**2 shapes_names = [str(x) for x in shape_mats] #nm_spacing = options['nm_spacing'] phi_sym = options['phi_symmetry'] n_theta_bins = options['n_theta_bins'] c_az = options['c_azimuth'] pol = options['pol'] # RCWA options rcwa_options = dict(LatticeTruncation='Circular', DiscretizedEpsilon=False, DiscretizationResolution=8, PolarizationDecomposition=False, PolarizationBasis='Default', LanczosSmoothing=False, SubpixelSmoothing=False, ConserveMemory=False, WeismannFormulation=False) user_options = options[ 'rcwa_options'] if 'rcwa_options' in options.keys() else {} rcwa_options.update(user_options) print(rcwa_options) theta_intv, phi_intv, angle_vector = make_angle_vector( n_theta_bins, phi_sym, c_az) if only_incidence_angle: thetas_in = np.array([options['theta_in']]) phis_in = np.array([options['phi_in']]) else: angles_in = angle_vector[:int(len(angle_vector) / 2), :] thetas_in = angles_in[:, 1] phis_in = angles_in[:, 2] # angle in degrees thetas_in = thetas_in * 180 / np.pi phis_in = phis_in * 180 / np.pi # initialise_S has to happen inside parallel job (get Pickle errors otherwise); # just pass relevant optical constants for each wavelength, like for RT angle_vector_0 = angle_vector[:, 0] if front_or_rear == "front": side = 1 else: side = -1 if options['parallel']: allres = Parallel(n_jobs=options['n_jobs'])(delayed(RCWA_wl)( wavelengths[i1] * 1e9, geom_list, layers_oc[i1], shapes_oc[i1], shapes_names, pol, thetas_in, phis_in, widths, size, orders, phi_sym, theta_intv, phi_intv, angle_vector_0, rcwa_options, detail_layer, side) for i1 in range(len(wavelengths))) else: allres = [ RCWA_wl(wavelengths[i1] * 1e9, geom_list, layers_oc[i1], shapes_oc[i1], shapes_names, pol, thetas_in, phis_in, widths, size, orders, phi_sym, theta_intv, phi_intv, angle_vector_0, rcwa_options, detail_layer, side) for i1 in range(len(wavelengths)) ] R = np.stack([item[0] for item in allres]) T = np.stack([item[1] for item in allres]) A_mat = np.stack([item[2] for item in allres]) full_mat = stack([item[3] for item in allres]) int_mat = stack([item[4] for item in allres]) #T_mat = np.stack([item[4] for item in allres]) #full_mat = np.hstack((R_mat, T_mat)) #full_mat = COO(full_mat) A_mat = COO(A_mat) if save: save_npz(savepath_RT, full_mat) save_npz(savepath_A, A_mat) #R_pfbo = np.stack([item[3] for item in allres]) #T_pfbo = np.stack([item[4] for item in allres]) #phi_rt = np.stack([item[5] for item in allres]) #theta_r = np.stack([item[6] for item in allres]) #theta_t = np.stack([item[7] for item in allres]) #R_pfbo_2 = np.stack([item[8] for item in allres]) #return {'R': R, 'T':T, 'A_layer': A_mat, 'full_mat': full_mat, 'int_mat': int_mat}#'R_pfbo': R_pfbo, 'T_pfbo': T_pfbo, 'phi_rt': phi_rt, 'theta_r': theta_r, 'theta_t': theta_t}#, 'R_pfbo_2': R_pfbo_2} return full_mat, A_mat # , R, T
def save_sparse(self, data, file_prefix='sparse_array_', **kwargs): assert isinstance( data, sparse.COO), 'Expected sparse.COO data type, got: {}'.format( type(data)) sparse.save_npz(self.path + file_prefix + self.filename[:-4] + '.npz', data)
def TMM(layers, incidence, transmission, surf_name, options, coherent=True, coherency_list=None, prof_layers=[], front_or_rear='front', save=True): """Function which takes a layer stack and creates an angular redistribution matrix. :param layers: A list with one or more layers. :param transmission: transmission medium :param incidence: incidence medium :param surf_name: name of the surface (to save the matrices generated. :param options: a list of options :param coherent: whether or not the layer stack is coherent. If None, it is assumed to be fully coherent :param coherency: a list with the same number of entries as the layers, either 'c' for a coherent layer or 'i' for an incoherent layer :param prof_layers: layers for which the absorption profile should be calculated (if None, do not calculate absorption profile at all) :param front_or_rear: a string, either 'front' or 'rear'; front incidence on the stack, from the incidence medium, or rear incidence on the stack, from the transmission medium. :return full_mat: R and T redistribution matrix :return A_mat: matrix describing absorption per layer """ def make_matrix_wl(wl): # binning into matrix, including phi RT_mat = np.zeros((len(theta_bins_in)*2, len(theta_bins_in))) A_mat = np.zeros((n_layers, len(theta_bins_in))) for i1 in range(len(theta_bins_in)): theta = theta_lookup[i1]#angle_vector[i1, 1] data = allres.loc[dict(angle=theta, wl=wl)] R_prob = np.real(data['R'].data.item(0)) T_prob = np.real(data['T'].data.item(0)) Alayer_prob = np.real(data['Alayer'].data) phi_out = phis_out[i1] #print(R_prob, T_prob) # reflection phi_int = phi_intv[theta_bins_in[i1]] phi_ind = np.digitize(phi_out, phi_int, right=True) - 1 bin_out_r = np.argmin(abs(angle_vector[:, 0] - theta_bins_in[i1])) + phi_ind #print(bin_out_r, i1+offset) RT_mat[bin_out_r, i1] = R_prob #print(R_prob) # transmission theta_t = np.abs(-np.arcsin((inc.n(wl * 1e-9) / trns.n(wl * 1e-9)) * np.sin(theta_lookup[i1])) + quadrant) #print('angle in, transmitted', angle_vector_th[i1], theta_t) # theta switches half-plane (th < 90 -> th >90 if ~np.isnan(theta_t): theta_out_bin = np.digitize(theta_t, theta_intv, right=True) - 1 phi_int = phi_intv[theta_out_bin] phi_ind = np.digitize(phi_out, phi_int, right=True) - 1 bin_out_t = np.argmin(abs(angle_vector[:, 0] - theta_out_bin)) + phi_ind RT_mat[bin_out_t, i1] = T_prob #print(bin_out_t, i1+offset) # absorption A_mat[:, i1] = Alayer_prob fullmat = COO(RT_mat) A_mat = COO(A_mat) return fullmat, A_mat structpath = os.path.join(results_path, options['project_name']) if not os.path.isdir(structpath): os.mkdir(structpath) savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz') savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz') prof_mat_path = os.path.join(results_path, options['project_name'], surf_name + front_or_rear + 'profmat.nc') if os.path.isfile(savepath_RT) and save: print('Existing angular redistribution matrices found') fullmat = load_npz(savepath_RT) A_mat = load_npz(savepath_A) if len(prof_layers) > 0: profile = xr.load_dataarray(prof_mat_path) return fullmat, A_mat, profile else: wavelengths = options['wavelengths']*1e9 # convert to nm theta_intv, phi_intv, angle_vector = make_angle_vector(options['n_theta_bins'], options['phi_symmetry'], options['c_azimuth']) angles_in = angle_vector[:int(len(angle_vector) / 2), :] thetas = np.unique(angles_in[:, 1]) n_angles = len(thetas) n_layers = len(layers) if front_or_rear == 'front': optlayers = OptiStack(layers, substrate=transmission, incidence=incidence) trns = transmission inc = incidence else: optlayers = OptiStack(layers[::-1], substrate=incidence, incidence=transmission) trns = incidence inc = transmission if len(prof_layers) > 0: profile = True z_limit = np.sum(np.array(optlayers.widths)) full_dist = np.arange(0, z_limit, options['nm_spacing']) layer_start = np.insert(np.cumsum(np.insert(optlayers.widths, 0, 0)), 0, 0) layer_end = np.cumsum(np.insert(optlayers.widths, 0, 0)) dist = [] for l in prof_layers: dist = np.hstack((dist, full_dist[np.all((full_dist >= layer_start[l], full_dist < layer_end[l]), 0)])) else: profile = False if options['pol'] == 'u': pols = ['s', 'p'] else: pols = [options['pol']] R = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='R') T = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='T') Alayer = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), n_layers)), dims=['pol', 'angle', 'wl', 'layer'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas, 'layer': range(1, n_layers + 1)}, name='Alayer') theta_t = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)), dims=['pol', 'wl', 'angle'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas}, name='theta_t') if profile: Aprof = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), len(dist))), dims=['pol', 'angle', 'wl', 'z'], coords={'pol': pols, 'wl': wavelengths, 'angle': thetas, 'z': dist}, name='Aprof') R_loop = np.empty((len(wavelengths), n_angles)) T_loop = np.empty((len(wavelengths), n_angles)) Alayer_loop = np.empty((n_angles, len(wavelengths), n_layers), dtype=np.complex_) th_t_loop = np.empty((len(wavelengths), n_angles)) if profile: Aprof_loop = np.empty((n_angles, len(wavelengths), len(dist))) tmm_struct = tmm_structure(optlayers, coherent=coherent, coherency_list=coherency_list, no_back_reflection=False) for i2, pol in enumerate(pols): for i3, theta in enumerate(thetas): res = tmm_struct.calculate(wavelengths, angle=theta, pol=pol, profile=profile, layers=prof_layers, nm_spacing = options['nm_spacing']) R_loop[:, i3] = np.real(res['R']) T_loop[:, i3] = np.real(res['T']) Alayer_loop[i3, :, :] = np.real(res['A_per_layer'].T) if profile: Aprof_loop[i3, :, :] = res['profile'] # sometimes get very small negative values (like -1e-20) R_loop[R_loop < 0] = 0 T_loop[T_loop < 0] = 0 Alayer_loop[Alayer_loop < 0] = 0 if front_or_rear == 'rear': Alayer_loop = np.flip(Alayer_loop, axis=2) print('flipping') R.loc[dict(pol=pol)] = R_loop T.loc[dict(pol=pol)] = T_loop Alayer.loc[dict(pol=pol)] = Alayer_loop theta_t.loc[dict(pol=pol)] = th_t_loop if profile: Aprof.loc[dict(pol=pol)] = Aprof_loop Aprof.transpose('pol', 'wl', 'angle', 'z') Alayer = Alayer.transpose('pol', 'wl', 'angle', 'layer') if profile: allres = xr.merge([R, T, Alayer, Aprof]) else: allres = xr.merge([R, T, Alayer]) if options['pol'] == 'u': allres = allres.reduce(np.mean, 'pol').assign_coords(pol='u').expand_dims('pol') # populate matrices if front_or_rear == "front": angle_vector_th = angle_vector[:int(len(angle_vector)/2),1] angle_vector_phi = angle_vector[:int(len(angle_vector)/2),2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) theta_lookup = angles_in[:,1] quadrant = np.pi else: angle_vector_th = angle_vector[int(len(angle_vector) / 2):, 1] angle_vector_phi = angle_vector[int(len(angle_vector) / 2):, 2] phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry']) theta_lookup = angles_in[:,1][::-1] quadrant = 0 phis_out[phis_out == 0] = 1e-10 theta_bins_in = np.digitize(angle_vector_th, theta_intv, right=True) -1 print(theta_bins_in) mats = [make_matrix_wl(wl) for wl in wavelengths] fullmat = stack([item[0] for item in mats]) A_mat = stack([item[1] for item in mats]) if save: save_npz(savepath_RT, fullmat) save_npz(savepath_A, A_mat) return fullmat, A_mat #, allres