def propagate_distribution(pomdp, D_ux, u_dim=None, x_dim=None):
'''evolve input/state distribution D_ux into output distribution D_xz
D_xz(x', z) = \sum_{x', u) P(X+ = x, Z = z | U = u X = x' ) D_ux(u, x')
'''
if u_dim is None:
u_dim = tuple(range(len(pomdp.M)))
if x_dim is None:
x_dim = (len(pomdp.M), )
if len(u_dim) != len(pomdp.M) or len(x_dim) != 1:
raise Exception('dimension problem')
if len(D_ux.shape) <= max(u_dim + x_dim) or len(
set(u_dim + x_dim)) < len(u_dim + x_dim) or sum(D_ux.data) != 1:
raise Exception('D_ux not a valid distribution')
T_uxXz = sparse.stack([sparse.stack([sparse.COO(pomdp.Tuz(m_tuple, z))
for z in range(pomdp.O)],
axis=-1)
for m_tuple in pomdp.m_tuple_iter()]) \
.reshape(pomdp.M + (pomdp.N, pomdp.N, pomdp.O))
T_zx = sparse.tensordot(D_ux,
T_uxXz,
axes=(u_dim + x_dim, range(len(pomdp.M) + 1)))
return sparse.COO(T_zx)
def __init__(self, dim, basis=None):
if basis is None:
self.dim = dim
self.basis = sparse.stack(gm.get_basis(dim, sparse=True))
else:
self.dim = basis[0].shape[0]
self.basis = COO.from_numpy(np.array(basis))
# Diagonal metric (since we're working with what are assumed to be
# orthogonal but not necessarily normalized basis vectors)
self.sq_norms = COO.from_numpy(
sparse.tensordot(
self.basis, self.basis,
([1, 2], [2, 1])).to_scipy_sparse().diagonal())
# Diagonal inverse metric
sq_norms_inv = COO.from_numpy(1 / self.sq_norms.todense())
# Dual basis obtained from the original basis by the inverse metric
self.dual = self.basis * sq_norms_inv[:,None,None]
# Structure coefficients for the Lie algebra showing how to represent a
# product of two basis elements as a complex-linear combination of basis
# elements
self.struct = sparse.tensordot(sparse.tensordot(self.basis, self.basis,
([2], [1])),
self.dual, ([1, 3], [2, 1]))
if isinstance(self.struct, np.ndarray):
# Sometimes sparse.tensordot returns numpy arrays. We want to force
# it to be sparse, since sparse.tensordot fails when passed two
# numpy arrays.
self.struct = COO.from_numpy(self.struct)
def test_stack(sd):
shape, axis, xx, yy, zz = sd
x = xx.todense()
y = yy.todense()
z = zz.todense()
assert_eq(np.stack([x, y, z], axis=axis), sparse.stack([xx, yy, zz], axis=axis))
def fit(self, x, y_multiclass, kernel=poly(1), C=0.001):
y_multiclass=y_multiclass.reshape(-1).astype(np.float64)
self.x = sparse.COO(x.astype(np.float64))
self.m = self.x.shape[0]
self.y_multiclass = y_multiclass
self.kernel = kernel
self.C = C
ys = [sparse.COO(self.cast(y_multiclass, k)) for k in range(self.n_svm)]
self.y_matrix = sparse.stack(ys,0)
del ys
for k in range(self.n_svm):
print("training ",k,"th SVM in ",self.n_svm)
y = self.y_matrix[k, :].reshape((-1,1))
yx = y * self.x
G = kernel(yx, yx) # Gram matrix
compensate = (sparse.eye(self.m)*1e-7).astype(np.float64)
G = (G + compensate)
objective = cp.Maximize(cp.sum(self.a[k])-(1/2)*cp.quad_form(self.a[k], G.tocsr()))
if not objective.is_dcp():
print("Not solvable!")
assert objective.is_dcp()
constraints = [self.a[k] <= C, cp.sum(cp.multiply(self.a[k],y.todense())) == 0] # box constraint
prob = cp.Problem(objective, constraints)
result = prob.solve()
x_pos = x[y.todense()[:,0]==1,:]
x_neg = x[y.todense()[:,0]==-1,:]
b_min = -np.min(self.wTx(k,x_pos)) if x_pos.shape[0]!=0 else 0
b_max = -np.max(self.wTx(k,x_neg)) if x_neg.shape[0]!=0 else 0
self.b[k,0] = (1/2)*(b_min + b_max)
self.a_matrix = np.stack([i.value.reshape(-1) for i in self.a],0)
self.a_matrix = sparse.COO(self.a_matrix)
def test_stack(shape, axis):
xx = sparse.random(shape, density=0.5, format="gcxs")
x = xx.todense()
yy = sparse.random(shape, density=0.5, format="gcxs")
y = yy.todense()
zz = sparse.random(shape, density=0.5, format="gcxs")
z = zz.todense()
assert_eq(np.stack([x, y, z], axis=axis), sparse.stack([xx, yy, zz], axis=axis))
def radial_bins(centerX, centerY, imageSizeX, imageSizeY,
radius=None, radius_inner=0, n_bins=None, normalize=False, use_sparse=None, dtype=None):
'''
Generate antialiased rings
'''
if radius is None:
radius = bounding_radius(centerX, centerY, imageSizeX, imageSizeY)
if n_bins is None:
n_bins = int(np.round(radius - radius_inner))
r, phi = polar_map(centerX, centerY, imageSizeX, imageSizeY)
r = r.flatten()
width = (radius - radius_inner) / n_bins
bin_area = np.pi * (radius**2 - (radius - width)**2)
if use_sparse is None:
use_sparse = bin_area / (imageSizeX * imageSizeY) < 0.1
if use_sparse:
jjs = np.arange(len(r), dtype=np.int64)
slices = []
for r0 in np.linspace(radius_inner, radius - width, n_bins) + width/2:
diff = np.abs(r - r0)
# The "0.5" ensures that the bins overlap and sum up to exactly 1
vals = np.maximum(0, np.minimum(1, width/2 + 0.5 - diff))
if use_sparse:
select = vals != 0
vals = vals[select]
if normalize: # Make sure each bin has a sum of 1
s = vals.sum()
if not np.isclose(s, 0):
vals /= s
slices.append(sparse.COO(shape=len(r), data=vals.astype(dtype), coords=(jjs[select],)))
else:
if normalize: # Make sure each bin has a sum of 1
s = vals.sum()
if not np.isclose(s, 0):
vals /= s
slices.append(vals.reshape((imageSizeY, imageSizeX)).astype(dtype))
# Patch a singularity at the center
if radius_inner < 0.5:
yy = int(np.round(centerY))
xx = int(np.round(centerX))
if use_sparse:
index = yy * imageSizeX + xx
diff = 1 - slices[0][index] - radius_inner
patch = sparse.COO(shape=len(r), data=[diff], coords=[index])
slices[0] += patch
else:
slices[0][yy, xx] = 1 - radius_inner
if use_sparse:
return sparse.stack(slices).reshape((-1, imageSizeY, imageSizeX))
else:
return np.stack(slices)
def convert_ndarray(value):
if isinstance(value, sparse.SparseArray):
return value
if isinstance(value, np.ndarray):
return sparse.COO(value)
try:
return sparse.COO(np.asarray(value))
except RuntimeError:
return sparse.stack([convert_ndarray(v) for v in value])
def make_D(alphas, thick, thetas):
"""
Makes the bulk absorption vector for the bulk material
:param alphas: absorption coefficient (m^{-1})
:param thick: thickness of the slab in m
:param thetas: incident thetas in angle_vector (second column)
:return:
"""
#print(alphas, abs(np.cos(thetas[None, :])))
diag = np.exp(-alphas[:, None] * thick / abs(np.cos(thetas[None, :])))
#print(diag)
D_1 = stack([COO(np.diag(x)) for x in diag])
return D_1
def test_upcast():
a = sparse.random((50, 50, 50),
density=0.1,
format="coo",
idx_dtype=np.uint8)
b = a.asformat("gcxs")
assert b.indices.dtype == np.uint16
a = sparse.random((8, 7, 6),
density=0.5,
format="gcxs",
idx_dtype=np.uint8)
b = sparse.random((6, 6, 6),
density=0.8,
format="gcxs",
idx_dtype=np.uint8)
assert sparse.concatenate((a, a)).indptr.dtype == np.uint16
assert sparse.stack((b, b)).indptr.dtype == np.uint16
def __init__(self,
langs,
order=1,
prediction_cutoff=.9,
base_path='./models/{}_o{}.pkl',
class_dict_path='./unique_bytes.npy'):
self.langs = langs
self.order = order
self.prediction_cutoff = prediction_cutoff
#Encode unique bytes
unique_bytes = np.load(class_dict_path)
byte_classes = dict()
for i, byt in enumerate(unique_bytes):
#correct off-by-one issue
byt = int(byt)
byt -= 1
byt = str(byt)
byte_classes[byt] = i
#Add 'other' class in case unexpected byte encountered.
byte_classes['other'] = i + 1
self.class_dict = byte_classes
print('loading models...')
#order of models matches order of languages
self.models = list()
for lang in langs:
path = base_path.format(lang, self.order)
try:
with open(path, 'rb') as f:
model = pickle.load(f)
except FileNotFoundError:
raise FileNotFoundError('No model found for language: ', lang)
self.models.append(model)
self.models = sparse.stack(self.models, axis=0)
print('Done!')
def build(self,
cohort,
cache_file='/tmp/store.csv',
nontemporal_cache_file='/tmp/store_ntmp.csv',
from_cached=False):
joined_sql = "{} order by {} asc".format(
" union all ".join(
f._sql_raw.format(cdm_schema=config.OMOP_CDM_SCHEMA,
cohort_table='{}.{}'.format(
cohort._schema_name,
cohort._cohort_table_name))
for f in self._temporal_features),
",".join([
self.unique_id_col, ## Order by unique_id
self.time_col,
self.feature_col
]))
if not from_cached:
copy_sql = """
copy
({query})
to
stdout
with
csv {head}
""".format(query=joined_sql, head="HEADER")
t = time.time()
conn = self._db.engine.raw_connection()
cur = conn.cursor()
store = open(cache_file, 'wb')
cur.copy_expert(copy_sql, store)
store.seek(0)
print(
'Data loaded to buffer in {0:.2f} seconds'.format(time.time() -
t))
t = time.time()
store = open(cache_file, 'rb')
self.concepts = set()
self.times = set()
self.seen_ids = set()
chunksize = int(2e6)
for chunk in pd.read_csv(store, chunksize=chunksize):
self.concepts = self.concepts.union(
set(chunk[self.feature_col].unique()))
self.times = self.times.union(set(chunk[self.time_col].unique()))
self.seen_ids = self.seen_ids.union(
set(chunk[self.unique_id_col].unique()))
self.times = sorted(list(self.times))
self.concepts = sorted(list(self.concepts))
self.seen_ids = sorted(list(self.seen_ids))
print('Got Unique Concepts and Timestamps in {0:.2f} seconds'.format(
time.time() - t))
t = time.time()
store.seek(0)
self.ids = cohort._cohort[self.unique_id_col].unique()
self.id_map = {
i: example_id
for i, example_id in enumerate(self.seen_ids)
}
self.id_map_rev = {
example_id: i
for i, example_id in enumerate(self.seen_ids)
}
self.concept_map = {
i: concept_name
for i, concept_name in enumerate(self.concepts)
}
self.concept_map_rev = {
concept_name: i
for i, concept_name in enumerate(self.concepts)
}
self.time_map = {i: t for i, t in enumerate(self.times)}
self.time_map_rev = {t: i for i, t in enumerate(self.times)}
print('Created Index Mappings in {0:.2f} seconds'.format(time.time() -
t))
t = time.time()
last = None
spm_stored = None
spm_arr = []
self.recorded_ids = set()
for chunk_num, chunk in enumerate(
pd.read_csv(store, chunksize=chunksize)):
first = chunk.iloc[0][self.unique_id_col]
vals = chunk[self.unique_id_col].unique()
indices = np.searchsorted(chunk[self.unique_id_col], vals)
self.recorded_ids = self.recorded_ids.union(set(vals))
chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply(
self.concept_map_rev.get)
chunk.loc[:, self.time_col] = chunk[self.time_col].apply(
self.time_map_rev.get)
df_split = [
chunk.iloc[indices[i]:indices[i + 1]]
for i in range(len(indices) - 1)
] + [chunk.iloc[indices[-1]:]]
def gen_sparr(sub_df):
sparr = coo_matrix(
(np.ones(len(sub_df[self.feature_col])),
(sub_df[self.feature_col], sub_df[self.time_col])),
shape=(len(self.concepts), len(self.times)))
return sparr
spm_local = [gen_sparr(s) for s in df_split]
if first == last:
spm_local[0] += spm_stored
else:
if spm_stored is not None:
spm_arr.append(spm_stored)
spm_arr += spm_local[:-1]
spm_stored = spm_local[-1]
# last = chunk.iloc[-1][sep_col]
last = chunk.iloc[-1][self.unique_id_col]
spm_arr.append(spm_stored)
print(len(spm_arr))
self._spm_arr = sparse.stack(
[sparse.COO.from_scipy_sparse(m) for m in spm_arr], 2)
print('Generated Sparse Representation of Data in {0:.2f} seconds'.
format(time.time() - t))
# Build nontemporal feature matrix
if len(self._nontemporal_features) > 0:
joined_sql = "{} order by {} asc".format(
" union all ".join(
f._sql_raw.format(cdm_schema=config.OMOP_CDM_SCHEMA,
cohort_table='{}.{}'.format(
cohort._schema_name,
cohort._cohort_table_name))
for f in self._nontemporal_features),
",".join([
self.unique_id_col, ## Order by unique_id
self.feature_col
]))
if not from_cached:
copy_sql = """
copy
({query})
to
stdout
with
csv {head}
""".format(query=joined_sql, head="HEADER")
t = time.time()
conn = self._db.engine.raw_connection()
cur = conn.cursor()
store = open(nontemporal_cache_file, 'wb')
cur.copy_expert(copy_sql, store)
store.seek(0)
print('Nontemporal data loaded to buffer in {0:.2f} seconds'.
format(time.time() - t))
t = time.time()
store = open(nontemporal_cache_file, 'rb')
self.ntmp_concepts = set()
self.ntmp_seen_ids = set()
chunksize = int(2e6)
for chunk in pd.read_csv(store, chunksize=chunksize):
self.ntmp_seen_ids = self.ntmp_seen_ids.union(
set(chunk[self.unique_id_col].unique()))
self.ntmp_concepts = self.ntmp_concepts.union(
set(chunk[self.feature_col].unique()))
self.ntmp_concepts = sorted(list(self.ntmp_concepts))
print('Got Unique Nontemporal Concepts in {0:.2f} seconds'.format(
time.time() - t))
t = time.time()
store.seek(0)
self.ntmp_id_map = {
i: example_id
for i, example_id in enumerate(self.ntmp_seen_ids)
}
self.ntmp_id_map_rev = {
example_id: i
for i, example_id in enumerate(self.ntmp_seen_ids)
}
self.ntmp_concept_map = {
i: concept_name
for i, concept_name in enumerate(self.ntmp_concepts)
}
self.ntmp_concept_map_rev = {
concept_name: i
for i, concept_name in enumerate(self.ntmp_concepts)
}
print(
'Created Nontemporal Index Mappings in {0:.2f} seconds'.format(
time.time() - t))
t = time.time()
ntmp_data = []
ntmp_unique_id = []
ntmp_feature_id = []
for chunk_num, chunk in enumerate(
pd.read_csv(store, chunksize=chunksize)):
chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply(
self.ntmp_concept_map_rev.get)
ntmp_data.append(chunk[self.ntmp_val_col])
ntmp_unique_id.append(chunk[self.unique_id_col])
ntmp_feature_id.append(chunk[self.feature_col])
ntmp_data = np.concatenate(ntmp_data)
ntmp_unique_id = np.concatenate(ntmp_unique_id)
ntmp_feature_id = np.concatenate(ntmp_feature_id)
sparr = coo_matrix((ntmp_data, (ntmp_unique_id, ntmp_feature_id)),
shape=(len(self.ids), len(self.ntmp_concepts)))
self._ntmp_spm = sparr
print(
'Generated Sparse Representation of Nontemporal Data in {0:.2f} seconds'
.format(time.time() - t))
def st(*XS):
return sp.stack(XS, axis=axis) if iscoo(XS[0]) else np.stack(XS,
axis=axis)
def test_upcast_2(a, b):
assert sparse.concatenate((a, a)).indptr.dtype == np.uint16
assert sparse.stack((b, b)).indptr.dtype == np.uint16
def RCWA(structure,
size,
orders,
options,
incidence,
transmission,
only_incidence_angle=False,
front_or_rear='front',
surf_name='',
detail_layer=False,
save=True):
""" Calculates the reflected, absorbed and transmitted intensity of the structure for the wavelengths and angles
defined using an RCWA method implemented using the S4 package.
:param structure: A solcore Structure object with layers and materials or a OptiStack object.
:param size: list with 2 entries, size of the unit cell (right now, can only be rectangular
:param orders: number of orders to retain in the RCWA calculations.
:param wavelength: Wavelengths (in nm) in which calculate the data.
:param theta: polar incidence angle (in degrees) of the incident light. Default: 0 (normal incidence)
:param phi: azimuthal incidence angle in degrees. Default: 0
:param pol: Polarisation of the light: 's', 'p' or 'u'. Default: 'u' (unpolarised).
:param transmission: semi-infinite transmission medium
:return: A dictionary with the R, A and T at the specified wavelengths and angle.
"""
# TODO: when non-zero incidence angle, not binned correctly in matrix (just goes in theta = 0)
# TODO: when doing unpolarized, why not just set s=0.5 p=0.5 in S4? (Maybe needs to be normalised differently). Also don't know if this is faster,
# or if internally it will still do s & p separately
# TODO: if incidence angle is zero, s and p polarization are the same so no need to do both
structpath = os.path.join(results_path, options['project_name'])
if not os.path.isdir(structpath):
os.mkdir(structpath)
savepath_RT = os.path.join(structpath,
surf_name + front_or_rear + 'RT.npz')
savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz')
prof_mat_path = os.path.join(results_path, options['project_name'],
surf_name + front_or_rear + 'profmat.nc')
if os.path.isfile(savepath_RT) and save:
print('Existing angular redistribution matrices found')
full_mat = load_npz(savepath_RT)
A_mat = load_npz(savepath_A)
else:
wavelengths = options['wavelengths']
if front_or_rear == 'front':
layers = structure
trns = transmission
inc = incidence
else:
layers = structure[::-1]
trns = incidence
inc = transmission
# write a separate function that makes the OptiStack structure into an S4 object, defined materials etc.
geom_list = [layer.geometry for layer in structure]
geom_list.insert(0, {}) # incidence medium
geom_list.append({}) # transmission medium
## Materials for the shapes need to be defined before you can do .SetRegion
shape_mats, geom_list_str = necessary_materials(geom_list)
shapes_oc = np.zeros((len(wavelengths), len(shape_mats)),
dtype=complex)
for i1, x in enumerate(shape_mats):
shapes_oc[:, i1] = (x.n(wavelengths) + 1j * x.k(wavelengths))**2
stack_OS = OptiStack(layers, no_back_reflection=False)
widths = stack_OS.get_widths()
layers_oc = np.zeros((len(wavelengths), len(structure) + 2),
dtype=complex)
layers_oc[:, 0] = (inc.n(wavelengths))**2 #+ 1j*inc.k(wavelengths))**2
layers_oc[:, -1] = (trns.n(wavelengths) + 1j * trns.k(wavelengths))**2
for i1, x in enumerate(layers):
layers_oc[:, i1 + 1] = (x.material.n(wavelengths) +
1j * x.material.k(wavelengths))**2
shapes_names = [str(x) for x in shape_mats]
#nm_spacing = options['nm_spacing']
phi_sym = options['phi_symmetry']
n_theta_bins = options['n_theta_bins']
c_az = options['c_azimuth']
pol = options['pol']
# RCWA options
rcwa_options = dict(LatticeTruncation='Circular',
DiscretizedEpsilon=False,
DiscretizationResolution=8,
PolarizationDecomposition=False,
PolarizationBasis='Default',
LanczosSmoothing=False,
SubpixelSmoothing=False,
ConserveMemory=False,
WeismannFormulation=False)
user_options = options[
'rcwa_options'] if 'rcwa_options' in options.keys() else {}
rcwa_options.update(user_options)
print(rcwa_options)
theta_intv, phi_intv, angle_vector = make_angle_vector(
n_theta_bins, phi_sym, c_az)
if only_incidence_angle:
thetas_in = np.array([options['theta_in']])
phis_in = np.array([options['phi_in']])
else:
angles_in = angle_vector[:int(len(angle_vector) / 2), :]
thetas_in = angles_in[:, 1]
phis_in = angles_in[:, 2]
# angle in degrees
thetas_in = thetas_in * 180 / np.pi
phis_in = phis_in * 180 / np.pi
# initialise_S has to happen inside parallel job (get Pickle errors otherwise);
# just pass relevant optical constants for each wavelength, like for RT
angle_vector_0 = angle_vector[:, 0]
if front_or_rear == "front":
side = 1
else:
side = -1
if options['parallel']:
allres = Parallel(n_jobs=options['n_jobs'])(delayed(RCWA_wl)(
wavelengths[i1] * 1e9, geom_list, layers_oc[i1], shapes_oc[i1],
shapes_names, pol, thetas_in, phis_in, widths, size, orders,
phi_sym, theta_intv, phi_intv, angle_vector_0, rcwa_options,
detail_layer, side) for i1 in range(len(wavelengths)))
else:
allres = [
RCWA_wl(wavelengths[i1] * 1e9, geom_list, layers_oc[i1],
shapes_oc[i1], shapes_names, pol, thetas_in, phis_in,
widths, size, orders, phi_sym, theta_intv, phi_intv,
angle_vector_0, rcwa_options, detail_layer, side)
for i1 in range(len(wavelengths))
]
R = np.stack([item[0] for item in allres])
T = np.stack([item[1] for item in allres])
A_mat = np.stack([item[2] for item in allres])
full_mat = stack([item[3] for item in allres])
int_mat = stack([item[4] for item in allres])
#T_mat = np.stack([item[4] for item in allres])
#full_mat = np.hstack((R_mat, T_mat))
#full_mat = COO(full_mat)
A_mat = COO(A_mat)
if save:
save_npz(savepath_RT, full_mat)
save_npz(savepath_A, A_mat)
#R_pfbo = np.stack([item[3] for item in allres])
#T_pfbo = np.stack([item[4] for item in allres])
#phi_rt = np.stack([item[5] for item in allres])
#theta_r = np.stack([item[6] for item in allres])
#theta_t = np.stack([item[7] for item in allres])
#R_pfbo_2 = np.stack([item[8] for item in allres])
#return {'R': R, 'T':T, 'A_layer': A_mat, 'full_mat': full_mat, 'int_mat': int_mat}#'R_pfbo': R_pfbo, 'T_pfbo': T_pfbo, 'phi_rt': phi_rt, 'theta_r': theta_r, 'theta_t': theta_t}#, 'R_pfbo_2': R_pfbo_2}
return full_mat, A_mat # , R, T
def build(self,
omop_cdm_schema=None,
cohort_table_name=None,
cohort_generation_script=None,
cohort_generation_kwargs=None,
first=None,
verbose=True,
outcome_col_name='y',
cache_file='/tmp/store.csv',
from_cached=False):
# Generate cohort here
with open(cohort_generation_script, 'r') as f:
cohort_generation_sql_raw = f.read()
# sep_col = self.id_col
joined_sql = """with {} as ({})
select
example_id,
person_id,
concept_name,
feature_start_date,
person_start_date,
person_end_date
from ({}) as t
order by {} asc""".format(
cohort_table_name,
cohort_generation_sql_raw.format(**cohort_generation_kwargs),
" union all ".join(
f._sql_raw.format(cdm_schema=omop_cdm_schema,
cohort_table=cohort_table_name)
for f in self._temporal_features),
",".join([
self.unique_id_col, ## Order by unique_id
# sep_col,
self.time_col,
self.feature_col
]))
# print(joined_sql)
if not from_cached:
# copy_sql = """
# copy
# ({query})
# to
# stdout
# with
# csv {head}
# """.format(
# query=joined_sql,
# head="HEADER"
# )
t = time.time()
conn = self._db.engine.raw_connection()
# cur = conn.cursor()
# store = open(cache_file,'wb')
# cur.copy_expert(copy_sql, store)
result = pd.read_sql(joined_sql, conn)
print(result.shape)
# print(result.columns)
result.to_csv(cache_file)
# store.seek(0)
print(
'Data loaded to buffer in {0:.2f} seconds'.format(time.time() -
t))
# Get outcomes separately
cohort = pd.read_sql(
cohort_generation_sql_raw.format(**cohort_generation_kwargs),
conn)
t = time.time()
store = open(cache_file, 'rb')
self.concepts = set()
self.times = set()
self.seen_ids = set()
chunksize = int(2e6)
for chunk in pd.read_csv(store, chunksize=chunksize):
self.concepts = self.concepts.union(
set(chunk[self.feature_col].unique()))
self.times = self.times.union(set(chunk[self.time_col].unique()))
# self.seen_ids = self.seen_ids.union(set(chunk[self.id_col].unique()))
self.seen_ids = self.seen_ids.union(
set(chunk[self.unique_id_col].unique()))
self.times = sorted(list(self.times))
self.concepts = sorted(list(self.concepts))
self.seen_ids = sorted(list(self.seen_ids))
print('Got Unique Concepts and Timestamps in {0:.2f} seconds'.format(
time.time() - t))
t = time.time()
store.seek(0)
# self.ids = cohort._cohort[self.id_col].unique()
# self.ids = cohort._cohort[self.unique_id_col].unique()
# self.id_map = {i:person_id for i,person_id in enumerate(self.seen_ids)}
# self.id_map_rev = {person_id:i for i,person_id in enumerate(self.seen_ids)}
self.id_map = {
i: example_id
for i, example_id in enumerate(self.seen_ids)
}
self.id_map_rev = {
example_id: i
for i, example_id in enumerate(self.seen_ids)
}
self.concept_map = {
i: concept_name
for i, concept_name in enumerate(self.concepts)
}
self.concept_map_rev = {
concept_name: i
for i, concept_name in enumerate(self.concepts)
}
self.time_map = {i: t for i, t in enumerate(self.times)}
self.time_map_rev = {t: i for i, t in enumerate(self.times)}
print('Created Index Mappings in {0:.2f} seconds'.format(time.time() -
t))
t = time.time()
last = None
spm_stored = None
spm_arr = []
self.recorded_ids = set()
for chunk_num, chunk in enumerate(
pd.read_csv(store, chunksize=chunksize)):
# first = chunk.iloc[0][sep_col]
first = chunk.iloc[0][self.unique_id_col]
# vals = chunk[sep_col].unique()
vals = chunk[self.unique_id_col].unique()
# indices = np.searchsorted(chunk[sep_col], vals)
indices = np.searchsorted(chunk[self.unique_id_col], vals)
self.recorded_ids = self.recorded_ids.union(set(vals))
chunk.loc[:, self.feature_col] = chunk[self.feature_col].apply(
self.concept_map_rev.get)
chunk.loc[:, self.time_col] = chunk[self.time_col].apply(
self.time_map_rev.get)
df_split = [
chunk.iloc[indices[i]:indices[i + 1]]
for i in range(len(indices) - 1)
] + [chunk.iloc[indices[-1]:]]
def gen_sparr(sub_df):
sparr = coo_matrix(
(np.ones(len(sub_df[self.feature_col])),
(sub_df[self.feature_col], sub_df[self.time_col])),
shape=(len(self.concepts), len(self.times)))
return sparr
spm_local = [gen_sparr(s) for s in df_split]
if first == last:
spm_local[0] += spm_stored
else:
if spm_stored is not None:
spm_arr.append(spm_stored)
spm_arr += spm_local[:-1]
spm_stored = spm_local[-1]
# last = chunk.iloc[-1][sep_col]
last = chunk.iloc[-1][self.unique_id_col]
spm_arr.append(spm_stored)
print(len(spm_arr))
self._spm_arr = sparse.stack(
[sparse.COO.from_scipy_sparse(m) for m in spm_arr], 2)
print('Generated Sparse Representation of Data in {0:.2f} seconds'.
format(time.time() - t))
return cohort
def generate_masks(reconstruct_shape,
mask_shape,
dtype,
lamb,
dpix,
semiconv,
semiconv_pix,
transformation=None,
center=None,
cutoff=1,
cutoff_freq=np.float32('inf'),
method='subpix'):
reconstruct_shape = np.array(reconstruct_shape)
dpix = np.array(dpix)
d_Kf = np.sin(semiconv) / lamb / semiconv_pix
d_Qp = 1 / dpix / reconstruct_shape
if center is None:
center = np.array(mask_shape) / 2
if transformation is None:
transformation = identity()
cy, cx = center
filter_center = circular(centerX=cx,
centerY=cy,
imageSizeX=mask_shape[1],
imageSizeY=mask_shape[0],
radius=semiconv_pix,
antialiased=True)
half_reconstruct = (reconstruct_shape[0] // 2 + 1, reconstruct_shape[1])
masks = []
for row in range(half_reconstruct[0]):
for column in range(half_reconstruct[1]):
# Do an fftshift of q and p
qp = np.array((row, column))
flip = qp > (reconstruct_shape / 2)
real_qp = qp.copy()
real_qp[flip] = qp[flip] - reconstruct_shape[flip]
if np.sum(real_qp**2) > cutoff_freq**2:
masks.append(empty_mask(mask_shape, dtype=dtype))
continue
# Shift of diffraction order relative to zero order
# without rotation in physical coordinates
real_sy_phys, real_sx_phys = real_qp * d_Qp
# We apply the transformation backwards to go
# from physical orientation to detector orientation,
# while the forward direction in center of mass analysis
# goes from detector coordinates to physical coordinates
# Afterwards, we transform from physical detector coordinates
# to pixel coordinates
sy, sx = ((real_sy_phys, real_sx_phys) @ transformation) / d_Kf
masks.append(
generate_mask(
cy=cy,
cx=cx,
sy=sy,
sx=sx,
filter_center=filter_center,
semiconv_pix=semiconv_pix,
cutoff=cutoff,
mask_shape=mask_shape,
dtype=dtype,
method=method,
))
# Since we go through masks in order, this gives a mask stack with
# flattened (q, p) dimension to work with dot product and mask container
masks = sparse.stack(masks)
return masks
def create_interaction_list(interaction_df,
num_individuals,
fps=3,
ringbuffer_size=5):
ts = interaction_df.timestamp.min()
rbs = ringbuffer_size
rbs_hp = (rbs // 2) + 1
# cumulative interactions over whole period
previous_interactions = sparse.COO([],
shape=(num_individuals,
num_individuals))
# frame sliding window
interaction_ringbuffer = [
sparse.COO([], shape=(num_individuals, num_individuals))
for i in range(rbs)
]
# cumulative interactions for all cameras within a 1/fps frame period
# == 1 frame combined for all cameras
current_interactions = sparse.COO([],
shape=(num_individuals, num_individuals))
interval_counter = 0
events = []
print("Number of events {}".format(len(interaction_df)), flush=True)
print("Number of timestamps {}".format(
len(interaction_df.timestamp.unique())),
flush=True)
for timestamp, group in list(
interaction_df.sort_values("timestamp").groupby("timestamp")):
# still within current time interval
if (timestamp - ts) < datetime.timedelta(milliseconds=int(900 / fps)):
pass
# end of current time interval
else:
# count as interaction if more than half of rbs consecutive frames had interactions
# == median filter over temporal dimension with kernel size rbs
if interval_counter >= rbs:
new_interactions = sparse.stack(interaction_ringbuffer).sum(
axis=0) > rbs_hp
stopped_interactions = (previous_interactions.astype(np.int) -
new_interactions.astype(np.int)) == 1
if stopped_interactions.sum() > 0:
for bee_id_a, bee_id_b in np.argwhere(
stopped_interactions):
events.append((timestamp, bee_id_a, bee_id_b))
previous_interactions = new_interactions
interaction_ringbuffer[interval_counter %
rbs] = current_interactions
# new time interval => reset adjacency matrix and timestamp
current_interactions = sparse.COO([],
shape=(num_individuals,
num_individuals))
ts = group.timestamp.min()
interval_counter += 1
# interaction adjacency matrix
adj_data = {(min(k), max(k)): 1 for k in tuple(group["bee_id"].values)}
adj = sparse.DOK(shape=(num_individuals, num_individuals),
data=adj_data)
# logical or => accumulate interactions from different cameras
# for current time interval (~1/fps of a second)
current_interactions += adj
current_interactions.clip(0, 1, current_interactions)
return events
def generate_masks(reconstruct_shape,
mask_shape,
dtype,
lamb,
dpix,
semiconv,
semiconv_pix,
transformation=None,
cy=None,
cx=None,
cutoff=1,
cutoff_freq=np.float32('inf'),
method='subpix'):
'''
Generate the trotter mask stack.
The y dimension is trimmed to size(y)//2 + 1 to exploit the inherent
symmetry of the mask stack.
Parameters
----------
reconstruct_shape : tuple(int)
Shape of the reconstructed area
mask_shape : tuple(int)
Shape of the detector
dtype : numpy dtype
dtype to use for the mask stack
lamb : float
Wavelength of the illuminating radiation in m
dpix : float or (float, float)
Scan step in m. Tuple (y, x) in case scan step is different in x and y direction.
semiconv : float
Semiconvergence angle of the illumination in radians
semiconv_pix : float
Semiconvergence angle in measured in detector pixel, i.e. radius of the zero order disk.
transformation : numpy.ndarray, optional
Matrix for affine transformation from the scan coordinate directions
to the detector coordinate directions. This does not include the scale, which is handled by
dpix, lamb, semiconv and semiconv_pix. It should only be used to rotate and flip
the coordinate system as necessary. See also
https://github.com/LiberTEM/LiberTEM/blob/master/src/libertem/corrections/coordinates.py
cy, cx : float, optional
Position of the optical axis on the detector in px, center of illumination.
Default: Center of the detector
cutoff : int, optional
Minimum number of pixels in the positive and negative trotter. This can be used to purge
very small trotters to reduce noise. Default is 1, i.e. no cutoff unless one trotter is
empty.
cutoff_freq: float
Trotters belonging to a spatial frequency higher than this value in reciprocal pixel
coordinates will be cut off.
method : str, optional
Can be :code:`'subpix'`(default) or :code:`'shift'` to switch between
:meth:`mask_pair_subpix` and :meth:`mask_pair_shift` to generate a trotter pair.
Returns
-------
masks : sparse.COO
Masks in sparse.pydata.org COO format. y and x frequency index are FFT shifted, i.e. the
zero frequency is at (0,0) and negative frequencies are in the far quadrant and reversed.
The y frequency index is cut in half with size(y)//2 + 1 to exploit the inherent symmetry
of a real-valued Fourier transform. The y and x index are then flattened to make it
suitable for using it with MaskContainer.
'''
reconstruct_shape = np.array(reconstruct_shape)
dpix = np.array(dpix)
d_Kf = np.sin(semiconv) / lamb / semiconv_pix
d_Qp = 1 / dpix / reconstruct_shape
if cy is None:
cy = mask_shape[0] / 2
if cx is None:
cx = mask_shape[1] / 2
if transformation is None:
transformation = identity()
filter_center = circular(centerX=cx,
centerY=cy,
imageSizeX=mask_shape[1],
imageSizeY=mask_shape[0],
radius=semiconv_pix,
antialiased=True)
half_reconstruct = (reconstruct_shape[0] // 2 + 1, reconstruct_shape[1])
masks = []
for row in range(half_reconstruct[0]):
for column in range(half_reconstruct[1]):
# Do an fftshift of q and p
qp = np.array((row, column))
flip = qp > (reconstruct_shape / 2)
real_qp = qp.copy()
real_qp[flip] = qp[flip] - reconstruct_shape[flip]
if np.sum(real_qp**2) > cutoff_freq**2:
masks.append(empty_mask(mask_shape, dtype=dtype))
continue
# Shift of diffraction order relative to zero order
# without rotation in physical coordinates
real_sy_phys, real_sx_phys = real_qp * d_Qp
# We apply the transformation backwards to go
# from physical orientation to detector orientation,
# while the forward direction in center of mass analysis
# goes from detector coordinates to physical coordinates
# Afterwards, we transform from physical detector coordinates
# to pixel coordinates
sy, sx = ((real_sy_phys, real_sx_phys) @ transformation) / d_Kf
masks.append(
generate_mask(
cy=cy,
cx=cx,
sy=sy,
sx=sx,
filter_center=filter_center,
semiconv_pix=semiconv_pix,
cutoff=cutoff,
dtype=dtype,
method=method,
))
# Since we go through masks in order, this gives a mask stack with
# flattened (q, p) dimension to work with dot product and mask container
masks = sparse.stack(masks)
return masks
def RT(group, incidence, transmission, surf_name, options, Fr_or_TMM = 0, front_or_rear = 'front',
n_absorbing_layers=0, calc_profile=[], only_incidence_angle=False, widths=[], save=True):
"""Calculates the reflection/transmission and absorption redistribution matrices for an interface using
either a previously calculated TMM lookup table or the Fresnel equations.
:param group: an RTgroup object containing the surface textures
:param incidence: incidence medium
:param: transmission: transmission medium
:param surf_name: name of the surface (to save matrices)
:param options: dictionary of options
:param Fr_or_TMM: whether to use the Fresnel equations (0) or a TMM lookup table (1)
:param front_or_rear: whether light is incident from the front or rear
:param for a structure with multiple interface layers, where a TMM lookuptable is used, the number of layers in
the interface
:param calc_profile: whether to save the relevant information to calculate the depth-dependent absorption
profile
:param only_incidence_angle: if True, the ray-tracing will only be performed for the incidence theta and phi
specified in the options.
:return out_mat: the R/T redistribution matrix at each wavelength, indexed as (wavelength, angle_bin_out, angle_bin_in)
:return A_mat: the absorption redistribution matrix (total absorption per layer), indexed as (wavelength, layer_out, angle_bin_in)
:return local_angle_mat: only if calc_profile = True. A matrix storing the local incidence angles for rays which were absorbed.
This is used to calculate absorption profiles using TMM.
"""
if Fr_or_TMM > 0 or save:
structpath = os.path.join(results_path, options['project_name']) # also need this to get lookup table
if save:
if not os.path.isdir(structpath):
os.mkdir(structpath)
savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz')
savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz')
prof_mat_path = os.path.join(results_path, options['project_name'],
surf_name + front_or_rear + 'profmat.nc')
if Fr_or_TMM > 0:
savepath_prof = os.path.join(structpath, surf_name + front_or_rear + 'Aprof.npz')
if os.path.isfile(savepath_RT) and save:
print('Existing angular redistribution matrices found')
allArrays = load_npz(savepath_RT)
absArrays = load_npz(savepath_A)
if Fr_or_TMM > 0 and os.path.isfile(savepath_prof):
local_angles = load_npz(savepath_prof)
if os.path.isfile(prof_mat_path):
prof_int = xr.load_dataset(prof_mat_path)
profile = prof_int['profile']
intgr = prof_int['intgr']
return allArrays, absArrays, local_angles, profile, intgr
else:
return allArrays, absArrays, local_angles
else:
return allArrays, absArrays
else:
wavelengths = options['wavelengths']
n_rays = options['n_rays']
nx = options['nx']
ny = options['ny']
n_angles = int(np.ceil(n_rays/(nx*ny)))
phi_sym = options['phi_symmetry']
n_theta_bins = options['n_theta_bins']
c_az = options['c_azimuth']
pol = options['pol']
nm_spacing = options['nm_spacing']
if front_or_rear == 'front':
side = 1
else:
side = -1
if Fr_or_TMM == 1:
lookuptable = xr.open_dataset(os.path.join(structpath, surf_name + '.nc'))
if front_or_rear == 'rear':
lookuptable = lookuptable.assign_coords(side=np.flip(lookuptable.side))
else:
lookuptable = None
theta_intv, phi_intv, angle_vector = make_angle_vector(n_theta_bins, phi_sym, c_az)
if only_incidence_angle:
print('Calculating matrix only for incidence theta/phi')
if options['theta_in'] == 0:
th_in = 0.0001
else:
th_in = options['theta_in']
angles_in = angle_vector[:int(len(angle_vector) / 2), :]
n_reps = int(np.ceil(n_angles / len(angles_in)))
thetas_in = np.tile(th_in, n_reps)
print('only inc angle' , thetas_in)
n_angles = n_reps
if options['phi_in'] == 'all':
# get relevant phis
phis_in = np.tile(options['phi_in'], n_reps)
else:
if options['phi_in'] == 0:
phis_in = np.tile(0.0001, n_reps)
else:
phis_in = np.tile(options['phi_in'], n_reps)
else:
if options['random_angles']:
thetas_in = np.random.random(n_angles)*np.pi/2
phis_in = np.random.random(n_angles)*2*np.pi
else:
angles_in = angle_vector[:int(len(angle_vector)/2),:]
if n_angles/len(angles_in) < 1:
warn('The number of rays is not sufficient to populate the redistribution matrix!')
n_reps = int(np.ceil(n_angles/len(angles_in)))
thetas_in = np.tile(angles_in[:,1], n_reps)[:n_angles]
phis_in = np.tile(angles_in[:,2], n_reps)[:n_angles]
if front_or_rear == 'front':
mats = [incidence]
else:
mats = [transmission]
for i1 in range(len(group.materials)):
mats.append(group.materials[i1])
if front_or_rear == 'front':
mats.append(transmission)
else:
mats.append(incidence)
# list of lists: first in tuple is front incidence
if front_or_rear == 'front':
surfaces = [x[0] for x in group.textures]
else:
surfaces = [x[1] for x in group.textures]
nks = np.empty((len(mats), len(wavelengths)), dtype=complex)
for i1, mat in enumerate(mats):
nks[i1] = mat.n(wavelengths) + 1j*mat.k(wavelengths)
h = max(surfaces[0].Points[:, 2])
x_lim = surfaces[0].Lx
y_lim = surfaces[0].Ly
if options['random_ray_position']:
xs = np.random.uniform(0, x_lim, nx)
ys = np.random.uniform(0, y_lim, ny)
else:
if options['avoid_edges']:
xs = np.linspace(x_lim / 4, x_lim - (x_lim / 4), nx)
ys = np.linspace(y_lim / 4, y_lim - (y_lim / 4), ny)
else:
xs = np.linspace(x_lim/101, x_lim-(x_lim/99), nx)
ys = np.linspace(y_lim/100, y_lim-(y_lim/102), ny)
print('n_th_in', len(thetas_in), len(xs))
if options['parallel']:
allres = Parallel(n_jobs=options['n_jobs'])(delayed(RT_wl)
(i1, wavelengths[i1], n_angles, nx, ny,
widths, thetas_in, phis_in, h,
xs, ys, nks, surfaces,
pol, phi_sym, theta_intv,
phi_intv, angle_vector, Fr_or_TMM, n_absorbing_layers,
lookuptable, calc_profile, nm_spacing, side)
for i1 in range(len(wavelengths)))
else:
allres = [RT_wl(i1, wavelengths[i1], n_angles, nx, ny, widths,
thetas_in, phis_in, h, xs, ys, nks, surfaces,
pol, phi_sym, theta_intv, phi_intv,
angle_vector, Fr_or_TMM, n_absorbing_layers, lookuptable, calc_profile, nm_spacing, side)
for i1 in range(len(wavelengths))]
allArrays = stack([item[0] for item in allres])
absArrays = stack([item[1] for item in allres])
if save:
save_npz(savepath_RT, allArrays)
save_npz(savepath_A, absArrays)
if Fr_or_TMM > 0:
local_angles = stack([item[2] for item in allres])
if save:
save_npz(savepath_prof, local_angles)
#make_profile_data(options, np.unique(angle_vector[:,1]), int(len(angle_vector) / 2),
# front_or_rear, surf_name, n_absorbing_layers, widths)
if len(calc_profile) > 0:
profile = xr.concat([item[3] for item in allres], 'wl')
intgr = xr.concat([item[4] for item in allres], 'wl')
intgr.name = 'intgr'
profile.name = 'profile'
allres = xr.merge([intgr, profile])
if save:
allres.to_netcdf(prof_mat_path)
save_npz(savepath_prof, local_angles)
return allArrays, absArrays, local_angles, profile, intgr
else:
return allArrays, absArrays, local_angles
else:
return allArrays, absArrays
def TMM(layers, incidence, transmission, surf_name, options,
coherent=True, coherency_list=None, prof_layers=[], front_or_rear='front', save=True):
"""Function which takes a layer stack and creates an angular redistribution matrix.
:param layers: A list with one or more layers.
:param transmission: transmission medium
:param incidence: incidence medium
:param surf_name: name of the surface (to save the matrices generated.
:param options: a list of options
:param coherent: whether or not the layer stack is coherent. If None, it is assumed to be fully coherent
:param coherency: a list with the same number of entries as the layers, either 'c' for a coherent layer or
'i' for an incoherent layer
:param prof_layers: layers for which the absorption profile should be calculated
(if None, do not calculate absorption profile at all)
:param front_or_rear: a string, either 'front' or 'rear'; front incidence on the stack, from the incidence
medium, or rear incidence on the stack, from the transmission medium.
:return full_mat: R and T redistribution matrix
:return A_mat: matrix describing absorption per layer
"""
def make_matrix_wl(wl):
# binning into matrix, including phi
RT_mat = np.zeros((len(theta_bins_in)*2, len(theta_bins_in)))
A_mat = np.zeros((n_layers, len(theta_bins_in)))
for i1 in range(len(theta_bins_in)):
theta = theta_lookup[i1]#angle_vector[i1, 1]
data = allres.loc[dict(angle=theta, wl=wl)]
R_prob = np.real(data['R'].data.item(0))
T_prob = np.real(data['T'].data.item(0))
Alayer_prob = np.real(data['Alayer'].data)
phi_out = phis_out[i1]
#print(R_prob, T_prob)
# reflection
phi_int = phi_intv[theta_bins_in[i1]]
phi_ind = np.digitize(phi_out, phi_int, right=True) - 1
bin_out_r = np.argmin(abs(angle_vector[:, 0] - theta_bins_in[i1])) + phi_ind
#print(bin_out_r, i1+offset)
RT_mat[bin_out_r, i1] = R_prob
#print(R_prob)
# transmission
theta_t = np.abs(-np.arcsin((inc.n(wl * 1e-9) / trns.n(wl * 1e-9)) * np.sin(theta_lookup[i1])) + quadrant)
#print('angle in, transmitted', angle_vector_th[i1], theta_t)
# theta switches half-plane (th < 90 -> th >90
if ~np.isnan(theta_t):
theta_out_bin = np.digitize(theta_t, theta_intv, right=True) - 1
phi_int = phi_intv[theta_out_bin]
phi_ind = np.digitize(phi_out, phi_int, right=True) - 1
bin_out_t = np.argmin(abs(angle_vector[:, 0] - theta_out_bin)) + phi_ind
RT_mat[bin_out_t, i1] = T_prob
#print(bin_out_t, i1+offset)
# absorption
A_mat[:, i1] = Alayer_prob
fullmat = COO(RT_mat)
A_mat = COO(A_mat)
return fullmat, A_mat
structpath = os.path.join(results_path, options['project_name'])
if not os.path.isdir(structpath):
os.mkdir(structpath)
savepath_RT = os.path.join(structpath, surf_name + front_or_rear + 'RT.npz')
savepath_A = os.path.join(structpath, surf_name + front_or_rear + 'A.npz')
prof_mat_path = os.path.join(results_path, options['project_name'],
surf_name + front_or_rear + 'profmat.nc')
if os.path.isfile(savepath_RT) and save:
print('Existing angular redistribution matrices found')
fullmat = load_npz(savepath_RT)
A_mat = load_npz(savepath_A)
if len(prof_layers) > 0:
profile = xr.load_dataarray(prof_mat_path)
return fullmat, A_mat, profile
else:
wavelengths = options['wavelengths']*1e9 # convert to nm
theta_intv, phi_intv, angle_vector = make_angle_vector(options['n_theta_bins'], options['phi_symmetry'], options['c_azimuth'])
angles_in = angle_vector[:int(len(angle_vector) / 2), :]
thetas = np.unique(angles_in[:, 1])
n_angles = len(thetas)
n_layers = len(layers)
if front_or_rear == 'front':
optlayers = OptiStack(layers, substrate=transmission, incidence=incidence)
trns = transmission
inc = incidence
else:
optlayers = OptiStack(layers[::-1], substrate=incidence, incidence=transmission)
trns = incidence
inc = transmission
if len(prof_layers) > 0:
profile = True
z_limit = np.sum(np.array(optlayers.widths))
full_dist = np.arange(0, z_limit, options['nm_spacing'])
layer_start = np.insert(np.cumsum(np.insert(optlayers.widths, 0, 0)), 0, 0)
layer_end = np.cumsum(np.insert(optlayers.widths, 0, 0))
dist = []
for l in prof_layers:
dist = np.hstack((dist, full_dist[np.all((full_dist >= layer_start[l], full_dist < layer_end[l]), 0)]))
else:
profile = False
if options['pol'] == 'u':
pols = ['s', 'p']
else:
pols = [options['pol']]
R = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)),
dims=['pol', 'wl', 'angle'],
coords={'pol': pols, 'wl': wavelengths, 'angle': thetas},
name='R')
T = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)),
dims=['pol', 'wl', 'angle'],
coords={'pol': pols, 'wl': wavelengths, 'angle': thetas},
name='T')
Alayer = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), n_layers)),
dims=['pol', 'angle', 'wl', 'layer'],
coords={'pol': pols,
'wl': wavelengths,
'angle': thetas,
'layer': range(1, n_layers + 1)}, name='Alayer')
theta_t = xr.DataArray(np.empty((len(pols), len(wavelengths), n_angles)),
dims=['pol', 'wl', 'angle'],
coords={'pol': pols, 'wl': wavelengths, 'angle': thetas},
name='theta_t')
if profile:
Aprof = xr.DataArray(np.empty((len(pols), n_angles, len(wavelengths), len(dist))),
dims=['pol', 'angle', 'wl', 'z'],
coords={'pol': pols,
'wl': wavelengths,
'angle': thetas,
'z': dist}, name='Aprof')
R_loop = np.empty((len(wavelengths), n_angles))
T_loop = np.empty((len(wavelengths), n_angles))
Alayer_loop = np.empty((n_angles, len(wavelengths), n_layers), dtype=np.complex_)
th_t_loop = np.empty((len(wavelengths), n_angles))
if profile:
Aprof_loop = np.empty((n_angles, len(wavelengths), len(dist)))
tmm_struct = tmm_structure(optlayers, coherent=coherent, coherency_list=coherency_list, no_back_reflection=False)
for i2, pol in enumerate(pols):
for i3, theta in enumerate(thetas):
res = tmm_struct.calculate(wavelengths, angle=theta, pol=pol, profile=profile, layers=prof_layers, nm_spacing = options['nm_spacing'])
R_loop[:, i3] = np.real(res['R'])
T_loop[:, i3] = np.real(res['T'])
Alayer_loop[i3, :, :] = np.real(res['A_per_layer'].T)
if profile:
Aprof_loop[i3, :, :] = res['profile']
# sometimes get very small negative values (like -1e-20)
R_loop[R_loop < 0] = 0
T_loop[T_loop < 0] = 0
Alayer_loop[Alayer_loop < 0] = 0
if front_or_rear == 'rear':
Alayer_loop = np.flip(Alayer_loop, axis=2)
print('flipping')
R.loc[dict(pol=pol)] = R_loop
T.loc[dict(pol=pol)] = T_loop
Alayer.loc[dict(pol=pol)] = Alayer_loop
theta_t.loc[dict(pol=pol)] = th_t_loop
if profile:
Aprof.loc[dict(pol=pol)] = Aprof_loop
Aprof.transpose('pol', 'wl', 'angle', 'z')
Alayer = Alayer.transpose('pol', 'wl', 'angle', 'layer')
if profile:
allres = xr.merge([R, T, Alayer, Aprof])
else:
allres = xr.merge([R, T, Alayer])
if options['pol'] == 'u':
allres = allres.reduce(np.mean, 'pol').assign_coords(pol='u').expand_dims('pol')
# populate matrices
if front_or_rear == "front":
angle_vector_th = angle_vector[:int(len(angle_vector)/2),1]
angle_vector_phi = angle_vector[:int(len(angle_vector)/2),2]
phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry'])
theta_lookup = angles_in[:,1]
quadrant = np.pi
else:
angle_vector_th = angle_vector[int(len(angle_vector) / 2):, 1]
angle_vector_phi = angle_vector[int(len(angle_vector) / 2):, 2]
phis_out = fold_phi(angle_vector_phi + np.pi, options['phi_symmetry'])
theta_lookup = angles_in[:,1][::-1]
quadrant = 0
phis_out[phis_out == 0] = 1e-10
theta_bins_in = np.digitize(angle_vector_th, theta_intv, right=True) -1
print(theta_bins_in)
mats = [make_matrix_wl(wl) for wl in wavelengths]
fullmat = stack([item[0] for item in mats])
A_mat = stack([item[1] for item in mats])
if save:
save_npz(savepath_RT, fullmat)
save_npz(savepath_A, A_mat)
return fullmat, A_mat #, allres
