def run(self):
_, cols = shape(self.pos_bm)
top_scores = ones(cols)*-inf
top_pos = ones(cols)*-inf
top_neg = ones(cols)*-inf
top_pattern = zeros(cols)
for running_int in range(self._start, self._end):
col_pattern = self._bin_array(running_int, cols)
if npsum(col_pattern) > self.max_cluster:
continue
pos_score = npany(self.pos_bm * col_pattern, axis=1)
neg_score = npany(self.neg_bm * col_pattern, axis=1)
pattern_score = npsum(pos_score) - npsum(neg_score)
if pattern_score > top_scores[sum(col_pattern)-1]:
top_scores[sum(col_pattern)-1] = pattern_score
top_pattern[sum(col_pattern)-1] = running_int
top_pos[sum(col_pattern)-1] = npsum(pos_score)
top_neg[sum(col_pattern)-1] = npsum(neg_score)
if running_int % 10000 == 0:
print((running_int - self._start)/(self._end - self._start))
# Write the result to the score and pattern matrices.
if self._score_queue:
self._score_queue.put({"score":top_scores,
"pos":top_pos,
"neg":top_neg,
"pattern":top_pattern})
def pareto_min(*args):
r"""Determine if observation is a Pareto point
Find the Pareto-efficient points that minimize the provided features.
Args:
xi (iterable OR gr.Intention()): Feature to minimize; use -X to maximize
Returns:
np.array of boolean: Indicates if observation is Pareto-efficient
"""
# Check invariants
lengths = map(len, args)
if len(set(lengths)) > 1:
raise ValueError("All arguments to pareto_min must be of equal length")
# Compute pareto points
costs = array([*args]).T
is_efficient = ones(costs.shape[0], dtype=bool)
for i, c in enumerate(costs):
is_efficient[i] = npall(npany(costs[:i] > c, axis=1)) and npall(
npany(costs[i + 1:] > c, axis=1))
return is_efficient
def pareto_min_rel(X_test, X_base=None):
r"""Determine if rows in X_test are optimal, compared to X_base
Finds the Pareto-efficient test-points that minimize the column values,
relative to a given set of base-points.
Args:
X_test (2d numpy array): Test point observations; rows are observations, columns are features
X_base (2d numpy array): Base point observations; rows are observations, columns are features
Returns:
array of boolean values: Indicates if test observation is Pareto-efficient, relative to base points
References:
Owen *Monte Carlo theory, methods and examples* (2013)
"""
# Compute Pareto points
is_efficient = ones(X_test.shape[0], dtype=bool)
if X_base is None:
for i, x in enumerate(X_test):
is_efficient[i] = npall(npany(X_test[:i] > x, axis=1)) and npall(
npany(X_test[i + 1:] > x, axis=1))
else:
for i, x in enumerate(X_test):
is_efficient[i] = not (npany(npall(x >= X_base, axis=1))
and npany(npany(x > X_base, axis=1)))
return is_efficient
def set_offset(self, value):
if value is not None:
value = asarray(value)
if npany(value != 0):
self._offset = value
else:
self._offset = None
def remove_dependent_cols(X, tol=1e-6, verbose=False):
r"""Remove dependent columns.
Return a matrix with dependent columns removed.
Parameters
----------
X : array_like
Matrix to might have dependent columns.
Returns
-------
array_like
Full column rank matrix.
"""
from scipy.linalg import qr
from numpy import abs as npabs
from numpy import any as npany
from numpy import where
R = qr(X, mode="r")[0][: X.shape[1], :]
I = npabs(R.diagonal()) > tol
if npany(~I) and verbose:
msg = "Columns " + str(where(~I)[0])
print(msg + " have been removed because linear dependence")
R = X[:, I]
else:
R = X.copy()
return R
def zScale(tau_box, tau, isotau_flag=False, nan_flag=True):
l = zeros_like(tau_box[:,:,0])
z = []
if isotau_flag:
isotau = empty_like(tau_box)
for i,t in zip(range(len(tau)),tau):
print('Step: '+str(i))
if isnan(t):
z+=[nan]
if isotau_flag:
isotau[:,:,i] = nan
else:
l = level(tau_box, t, l)
if isotau_flag:
isotau[:,:,i] = l
if not nan_flag:
if npany(isnan(l)):
z+=[nan]
if isotau_flag:
isotau[:,:,i] = nan
else:
z += [nanmean(l)]
else:
z += [nanmean(l)]
if isotau_flag:
return asarray(z), isotau
return asarray(z)
def set_offset(self, value):
if value is not None:
value = asarray(value)
if npany(value != 0):
self._offset = value
else:
self._offset = None
def is_pareto_efficient(costs, return_mask=True):
"""
Find the pareto-efficient points
:param costs: An (n_points, n_costs) array
:param return_mask: True to return a mask
:return: An array of indices of pareto-efficient points.
If return_mask is True, this will be an (n_points, ) boolean array
Otherwise it will be a (n_efficient_points, ) integer array of indices.
"""
is_efficient = arange(costs.shape[0])
n_points = costs.shape[0]
next_point_index = 0 # Next index in the is_efficient array to search for
while next_point_index < len(costs):
nondominated_point_mask = npany(costs < costs[next_point_index],
axis=1)
nondominated_point_mask[next_point_index] = True
# Remove dominated points
is_efficient = is_efficient[nondominated_point_mask]
costs = costs[nondominated_point_mask]
next_point_index = npsum(
nondominated_point_mask[:next_point_index]) + 1
if return_mask:
is_efficient_mask = zeros(n_points, dtype=bool)
is_efficient_mask[is_efficient] = True
return is_efficient_mask
else:
return is_efficient
def _negative_log_likelihood(params, freq, rec, T, weights, penalizer_coef):
if npany(asarray(params) <= 0.):
return np.inf
conditional_log_likelihood = ParetoNBDFitter._conditional_log_likelihood(params, freq, rec, T)
penalizer_term = penalizer_coef * sum(np.asarray(params) ** 2)
return -(weights * conditional_log_likelihood).mean() + penalizer_term
def convert(self, verbose=False, maxrows=None):
"""Method to loop through the data and convert it"""
# docstring is extended below
dataslice = slice(0, maxrows)
info = LoopInfo(total=len(self.data.ifiledata[0, dataslice]),
t0=datetime.now(),
verbose=verbose)
wrongvalues = 0
wrongarea = 0
for datatuple in izip(*self.data.ifiledata[:, dataslice]):
info.info()
fields = npall(
# normal gridcols
[
self.data.maskdata[self.gridcols.index(col)]
== datatuple[self.usecols.index(col)].astype(
self.data.maskdata[self.gridcols.index(col)].dtype)
for col in self.gridcols if col not in self.aliasdict
] +
# alias cols
[
self.data.maskdata[self.gridcols.index(col)]
== self.aliasdict[col][datatuple[self.usecols.index(col)]]
for col in self.aliasdict
],
axis=0)
itime = self.timefunc([
datatuple[self.usecols.index(col)]
for col in sorted(self.time.keys())
])
for catcol in self.catadddict:
for adderinstance in self.catadddict[catcol][tuple(
datatuple[self.usecols.index(col)] for col in catcol)]:
adderinstance.addfunc(
itime, fields, datatuple[self.usecols.index(
self.valcol)].astype(float))
for col in self.defaultadddict:
for adderinstance in self.defaultadddict[col][datatuple[
self.usecols.index(col)]]:
adderinstance.addfunc(
itime, fields, datatuple[self.usecols.index(
self.valcol)].astype(float))
if not npany(fields):
wrongvalues += 1
wrongarea += float(datatuple[self.usecols.index(self.valcol)])
if verbose:
print('\nNumber of wrong values: %i' % wrongvalues)
print('Missed Area [ha]: %6.4f' % wrongarea)
print('Missed Area: %1.3e %%' %
(wrongarea / sum(self.data.ifiledata[self.usecols.index(
self.valcol)].astype(float)) * 100.))
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = gammaln(r + freq) - gammaln(r) + r * log(alpha)
A_2 = gammaln(a + b) + gammaln(b + freq + 1) - gammaln(b) - gammaln(a + b + freq + 1)
A_3 = -(r + freq) * log(alpha + T)
A_4 = log(a) - log(b + freq) + (r + freq) * (log(alpha + T) - log(alpha + rec))
penalizer_term = penalizer_coef * sum(np.asarray(params) ** 2)
return -(A_1 + A_2 + A_3 + logaddexp(A_4, 0)).mean() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = special.gammaln(r + freq) - special.gammaln(r) + r * log(alpha)
A_2 = special.gammaln(a + b) + special.gammaln(b + freq + 1) - special.gammaln(b) - special.gammaln(a + b + freq + 1)
A_3 = -(r + freq) * log(alpha + T)
A_4 = log(a) - log(b + freq) + (r + freq) * (log(alpha + T) - log(alpha + rec))
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2 + A_3 + log(exp(A_4) + 1.)).sum() + penalizer_term
def latexify(self, var=None, idx=''):
template_dict = copy(self.objective_vars)
template_dict['idx'] = idx
if var is not None:
template_dict['var'] = var
if hasattr(self, 'offset') and self.offset is not None and npany(self.offset != 0):
template_dict['var'] = var + (r' - %(offset)s_{%(idx)s}' % template_dict)
obj = self.objective_template % template_dict
template_dict['obj'] = obj
if not self.quadratic.iszero:
return ' + '.join([obj, self.quadratic.latexify(var=var, idx=idx)])
return obj
def latexify(self, var=None, idx=''):
template_dict = copy(self.objective_vars)
template_dict['idx'] = idx
if var is not None:
template_dict['var'] = var
if hasattr(self, 'offset') and self.offset is not None and npany(self.offset != 0):
template_dict['var'] = (r'%(var)s - %(offset)s_{%(idx)s}' % template_dict)
obj = self.objective_template % template_dict
template_dict['obj'] = obj
if not self.quadratic.iszero:
return ' + '.join([obj, self.quadratic.latexify(var=var, idx=idx)])
return obj
def _negative_log_likelihood(params, freq, rec, T, weights, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = gammaln(r + freq) - gammaln(r) + r * log(alpha)
A_2 = (gammaln(a + b) + gammaln(b + freq + 1) - gammaln(b) -
gammaln(a + b + freq + 1))
A_3 = -(r + freq) * log(alpha + T)
A_4 = log(a) - log(b + freq) + (r + freq) * (log(alpha + T) -
log(alpha + rec))
penalizer_term = penalizer_coef * sum(np.asarray(params) ** 2)
return -(weights * (A_1 + A_2 + A_3 + logaddexp(A_4, 0))).mean() + penalizer_term
def _drop_missing(self) -> BoolArray:
missing = self.dependent.isnull.to_numpy()
missing |= self.exog.isnull.to_numpy()
missing |= self._absorb_inter.cat.isnull().any(1).to_numpy()
missing |= self._absorb_inter.cont.isnull().any(1).to_numpy()
for interact in self._interaction_list:
missing |= interact.isnull.to_numpy()
if npany(missing):
self.dependent.drop(missing)
self.exog.drop(missing)
self._absorb_inter.drop(missing)
for interact in self._interaction_list:
interact.drop(missing)
missing_warning(missing)
return missing
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = special.gammaln(r + freq) - special.gammaln(r) + r * log(alpha)
A_2 = special.gammaln(a + b) + special.gammaln(b + freq) - special.gammaln(b) - special.gammaln(a + b + freq)
A_3 = -(r + freq) * log(alpha + T)
d = vconcat[ones_like(freq), (freq > 0)]
A_4 = log(a) - log(b + freq - 1) - (r + freq) * log(rec + alpha)
A_4[isnan(A_4) | isinf(A_4)] = 0
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2 + misc.logsumexp(vconcat[A_3, A_4], axis=1, b=d)).sum() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = special.gammaln(r + freq) - special.gammaln(r) + r * log(alpha)
A_2 = special.gammaln(a + b) + special.gammaln(
b + freq + 1) - special.gammaln(b) - special.gammaln(a + b + freq +
1)
A_3 = -(r + freq) * log(alpha + T)
A_4 = log(a) - log(b + freq) + (r + freq) * (log(alpha + T) -
log(alpha + rec))
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2 + A_3 + log(exp(A_4) + 1.)).sum() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, weights,
penalizer_coef):
"""
Sums the conditional log-likelihood from the ``_conditional_log_likelihood`` function
and applies a ``penalizer_coef``.
"""
if npany(asarray(params) <= 0.0):
return np.inf
conditional_log_likelihood = ParetoNBDFitter._conditional_log_likelihood(
params, freq, rec, T)
penalizer_term = penalizer_coef * sum(np.asarray(params)**2)
return -(weights * conditional_log_likelihood).sum() / weights.mean(
) + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = special.gammaln(r + freq) - special.gammaln(r) + r * log(alpha)
A_2 = special.gammaln(a + b) + special.gammaln(
b + freq) - special.gammaln(b) - special.gammaln(a + b + freq)
A_3 = -(r + freq) * log(alpha + T)
d = vconcat[ones_like(freq), (freq > 0)]
A_4 = log(a) - log(b + freq - 1) - (r + freq) * log(rec + alpha)
A_4[isnan(A_4) | isinf(A_4)] = 0
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2 + misc.logsumexp(vconcat[A_3, A_4], axis=1,
b=d)).sum() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0.):
return np.inf
r, alpha, s, beta = params
x = freq
r_s_x = r + s + x
A_1 = special.gammaln(r + x) - special.gammaln(r) + r * log(alpha) + s * log(beta)
log_A_0 = ParetoNBDFitter._log_A_0(params, freq, rec, T)
A_2 = logaddexp(-(r+x)*log(alpha+T) - s*log(beta+T), log(s) + log_A_0 - log(r_s_x))
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2).sum() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0.):
return np.inf
r, alpha, s, beta = params
x = freq
r_s_x = r + s + x
A_1 = gammaln(r + x) - gammaln(r) + r * log(alpha) + s * log(beta)
log_A_0 = ParetoNBDFitter._log_A_0(params, freq, rec, T)
A_2 = logaddexp(-(r + x) * log(alpha + T) - s * log(beta + T),
log(s) + log_A_0 - log(r_s_x))
penalizer_term = penalizer_coef * sum(np.asarray(params)**2)
return -(A_1 + A_2).mean() + penalizer_term
def _negative_log_likelihood(params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0):
return np.inf
r, alpha, a, b = params
A_1 = gammaln(r + freq) - gammaln(r) + r * log(alpha)
A_2 = (gammaln(a + b) + gammaln(b + freq) - gammaln(b) -
gammaln(a + b + freq))
A_3 = -(r + freq) * log(alpha + T)
d = vconcat[ones_like(freq), (freq > 0)]
A_4 = log(a) - log(b + where(freq == 0, 1, freq) - 1) - \
(r + freq) * log(rec + alpha) if (a > 0 and (b + where(freq == 0, 1, freq) - 1) > 0 and (rec + alpha) > 0) else 0
A_4[isnan(A_4) | isinf(A_4)] = 0
penalizer_term = penalizer_coef * sum(np.asarray(params)**2)
return -(A_1 + A_2 + misc.logsumexp(vconcat[A_3, A_4], axis=1,
b=d)).mean() + penalizer_term
def random_walk(self, d=None, plot_h=None) -> None:
'''Let all population walk randomly'''
walk_left = self.move_per_day.copy()
# Every day, people start from home
pos = self.home.copy()
# Some travel less, some more
while npany(walk_left):
walk_left -= 1
# All randomly move an edge-length
pos += nparray(npround(
(nprandom.random(size=pos.shape) * 2 - 1) *
self.rms_v.T[:, None]) * (walk_left != 0).T[:, None],
dtype=pos.dtype)
# Can't jump beyond boundary
# So, reflect exploration
# pos = pos.clip(min=0, max=self.p_max-1)
pos = nparray(npnot(pos > (self.p_max-1)) * npabs(pos),
dtype=pos.dtype)\
+ nparray((pos > (self.p_max-1)) * (2 * (self.p_max - 1) - pos),
dtype=pos.dtype)
for indiv in range(self.pop_size):
# TODO: A ufunc or async map would have been faster
if walk_left[indiv]:
self.calc_exposure(indiv, pos)
if plot_h.contam_dots:
strain_persist = self.strain_types[-1].persistence
host_types = []
host_types.append(
(pos *
(npnot(self.active[:, None]) * self.susceptible[:, None] >
self.resist_def)).tolist())
host_types.append((pos * self.active[:, None]).tolist())
host_types.append((
pos *
(npnot(self.active[:, None]) *
(self.susceptible[:, None] <= self.resist_def))).tolist())
pathn_pers = []
for pers in range(int(strain_persist))[::-1]:
pathn_pers.append(
npnonzero(self.space_contam == (pers + 1)))
plot_h.update_contam(host_types, pathn_pers)
return
def listify(lst, dim):
"""
Flatten lists of indices and ensure bounded by a known dim.
Parameters
----------
lst : list
List of integer indices
dim : tuple
Bounds for indices
"""
if not all([l.dtype == int for l in lst]):
raise ValueError("indices must be integers")
if npany(asarray(lst) >= dim):
raise ValueError("indices out of bounds for axis with size %s" % dim)
return lst.flatten()
def _negative_log_likelihood(self, params, freq, rec, T, penalizer_coef):
if npany(asarray(params) <= 0.):
return np.inf
r, alpha, s, beta = params
x = freq
r_s_x = r + s + x
A_1 = special.gammaln(r + x) - special.gammaln(
r) + r * log(alpha) + s * log(beta)
A_0 = self._A_0(params, freq, rec, T)
A_2 = logaddexp(-(r + x) * log(alpha + T) - s * log(beta + T),
log(s) + log(A_0) - log(r_s_x))
penalizer_term = penalizer_coef * log(params).sum()
return -(A_1 + A_2).sum() + penalizer_term
def listify(lst, dim):
"""
Flatten lists of indices and ensure bounded by a known dim.
Parameters
----------
lst : list
List of integer indices
dim : tuple
Bounds for indices
"""
if not all([l.dtype == int for l in lst]):
raise ValueError("indices must be integers")
if npany(asarray(lst) >= dim):
raise ValueError("indices out of bounds for axis with size %s" % dim)
return lst.flatten()
def _negative_log_likelihood(self, params, freq, rec, T, weights,
penalizer_coef):
"""
Sums the conditional log-likelihood from the ``_conditional_log_likelihood`` function
and applies a ``penalizer_coef``.
"""
params = self._convert_parameters(params)
if npany(np.hstack(list(params)) <= 0.0):
return np.inf
conditional_log_likelihood = ParetoNBDwithCovariatesFitter._conditional_log_likelihood(
params, freq, rec, T)
penalizer_term = penalizer_coef * sum(
np.abs(np.hstack(list(params)))**2)
return -(weights * conditional_log_likelihood).sum() / weights.mean(
) + penalizer_term
def cluster_union(clusters: NDArray) -> NDArray:
"""
Compute a set of clusters that is nested within 2 clusters
Parameters
----------
clusters : ndarray
A nobs by 2 array of integer values of cluster group membership.
Returns
-------
ndarray
A nobs array of integer cluster group memberships
"""
sort_keys = lexsort(clusters.T)
locs = arange(clusters.shape[0])
lex_sorted = clusters[sort_keys]
sorted_locs = locs[sort_keys]
diff = npany(lex_sorted[1:] != lex_sorted[:-1], 1)
union = cumsum(r_[0, diff])
resort_locs = argsort(sorted_locs)
return union[resort_locs]
def _validate_locations(self):
dist = self.env._r(array(self.loc).reshape((3, 1)), self.mics.mpos)
if npany(dist < 1e-7):
warn("Source and microphone locations are identical.",
Warning,
stacklevel=2)
def query(ra, dec, radius=2., unit='arcmin', z=0., cosmo=None,
catalogs=None, return_single=True, squeeze=False,
return_values=('name','ra','dec','z','index','dist','dz')):
"""
Query different catalogs for clusters close to a given set of coordinates.
To-do's:
-possibility to return survey observables
Parameters
----------
ra : (array of) float or str
if float, should be the RA in decimal degrees; if str,
should be in hms format ('hh:mm:ss', requires astLib)
dec : (array of) float or str
if float, should be the Dec in decimal degrees; if str,
should be in dms format ('dd:mm:ss', requires astLib)
radius : float (default 2)
the search radius, in units given by argumetn "unit"
unit : {'arcmin', 'Mpc'}
the units of argument "radius". If Mpc, then argument "z"
must be larger than zero.
z : (array of) float (optional)
redshift(s) of the cluster(s). Must be >0 if unit=='Mpc'.
cosmo : module astro.cosmology (optional)
if the matching is done in physical distance then pass
this module to make sure all cosmological parameters are
used consistently with the parent code!
catalogs : str (optional)
list or comma-separated names of catalogs to be searched.
If not given, all available catalogs are searched. Allowed
values are:
* 'maxbcg' (Koester et al. 2007)
* 'gmbcg' (Hao et al. 2010)
* 'hecs2013' (Rines et al. 2013)
* 'hecs2016' (Rines et al. 2016) NOT YET
* 'orca' (Geach, Murphy & Bower 2011)
* 'psz1' (Planck Collaboration XXIX 2014)
* 'psz2' (Planck Collaboration XXVII 2016)
* 'redmapper' (Rykoff et al. 2014, v5.10)
* 'whl' (Wen, Han & Liu 2012, Wen & Han 2015)
return_single : bool
whether to return the single closest matching cluster (if
within the search radius) or all those falling within the
search radius
return_values : any subset of ('name','ra','dec','z','index','dist','dz')
what elements to return. 'index', 'dist' and 'dz' refer to
the index in the catalog and the distances of the matching
cluster(s) in arcmin and redshift space, respectively.
NOTE: altering the order of the elements in return_values
will have no effect in the order in which they are returned!
squeeze : bool
whether to return a list instead of a dictionary if only
one catalog is provided
Returns
-------
matches : dict
matching elements per catalog. Each requested catalog is
a key of this dictionary if more than one catalog was
searched or if squeeze==False. If only one catalog was
provided and squeeze==True, then return a list with
matching entry/ies.
withmatch : dict
for each searched catalog, contains hte indices of the
provided clusters for which at least one match was found.
The same formatting as for "matches" applies.
"""
available = ('maxbcg', 'gmbcg', 'hecs2013', 'orca', 'psz1', 'psz2',
'redmapper', 'whl')
# some formatting for convenience
if not hasattr(ra, '__iter__'):
ra = [ra]
dec = [dec]
if not hasattr(z, '__iter__'):
z = array([z])
# in the case of matching by physical radius, demand z > 0
if unit == 'Mpc' and npany(z <= 0):
msg = "ERROR: in catalogs.query:"
msg += " if unit=='Mpc' then z must be larger than 0"
print msg
exit()
if unit == 'Mpc':
if cosmo is None:
cosmo = cosmology
dproj = cosmo.dProj
# will this fail for numpy.string_?
if isinstance(ra[0], basestring):
ra = array([hms2decimal(x, ':') for x in ra])
dec = array([dms2decimal(y, ':') for y in dec])
if unit == 'arcmin':
radius = ones(ra.size) * radius# / 60
else:
radius = array([dproj(zi, radius, input_unit='Mpc', unit='arcmin')
for zi in z])
if catalogs is None:
catalogs = available
else:
try:
catalogs = catalogs.split(',')
# if this happens then we assume catalogs is already a list
except ValueError:
pass
for name in catalogs:
if name not in available:
msg = 'WARNING: catalog {0} not available'.format(name)
print msg
labels = {'maxbcg': 'maxBCG', 'gmbcg': 'GMBCG', 'hecs2013': 'HeCS',
'hecs2016': 'HeCS-SZ', 'orca': 'ORCA', 'psz1': 'PSZ1',
'psz2': 'PSZ2', 'redmapper': 'redMaPPer', 'whl': 'WHL'}
filenames = {'maxbcg': 'maxbcg/maxBCG.fits',
'gmbcg': 'gmbcg/GMBCG_SDSS_DR7_PUB.fit',
'hecs2013': 'hecs/2013/data.fits',
'orca': 'orca/fullstripe82.fits',
'psz1': osjoin('planck', 'PSZ-2013', 'PLCK-DR1-SZ',
'COM_PCCS_SZ-union_R1.11.fits'),
'psz2': osjoin('planck', 'PSZ-2015',
'HFI_PCCS_SZ-union_R2.08.fits'),
'redmapper': 'redmapper/redmapper_dr8_public' + \
'_v5.10_catalog.fits',
'whl': 'whl/whl2015.fits'}
columns = {'maxbcg': 'none,RAJ2000,DEJ2000,zph',
'gmbcg': 'OBJID,RA,DEC,PHOTOZ',
'hecs2013': 'Name,RAJ2000,DEJ2000,z',
'orca': 'ID,ra_bcg,dec_bcg,redshift',
'psz1': 'NAME,RA,DEC,REDSHIFT',
'psz2': 'NAME,RA,DEC,REDSHIFT',
'redmapper': 'NAME,RA,DEC,Z_LAMBDA',
'whl': 'WHL,RAJ2000,DEJ2000,zph'}
for cat in catalogs:
filenames[cat] = osjoin(path, filenames[cat])
columns[cat] = columns[cat].split(',')
matches = {}
withmatch = {}
for cat in available:
if cat not in catalogs:
continue
data = getdata(filenames[cat], ext=1)
aux = {}
for name in data.names:
aux[name] = data[name]
data = aux
# if the catalog doesn't give a name
if columns[cat][0] == 'none':
columns[cat][0] = 'Name'
data['Name'] = chararray(data[columns[cat][1]].size, itemsize=4)
data['Name'][:] = 'none'
data = [data[v] for v in columns[cat]]
name, xcat, ycat, zcat = data
colnames = 'name,ra,dec,z'.split(',')
close = [(abs(xcat - x) < 2*r/60.) & (abs(ycat - y) < 2*r/60.)
for x, y, r in izip(ra, dec, radius)]
withmatch[cat] = [j for j, c in enumerate(close) if name[c].size]
dist = [60 * calcAngSepDeg(xcat[j], ycat[j], x, y)
for j, x, y in izip(close, ra, dec)]
match = [(d <= r) for d, r in izip(dist, radius)]
withmatch[cat] = array([w for w, m in izip(count(), match)
if w in withmatch[cat] and name[m].size])
if return_single:
match = [argmin(d) if d.size else None for d in dist]
matches[cat] = {}
# keeping them all now because they may be needed for other properties
for name, x in izip(colnames, data):
matches[cat][name] = array([x[j][mj] for w, j, mj
in izip(count(), close, match)
if w in withmatch[cat]])
if 'index' in return_values:
matches[cat]['index'] = array([arange(xcat.size)[j][m]
for w, j, m in izip(count(),
close, match)
if w in withmatch[cat]])
if 'dist' in return_values:
matches[cat]['dist'] = array([d[m] for w, d, m
in izip(count(), dist, match)
if w in withmatch[cat]])
if unit == 'Mpc':
matches[cat]['dist'] *= array([dproj(zi, 1, unit='Mpc',
input_unit='arcmin')
for zi in matches[cat]['z']])
if 'dz' in return_values:
matches[cat]['dz'] = array([zcat[j][m] - zj for w, j, m, zj
in izip(count(), close, match, z)
if w in withmatch[cat]])
for key in matches[cat].keys():
if key not in return_values:
matches[cat].pop(key)
if not return_single and name[j][match].size == 1:
for key in matches[cat].keys():
matches[cat][key] = matches[cat][key][0]
if len(catalogs) == 1 and squeeze:
return matches[catalogs[0]], withmatch[catalogs[0]]
return matches, withmatch
try:
greyscale = config['greyscale']
except:
greyscale = 'true'
#output variables
vars = [
k for k in config.keys() if not npany([
k.startswith('base'),
k.startswith('MAX_LR'),
k.startswith('MIN_LR'),
k.startswith('DO_AUG'),
k.startswith('SHALLOW'),
k.startswith('res_folder'),
k.startswith('train_csvfile'),
k.startswith('csvfile'),
k.startswith('test_csvfile'),
k.startswith('name'),
k.startswith('greyscale'),
k.startswith('aux_in'),
k.startswith('dropout'),
k.startswith('N'),
k.startswith('scale'),
k.startswith('numclass')
])
]
vars = sorted(vars)
#this relates to 'mimo' and 'miso' modes that are planned for the future but not currently implemented
auxin = [k for k in config.keys() if k.startswith('aux_in')]
if len(auxin) > 0:
def qz(A, B, mode='complex'):
"""
QZ decompostion for generalized eigenvalues of a pair of matrices.
Parameters
----------
A : array-like
B : array-like
mode : str {'real','complex'}
Returns
-------
AA, BB, Q.T, and Z
Notes
-----
If either A or B is a matrix, matrices are returned
"""
A = asanyarray(A)
B = asanyarray(B)
if isinstance(A,matrix) or isinstance(B, matrix):
usematrix = True
else:
usematrix = False
#NOTE: the first arguments to both are the selector functions but
#sort = 'N' is hard coded in qz.pyf so both are ignored, but the
#arguments are checked
n = A.shape[1]
if mode == 'real':
# sort = lambda x,y,z: (x+y)/z <= 1.
(AA,BB,sdim,alphar,alphai,
beta, Q, Z, work, info) = dgges(lambda x,y,z: None,A ,B)
if mode == 'complex':
rwork = zeros(8*n)
(AA,BB,sdim,alpha,beta,
Q, Z, work, info) = zgges(lambda x,y: None,A ,B,rwork)
# note that it will mostly be given real arrays, so it should
# return real arrays, check that this is the case and return the reals
if not npany(iscomplex(AA)):
AA = AA.real
if not npany(iscomplex(BB)):
BB = BB.real
if not npany(iscomplex(Q)):
Q = Q.real
if not npany(iscomplex(Z)):
Z = Z.real
if info < 0:
raise ValueError("Incorrect value at pos. "+str(abs(i))+"in _gges")
elif info > 0 and info <= n:
raise LinAlgError("QZ iteration failed, but ALPHA(j) and BETA(j) \
should be correct for j = %s,...,%s" % (str(i),str(n))) # not zero-based
elif info > n: # shouldn't get to this because sort='N'
raise LinAlgError("Something other than QZ iteration failed. \
Return INFO = %s. See _GGES.f for more information." % str(info))
#return Q.T to follow matlab convention
if usematrix:
AA = matrix(AA)
BB = matrix(BB)
Q = matrix(Q)
Z = matrix(Z)
return AA,BB,Q.T,Z
def _check_constant(self) -> bool:
col_delta = ptp(self.exog.ndarray, 0)
has_constant = npany(col_delta == 0)
self._const_col = where(col_delta == 0)[0][0] if has_constant else None
return bool(has_constant)
def simulate(city,
logfile,
simul_pop,
med_eff: float = 0.,
med_recov: float = 0,
vac_res: float = 0,
vac_cov: float = 0.,
movement_restrict: int = 0,
contact_restrict: int = 0,
lockdown_chunk: int = 0,
lockdown_panic: int = 1,
seed_inf: int = 0,
zero_lock: bool = False,
intervention: bool = False,
early_action=False,
plot_h=None) -> tuple:
'''Recursive simulation of each day'''
vaccine_discovery_date = 0
drug_discovery_date = 0
while not vaccine_discovery_date:
for idx, k in enumerate(nprandom.random(size=(int(5 /
MED_DISCOVERY)))):
if k < MED_DISCOVERY:
vaccine_discovery_date = idx
while not drug_discovery_date:
for idx, k in enumerate(nprandom.random(size=(int(5 /
MED_DISCOVERY)))):
if k < MED_DISCOVERY:
drug_discovery_date = idx
lockdown = 0
next_lockdown = seed_inf * lockdown_panic
# Track infection trends
track: nparray = nparray([[]] * 0, dtype=npint64).reshape((0, 5))
days = 0
args = city.survey(simul_pop)
newsboard = (
"Total Population %d" % city.pop_size,
"ICUs Available: %d/%d" %
(city.infrastructure - args[3], city.infrastructure),
)
reaction = (zero_lock, early_action, intervention,
days > vaccine_discovery_date, days > drug_discovery_date)
plot_h.update_epidem(days, args, newsboard, lockdown, *reaction)
track = npappend(track, nparray(args).reshape((1, 5)), axis=0)
print(*args, file=logfile, flush=True)
city.pass_day(plot_h) # IT STARTS!
while npany(city.space_contam): # Absent from persons and places
if days == vaccine_discovery_date:
city.vaccine_resist = vac_res
city.vaccine_cov = vac_cov
if days == drug_discovery_date:
for idx, pathy in enumerate(city.strain_types):
if pathy is not None:
city.strain_types[idx].inf_per_day /= med_recov
city.strain_types[idx].cfr *= med_eff
city.inf_per_day /= med_recov
city.cfr *= med_eff
if early_action:
if not days:
# Restrict movement
city.rms_v //= movement_restrict
city.move_per_day //= contact_restrict
elif days == zero_lock:
# End of initial lockdown
city.rms_v *= movement_restrict
city.move_per_day *= contact_restrict
days += 1
args = city.survey(simul_pop)
newsboard = (
"Total Population %d" % city.pop_size,
"ICUs Available: %d/%d" %
(city.infrastructure - args[3], city.infrastructure),
)
if days > drug_discovery_date:
newsboard.append("Drug Discovered")
if days > vaccine_discovery_date:
newsboard.append("Vaccine Discovered")
track = npappend(track, nparray(args).reshape((1, 5)), axis=0)
print(*args, file=logfile, flush=True)
city.pass_day(plot_h)
reaction = (zero_lock, early_action, intervention,
days > vaccine_discovery_date, days > drug_discovery_date)
plot_h.update_epidem(days, args, newsboard, lockdown, *reaction)
if intervention and lockdown == 0 and (args[2] > next_lockdown):
next_lockdown *= lockdown_panic
# Panic by infection Spread
lockdown = 1
city.rms_v //= movement_restrict
city.move_per_day //= contact_restrict
if intervention and lockdown:
lockdown += 1
if intervention and lockdown > lockdown_chunk + 1:
# Business as usual
city.rms_v *= movement_restrict
city.move_per_day *= contact_restrict
lockdown = 0
args = city.survey(simul_pop)
newsboard = (
"Total Population %d" % city.pop_size,
"ICUs Available: %d/%d" %
(city.infrastructure - args[3], city.infrastructure),
)
if days > drug_discovery_date:
newsboard.append("Drug Discovered")
if days > vaccine_discovery_date:
newsboard.append("Vaccine Discovered")
track = npappend(track, nparray(args).reshape((1, 5)), axis=0)
print(*args, file=logfile, flush=True)
reaction = (zero_lock, early_action, intervention,
days > vaccine_discovery_date, days > drug_discovery_date)
plot_h.update_epidem(days, args, newsboard, lockdown, *reaction)
return
