def q():
env = gym.make("CartPole-v1")
rng = default_rng()
numBins = 25
bins = [
# np.linspace(-4.8, 4.8, numBins),
np.linspace(-5, 5, numBins),
np.linspace(-0.418, 0.418, numBins),
# np.linspace(-5, 5, numBins),
]
features = slice(1, 3)
# features = slice(0, 4)
qTable = rng.uniform(
low=0,
high=0,
size=([numBins] * len(bins) + [env.action_space.n]),
)
print("b:", bins)
# print("qt: ", qTable[0][0])
print([numBins] * len(bins) + [env.action_space.n])
cnt_l = []
eps = 1
for i_episode in range(10000):
observation = env.reset()
# print(observation[features])
discreteState = get_discrete_state(observation[features], bins,
len(bins))
cnt = 0 # how may movements cart has made
while True:
# env.render() # if running RL comment this out
cnt += 1
eps -= 0.02
eps = max(eps, 0.1)
if rng.random() > eps:
# Get action from Q table
action = np.argmax(qTable[discreteState])
else:
# Get random action
action = rng.integers(0, env.action_space.n)
# perform action on enviroment
newState, reward, done, info = env.step(action)
newDiscreteState = get_discrete_state(newState[features], bins,
len(bins))
maxFutureQ = np.max(
qTable[newDiscreteState]) # estimate of optiomal future value
currentQ = qTable[discreteState + (action, )] # old value
# formula to caculate all Q values
newQ = (1 - LR) * currentQ + LR * (reward + DISCOUNT * maxFutureQ)
# Update qTable with new Q value
qTable[discreteState + (action, )] = newQ
discreteState = newDiscreteState
if done:
print(f"Done: fell after: {cnt}")
cnt_l.append(cnt)
break
# print(cnt_l)
if len(cnt_l) > 100:
cnt_l.pop(0)
if sum(cnt_l) > 100 * 180:
print("Learned, lets test")
state = env.reset()
while True:
env.render()
discreteState = get_discrete_state(state[features], bins,
len(bins))
action = np.argmax(qTable[discreteState])
state, reward, done, info = env.step(action)
env.close()
# RA, 2021-02-19
"""
The minimum of n uniformly distributed rv.
"""
import numpy as np
import pandas as pd
from plox import Plox
from numpy.random import default_rng
rng = default_rng(12)
# Take n random variables X1, ..., Xn ~ [0, 1]
n = 10
# Define the random variable Y = min{X1, ..., Xn}
Y = (lambda: min(rng.uniform(size=n)))
# Its expected value is:
EY = 1 / (n + 1)
# Let's verify this for realizations of Y
YY = pd.Series(index=(1 + np.arange(2**19)), data=0)
YY = YY.transform(lambda i: Y())
# Empirical mean as the number of samples increases
YY = YY.cumsum() / YY.index
# Subsample for plotting
YY = YY[2**np.arange(0, np.floor(np.log2(len(YY))))]
import pandas as pd
import numpy as np
from bokeh import plotting as bk
import matplotlib.pyplot as plt
from numpy.random import default_rng
rng = default_rng(12345)
date_range = pd.date_range("2020-01-01", periods=50)
data = np.add.accumulate(rng.normal(0, 3, size=50))
series = pd.Series(data, index=date_range)
fig, ax = plt.subplots()
series.plot(ax=ax, title="Time series data")
ax.set_xlabel("date")
ax.set_ylabel("value")
fig.savefig("bokeh-plot.png", dpi=300)
bk.output_file("sample.html")
fig = bk.figure(title="Time series data",
x_axis_label="date",
x_axis_type="datetime",
y_axis_label="value")
fig.line(date_range, series)
bk.show(fig)
def get_stream(name, seed=None):
config_path = path.join(path.dirname(__file__), f'{name}.yaml')
stream_config = load_yaml(config_path)
config = load_default_config(stream_config)
recursive_update(config, stream_config)
return init_component(default_rng(seed), **config)['task_gen']
def _compute_states(
self,
input: np.ndarray,
forced_teacher: np.ndarray = None,
init_state: np.ndarray = None,
init_fb: np.ndarray = None,
seed: int = None,
verbose: bool = False,
**kwargs,
) -> Union[Tuple[np.ndarray, np.ndarray], np.ndarray]:
"""Compute all states generated from a single sequence of inputs.
Parameters
input {np.ndarray} -- Sequence of inputs.
Keyword Arguments:
forced_teacher {np.ndarray} -- Ground truth vectors to use as feedback
during training, if feedback is enabled.
(default: {None})
init_state {np.ndarray} -- Initialization vector for states.
(default: {None})
init_fb {np.ndarray} -- Initialization vector for feedback.
(default: {None})
wash_nr_time_step {int} -- Number of states to considered as transitory
when training. (default: {0})
input_id {int} -- Index of the input in the queue. Used for parallelization
of computations. (default: {None})
Raises:
RuntimeError: raised if no teachers are specifiyed for training
with feedback.
Returns:
Union[Tuple[np.ndarray, np.ndarray], np.ndarray] -- Index of the
input in queue
and computed states, or just states if no index is provided.
"""
if self.Wfb is not None and forced_teacher is None and self.Wout is None:
raise RuntimeError("Impossible to use feedback without readout "
"matrix or teacher forcing.")
check_input_lists(input, self.dim_in, forced_teacher, self.dim_out)
# to track successives internal states of the reservoir
states = np.zeros((len(input), self.N), dtype=self.typefloat)
# if a feedback matrix is available, feedback will be set to 0 or to
# a specific value.
if init_fb is not None:
last_feedback = init_fb.copy().reshape(1, -1)
else:
last_feedback = self.zero_feedback()
# State is initialized to 0 or to a specific value.
if init_state is None:
current_state = self.zero_state()
else:
current_state = init_state.copy().reshape(1, -1)
# random generator for training/running with additive noise
rng = default_rng(seed)
pbar = None
if kwargs.get("pbar") is not None:
pbar = kwargs.get("pbar")
elif verbose is True:
pbar = tqdm(total=input.shape[0])
# for each time step in the input
for t in range(input.shape[0]):
# compute next state from current state
current_state = self._get_next_state(
input[t, :],
feedback=last_feedback,
last_state=current_state,
noise_generator=rng,
)
# compute last feedback
if self.Wfb is not None:
# during training outputs are equal to teachers for feedback
if forced_teacher is not None:
last_feedback = forced_teacher[t, :]
# feedback of outputs, computed with Wout
else:
last_feedback = add_bias(current_state) @ self.Wout.T
states[t, :] = current_state
if pbar is not None:
pbar.update(1)
return states
def generate_dset_near_trajectory(self, trajectory, max_dist, samples,
orientation=False,
savename=None,
save_folder=Path('.')):
"""
Samples training points close to the supplied trajectory, if remove_duplicates is True,
it removes duplicated points
:return: dict with dataset
"""
Qs = []
rng = default_rng()
tentative_per_try = 50000
tries = 0
if orientation:
check_every = tentative_per_try*20
else:
check_every = tentative_per_try*5
while len(Qs) <= samples:
tries += tentative_per_try
tentative_Qs = rng.random([tentative_per_try, self.num_non_fixed_joints])
high_tiled = np.tile(np.array(self.upper_joint_limits), (tentative_per_try, 1))
low_tiled = np.tile(np.array(self.lower_joint_limits), (tentative_per_try, 1))
tentative_Qs = (high_tiled - low_tiled) * tentative_Qs + low_tiled
poses = self.computeForwardKinematics(tentative_Qs)
X = poses[:, 0]
Y = poses[:, 1]
distances = sqdist(X1=np.vstack((X, Y)).T, X2=trajectory[:, :2])
if orientation:
Theta = poses[:, 2]
circle_distances_min = np.array([np.sqrt(radial_squared_error(np.atleast_2d(candidate).T, np.atleast_2d(trajectory[:, 2]))).min() for candidate in Theta])
[Qs.append(q) for (q, p_wise_dists, min_ori_dist) in zip(tentative_Qs, distances, circle_distances_min)
if (p_wise_dists.min() <= max_dist) and (min_ori_dist <= 0.03)]
else:
[Qs.append(q) for (q, p_wise_dists) in zip(tentative_Qs, distances) if p_wise_dists.min() <= max_dist]
if tries % check_every == 0:
print(f'Generating dataset.\t Tries: {tries}\tCurrent samples: {len(Qs)}\\{samples}')
if len(Qs) > samples:
[Qs.pop() for idx in range(len(Qs)-samples)]
Qs = np.array(Qs)
poses = self.computeForwardKinematics(Qs)
X = poses[:, 0]
Y = poses[:, 1]
Theta = poses[:, 2]
# plt.plot([X, X + 0.2 * np.cos(Theta)],
# [Y, Y + 0.2 * np.sin(Theta)], 'k-')
# plt.scatter(X, Y) # Plotting the data set
# plt.xlabel("X Axis")
# plt.ylabel("Y Axis")
# plt.savefig(output_folder / f'Dataset of {samples} samples with {traj_name} trajectory.png')
data = np.vstack((X, Y, Theta)).T # X,Y,Theta
output = np.array(Qs)
return self.split_data_into_train_val_test(data, output, save_folder, fullspace=False, savename=savename)
_sage_const_4 = Integer(4)
_sage_const_0p5 = RealNumber('0.5')
_sage_const_20 = Integer(20)
_sage_const_16 = Integer(16)
_sage_const_100 = Integer(100)
_sage_const_0 = Integer(0)
import numpy as np
from numpy.random import default_rng
import sys
args = sys.argv
seed = int(args[_sage_const_1])
M = int(args[_sage_const_2])
N = int(args[_sage_const_3])
K = int(args[_sage_const_4])
rng = default_rng(seed)
# A = rng.random(size=(M, N), dtype=np.float64)
# B = rng.random(size=(N, K), dtype=np.float64)
remapping = np.vectorize(lambda x: (x - _sage_const_0p5) * _sage_const_20)
A = remapping(rng.random(size=(M, N), dtype=np.float64))
B = remapping(rng.random(size=(N, K), dtype=np.float64))
res_data = np.genfromtxt(
"./src/sim/build/C_res_{:d}_{:d}_{:d}_{:d}.out".format(M, N, K, seed),
comments="//",
dtype=str).reshape(M, K)
# res_data = np.genfromtxt("./src/sim/C_res.out".format(M,N,K,seed), comments="//", dtype = str).reshape(M,K)
str2num = np.vectorize(lambda x: int("0x" + x, int(_sage_const_16)))
res_num = str2num(res_data)
res_num.dtype = np.float64
R100 = RealField(_sage_const_100)
def gen_SimObs_from_sedgrid(
sedgrid,
sedgrid_noisemodel,
nsim=100,
compl_filter="max",
complcut=None,
magcut=None,
ranseed=None,
vega_fname=None,
weight_to_use="weight",
age_prior_model=None,
mass_prior_model=None,
):
"""
Generate simulated observations using the physics and observation grids.
The priors are sampled as they give the ensemble model for the stellar
and dust distributions (IMF, Av distribution etc.).
The physics model gives the SEDs based on the priors.
The observation model gives the noise, bias, and completeness all of
which are used in simulating the observations.
Currently written to only work for the toothpick noisemodel.
Parameters
----------
sedgrid: grid.SEDgrid instance
model grid
sedgrid_noisemodel: beast noisemodel instance
noise model data
nsim : int
number of observations to simulate
compl_filter : str
Filter to use for completeness (required for toothpick model).
Set to max to use the max value in all filters.
complcut : float (defualt=None)
completeness cut for only including model seds above the cut
where the completeness cut ranges between 0 and 1.
magcut : float (defualt=None)
faint-end magnitude cut for only including model seds brighter
than the given magnitude in compl_filter.
ranseed : int
used to set the seed to make the results reproducable,
useful for testing
vega_fname : string
filename for the vega info, useful for testing
weight_to_use : string (default='weight')
Set to either 'weight' (prior+grid), 'prior_weight', 'grid_weight',
or 'uniform' (this option is valid only when nsim is supplied) to
choose the weighting for SED selection.
age_prior_model : dict
age prior model in the BEAST dictonary format
mass_prior_model : dict
mass prior model in the BEAST dictonary format
Returns
-------
simtable : astropy Table
table giving the simulated observed fluxes as well as the
physics model parmaeters
"""
n_models, n_filters = sedgrid.seds.shape
flux = sedgrid.seds
# get the vega fluxes for the filters
_, vega_flux, _ = Vega(source=vega_fname).getFlux(sedgrid.filters)
# cache the noisemodel values
model_bias = sedgrid_noisemodel["bias"]
model_unc = np.fabs(sedgrid_noisemodel["error"])
model_compl = sedgrid_noisemodel["completeness"]
# only use models that have non-zero completeness in all filters
# zero completeness means the observation model is not defined for that filters/flux
# if complcut is provided, only use models above that completeness cut
if complcut is not None:
finalcomplcut = complcut
else:
finalcomplcut = 0.0
ast_defined = model_compl > finalcomplcut
sum_ast_defined = np.sum(ast_defined, axis=1)
goodobsmod = sum_ast_defined >= n_filters
# completeness from toothpick model so n band completeness values
# require only 1 completeness value for each model
# max picked to best "simulate" how the photometry detection is done
if compl_filter.lower() == "max":
model_compl = np.max(model_compl, axis=1)
else:
short_filters = [
filter.split(sep="_")[-1].upper() for filter in sedgrid.filters
]
if compl_filter.upper() not in short_filters:
raise NotImplementedError(
"Requested completeness filter not present:" +
compl_filter.upper() + "\nPossible filters:" +
"\n".join(short_filters))
filter_k = short_filters.index(compl_filter.upper())
print("Completeness from %s" % sedgrid.filters[filter_k])
model_compl = model_compl[:, filter_k]
# if magcut is provided, only use models brighter than the magnitude cut
# in addition to the non-zero completeness criterion
if magcut is not None:
fluxcut_compl_filter = 10**(-0.4 * magcut) * vega_flux[filter_k]
goodobsmod = (goodobsmod) & (flux[:, filter_k] >= fluxcut_compl_filter)
# initialize the random number generator
rangen = default_rng(ranseed)
# if the age and mass prior models are given, use them to determine the
# total number of stars to simulate
model_indx = np.arange(n_models)
if (age_prior_model is not None) and (mass_prior_model is not None):
nsim = 0
# logage_range = [min(sedgrid["logA"]), max(sedgrid["logA"])]
mass_range = [min(sedgrid["M_ini"]), max(sedgrid["M_ini"])]
# compute the total mass and average mass of a star given the mass_prior_model
nmass = 100
masspts = np.logspace(np.log10(mass_range[0]), np.log10(mass_range[1]),
nmass)
mass_prior = PriorMassModel(mass_prior_model)
massprior = mass_prior(masspts)
totmass = np.trapz(massprior, masspts)
avemass = np.trapz(masspts * massprior, masspts) / totmass
# compute the mass of the remaining stars at each age and
# simulate the stars assuming everything is complete
gridweights = sedgrid[weight_to_use]
gridweights = gridweights / np.sum(gridweights)
grid_ages = np.unique(sedgrid["logA"])
age_prior = PriorAgeModel(age_prior_model)
ageprior = age_prior(grid_ages)
bin_boundaries = compute_bin_boundaries(grid_ages)
bin_widths = np.diff(10**(bin_boundaries))
totsim_indx = np.array([], dtype=int)
for cage, cwidth, cprior in zip(grid_ages, bin_widths, ageprior):
gmods = sedgrid["logA"] == cage
cur_mass_range = [
min(sedgrid["M_ini"][gmods]),
max(sedgrid["M_ini"][gmods]),
]
gmass = (masspts >= cur_mass_range[0]) & (masspts <=
cur_mass_range[1])
curmasspts = masspts[gmass]
curmassprior = massprior[gmass]
totcurmass = np.trapz(curmassprior, curmasspts)
# compute the mass remaining at each age -> this is the mass to simulate
simmass = cprior * cwidth * totcurmass / totmass
nsim_curage = int(round(simmass / avemass))
# simluate the stars at the current age
curweights = gridweights[gmods]
if np.sum(curweights) > 0:
curweights /= np.sum(curweights)
cursim_indx = rangen.choice(model_indx[gmods],
size=nsim_curage,
p=curweights)
totsim_indx = np.concatenate((totsim_indx, cursim_indx))
nsim += nsim_curage
# totsimcurmass = np.sum(sedgrid["M_ini"][cursim_indx])
# print(cage, totcurmass / totmass, simmass, totsimcurmass, nsim_curage)
totsimmass = np.sum(sedgrid["M_ini"][totsim_indx])
print(f"number total simulated stars = {nsim}; mass = {totsimmass}")
compl_choice = rangen.random(nsim)
compl_indx = model_compl[totsim_indx] >= compl_choice
sim_indx = totsim_indx[compl_indx]
totcompsimmass = np.sum(sedgrid["M_ini"][sim_indx])
print(
f"number of simulated stars w/ completeness = {len(sim_indx)}; mass = {totcompsimmass}"
)
else: # total number of stars to simulate set by command line input
if weight_to_use == "uniform":
# sample to get the indices of the picked models
sim_indx = rangen.choice(model_indx[goodobsmod], nsim)
else:
gridweights = sedgrid[weight_to_use][goodobsmod] * model_compl[
goodobsmod]
gridweights = gridweights / np.sum(gridweights)
# sample to get the indexes of the picked models
sim_indx = rangen.choice(model_indx[goodobsmod],
size=nsim,
p=gridweights)
print(f"number of simulated stars = {nsim}")
# setup the output table
ot = Table()
qnames = list(sedgrid.keys())
# simulated data
for k, filter in enumerate(sedgrid.filters):
simflux_wbias = flux[sim_indx, k] + model_bias[sim_indx, k]
simflux = rangen.normal(loc=simflux_wbias,
scale=model_unc[sim_indx, k])
bname = filter.split(sep="_")[-1].upper()
fluxname = f"{bname}_FLUX"
colname = f"{bname}_RATE"
magname = f"{bname}_VEGA"
ot[fluxname] = Column(simflux)
ot[colname] = Column(ot[fluxname] / vega_flux[k])
pindxs = ot[colname] > 0.0
nindxs = ot[colname] <= 0.0
ot[magname] = Column(ot[colname])
ot[magname][pindxs] = -2.5 * np.log10(ot[colname][pindxs])
ot[magname][nindxs] = 99.999
# add in the physical model values in a form similar to
# the output simulated (physics+obs models) values
# useful if using the simulated data to interpolate ASTs
# (e.g. for MATCH)
fluxname = f"{bname}_INPUT_FLUX"
ratename = f"{bname}_INPUT_RATE"
magname = f"{bname}_INPUT_VEGA"
ot[fluxname] = Column(flux[sim_indx, k])
ot[ratename] = Column(ot[fluxname] / vega_flux[k])
pindxs = ot[ratename] > 0.0
nindxs = ot[ratename] <= 0.0
ot[magname] = Column(ot[ratename])
ot[magname][pindxs] = -2.5 * np.log10(ot[ratename][pindxs])
ot[magname][nindxs] = 99.999
# model parmaeters
for qname in qnames:
ot[qname] = Column(sedgrid[qname][sim_indx])
return ot
def test_seed(self):
for args in [(), (None,), (1234,), ([1234, 5678],)]:
rg = default_rng(*args)
assert isinstance(rg.bit_generator, PCG64)
def autogen_points(input_seg,
count,
roi,
body,
tbars,
use_skeleton,
random_seed=None,
minimum_distance=0):
"""
Generate a list of points within the input segmentation, based on the given criteria.
See the main help text below for details.
"""
if tbars and not body:
sys.exit(
"If you want to auto-generate tbar points, please specify a body.")
if not tbars and not count:
sys.exit(
"You must supply a --count unless you are generating all tbars of a body."
)
if use_skeleton:
if not body:
sys.exit(
"You must supply a body ID if you want to use a skeleton.")
if tbars:
sys.exit(
"You can't select both tbar points and skeleton points. Pick one or the other."
)
if not count and minimum_distance > 0:
sys.exit(
"You must supply a --count if you want skeleton point samples to respect the minimum distance."
)
if not count and not roi and minimum_distance == 0:
logger.warning(
"You are using all nodes of a skeleton without any ROI filter! Is that what you meant?"
)
rng = default_rng(random_seed)
if tbars:
logger.info(f"Fetching synapses for body {body}")
syn_df = fetch_annotation_label(*input_seg[:2],
'synapses',
body,
format='pandas')
tbars = syn_df.query('kind == "PreSyn"')[[*'zyx']]
if roi:
logger.info(f"Filtering tbars for roi {roi}")
determine_point_rois(*input_seg[:2], [roi], tbars)
tbars = tbars.query('roi == @roi')[[*'zyx']]
if minimum_distance:
logger.info(
f"Pruning close points from {len(tbars)} total tbar points")
tbars = prune_close_pairs(tbars, minimum_distance, rng)
logger.info(f"After pruning, {len(tbars)} tbars remain.")
if count:
count = min(count, len(tbars))
logger.info(f"Sampling {count} tbars")
choices = rng.choice(tbars.index, size=count, replace=False)
tbars = tbars.loc[choices]
logger.info(f"Returning {len(tbars)} tbar points")
return tbars
elif use_skeleton:
assert body
logger.info(f"Fetching skeleton for body {body}")
skeleton_instance = f'{input_seg[2]}_skeletons'
swc = fetch_key(*input_seg[:2], skeleton_instance, f'{body}_swc')
skeleton_df = swc_to_dataframe(swc)
skeleton_df['x'] = skeleton_df['x'].astype(int)
skeleton_df['y'] = skeleton_df['y'].astype(int)
skeleton_df['z'] = skeleton_df['z'].astype(int)
if roi:
logger.info(f"Filtering skeleton for roi {roi}")
determine_point_rois(*input_seg[:2], [roi], skeleton_df)
skeleton_df = skeleton_df.query('roi == @roi')[[*'zyx']]
if minimum_distance:
assert count
# Distance-pruning is very expensive on a huge number of close points.
# If skeleton is large, first reduce the workload by pre-selecting a
# random sample of skeleton points, and prune more from there.
if len(skeleton_df) > 10_000:
# FIXME: random_state can't use rng until I upgrade to pandas 1.0
skeleton_df = skeleton_df.sample(min(4 * count,
len(skeleton_df)),
random_state=None)
logger.info(
f"Pruning close points from {len(skeleton_df)} skeleton points"
)
prune_close_pairs(skeleton_df, minimum_distance, rng)
logger.info(
f"After pruning, {len(skeleton_df)} skeleton points remain.")
if count:
count = min(count, len(skeleton_df))
logger.info(f"Sampling {count} skeleton points")
choices = rng.choice(skeleton_df.index, size=count, replace=False)
skeleton_df = skeleton_df.loc[choices]
logger.info(f"Returning {len(skeleton_df)} skeleton points")
return skeleton_df
elif body:
assert count
if roi:
# TODO: intersect the ranges with the ROI.
raise NotImplementedError(
"Sorry, I haven't yet implemented support for "
"body+roi filtering. Pick one or the other, "
"or ask Stuart to fix this.")
logger.info(f"Fetching sparsevol for body {body}")
ranges = fetch_sparsevol(*input_seg, body, format='ranges')
logger.info("Sampling from sparsevol")
if minimum_distance > 0:
# Sample 4x extra so we still have enough after pruning.
points = sample_points_from_ranges(ranges, 4 * count, rng)
else:
points = sample_points_from_ranges(ranges, count, rng)
points = pd.DataFrame(points, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(f"Pruning close points from {len(points)} body points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(f"After pruning, {len(points)} body points remain")
points = points.iloc[:count]
logger.info(f"Returning {len(points)} body points")
return points
elif roi:
assert count
logger.info(f"Fetching roi {roi}")
roi_ranges = fetch_roi_roi(*input_seg[:2], roi, format='ranges')
logger.info("Sampling from ranges")
if minimum_distance > 0:
# Sample 4x extra so we can prune some out if necessary.
points_s5 = sample_points_from_ranges(roi_ranges, 4 * count, rng)
else:
points_s5 = sample_points_from_ranges(roi_ranges, count, rng)
corners_s0 = points_s5 * (2**5)
points_s0 = rng.integers(corners_s0, corners_s0 + (2**5))
points = pd.DataFrame(points_s0, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(f"Pruning close points from {len(points)} roi points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(
f"After pruning, points from {len(points)} roi points remain")
points = points.iloc[:count]
logger.info(f"Returning {len(points)} roi points")
return points
else:
# No body or roi specified, just sample from the whole non-zero segmentation area
assert count
logger.info("Sampling random points from entire input segmentation")
logger.info("Fetching low-res input volume")
box_s6 = round_box(fetch_volume_box(*input_seg), 2**6, 'out') // 2**6
seg_s6 = fetch_labelmap_voxels(*input_seg, box_s6, scale=6)
mask_s6 = seg_s6.astype(bool)
logger.info("Encoding segmentation as ranges")
seg_ranges = runlength_encode_mask_to_ranges(mask_s6, box_s6)
logger.info("Sampling from ranges")
if minimum_distance > 0:
# Sample 4x extra so we can prune some out if necessary.
points_s6 = sample_points_from_ranges(seg_ranges, 4 * count, rng)
else:
points_s6 = sample_points_from_ranges(seg_ranges, count, rng)
corners_s0 = points_s6 * (2**6)
points_s0 = rng.integers(corners_s0, corners_s0 + (2**6))
points = pd.DataFrame(points_s0, columns=[*'zyx'])
if minimum_distance > 0:
logger.info(
f"Pruning close points from {len(points)} segmentation points")
prune_close_pairs(points, minimum_distance, rng)
logger.info(
f"After pruning, points from {len(points)} segmentation points remain"
)
points = points.iloc[:count]
logger.info(f"Returning {len(points)} segmentation points")
return points
if (np.linalg.norm(point - prob_distribution.center_point) <=
prob_distribution.width / 2):
break
elif prob_distribution.name == "3d_cube":
point = rng.random(
3) * prob_distribution.width + prob_distribution.center_point
else:
raise ValueError(
f"unknown name of point distribution: {prob_distribution.name}")
return point
PROBABILITY_DISTRIBUTIONS = {
"IC fixed-size": random_ic_fixed_size_profile,
"IC": random_ic_profile,
"Truncated Mallows": random_mallows_profile,
"Urn fixed-size": random_urn_fixed_size_profile,
"Urn": random_urn_profile,
"Truncated Urn": random_truncated_urn_profile,
"Euclidean VCR": random_euclidean_vcr_profile,
"Euclidean fixed-size": random_euclidean_fixed_size_profile,
"Euclidean Threshold": random_euclidean_threshold_profile,
"Resampling": random_resampling_profile,
"Disjoint Resampling": random_disjoint_resampling_profile,
"Noise": random_noise_model_profile,
}
PROBABILITY_DISTRIBUTION_IDS = tuple(PROBABILITY_DISTRIBUTIONS.keys())
rng = default_rng() # random number generator
def generate_matrix(size, dtype):
from numpy.random import default_rng
rng = default_rng(42)
A = rng.random((size, size), dtype=dtype)
return (0.5 * A @ A.T).copy()
def weights_init(size, map_depth, dist_sigma=1.0):
generator = default_rng()
weights = generator.normal(scale=dist_sigma, size=(size, size, map_depth))
return weights
import numpy as np
import pandas as pd
import seaborn as sns
from nbex.interactive import session
from numpy.random import default_rng
from ml_for_programmers.config import Config
# %%
config = Config()
# %%
sns.set_theme(style="darkgrid")
# %%
rng = default_rng(seed=42)
# %%
def generate_linear_relation(x,
slope=1.5,
offset=1,
random_scale=1.0,
rng=rng):
random_offsets = rng.normal(size=x.shape, scale=random_scale)
y_linear = slope * x + offset
return y_linear + random_offsets
# %%
def linear_salaries_for_ages(ages,
def generator():
# Instantiate a random generator
rng = default_rng()
ppn_bs = round(batch_size / 5)
lm_bs = batch_size - 3 * ppn_bs
num = (steps_per_epoch // 3) * ppn_bs
neg_lm_bb = np.zeros((ppn_bs + lm_bs, 5), np.float32)
pos_par_neg_lm = np.zeros((3 * ppn_bs, 11), np.float32)
batch_class = np.zeros((batch_size, 3), np.float32)
batch_class[:ppn_bs, :2] = 1
batch_class[2 * ppn_bs:3 * ppn_bs, ::2] = 1
# Each iteration of this loop is an epoch
while True:
random_pos = rng.permutation(6)
random_par = rng.permutation(3)
random_neg = rng.permutation(3)
random_lm = rng.permutation(3)
# Each iteration of this loop is one third of an epoch
for i in range(3):
# Load positive data
pos_img = list()
pos_bb = list()
for j in range(2):
img = np.load(data_folder + '/train/pos_img_' +
str(random_pos[2 * i + j]) + '.npy')
anno = np.load(data_folder + '/train/pos_bb_' +
str(random_pos[2 * i + j]) + '.npy')
num_img = img.shape[0]
random_indices = rng.permutation(num_img)
img = img[random_indices]
anno = anno[random_indices]
pos_img.append(img[:num // 2])
pos_bb.append(anno[:num // 2])
del img
pos_img = np.concatenate(pos_img)
pos_bb = np.concatenate(pos_bb)
# Load part face data
par_img = np.load(data_folder + '/train/par_img_' +
str(random_par[i]) + '.npy')
par_bb = np.load(data_folder + '/train/par_bb_' +
str(random_par[i]) + '.npy')
num_img = par_img.shape[0]
random_indices = rng.permutation(num_img)
par_img = par_img[random_indices][:num]
par_bb = par_bb[random_indices][:num]
# Load negative data
neg_img = np.load(data_folder + '/train/neg_img_' +
str(random_neg[i]) + '.npy')
num_img = neg_img.shape[0]
random_indices = rng.permutation(num_img)
neg_img = neg_img[random_indices][:num]
# Load landmark data
lm_img = np.load(data_folder + '/train/lm_img_' +
str(random_lm[i]) + '.npy')
lm_lm = np.load(data_folder + '/train/lm_lm_' +
str(random_lm[i]) + '.npy')
num_img = lm_img.shape[0]
random_indices = rng.permutation(num_img)
lm_img = lm_img[random_indices]
lm_lm = lm_lm[random_indices]
# Each iteration of this loop is a batch
for j in range(steps_per_epoch // 3):
batch_img = np.concatenate(
(pos_img[j * ppn_bs:(j + 1) * ppn_bs],
par_img[j * ppn_bs:(j + 1) * ppn_bs],
neg_img[j * ppn_bs:(j + 1) * ppn_bs],
lm_img[j * lm_bs:(j + 1) * lm_bs]))
batch_bb = np.concatenate(
(pos_bb[j * ppn_bs:(j + 1) * ppn_bs],
par_bb[j * ppn_bs:(j + 1) * ppn_bs], neg_lm_bb))
batch_lm = np.concatenate(
(pos_par_neg_lm, lm_lm[j * lm_bs:(j + 1) * lm_bs]))
yield batch_img, (batch_class, batch_bb, batch_lm)
for i, dataset in enumerate(datasets):
dataset.path = f"{prefix}_{i:03d}.csv"
dataset.content.to_csv(f"{base_cache_path}/{dataset.path}")
dataset.content = None
with open(f"{base_config_path}/{prefix}_{i}.json",
'w',
encoding='utf-8') as f:
json.dump(dataset.to_dict(), f)
if __name__ == '__main__':
base_config_path: str = "../../data/synthetic"
base_cache_path: str = "../../data/cache"
random_state = 42
rng = default_rng(random_state)
def _generate_blobs():
print("Generating blobs")
blobs_count = 200
blobs_datasets: List[Dataset] = []
for _ in range(blobs_count):
params = {
"n_samples": rng.integers(low=100, high=2000),
"n_features": rng.integers(low=20, high=250),
"n_clusters": rng.integers(low=2, high=20),
"random_state": random_state
}
dataset: Dataset = generate_blobs(**params)
blobs_datasets.append(dataset)
save_datasets(base_config_path,
def main():
# Create train, validati¿n and test sets from STEAD
# and non seismic geophone dataset files
# Args
parser = argparse.ArgumentParser(description='Dataset creation parameters')
parser.add_argument('--source_file', default='../Data/STEAD/STEAD.hdf5',
help='Source HDF5 file path')
parser.add_argument('--train_file', default='Train_data_v3.hdf5',
help='Output train HDF5 file path')
parser.add_argument('--val_file', default='Validation_data_v3.hdf5',
help='Output validation HDF5 file path')
parser.add_argument('--test_file', default='Test_data_v2.hdf5',
help='Output test HDF5 file path')
parser.add_argument('--train_traces', type=int, default=52500,
help='Number of training seismic traces to copy')
parser.add_argument('--train_noise', type=int, default=26250,
help='Number of training noise traces to copy')
parser.add_argument('--val_traces', type=int, default=7500,
help='Number of validation seismic traces to copy')
parser.add_argument('--val_noise', type=int, default=3750,
help='Number of validation noise traces to copy')
parser.add_argument('--test_traces', type=int, default=10000,
help='Number of test seismic traces to copy')
parser.add_argument('--test_noise', type=int, default=10000,
help='Number of test noise traces to copy')
args = parser.parse_args()
# Init rng
rng = default_rng()
with open('fault.txt', 'r') as f:
ln = f.readline()
faulty = np.asarray(list(map(int, ln.strip().split(','))))
# Load geophone data
coompana_traces = read_coompana()
lesser_traces = read_lesser_antilles_airgun()
nc_traces = read_north_carolina_airgun()
# NUMERO DE TRAZAS DE ENTRENAMIENTO, VALIDACION Y PRUEBA GEO
n_geo_train = 8750
n_geo_val = 1250
n_geo_test = 10000
# Split geophone data
coompana_traces_train = coompana_traces[:n_geo_train]
coompana_traces_val = coompana_traces[n_geo_train:n_geo_train + n_geo_val]
coompana_traces_test = coompana_traces[n_geo_train + n_geo_val:]
lesser_traces_train = lesser_traces[:n_geo_train]
lesser_traces_val = lesser_traces[n_geo_train:n_geo_train + n_geo_val]
lesser_traces_test = lesser_traces[n_geo_train + n_geo_val:
n_geo_train + n_geo_val + n_geo_test]
nc_traces_train = nc_traces[:n_geo_train]
nc_traces_val = nc_traces[n_geo_train:n_geo_train + n_geo_val]
nc_traces_test = nc_traces[n_geo_train + n_geo_val:
n_geo_train + n_geo_val + n_geo_test]
# Read the hdf5 source file
with h5py.File(args.source_file, 'r') as source:
# Retrieve file groups
src_seis = source['earthquake']['local']
src_ns = source['non_earthquake']['noise']
# Total number of traces to copy
seis2copy = args.train_traces + args.val_traces + args.test_traces
ns2copy = args.train_noise + args.val_noise + args.test_noise
# Indexes of traces to copy
seismic_ids = rng.choice(len(src_seis), size=seis2copy, replace=False)
noise_ids = rng.choice(len(src_ns), size=ns2copy, replace=False)
# Check faulty datasets selected
for val in faulty:
# If faulty selected
if val in seismic_ids:
# Delete from array
idx = np.argwhere(seismic_ids == val)
seismic_ids = np.delete(seismic_ids, idx)
# Select new one
new_val = rng.choice(len(src_seis), size=1)
# Check if is already in array
while new_val in seismic_ids:
new_val = rng.choice(len(src_seis), size=1)
# Append to array
seismic_ids = np.append(seismic_ids, new_val)
# Indexes of traces to copy to train dataset
train_seis_ids = seismic_ids[:args.train_traces]
train_noise_ids = noise_ids[:args.train_noise]
# Indexes of traces to copy to validation dataset
val_seis_ids = seismic_ids[args.train_traces:args.train_traces + args.val_traces]
val_noise_ids = noise_ids[args.train_noise:args.train_noise + args.val_noise]
# Indexes of traces to copy to test dataset
test_seis_ids = seismic_ids[args.train_traces + args.val_traces:args.train_traces + args.val_traces+args.test_traces]
test_noise_ids = noise_ids[args.train_noise + args.val_noise:args.train_noise + args.val_noise+args.test_noise]
# ARMAR DATASET DE ENTRENAMIENTO Y VALIDACION
with h5py.File('../Data/STEAD/' + args.train_file, 'w') as train_dst, \
h5py.File('../Data/STEAD/' + args.val_file, 'w') as val_dst:
# Create new train file groups
train_dst_wv = train_dst.create_group('earthquake/local')
train_dst_ns = train_dst.create_group('non_earthquake/noise')
# Create new val file groups
val_dst_wv = val_dst.create_group('earthquake/local')
val_dst_ns = val_dst.create_group('non_earthquake/noise')
# For every dataset in source seismic group
for idx, dset in enumerate(src_seis):
if idx in train_seis_ids:
# Retrieve dataset object
data = src_seis[dset]
# Copy seismic trace to new train file
train_dst_wv.copy(data, dset)
if idx in val_seis_ids:
# Retrieve dataset object
data = src_seis[dset]
# Copy seismic trace to new train file
val_dst_wv.copy(data, dset)
# For every dataset in source noise group
for idx, dset in enumerate(src_ns):
if idx in train_noise_ids:
# Retrieve dataset object
data = src_ns[dset]
# Copy noise trace to new noise file
train_dst_ns.copy(data, dset)
if idx in val_noise_ids:
# Retrieve dataset object
data = src_ns[dset]
# Copy seismic trace to new train file
val_dst_ns.copy(data, dset)
# AGREGAR SEÑALES DE GEOFONOS
# Coompana
add_traces2group('Coompana', coompana_traces_train, train_dst_ns)
add_traces2group('Coompana', coompana_traces_val, val_dst_ns)
# Lesser
add_traces2group('Lesser', lesser_traces_train, train_dst_ns)
add_traces2group('Lesser', lesser_traces_val, val_dst_ns)
# NC
add_traces2group('NC', nc_traces_train, train_dst_ns)
add_traces2group('NC', nc_traces_val, val_dst_ns)
# ARMAR DATASETS DE PRUEBA
with h5py.File('../Data/STEAD/STEAD_Seis_TEST.hdf5', 'w') as test_dst:
# Create new test file groups
test_dst_wv = test_dst.create_group('earthquake/local')
test_dst_ns = test_dst.create_group('non_earthquake/noise')
# For every dataset in source seismic group
for idx, dset in enumerate(src_seis):
if idx in test_seis_ids:
# Retrieve dataset object
data = src_seis[dset]
# Copy seismic trace to new train file
test_dst_wv.copy(data, dset)
with h5py.File('../Data/STEAD/STEAD_NSeis_TEST.hdf5', 'w') as test_dst:
# Create new test file groups
test_dst_wv = test_dst.create_group('earthquake/local')
test_dst_ns = test_dst.create_group('non_earthquake/noise')
# For every dataset in source noise group
for idx, dset in enumerate(src_ns):
if idx in test_noise_ids:
# Retrieve dataset object
data = src_ns[dset]
# Copy seismic trace to new train file
test_dst_ns.copy(data, dset)
with h5py.File('../Data/STEAD/GEO_TEST.hdf5', 'w') as test_dst:
# Create new test file groups
test_dst_wv = test_dst.create_group('earthquake/local')
test_dst_ns = test_dst.create_group('non_earthquake/noise')
# Coompana
add_traces2group('Coompana', coompana_traces_test, test_dst_ns)
# Lesser
add_traces2group('Lesser', lesser_traces_test, test_dst_ns)
# NC
add_traces2group('NC', nc_traces_test, test_dst_ns)
def __init__(self,
dist_type: Any,
constant: Optional[float] = None,
data: Optional[ndarray] = None,
filename: Optional[str] = None,
dist_name: Optional[str] = None,
**kwargs):
"""
Constructor of Distribution class.
It specifies which type of distribution is going to be used
and its corresponding parameters.
Parameters
----------
dist_type : {None, 'constant', 'empirical', 'weights', 'numpy'}
None : it allows a `Distribution` object to be set in such
way that it always return None.
'constant' : it numerically implements a "Dirac delta"
function, i.e. all points will have the same value
specified by the parameter `constant`
'empirical' : build distributions from empirical data,
estimating the overall shape of the distribution using
the KDE approach available via Scikit-Learn. If the data is
stored in a file, then 'filename' must be passed to specify
the path to the file and the data inside that file must be
formatted without header in this way:
.. code-block::
data_0
data_1
data_2
...
'weights' : it needs the histogram (xi, pi) data
in order to generate a random sample from
this given array of points and their corresponding
probability weights.
It uses Numpy Random Choices [2]_. If the data is
stored in a file, then 'filename' must be passed to specify
the path to the file and the data inside that file must be
formatted without header in this way:
.. code-block::
x_0, p_0
x_1, p_1
x_2, p_2
...
'numpy' : it uses the distributions implemented in
Numpy Random Distributions [3]_
constant : float, optional
It specifies the constant value to which all point are
mapped.
data : ndarray, optional
It corresponds
filename : str, optional
It specifies the path for the required data in order to build
a distribution from this data. It is required by
`dist_type = 'empirical'` or `dist_type = 'weights'`.
dist_name : str, optional
It specifies the distribution to use from numpy when
`dist_type = 'numpy'`.
**kwargs : dict, optional
Extra arguments that must to be passed to the
Scikit-Learn's Kernel Density constructor [1]_ or
to the Numpy Random Distributions [3]_, except the parameter
`size`. In the latter case, keyword argument must not be
passed since it is used directly in the sampling methods.
Raises
------
ValueError
TODO: when ?
SystemError
TODO: when ?
References
----------
.. [1] [Scikit-learn: Kernel Density Estimation](https://scikit-learn.org/stable/modules/density.html#kernel-density)
.. [2] [Numpy Random Choice](https://numpy.org/doc/stable/reference/random/generated/numpy.random.Generator.choice.html#numpy.random.Generator.choice)
.. [3] [Numpy Random Distributions](https://numpy.org/doc/stable/reference/random/generator.html#distributions)
Examples
--------
TODO: include some exhaustive examples here using each dist_type
"""
self.dist_type = dist_type
self.seed = int(time())
try:
if self.dist_type is None:
# Allows a `Distribution` object set to None
self.constant = None
elif self.dist_type == "constant":
# "Dirac delta"-like function
if constant is not None:
self.constant = constant
else:
raise ValueError(
"Parameter `constant` should not be None when "
"`dist_type` = 'constant'")
elif self.dist_type == "empirical":
# Using KernelDensity estimator from Scikit-learn
if data is not None and filename is None:
pass
elif data is None and filename is not None:
self.filename = filename
data = genfromtxt(self.filename)
else:
raise ValueError(
"The data is required and must be provided through "
"the parameter `data` or `filename` when "
"`dist_type` = 'empirical', then one of them should "
"not be None")
# Check data has the right dimension
if data.ndim == 1:
self.kwargs = kwargs
self.kd_estimator = KernelDensity(**self.kwargs).fit(
data.reshape(-1, 1))
else:
raise ValueError(
"The data provided should be a 1-D array.")
elif self.dist_type == "weights":
# Using numpy.random.choice
if data is not None and filename is None:
pass
elif data is None and filename is not None:
self.filename = filename
data = genfromtxt(self.filename, delimiter=",")
else:
raise ValueError(
"The data is required and must be provided through "
"the parameter `data` or `filename` when "
"`dist_type` = 'weights', then one of them should "
"not be None")
# Check data has the right dimension
if data.ndim == 2:
self.xi = data[:, 0]
self.pi = data[:, 1]
self.random_number_generator = \
random.default_rng(seed=self.seed)
else:
raise ValueError(
"The data provided should be a 2-D array.")
elif self.dist_type == "numpy":
# Using numpy.random
self.kwargs = kwargs
self.random_number_generator = \
random.default_rng(seed=self.seed)
if dist_name in dir(random):
self.numpy_distribution = getattr(
self.random_number_generator, dist_name)
else:
raise ValueError(
f"Distribution '{dist_name}' is not "
"implemented in numpy.random. See: "
"https://numpy.org/doc/stable/reference/random/"
"generator.html#distributions")
else:
raise ValueError(
"'dist_type' is not in "
"{None, 'constant', 'empirical', 'weights', 'numpy'}")
except Exception as error:
self.manage_exception(exception=error,
message="Error initializing distribution.")
def random_clifford(num_qubits, seed=None):
"""Return a random Clifford operator.
The Clifford is sampled using the method of Reference [1].
Args:
num_qubits (int): the number of qubits for the Clifford
seed (int or np.random.Generator): Optional. Set a fixed seed or
generator for RNG.
Returns:
Clifford: a random Clifford operator.
Reference:
1. S. Bravyi and D. Maslov, *Hadamard-free circuits expose the
structure of the Clifford group*.
`arXiv:2003.09412 [quant-ph] <https://arxiv.org/abs/2003.09412>`_
"""
if seed is None:
rng = np.random.default_rng()
elif isinstance(seed, np.random.Generator):
rng = seed
else:
rng = default_rng(seed)
had, perm = _sample_qmallows(num_qubits, rng)
gamma1 = np.diag(rng.integers(2, size=num_qubits, dtype=np.int8))
gamma2 = np.diag(rng.integers(2, size=num_qubits, dtype=np.int8))
delta1 = np.eye(num_qubits, dtype=np.int8)
delta2 = delta1.copy()
_fill_tril(gamma1, rng, symmetric=True)
_fill_tril(gamma2, rng, symmetric=True)
_fill_tril(delta1, rng)
_fill_tril(delta2, rng)
# For large num_qubits numpy.inv function called below can
# return invalid output leading to a non-symplectic Clifford
# being generated. This can be prevented by manually forcing
# block inversion of the matrix.
block_inverse_threshold = 50
# Compute stabilizer table
zero = np.zeros((num_qubits, num_qubits), dtype=np.int8)
prod1 = np.matmul(gamma1, delta1) % 2
prod2 = np.matmul(gamma2, delta2) % 2
inv1 = _inverse_tril(delta1, block_inverse_threshold).transpose()
inv2 = _inverse_tril(delta2, block_inverse_threshold).transpose()
table1 = np.block([[delta1, zero], [prod1, inv1]])
table2 = np.block([[delta2, zero], [prod2, inv2]])
# Apply qubit permutation
table = table2[np.concatenate([perm, num_qubits + perm])]
# Apply layer of Hadamards
inds = had * np.arange(1, num_qubits + 1)
inds = inds[inds > 0] - 1
lhs_inds = np.concatenate([inds, inds + num_qubits])
rhs_inds = np.concatenate([inds + num_qubits, inds])
table[lhs_inds, :] = table[rhs_inds, :]
# Apply table
table = np.mod(np.matmul(table1, table), 2).astype(bool)
# Generate random phases
phase = rng.integers(2, size=2 * num_qubits).astype(bool)
return Clifford(StabilizerTable(table, phase))
"""
Methods to create random operators.
"""
import numpy as np
from numpy.random import default_rng
from qiskit.quantum_info.operators import Operator, Stinespring
from qiskit.exceptions import QiskitError
# pylint: disable=unused-import
from .symplectic.random import random_clifford
from .symplectic.random import random_pauli_table
from .symplectic.random import random_stabilizer_table
DEFAULT_RNG = default_rng()
def random_unitary(dims, seed=None):
"""Return a random unitary Operator.
The operator is sampled from the unitary Haar measure.
Args:
dims (int or tuple): the input dimensions of the Operator.
seed (int or np.random.Generator): Optional. Set a fixed seed or
generator for RNG.
Returns:
Operator: a unitary operator.
"""
popSize = 500
epsMin = 1
# Frequency-Loss Model
λ, β, δ = 4, 2, 0.2
θ_True = λ, β, δ
sev = "frequency dependent exponential"
freq = "poisson"
psi = abc.Psi("sum") # Aggregation process
# Simulate some data to fit
sample_sizes = [50, 250]
T = sample_sizes[-1]
rg = rnd.default_rng(123)
freqs, sevs = abc.simulate_claim_data(rg, T, freq, sev, θ_True)
xData = abc.compute_psi(freqs, sevs, psi)
# Specify model to fit
params = ("λ", "β", "δ")
prior = abc.IndependentUniformPrior([(0, 10), (0, 20), (-1, 1)], params)
model = abc.Model("poisson", "frequency dependent exponential", psi, prior)
abcFits = []
for ss in sample_sizes:
fit = abc.smc(numIters,
popSize,
xData[:ss],
model,
def setUp(self): self.D, self.F, self.CR = 10, 0.9, 0.3 self.x, self.task = default_rng().uniform(10, 50, self.D), Task(self.D, nFES=230, nGEN=None, benchmark=MyBenchmark()) self.s1, self.s2 = SolutionjDE(task=self.task, e=False), SolutionjDE(x=self.x, CR=self.CR, F=self.F)
def _get_next_state(
self,
single_input: np.ndarray,
feedback: np.ndarray = None,
last_state: np.ndarray = None,
noise_generator: Generator = None,
) -> np.ndarray:
"""Given a state vector x(t) and an input vector u(t),
compute the state vector x(t+1).
Parameters
----------
single_input: np.ndarray
Input vector u(t)
feedback: numpy.ndarray, optional
Feedback vector if enabled.
last_state: numpy.ndarray, optional
Current state to update x(t). Default to 0 vector.
Raises
------
RuntimeError: feedback is enabled but no feedback vector is available.
Returns
-------
numpy.ndarray
Next state x(t+1)
"""
if noise_generator is None:
noise_generator = default_rng()
# first initialize the current state of the ESN
if last_state is None:
x = self.zero_state()
else:
x = np.asarray(last_state, dtype=self.typefloat).reshape(1, -1)
u = np.asarray(single_input, dtype=self.typefloat).reshape(1, -1)
# add bias
if self._input_bias:
u = add_bias(u)
# prepare noise sequence
noise_in = self.noise_in * noise_generator.uniform(-1, 1, size=u.shape)
noise_rc = self.noise_rc * noise_generator.uniform(-1, 1, size=x.shape)
# linear transformation
x1 = (u + noise_in) @ self.Win.T + x @ self.W
# add feedback if requested
if self.Wfb is not None:
if feedback is None:
feedback = self.zero_feedback()
noise_out = self.noise_out * noise_generator.uniform(
-1, 1, size=feedback.shape)
fb = self.fbfunc(np.asarray(feedback)).reshape(1, -1)
x1 += (fb + noise_out) @ self.Wfb.T
# previous states memory leak and non-linear transformation
x1 = (1 - self.lr) * x + self.lr * (self.activation(x1) + noise_rc)
# return the next state computed
return x1
def main():
parser = argparse.ArgumentParser(
description=
'Create Positives, Negatives and Part Faces from the WIDER FACE dataset for P-Net.'
)
parser.add_argument(
'-a',
'--annotation-file',
type=str,
help='The path to the annotation file of the WIDER FACE dataset',
required=True)
parser.add_argument(
'-i',
'--images-directory',
type=str,
help='The path to the folder that contains WIDER FACE dataset',
required=True)
parser.add_argument(
'-o',
'--output-directory',
type=str,
help='The path to the folder where you want to save your output',
default='./data/temp')
args = parser.parse_args()
# Instantiate a random generator
rng = default_rng()
# Prepare for saving
if not path.exists(args.output_directory):
os.makedirs(args.output_directory)
# These are for storing the images
pos_images = list()
par_images = list()
neg_images = list()
# These are for storing the annotations of bounding box regression task
pos_bboxes = list()
par_bboxes = list()
# Start reading the file
window = np.zeros(3, np.int32)
f = open(args.annotation_file)
for i in tqdm(
range(12880)
): # Because there are a total of 12880 images in the Wider Face dataset
# Read the file line by line
line = f.readline()
# Obtain the number of faces in this image
num_faces = int(f.readline())
# Skip the image if it does not contain any faces
if num_faces == 0:
f.readline()
continue
# Obtain bounding boxes while taking into account invalid faces
bboxes = np.zeros((num_faces, 4), np.int32)
invalid = []
for j in range(num_faces):
face = f.readline()
if face[-7] == '1':
invalid.append(j)
continue
bboxes[j] = face[:-14].split(' ')
# Remove invalid faces
if len(invalid) > 0:
bboxes = np.delete(bboxes, invalid, 0)
if bboxes.shape[0] == 0:
continue
num_faces = bboxes.shape[0]
# Read the image
img = cv2.imread(args.images_directory + '/' + line[:-1])[:, :, ::-1]
# Create positives, part faces and negatives
for bbox in bboxes:
found_part_face = False
found_positive = False
find_negative = False
for i in range(30):
if found_positive == True and found_part_face == True:
break
window[2] = max(bbox[2], bbox[3])
window[2] += ((rng.random() * 0.2 - 0.1) * window[2]).astype(
np.int32)
window[2] = max(window[2], 12)
max_y, max_x = np.array(img.shape[:-1]) - window[2]
if max_x < 0 or max_y < 0:
continue
bbox_center = bbox[:2] + bbox[2:] // 2
if found_part_face == False:
window_center = bbox_center + (
(rng.random() * 2 - 1) * window[2]).astype(np.int32)
window[:2] = np.clip(window_center - window[2] // 2, 0,
np.array([max_x, max_y]))
else:
window_center = bbox_center + ((rng.random() * 0.8 - 0.4) *
window[2]).astype(np.int32)
window[:2] = np.clip(window_center - window[2] // 2, 0,
np.array([max_x, max_y]))
iou = find_iou(window, bbox.reshape(1, 4))[0]
# If the window is a part face
if 0.4 <= iou and iou <= 0.65 and found_part_face == False:
# Prepare bounding box regression annotation
par_bboxes.append(create_bbr_annotation(window, bbox))
# Crop out the window and resize if necessary
par_images.append(crop_and_resize(img, window, 12))
# Move on to the next iteration
found_part_face = True
continue
# If the window is a positive
if iou > 0.65 and found_positive == False:
# Prepare bounding box regression annotation
pos_bboxes.append(create_bbr_annotation(window, bbox))
# Crop out the window and resize if necessary
pos_images.append(crop_and_resize(img, window, 12))
# Move on to the next iteration
found_positive = True
continue
if rng.random() < 0.05 and find_negative == False:
find_negative = True
if rng.random() < 0.15:
ious = find_iou(window, bboxes)
if np.max(ious) < 0.3:
# Crop out the window and resize if necessary
neg_images.append(crop_and_resize(img, window, 12))
# Get 8 negatives out of each image
num_neg = 0
for i in range(15):
window[2] = rng.integers(12, np.min(img.shape[:-1]), endpoint=True)
max_y, max_x = np.array(img.shape[:-1]) - window[2]
window[0] = rng.integers(0, max_x + 1)
window[1] = rng.integers(0, max_y + 1)
ious = find_iou(window, bboxes)
if np.max(ious) < 0.3:
# Crop out the window and resize if necessary
neg_images.append(crop_and_resize(img, window, 12))
num_neg += 1
if num_neg == 8:
break
# Always close the file
f.close()
# Save the images
images = np.concatenate((np.asarray(pos_images), np.asarray(par_images),
np.asarray(neg_images)),
axis=0)
np.save(args.output_directory + '/pos_par_neg_images.npy', images)
# Save the bounding box annotations
np.save(args.output_directory + '/part_faces.npy', np.asarray(par_bboxes))
np.save(args.output_directory + '/positives.npy', np.asarray(pos_bboxes))
def generate(
self,
nb_timesteps: int,
warming_inputs: np.ndarray = None,
init_state: np.ndarray = None,
init_fb: np.ndarray = None,
verbose: bool = False,
init_inputs: np.ndarray = None,
seed: int = None,
return_init: bool = None,
) -> Tuple[np.ndarray, np.ndarray]:
"""Run the ESN on generative mode.
After the `̀warming_inputs` are consumed, new outputs are
used as inputs for the next nb_timesteps, i.e. the
ESN is feeding himself with its own outputs.
Note that this mode can only work if the ESN is trained
on a regression task. The outputs of the ESN must be
the same kind of data as its input.
To train an ESN on generative mode, use the :py:func:`ESN.train`
method to train the ESN on a regression task (for
instance, predict the future data point t+1 of a timeseries
give the data at time t).
Parameters
----------
nb_timesteps: int
Number of timesteps of data to generate
from the intial input.
warming_inputs: numpy.ndarray
Input data used to initiate generative mode.
This data is meant to "seed" the ESN internal
states with some real information, before it runs
on its own created outputs.
init_state: numpy.ndarray, optional:
State initialization vector for the reservoir.
By default, internal state of the reservoir is initialized to 0.
init_fb: numpy.ndarray, optional
Feedback initialization vector for the reservoir, if feedback is
enabled. By default, feedback is initialized to 0.
verbose: bool, optional
init_intputs: list of numpy.ndarray, optional
Same as ``warming_inputs̀``.
Kept for compatibility with previous version. Deprecated
since 0.2.2, will be removed soon.
return_init: bool, optional
Kept for compatibility with previous version. Deprecated
since 0.2.2, will be removed soon.
Returns
-------
tuple of numpy.ndarray
Generated outputs, generated states, warming outputs, warming states
Generated outputs are the timeseries predicted by the ESN from
its own predictions over time. Generated states are the
corresponding internal states.
Warming outputs are the predictions made by the ESN based on the
warming inputs passed as parameters. These predictions are prior
to the generated outputs. Warming states are the corresponding
internal states. In the case no warming inputs are provided, warming
outputs and warming states are None.
"""
if warming_inputs is None and init_state is None and init_inputs is None:
raise ValueError("at least one of the parameter 'warming_input' "
"or 'init_state' must not be None. Impossible "
"to generate from scratch.")
if return_init is not None:
warnings.warn("Deprecation warning : return_init parameter "
"is deprecated since 0.2.2 and will be removed.")
# for additive noise in the reservoir
# 2 separate seeds made from one: one for the warming
# (if needed), one for the generation
seed = seed if seed is not None else self.seed
ss = SeedSequence(seed)
child_seeds = ss.spawn(2)
if warming_inputs is not None or init_inputs is not None:
if init_inputs is not None:
warnings.warn("Deprecation warning : init_inputs parameter "
"is deprecated since 0.2.2 and will be removed. "
"Please use warming_inputs instead.")
warming_inputs = init_inputs
if verbose:
print(f"Generating {nb_timesteps} timesteps from "
f"{warming_inputs.shape[0]} inputs.")
print("Computing initial states...")
warming_states = self._compute_states(
warming_inputs,
init_state=init_state,
init_fb=init_fb,
seed=child_seeds[0],
)
# initial state (at begining of generation)
s0 = warming_states[-1, :].reshape(1, -1)
warming_outputs = self.compute_outputs([warming_states])[0]
# intial input (at begining of generation)
u1 = warming_outputs[-1, :].reshape(1, -1)
if init_fb is not None:
# initial feedback (at begining of generation)
fb0 = warming_outputs[-2, :].reshape(1, -1)
else:
fb0 = None
else:
warming_outputs, warming_states = None, None
# time is often first axis but compute_outputs await
# for time in second axis, so the reshape :
s0 = init_state.reshape(1, -1)
if init_fb is not None:
fb0 = init_fb.reshape(1, -1)
else:
fb0 = None
u1 = self.compute_outputs([s0])[0][-1, :].reshape(1, -1)
states = np.zeros((nb_timesteps, self.N))
outputs = np.zeros((nb_timesteps, self.dim_out))
if verbose:
track = tqdm
else:
def track(x, text):
return x
# for additive noise in the reservoir
rg = default_rng(child_seeds[1])
for i in track(range(nb_timesteps), "Generate"):
# from new input u1 and previous state s0
# compute next state s1 -> s0
s1 = self._get_next_state(single_input=u1,
feedback=fb0,
last_state=s0,
noise_generator=rg)
s0 = s1[-1, :].reshape(1, -1)
states[i, :] = s0
if fb0 is not None:
fb0 = u1.copy()
# from new state s1 compute next input u2 -> u1
u1 = self.compute_outputs([s0])[0][-1, :].reshape(1, -1)
outputs[i, :] = u1
return outputs, states, warming_outputs, warming_states
def calc_beta():
rng = default_rng()
a = (11.0 / 20.0) * 10
b = (23.0 / 8.0) * 10
r = beta.rvs(a, b, loc=a, scale=b - a, size=10)
print(r)
import matplotlib.pyplot as plt
from numpy.random import default_rng
rng = default_rng()
num = int(input("Please enter number of rain drops: "))
x = rng.uniform(-1, 1, num)
y = rng.uniform(-1, 1, num)
r = 1
circle1 = plt.Circle((0, 0), r, fill=False)
ax = plt.gca()
ax.add_artist(circle1)
x_in = []
x_out = []
y_in = []
y_out = []
num_in = 0
for i in range(num):
if x[i] * x[i] + y[i] * y[i] <= r * r:
x_in.append(x[i])
y_in.append(y[i])
num_in += 1
else:
x_out.append(x[i])
y_out.append(y[i])
ax.plot(x_in, y_in, '.', color='r')
ax.plot(x_out, y_out, '.', color='b')
def random_char_data(seed=None):
"""Generate random non-magical human character data with no gifts/flaws by optional seed"""
from numpy.random import default_rng
if seed == None:
rng = default_rng()
else:
rng = default_rng(seed)
ranking = rng.shuffle([1, 2, 3, 5])
# Race
race = "Human"
# Social Class
soclass = {
1: (250, "Landed Nobility"),
2: (100, "Landless Nobility"),
3: (50, "High Freeman"),
4: (15, "Low Freeman"),
5: (5, "Peasant"),
6: (0, "Slave"),
}[ranking[0]]
# Attribute Points
atpoint = {
1: 47,
2: 43,
3: 39,
4: 35,
5: 31,
6: 27,
}[ranking[1]]
# Skill Defaults
(skillp1, skillp2) = {
1: (6, 6),
2: (6, 7),
3: (7, 7),
4: (8, 8),
5: (9, 9),
6: (9, None),
}[ranking[2]]
# Proficiency Points
profpoint = {
1: 14,
2: 9,
3: 6,
4: 4,
5: 2,
6: 0,
}[ranking[3]]
if profpoint > 9:
profmax = 8
else:
profmax = 7
# Gifts/Flaws
giflaw = None
# Attributes
atts = 2 * np.ones(10)
high = rng.integers(0, 10)
atts[high] = rng.integers(
np.ceil(atpoint / 10), np.amin([7, atpoint - np.sum(atts)])
) # min for high must allow space for distribution of all points smaller than it and >2
big = atts[high]
while np.sum(atts) < atpoints:
pick = rng.integers(0, 10)
if atts[pick] >= np.amin([7, atpoint - np.sum(atts), big]):
continue
if pick == high:
atts[high] = rng.integers(atts[pick] + 1,
np.amin([7, atpoint - np.sum(atts)]))
big = atts[high]
else:
atts[pick] = rng.integers(
atts[pick] + 1, np.amin([7, atpoint - np.sum(atts), big]))
# Skills not implemented as they don't impact combat strongly
# Proficiency
profs = np.zeros(18) # Excluding Spear & Trident
high = rng.integers(0, 18)
profs[high] = rng.integers(1, np.min([profmax, profpoints]))
while np.sum(profs) < profpoints:
pick = rng.integers(0, 18)
profs[pick] = rng.integers(profs[pick], np.amin([profmax, profpoints]))
# somehow put in default values as min???
# Equipment
return ("Non-Magical Human", )
def simulate_from_founders_with_sex(cls,
n_founders,
n_generations,
avg_offspring=2,
avg_immigrants=2,
seed=None):
ped = cls()
ped.generations = n_generations
rng = rnd.default_rng(seed)
current_males, current_females = [], []
next_males, next_females = [], []
id_counter = count(1)
# assign sex to the founder generation
for _ in range(n_founders):
ind_id = next(id_counter)
male = rng.random() < 0.5
if male:
current_males.append(ind_id)
ped.graph.add_node(ind_id, time=n_generations, sex=1)
else:
current_females.append(ind_id)
ped.graph.add_node(ind_id, time=n_generations, sex=2)
for t in range(n_generations - 1, -1, -1):
# pad the arrays if we have uneven sex ratio
diff = len(current_males) - len(current_females)
if diff > 0:
for _ in range(diff):
ind_id = next(id_counter)
current_females.append(ind_id)
ped.graph.add_node(ind_id, time=t + 1, sex=2)
elif diff < 0:
for _ in range(-diff):
ind_id = next(id_counter)
current_males.append(ind_id)
ped.graph.add_node(ind_id, time=t + 1, sex=1)
# Pick couples
while len(current_males) and len(current_females):
father = rnd.choice(current_males)
mother = rnd.choice(current_females)
current_males.remove(father)
current_females.remove(mother)
n_children = rng.poisson(avg_offspring)
for ch in range(n_children):
child_id = next(id_counter)
child_male = rng.random() < 0.5
if child_male:
next_males.append(child_id)
ped.add_child(child_id, mother, father, time=t, sex=1)
else:
next_females.append(child_id)
ped.add_child(child_id, mother, father, time=t, sex=2)
# add extra out-of-family individuals - but not in the present
if t > 1:
n_immigrants = rnd.poisson(avg_immigrants)
for _ in range(n_immigrants):
ind_id = next(id_counter)
ind_male = rng.random() < 0.5
if ind_male:
next_males.append(ind_id)
ped.add_individual(ind_id, t, sex=1)
else:
next_females.append(ind_id)
ped.add_individual(ind_id, t, sex=2)
if not (next_males or next_females):
raise (RuntimeError('Simulation terminated at time t=' +
str(t) + ', (' + str(n_generations - t) +
' generations from founders)'))
current_males = next_males
current_females = next_females
next_males = []
next_females = []
if not nx.is_weakly_connected(ped.graph):
# if multiple subgraphs, return largest
largest = max(nx.weakly_connected_components(ped.graph), key=len)
ped.graph = nx.subgraph(ped.graph, largest)
return ped
def create_data():
rng = default_rng(2)
N = 20
x = linspace(0, 10, N)
y = sin(x) + rng.normal(loc=0.1, scale=0.1, size=N)
return x.reshape([N, 1]), y