def test_call_within_randomstate(self):
# Check that custom BitGenerator does not call into global state
res = np.array([1, 8, 0, 1, 5, 3, 3, 8, 1, 4])
for i in range(3):
mt19937 = Generator(MT19937(i))
m = Generator(MT19937(4321))
# If m.state is not honored, the result will change
assert_array_equal(m.choice(10, size=10, p=np.ones(10) / 10.), res)
def set_state(self, state):
"""Update state information, accepts numpy or IDL format"""
if state is not None:
if isinstance(state, (tuple, list, ndarray)):
if len(state) == 628: # If seed is iterable and has len is 628 then
state = self._idl2numpy( state ) # Convert seed to numpy state
else:
state = MT19937( state[0] ).state #Convert seed to numpy state
else:
state = MT19937( state ).state
self.bit_generator.state = state # Update the state
def __init__(self,
dim,
bounds,
popsize=31,
stop_fitness=None,
keep=200,
F=0.5,
Cr=0.9,
rg=Generator(MT19937()),
logger=None):
self.dim, self.lower, self.upper = _check_bounds(bounds, dim)
self.popsize = popsize
self.stop_fitness = stop_fitness
self.keep = keep
self.rg = rg
self.F0 = F
self.Cr0 = Cr
self.stop = 0
self.iterations = 0
self.evals = 0
self.p = 0
self.improves = deque()
self._init()
if not logger is None:
self.logger = logger
self.best_y = mp.RawValue(ct.c_double, 1E99)
self.n_evals = mp.RawValue(ct.c_long, 0)
self.time_0 = time()
def retry(store,
prob,
algo,
num_retries,
value_limit=math.inf,
popsize=1,
workers=mp.cpu_count()):
try:
import pygmo as pg
except ImportError as e:
raise ImportError(
"Please install PYGMO (pip install pygmo) to use PAGMO optimizers"
) from e
sg = SeedSequence()
rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)]
proc = [
Process(target=_retry_loop,
args=(pid, rgs, store, prob, algo, num_retries, value_limit,
popsize, pg)) for pid in range(workers)
]
[p.start() for p in proc]
[p.join() for p in proc]
store.sort()
store.dump()
return OptimizeResult(x=store.get_x_best(),
fun=store.get_y_best(),
nfev=store.get_count_evals(),
success=True)
def minimize(self,
fun,
bounds,
guess=None,
sdevs=None,
rg=Generator(MT19937()),
store=None):
dim = len(bounds.lb)
popsize = 31 if self.popsize is None else self.popsize
es = de.DE(dim,
bounds,
popsize=popsize,
rg=rg,
keep=self.keep,
F=self.f,
Cr=self.cr)
es.fun = fun #remove
max_evals = self.max_eval_num(store)
while es.evals < max_evals:
xs = es.ask()
ys = [fun(x) for x in xs]
stop = es.tell(ys, xs)
if stop != 0:
break
return es.best_x, es.best_value, es.evals
def split(X, y, s, r):
rs = RandomState(MT19937(SeedSequence(r)))
i = X.index.to_numpy()
np.random.shuffle(i, )
P = np.rint(np.multiply(i.size, s))
P[-1] = len(i) - np.sum(P[:-1])
P = np.cumsum(P)
P = P - 1
P = [int(p) for p in P]
last = -1
X_splitted = []
for j, p in enumerate(P):
X_splitted.append(X[(last + 1):p])
last = p
last = -1
y_splitted = []
for j, p in enumerate(P):
y_splitted.append(y[(last + 1):p])
last = p
return (X_splitted + y_splitted)
def __init__(self, *args, **kwargs):
super(NumpyMt19937, self).__init__(*args, **kwargs)
self.name = "Numpy_Mt19937"
self.seed = 0
self.min = 0
self.max = 999999
self.random = Generator(MT19937(self.seed))
def __batch_generator(self, X: np.ndarray, y: np.ndarray):
batch_size = min(X.shape[0], max(1, self.batch_size))
steps = np.arange(0, X.shape[0], batch_size)
if self.shuffle == True:
np.random.Generator(MT19937()).shuffle(steps)
for start in steps:
end = start + batch_size
yield (X[start:end], y[start:end])
def test_shuffle_mixed_dimension(self):
# Test for trac ticket #2074
for t in [[1, 2, 3, None], [(1, 1), (2, 2), (3, 3), None],
[1, (2, 2), (3, 3), None], [(1, 1), 2, 3, None]]:
mt19937 = Generator(MT19937(12345))
shuffled = list(t)
mt19937.shuffle(shuffled)
assert_array_equal(shuffled, [t[2], t[0], t[3], t[1]])
def test_call_within_randomstate(self):
# Check that custom RandomState does not call into global state
m = Generator(MT19937()) # mt19937.RandomState()
res = np.array([0, 8, 7, 2, 1, 9, 4, 7, 0, 3])
for i in range(3):
mt19937.bit_generator.seed(i)
m.bit_generator.seed(4321)
# If m.state is not honored, the result will change
assert_array_equal(m.choice(10, size=10, p=np.ones(10)/10.), res)
def _retry(minimizer):
sg = SeedSequence()
rgs = [Generator(MT19937(s)) for s in sg.spawn(minimizer.workers)]
procs = [
Process(target=_retry_loop, args=(pid, rgs, minimizer))
for pid in range(minimizer.workers)
]
[p.start() for p in procs]
return procs
def test_shuffle_mixed_dimension(self):
# Test for trac ticket #2074
for t in [[1, 2, 3, None], [(1, 1), (2, 2), (3, 3), None],
[1, (2, 2), (3, 3), None], [(1, 1), 2, 3, None]]:
mt19937 = Generator(MT19937(12345))
shuffled = np.array(t, dtype=object)
mt19937.shuffle(shuffled)
expected = np.array([t[2], t[0], t[3], t[1]], dtype=object)
assert_array_equal(np.array(shuffled, dtype=object), expected)
def minimize(self,
fun,
bounds,
guess=None,
sdevs=None,
rg=Generator(MT19937()),
store=None):
choice = rg.integers(0, len(self.optimizers))
opt = self.optimizers[choice]
return opt.minimize(fun, bounds, guess, sdevs, rg, store)
def minimize(self,
fun,
bounds,
guess=None,
sdevs=None,
rg=Generator(MT19937()),
store=None):
ret = shgo(fun,
bounds=list(zip(bounds.lb, bounds.ub)),
options={'maxfev': self.max_eval_num(store)})
return ret.x, ret.fun, ret.nfev
def minimize(self,
fun,
bounds,
guess=None,
sdevs=None,
rg=Generator(MT19937()),
store=None):
if guess is None:
guess = rg.uniform(bounds.lb, bounds.ub)
ret = minimize(fun, x0=guess, bounds=bounds)
return ret.x, ret.fun, ret.nfev
def inverse_shuffle(input_rgb, r1, r2):
rs_row = RandomState(MT19937(SeedSequence(r1)))
rs_col = RandomState(MT19937(SeedSequence(r2)))
row_size = len(input_rgb)
col_size = len(input_rgb[0])
shuf_rgb = np.zeros(input_rgb.shape)
rows = list(range(row_size))
for i in range(row_size):
row = rs_row.choice(rows)
rows.remove(row)
cols = list(range(col_size))
for j in range(col_size):
col = rs_col.choice(cols)
cols.remove(col)
shuf_rgb[i][j] = input_rgb[row][col]
return shuf_rgb
def UpdateNormal1D(i, d=0.01, n=1, Mb=100, mb= -100, rs=0):
i = np.array(i)
if not rs:
rseed = random.randint(1000, 9999)
rs = RandomState(MT19937(SeedSequence(rseed)))
Ix = rs.randint(0, len(i),n) # faster than np.random.choice
z = np.zeros(i.shape) + i
z[Ix] = z[Ix] + rs.normal(0, d, n)
z[z > Mb] = Mb - (z[z>Mb] - Mb)
z[z < mb] = mb + (mb - z[z<mb])
hastings = 0
return z, Ix, hastings
def UpdateNormalNormalized(i, d=0.01, n=1, Mb=100, mb= -100, rs=0):
i = np.array(i)
if not rs:
rseed = random.randint(1000, 9999)
rs = RandomState(MT19937(SeedSequence(rseed)))
Ix = rs.randint(0, i.shape[0],n) # faster than np.random.choice
Iy = rs.randint(0, i.shape[1],n)
z = np.zeros(i.shape) + i
z[Ix,Iy] = z[Ix,Iy] + rs.normal(0, d[Ix,Iy], n)
z = z/np.sum(z)
hastings = 0
return z, (Ix, Iy), hastings
def main(outdir):
rng = RandomState(MT19937(SeedSequence(config.seed)))
berlin_holidays = holidays.DE(prov="BW")
num_employees = 20000
num_jobsites = 200
num_areas = 20
num_qualifications = 40
num_shifts = 3
num_days = 356
num_orders = 1000
df = pd.DataFrame.from_dict({
"Einsatzort":
rng.randint(0, num_jobsites, num_orders),
"Qualifikation":
rng.randint(0, num_qualifications, num_orders),
"Schicht":
rng.randint(0, num_shifts, num_orders),
"Tag":
rng.randint(0, num_days, num_orders),
})
df["Tag"] = df["Tag"].apply(
lambda day: datetime(2019, 1, 1) + timedelta(day))
df["Wochentag"] = df["Tag"].apply(lambda day: day.strftime("%a"))
df["Feiertag"] = df["Tag"].apply(lambda day: day in berlin_holidays)
# grouping of jobsites into areas
area_splits = np.cumsum(rng.randint(1, 10, num_areas))
area_splits = (area_splits.T / area_splits.max() *
num_jobsites).astype(int)
df["Ort"] = df["Einsatzort"].apply(
lambda jobsite_id: np.argmax(area_splits > jobsite_id))
offers = []
for _ in range(len(df)):
offers.append(
rng.choice(range(num_employees),
replace=False,
size=rng.randint(1, 6)).tolist())
df["Mitarbeiter ID"] = offers
train, test = train_test_split(df)
train.to_csv(os.path.join(outdir, "train.tsv"), index=False, sep="\t")
test.to_csv(os.path.join(outdir, "test_truth.tsv"), index=False, sep="\t")
test[[
"Einsatzort", "Qualifikation", "Schicht", "Tag", "Wochentag",
"Feiertag", "Ort"
]].to_csv(os.path.join(outdir, "test_publish.tsv"), index=False, sep="\t")
def mo_retry(fun, weight_bounds, ncon, y_exp, store, optimize, num_retries, value_limits,
workers=mp.cpu_count()):
sg = SeedSequence()
rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)]
proc=[Process(target=_retry_loop,
args=(pid, rgs, fun, weight_bounds, ncon, y_exp,
store, optimize, num_retries, value_limits)) for pid in range(workers)]
[p.start() for p in proc]
[p.join() for p in proc]
store.sort()
store.dump()
return store.get_xs()
def test_multivariate_normal_basic_stats():
# numpy/numpy#15871
random = Generator(MT19937(12345))
n_s = 1000
mean = np.array([1, 2])
cov = np.array([[2, 1], [1, 2]])
s = random.multivariate_normal(mean, cov, size=(n_s,))
s_center = s - mean
cov_emp = (s_center.T @ s_center) / (n_s - 1)
# these are pretty loose and are only designed to detect major errors
assert np.all(np.abs(s_center.mean(-2)) < 0.1)
assert np.all(np.abs(cov_emp - cov) < 0.2)
def test_permutation_subclass(self):
class N(np.ndarray):
pass
mt19937 = Generator(MT19937(1))
orig = np.arange(3).view(N)
perm = mt19937.permutation(orig)
assert_array_equal(perm, np.array([2, 0, 1]))
assert_array_equal(orig, np.arange(3).view(N))
class M:
a = np.arange(5)
def __array__(self):
return self.a
mt19937 = Generator(MT19937(1))
m = M()
perm = mt19937.permutation(m)
assert_array_equal(perm, np.array([4, 1, 3, 0, 2]))
assert_array_equal(m.__array__(), np.arange(5))
def retry(fun, store, optimize, num_retries, value_limit = math.inf,
workers=mp.cpu_count(), stop_fitness = -math.inf):
sg = SeedSequence()
rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)]
proc=[Process(target=_retry_loop,
args=(pid, rgs, fun, store, optimize, num_retries, value_limit, stop_fitness)) for pid in range(workers)]
[p.start() for p in proc]
[p.join() for p in proc]
store.sort()
store.dump()
return OptimizeResult(x=store.get_x_best(), fun=store.get_y_best(),
nfev=store.get_count_evals(), success=True)
def _get_generator(seed: Union[int, Generator]) -> Generator:
if isinstance(seed, Generator):
return seed
# provided to support legacy RandomState generator
# of Numpy. It is not the best thing to do however
# and recommend the user to keep using integer seeds
# and proper Numpŷ Generator API.
if isinstance(seed, RandomState):
mt19937 = MT19937()
mt19937.state = seed.get_state()
return Generator(mt19937)
else:
return default_rng(seed)
def multiplier_proposal_vector(q, d=1.05, f=1, rs=0):
if not rs:
rseed = random.randint(1000, 9999)
rs = RandomState(MT19937(SeedSequence(rseed)))
S = q.shape
ff = rs.binomial(1,f,S)
u = rs.random(S)
l = 2 * np.log(d)
m = np.exp(l * (u - .5))
m[ff==0] = 1.
new_q = q * m
U=np.sum(np.log(m))
return new_q, 0, U
def test_shuffle_of_array_of_objects(self):
# Test that permuting an array of objects will not cause
# a segfault on garbage collection.
# See gh-7719
mt19937 = Generator(MT19937(1234))
a = np.array([np.arange(1), np.arange(4)])
for _ in range(1000):
mt19937.shuffle(a)
# Force Garbage Collection - should not segfault.
import gc
gc.collect()
def test_choice_sum_of_probs_tolerance(self):
# The sum of probs should be 1.0 with some tolerance.
# For low precision dtypes the tolerance was too tight.
# See numpy github issue 6123.
mt19937 = Generator(MT19937(1234))
a = [1, 2, 3]
counts = [4, 4, 2]
for dt in np.float16, np.float32, np.float64:
probs = np.array(counts, dtype=dt) / sum(counts)
c = mt19937.choice(a, p=probs)
assert_(c in a)
with pytest.raises(ValueError):
mt19937.choice(a, p=probs * 0.9)
def minimize(self,
fun,
bounds,
guess=None,
sdevs=None,
rg=Generator(MT19937()),
store=None):
ret = dual_annealing(fun,
bounds=list(zip(bounds.lb, bounds.ub)),
maxfun=self.max_eval_num(store),
no_local_search=self.no_local_search,
x0=guess,
seed=int(rg.uniform(0, 2**32 - 1)))
return ret.x, ret.fun, ret.nfev
def test_shuffle_of_array_of_different_length_strings(self):
# Test that permuting an array of different length strings
# will not cause a segfault on garbage collection
# Tests gh-7710
mt19937 = Generator(MT19937(1234))
a = np.array(['a', 'a' * 1000])
for _ in range(100):
mt19937.shuffle(a)
# Force Garbage Collection - should not segfault.
import gc
gc.collect()
def generate_image(size, prng):
allowed_prngs = [
"java",
"python",
"numpy",
"Xoroshiro128",
"MT19937",
"Philox",
"SFC64",
"Xorshift1024",
"ThreeFry",
]
if prng not in allowed_prngs:
raise ValueError(f"prng={prng} is not in {allowed_prngs}")
arr = np.zeros((size, size))
for i in range(size):
if prng == "python":
random.seed(i)
elif prng == "numpy":
np.random.seed(i)
elif prng == "java":
rnd = javarandom.Random(i)
elif prng == "Xoroshiro128":
rnd = RandomGenerator(Xoroshiro128())
elif prng == "Xorshift1024":
rnd = RandomGenerator(Xorshift1024())
elif prng == "ThreeFry":
rnd = RandomGenerator(ThreeFry())
elif prng == "MT19937":
rnd = Generator(MT19937())
elif prng == "Philox":
rnd = Generator(Philox())
elif prng == "SFC64":
rnd = Generator(SFC64())
for j in range(size):
if prng == "python":
random_number = random.random()
elif prng == "numpy":
random_number = np.random.random()
elif prng == "java":
random_number = rnd.nextDouble()
elif prng in ["Xoroshiro128", "Xorshift1024", "ThreeFry"]:
random_number = rnd.random_sample()
elif prng in ["MT19937", "Philox", "SFC64"]:
random_number = rnd.random()
arr[j, i] = random_number
print("{}\t{}\t{}".format(i, arr[0, i], arr[1, i]))
imageio.imwrite(f"1000-random-numbers-{prng}.png", arr)
