def reseed(self, seed_seq=None): """ Get new random number generator. Parameters ---------- seed_seq : np.random.SeedSequence, rlberry.seeding.Seeder or int, default : None Seed sequence from which to spawn the random number generator. If None, generate random seed. If int, use as entropy for SeedSequence. If seeder, use seeder.seed_seq """ # if None, new seed sequence if seed_seq is None: seed_seq = SeedSequence() # if SeedSequence, do nothing elif isinstance(seed_seq, SeedSequence): seed_seq = seed_seq # if Seeder, get Seeder.seed_seq elif isinstance(seed_seq, Seeder): seed_seq = seed_seq.seed_seq # if integer, new SeedSequence else: seed_seq = SeedSequence(seed_seq) # spawn seed_seq = seed_seq.spawn(1)[0] self.seed_seq = seed_seq self.rng = default_rng(self.seed_seq)
def set_global_seed(entropy=42): """ rlberry has a global seed from which we can obtain different random number generators with (close to) independent outcomes. Important: In each new process/thread, set_global_seed must be called with a new seed. To do: Check (torch seeding): https://github.com/pytorch/pytorch/issues/7068#issuecomment-487907668 Parameters --------- entropy: int, SeedSequence optional The entropy for creating the global SeedSequence, or a SeedSequence """ global _GLOBAL_ENTROPY, _GLOBAL_SEED_SEQ, _GLOBAL_RNG if isinstance(entropy, SeedSequence): seedseq = entropy _GLOBAL_ENTROPY = seedseq.entropy _GLOBAL_SEED_SEQ = seedseq else: _GLOBAL_ENTROPY = entropy _GLOBAL_SEED_SEQ = SeedSequence(entropy) _GLOBAL_RNG = get_rng() # seed torch if _TORCH_INSTALLED: torch.manual_seed( _GLOBAL_SEED_SEQ.generate_state(1, dtype=np.uint32)[0])
def __init__(self, seed_seq=None, spawn_seed_seq=True): """ Parameters ---------- seed_seq : np.random.SeedSequence, rlberry.seeding.Seeder or int, default : None Seed sequence from which to spawn the random number generator. If None, generate random seed. If int, use as entropy for SeedSequence. If seeder, use seeder.seed_seq spawn_seed_seq : bool, default : True If True, uses seed_seq to spawn a new seed sequence (strongly recommended) for the Seeder. If False, uses the input seed_seq to define the Seeder. Warning: Setting to false can lead to unexpected behavior. This argument is only used internally in rlberry, in Seeder.spawn(), to avoid unnecessary spawning. """ super().__init__() if seed_seq is None: seed_seq = SeedSequence() elif isinstance(seed_seq, SeedSequence): seed_seq = seed_seq elif isinstance(seed_seq, Seeder): seed_seq = seed_seq.seed_seq else: # integer seed_seq = SeedSequence(seed_seq) if spawn_seed_seq: seed_seq = seed_seq.spawn(1)[0] self.seed_seq = seed_seq self.rng = default_rng(self.seed_seq)
def _create_cache(t0, t1, w0, w1, entropy, pool_size, k): ts = [t0, t1] ws = [w0, w1] parent = SeedSequence(entropy=entropy, pool_size=pool_size) seeds = [parent] for level in range(1, k + 1): new_ts = [] new_ws = [] new_seeds = [] for i, parent in enumerate(seeds): seedv, seedl, seedr = parent.spawn(3) new_seeds.extend([seedl, seedr]) t0, t1 = ts[i], ts[i + 1] w0, w1 = ws[i], ws[i + 1] t = (t0 + t1) / 2 w = utils.brownian_bridge(t0=t0, t1=t1, w0=w0, w1=w1, t=t, seed=seedv) new_ts.extend([ts[i], t]) new_ws.extend([ws[i], w]) new_ts.append(ts[-1]) new_ws.append(ws[-1]) ts = new_ts ws = new_ws seeds = new_seeds return ts, ws, seeds
def generate_random_configurations(box_length, n_part, n_dim, n_ensemble, seed_entropy): """ Generates a set of configurations in an ensemble Parameters ---------- box_length : float length of the box n_part : int number of particles n_dim : int dimension of the system seed_coords : int seed for random number generator n_ensemble : int number of configurations to generate Returns ------- list of configurations : list of numpy arrays list of configurations in the ensemble seeds : list of ints list of seeds for random number generator for each configuration """ sq = SeedSequence(seed_entropy) seeds = sq.spawn(n_ensemble) coords_list = [] for seed in seeds: initial_coords = generate_random_configuration_single( box_length, n_part, n_dim, seed) coords_list.append(initial_coords) return (coords_list, seeds)
def retry(store, prob, algo, num_retries, value_limit=math.inf, popsize=1, workers=mp.cpu_count()): try: import pygmo as pg except ImportError as e: raise ImportError( "Please install PYGMO (pip install pygmo) to use PAGMO optimizers" ) from e sg = SeedSequence() rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)] proc = [ Process(target=_retry_loop, args=(pid, rgs, store, prob, algo, num_retries, value_limit, popsize, pg)) for pid in range(workers) ] [p.start() for p in proc] [p.join() for p in proc] store.sort() store.dump() return OptimizeResult(x=store.get_x_best(), fun=store.get_y_best(), nfev=store.get_count_evals(), success=True)
def __init__(self, seed=None, comm=MPI.COMM_WORLD): """Create independent random number generators in parallel Optional keyword arguments: seed=None: seed the Gnerator to get a reproducible stream. comm=MPI.COMM_WORLD: The MPI communicator Creates an independent np.random.Generator in each MPI process. This generator can be retrived with the __call__ method, e.g. from KSFD import Generator ... kgen = Generator(seed) rng = kgen() Also, the class method get_rng() will retrieve the process-wide npp.random.Generator, so that you don't need to carry the Generator instance around with you: rng = Generator.get_rng() """ if seed is None and self._rng is not None: # # already set -- nothing to do # return size = comm.size rank = comm.rank ss = SeedSequence(seed) seeds = ss.spawn(size) type(self)._seeds = seeds type(self)._rng = default_rng(seeds[rank]) return
def run(self): ### 1.- Especificar limites maximos y minimos self.valid_args_limit() chunk = int(np.ceil(self.maxiter / self.workers)) seq = SeedSequence() random = seq.spawn(self.workers) if self.debug: print() print('maxiter {} chunk {} workers {} threads {} cats {}'.format( self.maxiter, chunk, self.workers, self.n_threads, self.n_cats)) print() print('exec pstree -p', os.getpid()) best = None if self.workers == 1: best = self.__worker__(0, default_rng(random[0]), self.maxiter, None) else: best = [[]] * 4 best[self.BEST_FUNC_TEST_VALUE] = np.inf shared_best = Array('d', [0.0] * (self.dimension + 4), lock=True) shared_best[self.BEST_FUNC_TEST_VALUE] = np.Inf jobs = [] for pid in range(self.workers): p = Process(target=self.__worker__, args=(pid, default_rng(random[pid]), chunk, shared_best)) jobs.append(p) p.start() for job in jobs: job.join() best[self.BEST_PROCESS_ID] = shared_best[self.BEST_PROCESS_ID] best[self.BEST_CAT_INDEX] = shared_best[self.BEST_CAT_INDEX] best[self.BEST_FUNC_TEST_VALUE] = shared_best[ self.BEST_FUNC_TEST_VALUE] for i in range(self.dimension): best[self.BEST_CAT_POSITION].append( shared_best[self.BEST_CAT_POSITION + i]) # Se parchea la salida si se esta maximizando para optener el valor real: -f(x) if self.maximize: best[self.BEST_FUNC_TEST_VALUE] = -best[self.BEST_FUNC_TEST_VALUE] if self.debug: print() return best
def _retry(minimizer): sg = SeedSequence() rgs = [Generator(MT19937(s)) for s in sg.spawn(minimizer.workers)] procs = [ Process(target=_retry_loop, args=(pid, rgs, minimizer)) for pid in range(minimizer.workers) ] [p.start() for p in procs] return procs
def mo_retry(fun, weight_bounds, ncon, y_exp, store, optimize, num_retries, value_limits, workers=mp.cpu_count()): sg = SeedSequence() rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)] proc=[Process(target=_retry_loop, args=(pid, rgs, fun, weight_bounds, ncon, y_exp, store, optimize, num_retries, value_limits)) for pid in range(workers)] [p.start() for p in proc] [p.join() for p in proc] store.sort() store.dump() return store.get_xs()
def retry(fun, store, optimize, num_retries, value_limit = math.inf, workers=mp.cpu_count(), stop_fitness = -math.inf): sg = SeedSequence() rgs = [Generator(MT19937(s)) for s in sg.spawn(workers)] proc=[Process(target=_retry_loop, args=(pid, rgs, fun, store, optimize, num_retries, value_limit, stop_fitness)) for pid in range(workers)] [p.start() for p in proc] [p.join() for p in proc] store.sort() store.dump() return OptimizeResult(x=store.get_x_best(), fun=store.get_y_best(), nfev=store.get_count_evals(), success=True)
def __init__(self, n, seed=None, threads=None): if threads is None: threads = multiprocessing.cpu_count() self.threads = threads seq = SeedSequence(seed) self._random_generators = [default_rng(s) for s in seq.spawn(threads)] self.n = n self.executor = concurrent.futures.ThreadPoolExecutor(threads) self.values = np.empty(n) self.step = np.ceil(n / threads).astype(np.int_)
def _generate_args(data_dir): vehicle_data = osp.join(data_dir, "Vehicle Data", "Simulation Snapshot") ss = SeedSequence(SEED) seeds = ss.spawn(N_FILES) for chunk in range(N_FILES): fname = osp.join(vehicle_data, "Snapshot_" + str(chunk * 1000000) + ".csv") sys.stderr.write("Processing chunk {}...\n".format(chunk)) sys.stderr.flush() yield (seeds[chunk], fname)
def calculate_variable_importance(explainer, type, loss_function, variables, N, B, label, processes, keep_raw_permutations, random_state): if processes == 1: result = [None] * B for i in range(B): result[i] = loss_after_permutation(explainer.data, explainer.y, explainer.model, explainer.predict_function, loss_function, variables, N, np.random) else: # Create number generator for each iteration ss = SeedSequence(random_state) generators = [default_rng(s) for s in ss.spawn(B)] pool = mp.Pool(processes) result = pool.starmap_async( loss_after_permutation, [(explainer.data, explainer.y, explainer.model, explainer.predict_function, loss_function, variables, N, generators[i]) for i in range(B)]).get() pool.close() raw = pd.concat(result, sort=True) result = raw.mean().sort_values().reset_index() result['label'] = label result.rename(columns={ 0: 'dropout_loss', 'index': 'variable' }, inplace=True) if type == "ratio": result.loc[:, 'dropout_loss'] = result.loc[:, 'dropout_loss'] / result.loc[ result.variable == '_full_model_', 'dropout_loss'].values if type == "difference": result.loc[:, 'dropout_loss'] = result.loc[:, 'dropout_loss'] - result.loc[ result.variable == '_full_model_', 'dropout_loss'].values raw_permutations = raw.reset_index( drop=True) if keep_raw_permutations else None return result, raw_permutations
def test_reference_data(): """ Check that SeedSequence generates data the same as the C++ reference. https://gist.github.com/imneme/540829265469e673d045 """ inputs = [ [3735928559, 195939070, 229505742, 305419896], [3668361503, 4165561550, 1661411377, 3634257570], [164546577, 4166754639, 1765190214, 1303880213], [446610472, 3941463886, 522937693, 1882353782], [1864922766, 1719732118, 3882010307, 1776744564], [4141682960, 3310988675, 553637289, 902896340], [1134851934, 2352871630, 3699409824, 2648159817], [1240956131, 3107113773, 1283198141, 1924506131], [2669565031, 579818610, 3042504477, 2774880435], [2766103236, 2883057919, 4029656435, 862374500], ] outputs = [ [3914649087, 576849849, 3593928901, 2229911004], [2240804226, 3691353228, 1365957195, 2654016646], [3562296087, 3191708229, 1147942216, 3726991905], [1403443605, 3591372999, 1291086759, 441919183], [1086200464, 2191331643, 560336446, 3658716651], [3249937430, 2346751812, 847844327, 2996632307], [2584285912, 4034195531, 3523502488, 169742686], [959045797, 3875435559, 1886309314, 359682705], [3978441347, 432478529, 3223635119, 138903045], [296367413, 4262059219, 13109864, 3283683422], ] outputs64 = [ [2477551240072187391, 9577394838764454085], [15854241394484835714, 11398914698975566411], [13708282465491374871, 16007308345579681096], [15424829579845884309, 1898028439751125927], [9411697742461147792, 15714068361935982142], [10079222287618677782, 12870437757549876199], [17326737873898640088, 729039288628699544], [16644868984619524261, 1544825456798124994], [1857481142255628931, 596584038813451439], [18305404959516669237, 14103312907920476776], ] for seed, expected, expected64 in zip(inputs, outputs, outputs64): expected = np.array(expected, dtype=np.uint32) ss = SeedSequence(seed) state = ss.generate_state(len(expected)) assert_array_equal(state, expected) state64 = ss.generate_state(len(expected64), dtype=np.uint64) assert_array_equal(state64, expected64)
def divide_in_partitions(l: List[T], k: int, seed: int) -> List[FrozenSet[T]]: """ Divide the given list into k random partitions """ rng = Generator(PCG64(SeedSequence(seed))) rng.shuffle(l) return [frozenset(l[j::k]) for j in range(k)]
def split(X, y, s, r): rs = RandomState(MT19937(SeedSequence(r))) i = X.index.to_numpy() np.random.shuffle(i, ) P = np.rint(np.multiply(i.size, s)) P[-1] = len(i) - np.sum(P[:-1]) P = np.cumsum(P) P = P - 1 P = [int(p) for p in P] last = -1 X_splitted = [] for j, p in enumerate(P): X_splitted.append(X[(last + 1):p]) last = p last = -1 y_splitted = [] for j, p in enumerate(P): y_splitted.append(y[(last + 1):p]) last = p return (X_splitted + y_splitted)
def _set_rng(self): """ Initialize random generator stream. For seeded runs, sets the state reproducibly. """ # TODO: checkpointing save of self._rng.bit_generator.state per process if mpi.is_main_process(): seed = getattr(self, "seed", None) if seed is not None: self.mpi_warning("This run has been SEEDED with seed %s", seed) ss = SeedSequence(seed) child_seeds = ss.spawn(mpi.size()) else: child_seeds = None ss = mpi.scatter(child_seeds) self._entropy = ss.entropy # for debugging store for reproducibility self._rng = default_rng(ss)
def set_global_seed(seed=42): """ rlberry has a global seed from which we can obtain different random number generators with (close to) independent outcomes. Important: If the global seed is altered by the user, it should be done only once, after importing rlberry for the first time. This is to ensure that all random number generators are children of the same SeedSequence. To do: Check (torch seeding): https://github.com/pytorch/pytorch/issues/7068#issuecomment-487907668 """ global _GLOBAL_SEED, _GLOBAL_SEED_SEQ _GLOBAL_SEED = seed _GLOBAL_SEED_SEQ = SeedSequence(_GLOBAL_SEED) # get state (for seeding) # rng_libs = get_rng() # state = rng_libs.__getstate__()['state']['state'] # seed torch if _TORCH_INSTALLED: torch.manual_seed(seed)
def MultiDist(n, mu, sigma): rand_gen = Generator(PCG64(SeedSequence())) dist_1 = rand_gen.exponential(1, n) mu_array = np.full(n, mu) mu_array = mu_array * dist_1 dist_2 = rand_gen.normal(mu_array, sigma) plt.hist(dist_2, bins='auto') plt.show()
def test_seedsequence(): from numpy.random.bit_generator import (ISeedSequence, ISpawnableSeedSequence, SeedlessSeedSequence) s1 = SeedSequence(range(10), spawn_key=(1, 2), pool_size=6) s1.spawn(10) s2 = SeedSequence(**s1.state) assert_equal(s1.state, s2.state) assert_equal(s1.n_children_spawned, s2.n_children_spawned) # The interfaces cannot be instantiated themselves. assert_raises(TypeError, ISeedSequence) assert_raises(TypeError, ISpawnableSeedSequence) dummy = SeedlessSeedSequence() assert_raises(NotImplementedError, dummy.generate_state, 10) assert len(dummy.spawn(10)) == 10
def __init__(self, fname): self.vars = Variables(fname) self.seed = SeedSequence() self.rand_gen = Generator(PCG64(self.seed)) self.vector_initial = Normalise(self.rand_gen.uniform(-1,1,3)) self.pos_initial = np.array([0, 0, 0]) self.time = np.full(self.vars.sample_total, self.vars.base_time) self.time = np.append([0], self.time) self.time = np.cumsum(self.time) self.time = np.reshape(self.time, (self.time.size, 1))
def inverse_shuffle(input_rgb, r1, r2): rs_row = RandomState(MT19937(SeedSequence(r1))) rs_col = RandomState(MT19937(SeedSequence(r2))) row_size = len(input_rgb) col_size = len(input_rgb[0]) shuf_rgb = np.zeros(input_rgb.shape) rows = list(range(row_size)) for i in range(row_size): row = rs_row.choice(rows) rows.remove(row) cols = list(range(col_size)) for j in range(col_size): col = rs_col.choice(cols) cols.remove(col) shuf_rgb[i][j] = input_rgb[row][col] return shuf_rgb
def run_experiment(c): # timestamp to identify the experiment timestamp = datetime.now() # multiprocessing initialization manager = mp.Manager() queue = manager.Queue() pool = mp.Pool(mp.cpu_count() + 1) # calculate number of total iterations to show progress num_iterations_total = len(c['DATASETS']) * len( c['OPTIMIZERS']) * c['NUM_ITERATIONS'] # start process which listens to the queue and writes new results to file result_writer = pool.apply_async( queue_listener, (queue, c, timestamp, num_iterations_total)) # create independent random generator objects (streams) for every iteration seed_sequence = SeedSequence(12345) child_seeds = seed_sequence.spawn(num_iterations_total) random_streams = iter([default_rng(s) for s in child_seeds]) # create all the workers which each compute one iteration iterations = [] for dataset in c['DATASETS']: for opt_name, opt_params in c['OPTIMIZERS'].items(): for iteration_idx in range(c['NUM_ITERATIONS']): rng = next(random_streams) iteration = pool.apply_async( run_iteration, (c, dataset, iteration_idx, opt_name, opt_params, timestamp, queue, rng)) iterations.append(iteration) # collect results from the workers through the pool result queue for iteration in iterations: iteration.get() # now we are done, kill the listener queue.put('kill') pool.close() pool.join()
def UpdateNormalNormalized(i, d=0.01, n=1, Mb=100, mb= -100, rs=0): i = np.array(i) if not rs: rseed = random.randint(1000, 9999) rs = RandomState(MT19937(SeedSequence(rseed))) Ix = rs.randint(0, i.shape[0],n) # faster than np.random.choice Iy = rs.randint(0, i.shape[1],n) z = np.zeros(i.shape) + i z[Ix,Iy] = z[Ix,Iy] + rs.normal(0, d[Ix,Iy], n) z = z/np.sum(z) hastings = 0 return z, (Ix, Iy), hastings
def UpdateNormal1D(i, d=0.01, n=1, Mb=100, mb= -100, rs=0): i = np.array(i) if not rs: rseed = random.randint(1000, 9999) rs = RandomState(MT19937(SeedSequence(rseed))) Ix = rs.randint(0, len(i),n) # faster than np.random.choice z = np.zeros(i.shape) + i z[Ix] = z[Ix] + rs.normal(0, d, n) z[z > Mb] = Mb - (z[z>Mb] - Mb) z[z < mb] = mb + (mb - z[z<mb]) hastings = 0 return z, Ix, hastings
def main(outdir): rng = RandomState(MT19937(SeedSequence(config.seed))) berlin_holidays = holidays.DE(prov="BW") num_employees = 20000 num_jobsites = 200 num_areas = 20 num_qualifications = 40 num_shifts = 3 num_days = 356 num_orders = 1000 df = pd.DataFrame.from_dict({ "Einsatzort": rng.randint(0, num_jobsites, num_orders), "Qualifikation": rng.randint(0, num_qualifications, num_orders), "Schicht": rng.randint(0, num_shifts, num_orders), "Tag": rng.randint(0, num_days, num_orders), }) df["Tag"] = df["Tag"].apply( lambda day: datetime(2019, 1, 1) + timedelta(day)) df["Wochentag"] = df["Tag"].apply(lambda day: day.strftime("%a")) df["Feiertag"] = df["Tag"].apply(lambda day: day in berlin_holidays) # grouping of jobsites into areas area_splits = np.cumsum(rng.randint(1, 10, num_areas)) area_splits = (area_splits.T / area_splits.max() * num_jobsites).astype(int) df["Ort"] = df["Einsatzort"].apply( lambda jobsite_id: np.argmax(area_splits > jobsite_id)) offers = [] for _ in range(len(df)): offers.append( rng.choice(range(num_employees), replace=False, size=rng.randint(1, 6)).tolist()) df["Mitarbeiter ID"] = offers train, test = train_test_split(df) train.to_csv(os.path.join(outdir, "train.tsv"), index=False, sep="\t") test.to_csv(os.path.join(outdir, "test_truth.tsv"), index=False, sep="\t") test[[ "Einsatzort", "Qualifikation", "Schicht", "Tag", "Wochentag", "Feiertag", "Ort" ]].to_csv(os.path.join(outdir, "test_publish.tsv"), index=False, sep="\t")
def dot_jrrp_handler( update: Update, context: CallbackContext, argv: Tuple[str], ) -> None: username = update.effective_user.username temp = f"{username}{arrow.utcnow().to('utf-8').isocalendar()}" hash_ = int(hashlib.md5(temp.encode()).hexdigest(), 16) % sys.maxsize # logging.info(f"in jrrp_handler: {temp=}, {hash_=}") rp = default_rng(SeedSequence(hash_)).integers(0, 100, endpoint=True) chat_id = update.effective_chat.id msg = f"@{username} 今天的人品值是:**{rp}**。" context.bot.send_message(chat_id, text=msg, parse_mode=ParseMode.MARKDOWN)
def multiplier_proposal_vector(q, d=1.05, f=1, rs=0): if not rs: rseed = random.randint(1000, 9999) rs = RandomState(MT19937(SeedSequence(rseed))) S = q.shape ff = rs.binomial(1,f,S) u = rs.random(S) l = 2 * np.log(d) m = np.exp(l * (u - .5)) m[ff==0] = 1. new_q = q * m U=np.sum(np.log(m)) return new_q, 0, U
def create_bit_generator(seed=None, stream=0): """Creates an instance of a ``BIT_GENERATOR``. Parameters ---------- seed : int, optional The seed to use. If seed is None (the default), will create a seed using :py:func:`create_seed`. stream : int, optional The stream to create the bit generator for. This allows multiple generators to exist with the same seed, but that produce different sets of random numbers. Default is 0. Returns ------- BIT_GENERATOR : The bit generator initialized with the given seed and stream. """ # create the seed sequence seedseq = SeedSequence(create_seed(seed)) if stream > 0: seedseq = seedseq.spawn(stream + 1)[stream] return BIT_GENERATOR(seedseq)