def __init__(self, max_size): from multiprocessing import Lock, RawArray, RawValue self._max_size = max_size self._array = RawArray('c', max_size) self._pos = RawValue('L') self._size = RawValue('L') self._locks = Lock(), Lock(), Lock()
def __init__(self, struct, size=20): """ struct - a ctypes object (value, array or struct) size - number of slots in the buffer. If the number of slots exceeds the number of python ints which will fit in an os.pipe buffer, this will block indefinitely. I don't see a way round this. On the plus side, at least it fails early, rather than on a call to the put method. """ buf = RawArray(struct, int(size)) self.buffer = buf #self.buffer = numpy.frombuffer(buf, dtype=numpy.dtype(buf._type_)) stock_out, stock_in = Pipe(duplex=False) queue_out, queue_in = Pipe(duplex=False) stock_out_lock = Lock() stock_in_lock = Lock() queue_out_lock = Lock() queue_in_lock = Lock() self.stock_closed = RawValue('h', 0) self.queue_closed = RawValue('h', 0) for i in xrange(size): stock_in.send(i) self.map = {} self._put_obj = (stock_out, stock_out_lock, queue_in, queue_in_lock) self._ret_obj = (stock_in, stock_in_lock) self._get_obj = (queue_out, queue_out_lock)
def __init__(self, num_actions, alg_type, opt_type=None, lr=0): # Net if alg_type in ['q', 'sarsa']: self.var_shapes = [ (8, 8, 4, 16), (16), (4, 4, 16, 32), (32), (2592, 256), #(3872, 256) if PADDING = "SAME" (256), (256, num_actions), (num_actions) ] self.size = 0 for shape in self.var_shapes: self.size += np.prod(shape) if opt_type == "adam": self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) elif opt_type == "rmsprop": self.vars = self.malloc_contiguous( self.size, np.ones(self.size, dtype=np.float)) else: #momentum self.vars = self.malloc_contiguous(self.size) else: # no lstm self.var_shapes = [ (8, 8, 4, 16), (16), (4, 4, 16, 32), (32), (2592, 256), #(3872, 256) (256), (256, num_actions), (num_actions), (256, 1), (1) ] self.size = 0 for shape in self.var_shapes: self.size += np.prod(shape) if opt_type == "adam": self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) if opt_type == "rmsprop": self.vars = self.malloc_contiguous( self.size, np.ones(self.size, dtype=np.float)) elif opt_type == "momentum": self.vars = self.malloc_contiguous(self.size) else: self.vars = self.malloc_contiguous(self.size)
def __init__(self, _simulationCommunicator=None): print("ImageProcessor object created") self.simulationCommunicator = _simulationCommunicator #wartosci-rezultaty przetwarzania obrazu self.result_x = RawValue('f', 0.0) self.result_y = RawValue('f', 0.0) self.key = RawValue('i', 0) self.corners = np.zeros((4, 2), np.int32) #pozycje rogow plyty self.obstacle_map = RawArray('i', ImageProcessor.obstacle_map_size**2) self.obstacle_map_update_counter = 0
def test_multipush(self): '''Tests behavior when multiple servers are pushing jobs simultaneously.''' total_completed = RawValue('i') total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,), num_pushers=config.NUM_PUSHERS) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS * config.NUM_PUSHERS, get_timeout(len(config.WORKER_ADDRESSES))) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS * config.NUM_PUSHERS))
def test_volume(self): '''Tests the ability to handle various traffic patterns (trickle, normal, and spike by default).''' total_completed = RawValue('i') for pattern in config.USAGE_PATTERNS: total_completed.value = 0 total_jobs = pattern.sets * pattern.set_reps start_workers, kill_workers = testing_lib.construct_worker_pool(len(config.WORKER_ADDRESSES), config.WORKER_ADDRESSES, send_jobs, (pattern,), on_recv_result, (total_completed,)) start_workers() if not testing_lib.check_for_completion(total_completed, total_jobs, get_timeout(pattern, len(config.WORKER_ADDRESSES))): self.fail('Failed on usage pattern: %s' % str(pattern)) kill_workers()
def test_files(self): '''Tests that workers can correctly store files in a central location.''' total_completed = RawValue('i') total_completed.value = 0 if not os.path.exists(os.path.join(os.path.dirname(__file__), 'testing_files/')): os.mkdir('testing_files') start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed,)) start_workers() if not testing_lib.check_for_completion(total_completed, config.NUM_FILES, get_timeout(len(config.WORKER_ADDRESSES))): self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_FILES)) kill_workers() os.rmdir('testing_files')
def test_mem(self): '''Tests that memory required to transfer jobs does not exceed the given amount.''' total_completed = RawValue('i') print 'Constructing test string (this could take some time).' test_str = generate_str(config.STR_LEN) print 'Done. Starting test.' total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (test_str,), on_recv_result, (total_completed,)) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_STRINGS, get_timeout()) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_STRINGS))
def test_multiprocessing(self): '''Tests that jobs will execute properly on multiple processes simultaneously.''' total_completed = RawValue('i') job_processors = MultiQueue() total_completed.value = 0 start_workers, kill_workers = testing_lib.construct_worker_pool(config.num_local_workers(), config.WORKER_ADDRESSES, send_jobs, (), on_recv_result, (total_completed, job_processors)) start_workers() completion = testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))) kill_workers() if not completion: self.fail('Not all jobs received: %d / %d' % (total_completed.value, config.NUM_JOBS)) if not check_load_balance(job_processors): self.fail('Not all workers utilized.')
def __new__(cls, *args, **kwargs): """Make this a singleton class.""" if cls._instance is None: cls._instance = super().__new__(cls) cls._instance.val = RawValue("i", 1) cls._instance.lock = threading.Lock() return cls._instance
def fasta(n): alu = sub( r'\s+', '', """ GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGA TCACCTGAGGTCAGGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACT AAAAATACAAAAATTAGCCGGGCGTGGTGGCGCGCGCCTGTAATCCCAGCTACTCGGGAG GCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTTGCAGTGAGCCGAGATCGCG CCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA """) iub = list( zip_longest('acgtBDHKMNRSVWY', (.27, .12, .12, .27), fillvalue=.02)) homosapiens = list( zip('acgt', (0.3029549426680, 0.1979883004921, 0.1975473066391, 0.3015094502008))) seed = RawValue('f', 42) width = 60 tasks = [ (copy_from_sequence, [b'>ONE H**o sapiens alu\n', alu, n * 2, width]), (random_selection, [b'>TWO IUB ambiguity codes\n', iub, n * 3, width, seed]), (random_selection, [b'>THREE H**o sapiens frequency\n', homosapiens, n * 5, width, seed]), ] for func, args in tasks: func(*args) output_file.close()
class ThreadedTraceParser: """ Trace parser that scans a trace using multiple processes The scanning processes are forked from the current parser and each one is allocated a range of the trace. The parsed data items are piped through one or more queues. The current process pulls data from the queue(s) and stores it in the dataset XXX: experimental/broken """ def __init__(self, path, parser, threads=2): self.trace = RawValue(py_object, None) """Trace in shared memory""" self.parser = parser """ Callback that handles each trace block, this is run in separate processes """ self.n_threads = threads """Number of subprocesses that are spawned""" self.path = path """Trace path""" if not os.path.exists(path): raise IOError("File not found %s" % path) self.trace = pct.trace.open(path) if self.trace is None: raise IOError("Can not open trace %s" % path) def __len__(self): if self.trace: return self.trace.size() return 0 def parse(self, *args, **kwargs): start = kwargs.pop("start", 0) end = kwargs.pop("end", len(self)) block_size = math.floor((end - start) / self.n_threads) start_indexes = np.arange(start, end - block_size + 1, block_size) end_indexes = np.arange(start + block_size, end + 1, block_size) - 1 # the last process consumes any remaining entries left by the # rounding of block_size end_indexes[-1] = end procs = [] for idx_start, idx_end in zip(start_indexes, end_indexes): print(idx_start, idx_end) p = Process(target=self.parser, args=(self.path, idx_start, idx_end)) procs.append(p) for p in procs: p.start() for p in procs: p.join()
def __init__(self, path, parser, threads=2): self.trace = RawValue(py_object, None) """Trace in shared memory""" self.parser = parser """ Callback that handles each trace block, this is run in separate processes """ self.n_threads = threads """Number of subprocesses that are spawned""" self.path = path """Trace path""" if not os.path.exists(path): raise IOError("File not found %s" % path) self.trace = pct.trace.open(path) if self.trace is None: raise IOError("Can not open trace %s" % path)
def __init__(self, maxlen, shape, dtype=np.float32, data=None): """ A buffer object, when full restarts at the initial position :param maxlen: (int) the max number of numpy objects to store :param shape: (tuple) the shape of the numpy objects you want to store :param dtype: (str) the name of the type of the numpy object you want to store """ self.maxlen = maxlen self.start = RawValue('L') self.length = RawValue('L') self.shape = shape if data is None: self.data = np.zeros((maxlen, ) + shape, dtype=dtype) else: assert data.shape == (maxlen, ) + shape assert data.dtype == dtype self.data = data
def __init__(self, params, opt_type=None, lr=0, step=0): self.var_shapes = [var.get_shape().as_list() for var in params] self.size = sum([np.prod(shape) for shape in self.var_shapes]) self.step = RawValue(ctypes.c_int, step) if opt_type == 'adam': self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) elif opt_type == 'adamax': self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) elif opt_type == 'rmsprop': self.vars = self.malloc_contiguous( self.size, np.ones(self.size, dtype=np.float)) elif opt_type == 'momentum': self.vars = self.malloc_contiguous(self.size) else: self.vars = self.malloc_contiguous(self.size)
def test_threading(self): '''Tests that jobs will execute properly on multiple threads simultaneously.''' total_completed = RawValue('i') job_processors = MultiQueue() def push(vent_port, sink_port, worker_pool): worker, close, run_job = parallel.construct_worker(worker_pool, {'vent_port': vent_port, 'sink_port': sink_port}) for i in range(config.NUM_JOBS): run_job(wait_job, (config.WAIT_TIME)) worker(on_recv_result, (total_completed, job_processors)) total_completed.value = 0 port = 5000 thread.start_new_thread(push, (port, port + 1, config.WORKER_ADDRESSES)) for i in range(config.num_local_workers() - 1): port += 2 thread.start_new_thread(testing_lib.work, (port, port + 1, config.WORKER_ADDRESSES)) if not testing_lib.check_for_completion(total_completed, config.NUM_JOBS, get_timeout(len(config.WORKER_ADDRESSES))): self.fail('Not all jobs received: %d / %d' % (total_completed.value, len(config.WORKER_ADDRESSES))) if not check_load_balance(job_processors): self.fail('Not all workers utilized.')
def __init__(self, struct, size=20): size = int(size) + 1 self._size = size self.buffer = Array(struct, size) #dt = numpy.dtype(struct) #self.buffer = numpy.frombuffer(self._buffer, dtype=dt) self.stock = RawArray('I', size) self.queue = RawArray('I', size) self.stock_write = RawValue('I', 0) self.stock_read = RawValue('I', 0) self.queue_write = RawValue('I', 0) self.queue_read = RawValue('I', 0) self.stock[:] = xrange(size) self.stock_write.value = size - 1 self.stock_lock = Condition(Lock()) self.queue_lock = Condition(Lock()) self.map = {}
def fasta(n): alu = sub(r'\s+', '', """ GGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGA TCACCTGAGGTCAGGAGTTCGAGACCAGCCTGGCCAACATGGTGAAACCCCGTCTCTACT AAAAATACAAAAATTAGCCGGGCGTGGTGGCGCGCGCCTGTAATCCCAGCTACTCGGGAG GCTGAGGCAGGAGAATCGCTTGAACCCGGGAGGCGGAGGTTGCAGTGAGCCGAGATCGCG CCACTGCACTCCAGCCTGGGCGACAGAGCGAGACTCCGTCTCAAAAA """) iub = list(zip_longest('acgtBDHKMNRSVWY', (.27, .12, .12, .27), fillvalue=.02)) homosapiens = list(zip('acgt', (0.3029549426680, 0.1979883004921, 0.1975473066391, 0.3015094502008))) seed = RawValue('f', 42) width = 60 tasks = [ (copy_from_sequence, [b'>ONE H**o sapiens alu\n', alu, n * 2, width]), (random_selection, [b'>TWO IUB ambiguity codes\n', iub, n * 3, width, seed]), (random_selection, [b'>THREE H**o sapiens frequency\n', homosapiens, n * 5, width, seed]), ] if cpu_count() < 2: for func, args in tasks: func(*args) else: written_1 = acquired_lock() seeded_2 = acquired_lock() written_2 = acquired_lock() locks_sets = [ (None, written_1), (None, seeded_2, written_1, written_2), (seeded_2, None, written_2, None), ] processes = [ started_process(target, args + [locks_sets[i]]) for i, (target, args) in enumerate(tasks) ] for p in processes: p.join() output_file.close()
def __init__(self, num_actions, alg_type, network, opt_type=None, lr=0): self.var_shapes = [ var.get_shape().as_list() for var in tf.global_variables() ] self.size = 0 for shape in self.var_shapes: self.size += np.prod(shape) if opt_type == 'adam': self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) elif opt_type == 'adamax': self.ms = self.malloc_contiguous(self.size) self.vs = self.malloc_contiguous(self.size) self.lr = RawValue(ctypes.c_float, lr) elif opt_type == 'rmsprop': self.vars = self.malloc_contiguous( self.size, np.ones(self.size, dtype=np.float)) elif opt_type == 'momentum': self.vars = self.malloc_contiguous(self.size) else: self.vars = self.malloc_contiguous(self.size)
def __init__(self, conf={}): self.conf = DEFAULT_CONF self.conf.update(conf) # Init one-to-one mapped variables self.net_man = None self.state_man = None self.traffic_gen = None self.bw_ctrl = None self.sampler = None self.input_file = None self.terminated = False self.reward = RawValue('d', 0) # set the id of this environment self.short_id = dc_utils.generate_id() if self.conf["parallel_envs"]: self.conf["topo_conf"]["id"] = self.short_id # initialize the topology self.topo = TopoFactory.create(self.conf["topo"], self.conf["topo_conf"]) # Save the configuration we have, id does not matter here dc_utils.dump_json(path=self.conf["output_dir"], name="env_config", data=self.conf) dc_utils.dump_json(path=self.conf["output_dir"], name="topo_config", data=self.topo.conf) # set the dimensions of the state matrix self._set_gym_matrices() # Set the active traffic matrix self._set_traffic_matrix(self.conf["tf_index"], self.conf["input_dir"], self.topo) # each unique id has its own sub folder if self.conf["parallel_envs"]: self.conf["output_dir"] += f"/{self.short_id}" # check if the directory we are going to work with exists dc_utils.check_dir(self.conf["output_dir"]) # handle unexpected exits scenarios gracefully atexit.register(self.close)
def run(self, ensemble, ncores=None, pairwise_align=False, align_subset_coordinates=None, mass_weighted=True, metadata=True): ''' Run the conformational distance matrix calculation. **Arguments:** `ensemble` : encore.Ensemble.Ensemble object Ensemble object for which the conformational distance matrix will be computed. `pairwise_align` : bool Whether to perform pairwise alignment between conformations `align_subset_coordinates` : numpy.array or None Use these coordinates for superimposition instead of those from ensemble.superimposition_coordinates `mass_weighted` : bool Whether to perform mass-weighted superimposition and metric calculation `metadata` : bool Whether to build a metadata dataset for the calculated matrix `ncores` : int Number of cores to be used for parallel calculation **Returns:** `cond_dist_matrix` : encore.utils.TriangularMatrix object Conformational distance matrix in triangular representation. ''' # Decide how many cores have to be used. Since the main process is stopped while the workers do their job, ncores workers will be spawned. if not ncores: ncores = cpu_count() if ncores < 1: ncores = 1 # framesn: number of frames framesn = len(ensemble.coordinates) # Prepare metadata recarray if metadata: metadata = array( [(gethostname(), getuser(), str(datetime.now()), ensemble.topology_filename, framesn, pairwise_align, ensemble.superimposition_selection_string, mass_weighted)], dtype=[('host', object), ('user', object), ('date', object), ('topology file', object), ('number of frames', int), ('pairwise superimposition', bool), ('superimposition subset', object), ('mass-weighted', bool)]) # Prepare alignment subset coordinates as necessary subset_coords = None if pairwise_align: subset_selection = ensemble.superimposition_selection if align_subset_coordinates == None: subset_coords = align_subset_coordinates else: subset_coords = ensemble.superimposition_coordinates # Prepare masses as necessary subset_masses = None if mass_weighted: masses = ensemble.atom_selection.masses if pairwise_align: subset_masses = subset_selection.masses else: masses = ones((ensemble.coordinates[0].shape[0])) if pairwise_align: subset_masses = ones((subset_coords[0].shape[0])) # matsize: number of elements of the triangular matrix, diagonal elements included. matsize = framesn * (framesn + 1) / 2 # Calculate the number of matrix elements that each core has to calculate as equally as possible. if ncores > matsize: ncores = matsize runs_per_worker = [matsize / int(ncores) for x in range(ncores)] unfair_work = matsize % ncores for i in range(unfair_work): runs_per_worker[i] += 1 # Splice the matrix in ncores segments. Calculate the first and the last (i,j) # matrix elements of the slices that will be assigned to each worker. Each of them will proceed in a column-then-row order # (e.g. 0,0 1,0 1,1 2,0 2,1 2,2 ... ) i = 0 a = [0, 0] b = [0, 0] tasks_per_worker = [] for n in range(len(runs_per_worker)): while i * (i - 1) / 2 < sum(runs_per_worker[:n + 1]): i += 1 b = [ i - 2, sum(runs_per_worker[0:n + 1]) - (i - 2) * (i - 1) / 2 - 1 ] tasks_per_worker.append((tuple(a), tuple(b))) if b[0] == b[1]: a[0] = b[0] + 1 a[1] = 0 else: a[0] = b[0] a[1] = b[1] + 1 # Allocate for output matrix distmat = Array(c_float, matsize) # Prepare progress bar stuff and run it pbar = AnimatedProgressBar(end=matsize, width=80) partial_counters = [RawValue('i', 0) for i in range(ncores)] # Initialize workers. Simple worker doesn't perform fitting, fitter worker does. if pairwise_align: workers = [ Process(target=self._fitter_worker, args=(tasks_per_worker[i], ensemble.coordinates, subset_coords, masses, subset_masses, distmat, partial_counters[i])) for i in range(ncores) ] else: workers = [ Process(target=self._simple_worker, args=(tasks_per_worker[i], ensemble.coordinates, masses, distmat, partial_counters[i])) for i in range(ncores) ] workers += [ Process(target=self._pbar_updater, args=(pbar, partial_counters, matsize)) ] # Start & join the workers for w in workers: w.start() for w in workers: w.join() # When the workers have finished, return a TriangularMatrix object return TriangularMatrix(distmat, metadata=metadata)
def __init__(self): # RawValue because we don't need it to create a Lock: self.val = RawValue('d', 0) self.num = RawValue('i', 0) self.lock = Lock()
def __init__(self, initial_value=0): self.progress_marker = RawValue(c_ulonglong, initial_value) self.lock = Lock()
def __init__(self, initval=0): self.val = RawValue('i', initval) self.lock = Lock()
print q.get_nowait() except: print "Queue empty !" i = INT(0); v1=RawValue('i', 0) v2=RawValue('i', 0) p2 = INTProcess(i, v1, v2); p2.start() p2.join() print i.getI() print i.getV() print v1.value print v2.value v = RawValue('i', 0) l = Lock() nbrProcs = 10 procs = [LockProcess(v, l, False) for j in range(nbrProcs)] for j in range(nbrProcs): procs[j].start() for j in range(nbrProcs): procs[j].join() print v.value v.value = 0 procs2 = [LockProcess(v, l, True) for j in range(nbrProcs)] for j in range(nbrProcs): procs2[j].start() for j in range(nbrProcs): procs2[j].join() print v.value
import numpy as np import csv import pandas as pd import re import linecache import time import matplotlib.pyplot as plt import matplotlib from multiprocessing import Pool, sharedctypes, RawArray, RawValue from MS.IBIS import * from MS.LPJ import * from MS.Biome_BGC import * from CMIP import * plt.switch_backend('tkagg') counter = RawValue('i') counter.value = 0 failed_counter = RawValue('i') failed_counter.value = 0 test = False options, args = getopt.getopt(sys.argv[1:], '', ['model=', 'test=']) for opt in options: if (opt[0] == '--model'): if opt[1] == 'IBIS': ms_daily = IBIS('daily') ms_month = IBIS('month') elif opt[1] == 'Biome-BGC': ms_daily = Biome_BGC('daily') ms_month = Biome_BGC('month') elif opt[1] == 'LPJ': ms_daily = LPJ('daily')
"metricName": "daily-average-GPP" }, { "id": "adnpptot", "type": "", "description": "daily average NPP", "scale": 1000.0, "offset": 0.0, "unit": "kgC m-2 d-1", "metricName": "daily-average-NPP" }] folder = DATA_HOME + '/IBIS_Data/5b9012e4c29ca433443dcfab/outputs' srcSuffix = '.daily.txt' distSuffix = '.month.txt' counter = RawValue('i') counter.value = 0 def convert2Month(i): # try: counter.value += 1 print('counter: %s %5.2f%% site index: %s' % (str(counter.value), counter.value * 100 / 40595, str(i))) srcPath = '%s/%s%s' % (folder, str(i), srcSuffix) distPath = '%s/%s%s' % (folder, str(i), distSuffix) if path.exists(distPath): fsize = path.getsize(distPath) if fsize > 100000: # 100k rerun = False else:
def __init__(self): self.val = RawValue('i', 0) self.lock = Lock()
def __init__(self, normalize_inputs, T, gamma, more_obs, average_rewards): if os.geteuid() != 0: exit( "You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'. Exiting." ) clean() os.system("systemctl start ovsdb-server.service") os.system("systemctl start ovs-vswitchd.service") self.T = T self.average_rewards = average_rewards self.normalize_inputs = normalize_inputs self.GAMMA = gamma self.max_steps = 10000000 self.n_agent = 16 self.nD = self.n_agent self.n_signal = 4 self.n_episode = 300 #for around 3M transitions self.max_u = None self.input_size = 96 self.n_actions = 1 #number of dim per agent self.conf = DEFAULT_CONF # self.conf.update(conf) if more_obs: self.conf.update( {"state_model": ["backlog", "d_backlog", "olimit", "drops"]}) self.input_size = 336 # Init one-to-one mapped variables self.net_man = None self.state_man = None self.traffic_gen = None self.bw_ctrl = None self.sampler = None self.input_file = None self.terminated = False self.reward = RawValue('d', 0) # set the id of this environment self.short_id = dc_utils.generate_id() if self.conf["parallel_envs"]: self.conf["topo_conf"]["id"] = self.short_id # initialize the topology self.topo = TopoFactory.create(self.conf["topo"], self.conf["topo_conf"]) # Save the configuration we have, id does not matter here dc_utils.dump_json(path=self.conf["output_dir"], name="env_config", data=self.conf) dc_utils.dump_json(path=self.conf["output_dir"], name="topo_config", data=self.topo.conf) # set the dimensions of the state matrix self._set_gym_matrices() # Set the active traffic matrix self._set_traffic_matrix(self.conf["tf_index"], self.conf["input_dir"], self.topo) # each unique id has its own sub folder if self.conf["parallel_envs"]: self.conf["output_dir"] += f"/{self.short_id}" # check if the directory we are going to work with exists dc_utils.check_dir(self.conf["output_dir"]) # handle unexpected exits scenarios gracefully atexit.register(self.close) self.compute_neighbors = False self.neighbors_size = 4 # max number of neighbor self.compute_neighbors_last = np.array([[1, 2, 3], [0, 2, 3], [0, 1, 3], [0, 1, 2], [5, 6, 7], [4, 6, 7], [4, 5, 7], [4, 5, 6], [9, 10, 11], [8, 10, 11], [8, 9, 11], [8, 9, 10], [13, 14, 15], [12, 14, 15], [12, 13, 15], [12, 13, 14]]) self.compute_neighbors_last_index = [ list(range(len(self.compute_neighbors_last[i]))) for i in range(self.n_agent) ] if normalize_inputs: self.obs_rms = RunningMeanStd(shape=self.input_size) self.fileresults = open('learning.data', "w")
if not self.is_locked(): raise IllegalLockStateException() self.__exit_lock() if not self.is_locked(): return self.redis_client.delete(self.lock_key) return True # main 中是lock 的测试 if __name__ == "__main__": from multiprocessing import Process, RawValue import os redis_client = redis.StrictRedis("localhost", 6379, 0) redis_lock = RedisLock(_lock_key="test_lock", _redis_client=redis_client) x = RawValue("i", 0) def test_lock(shared_x): n = 0 while n < 3: if redis_lock.try_lock(3): redis_lock.lock() n += 1 shared_x.value += 1 print "{0} {1} lock success and do {2}".format( os.getpid(), threading.currentThread().name, shared_x.value) # print "do lock thing" time.sleep(0.1) redis_lock.unlock() redis_lock.unlock()
def __init__(self, value=0): # RawValue because we don't need it to create a Lock: self.val = RawValue('i', value) self.lock = Lock()
def __init__(self, value=0): self.val = RawValue('i', value) self.lock = threading.Lock()
def __init__(self, value=0): self.val = RawValue('i', value) self.lock = Lock()
def reset(self): with self.lock: self.val = RawValue('i', self.initial_value)
def __init__(self, initval=0): self.val = RawValue('i', initval) self.last_step_update_target = RawValue('i', initval) self.lock = Lock()
def __init__(self): print("SimulationCommunicator object created") #wartosci-rezultaty dzialania symulacji self.ball_x = RawValue('f', 0.0) self.ball_y = RawValue('f', 0.0) self.servo_x = RawValue('i', 0) self.servo_y = RawValue('i', 0) self.corner_tl_x = RawValue('f', 0.0) self.corner_tl_y = RawValue('f', 0.0) self.corner_tr_x = RawValue('f', 0.0) self.corner_tr_y = RawValue('f', 0.0) self.corner_br_x = RawValue('f', 0.0) self.corner_br_y = RawValue('f', 0.0) self.corner_bl_x = RawValue('f', 0.0) self.corner_bl_y = RawValue('f', 0.0) self.cameraFrame = RawArray('i', 3 * 256**2) #zmienne wartosci self.servo_actual_pos = [0, 0] #aktualna pozycja serwa self.servo_target_pos = [0, 0] #docelowa pozycja serwa self.refreshDeltaTime = 1 / 60 self.frameReadTargetDelta = 1 / 40 self.frameReadLastTime = 0.0 self.capturedFrame = np.zeros((256, 256, 3))