def __init__(self, manager_params, browser_params): self.manager_params = manager_params self.browser_params = browser_params self.listener_address = None self.listener_process = None self.status_queue = Queue() self.completion_queue = Queue() self.shutdown_queue = Queue() self._last_status = None self._last_status_received = None self.logger = logging.getLogger('openwpm')
def thread_samples(adj, mSamp): if (mSamp <= 1): lst = [] temp_que = Queue() # Use thread to speed up processing (multithreading) temp_thread = threading.Thread( None, target=make_sample, name=("1: " + str(mSamp)), args=[adj, mSamp, temp_que], daemon=True, ) # temp_thread = threading.Thread(None, target=make_sample, name=("2:" + # str( # max_Sample)), # args=[adj, max_Sample, queue], ) temp_thread.start() temp_thread.join() time_samp_node = temp_que.get( ) # the resulting (samples,thread_time) temp_que.task_done() lst.append(time_samp_node) return (lst, True) else: rec_val = thread_samples(adj, mSamp - 1) temp_que = Queue() # Use thread to speed up processing (multithreading) temp_thread = threading.Thread( None, target=make_sample, name=("1: " + str(mSamp)), args=[adj, mSamp, temp_que], daemon=True, ) # temp_thread = threading.Thread(None, target=make_sample, name=("2:" + # str( # max_Sample)), # args=[adj, max_Sample, queue], ) temp_thread.start() temp_thread.join() time_samp_node = temp_que.get( ) # the resulting (samples,thread_time) temp_que.task_done() # s = queue.get() # Use thread to speed up processing (multithreading) if (rec_val[1]): rec_lst = rec_val[0] rec_lst.append(time_samp_node) ret = (rec_lst, True) # Return the begining of a (time x sample) x boolean tuple list return ret
def __init__( self, manager_params: ManagerParamsInternal, browser_params: List[BrowserParamsInternal], ): self.manager_params = manager_params self.browser_params = browser_params self.listener_address = None self.listener_process = None self.status_queue = Queue() self.completion_queue = Queue() self.shutdown_queue = Queue() self._last_status = None self._last_status_received = None self.logger = logging.getLogger("openwpm")
def run(self): setproctitle('python.DataStreamRecorder.{0}'.format(self._name)) try: logging.debug("Starting data recording on {0}".format(self.name)) self._tokens_q.put(("return", self.name)) while True: if not self._cmds_q.empty(): cmd = self._cmds_q.get() if cmd[0] == 'stop': break elif cmd[0] == 'pause': self._recording = False if self._saving_cache: self._save_cache(self._cur_data_segment) self._cur_data_segment += 1 self._data_qs.append(Queue()) elif cmd[0] == 'reset_data_segment': self._start_data_segment = self._cur_data_segment elif cmd[0] == 'resume': self._recording = True elif cmd[0] == 'save': self._save_data(cmd[1], cmd[2], cmd[3]) elif cmd[0] == 'params': self._args = cmd[1] self._kwargs = cmd[2] if self._recording and not self._ok_q.empty(): timestamp = self._ok_q.get() self._tokens_q.put(("take", self.name)) data = self._data_sampler_method(*self._args, **self._kwargs) cur_data_q = self._data_qs[self._cur_data_segment] if self._saving_cache and cur_data_q.qsize( ) == self._save_every: self._save_cache(self._cur_data_segment) cur_data_q = Queue() self._data_qs.append(cur_data_q) self._cur_data_segment += 1 cur_data_q.put((timestamp, data)) self._tokens_q.put(("return", self.name)) except KeyboardInterrupt: logging.debug("Shutting down data streamer on {0}".format( self.name)) sys.exit(0)
def __init__(self, num_cores): self.num_cores = num_cores self.names = set() self.global_queue = Queue() self.pool = [] self.channels = [] self.__terminated = False atexit.register(self.__del__) queues = [Queue() for _ in range(num_cores)] for i in range(num_cores): pipe_master, pipe_slave = Pipe() state = Value('i', 0) process = MRServer(i, queues, pipe_slave, state, self.global_queue) self.channels.append(Channel(queues[i], pipe_master, state)) self.pool.append(process) process.start()
def test_multiprocess_func_does_something(): q = Queue() multiprocess(lambda q, x: q.push(x), [(q, 1), (q, 2)]) arr = [] while not q.empty(): arr.append(q.pop()) assert (set([1, 2]) == set(arr))
def _solve(self, X, Y, batch_size): ''' Solve the multi-objective problem by multiple scalarized single-objective solvers. ''' # generate scalarization weights weights = np.random.random((batch_size, self.problem.n_obj)) weights /= np.expand_dims(np.sum(weights, axis=1), 1) # initial solutions X = np.vstack([X, lhs(X.shape[1], batch_size)]) F = self.problem.evaluate(X, return_values_of=['F']) # optimization xs, ys = [], [] queue = Queue() n_active_process = 0 for i in range(batch_size): x0 = X[np.argmin(augmented_tchebicheff(F, weights[i]))] Process(target=optimization, args=(self.problem, x0, weights[i], queue)).start() n_active_process += 1 if n_active_process >= self.n_process: x, y = queue.get() xs.append(x) ys.append(y) n_active_process -= 1 # gather result for _ in range(n_active_process): x, y = queue.get() xs.append(x) ys.append(y) return np.array(xs), np.array(ys)
def fit(self, X, Y): assert not self.fit_done assert len(X) == len(Y) possible_labels = list(set(y_val for y in Y for y_val in y)) job_labels = np.array_split(possible_labels, self.n_jobs) with Manager() as manager: X_proxy = manager.list(X) Y_proxy = manager.list(Y) output_queue = Queue() processes = [ Process(target=sequential_execute, args=(output_queue, get_binary_clf_from_multilabel, [{ 'X': X_proxy, 'Y': Y_proxy, 'label': lbl, 'return_label': True } for lbl in job])) for job in job_labels ] [p.start() for p in processes] results = [output_queue.get() for lbl in possible_labels] # needs to be flattened [p.join() for p in processes] self.classifiers = dict(results) self.fit_done = True
def _launch_aggregators(self): """Launch the necessary data aggregators""" if self.manager_params["output_format"] == "local": self.data_aggregator = SqliteAggregator.SqliteAggregator( self.manager_params, self.browser_params) elif self.manager_params["output_format"] == "s3": self.data_aggregator = S3Aggregator.S3Aggregator( self.manager_params, self.browser_params) else: raise Exception("Unrecognized output format: %s" % self.manager_params["output_format"]) self.data_aggregator.launch() self.manager_params[ 'aggregator_address'] = self.data_aggregator.listener_address # open connection to aggregator for saving crawl details self.sock = clientsocket(serialization='dill') self.sock.connect(*self.manager_params['aggregator_address']) # TODO refactor ldb aggregator to use new base classes if self.ldb_enabled: self.ldb_status_queue = Queue() self.ldb_aggregator = Process( target=LevelDBAggregator.LevelDBAggregator, args=(self.manager_params, self.ldb_status_queue)) self.ldb_aggregator.daemon = True self.ldb_aggregator.start() # socket location: (address, port) self.manager_params['ldb_address'] = self.ldb_status_queue.get()
def main(args): resultQ = Queue() plugins_folder = PLUGINS_FOLDER if args.vipgo: plugins_folder = os.path.join( VIPGO_PATH, 'src/wp-content/plugins/newscorpau-plugins') plugin_names = list( filter(lambda x: os.path.isdir(os.path.join(plugins_folder, x)), os.listdir(plugins_folder))) if args.plugins: assert (set(args.plugins).issubset(set(plugin_names))) plugin_names = args.plugins print( f'Setting branch "{args.branch}" for {len(plugin_names)} plugins in "{plugins_folder}"...\n' ) print(sorted(plugin_names)) multiprocess(worker, [( plugin_name, plugins_folder, resultQ, args.branch, args.force, ) for plugin_name in plugin_names]) results = flush_results(resultQ) print('done') print('RESULTS\n' + '=' * 64) print('\n'.join(results))
def classic_gen_time_Sample2(max_Samp, sample_time_lst, queue): if max_Samp <= 0: queue.put(sample_time_lst) return sample_time_lst else: temp_que = Queue() # Use thread to speed up processing (multithreading) temp_thread = threading.Thread( None, target=make_sample, name=("2: " + str(max_Samp)), args=[adj, max_Samp, temp_que], daemon=True, ) temp_thread.start() s = temp_que.get() # the resulting (samples,thread_time) temp_thread.join() # s = queue.get() total_n = s[0] print("2", total_n, max_Samp) sample_time_lst.append((total_n, max_Samp)) ret = classic_gen_time_Sample2(max_Samp - 1, sample_time_lst, queue) queue.put(ret) return ret
def _launch_loggingserver(self): """ sets up logging server """ self.logging_status_queue = Queue() loggingserver = Process(target=MPLogger.loggingserver, args=(self.manager_params['log_file'], self.logging_status_queue, )) loggingserver.daemon = True loggingserver.start() return loggingserver
def __init__(self, difficulty=1, pool_size=4, queue_size=16 ): # NOTE: `queue_size` must be larger than 2x `pool_size`!!! self._args_queue = Queue(queue_size) # pipes for data dispatching self._args_pipes = [Pipe() for _ in range(pool_size)] self._return_queue = Queue(queue_size) self._envs = [ self._factory_env(difficulty=difficulty) for _ in range(pool_size) ] self._service_process = Process(target=self._service_job, args=(self._envs, self._args_pipes, self._args_queue, self._return_queue)) self._service_process.start() print('[LOG] environment service started!')
def launch(self, listener_process_runner): """Launch the aggregator listener process""" self.status_queue = Queue() self.listener_process = Process(target=listener_process_runner, args=(self.manager_params, self.status_queue)) self.listener_process.daemon = True self.listener_process.start() self.listener_address = self.status_queue.get()
def test_lambda_pickling(): f = lambda q: q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.do_tasks() m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def __init__(self, num_workers): self.num_workers = num_workers self.problem = None self.total_tasks = 0 self.total_groups = 0 self.max_group_size = 0 self.tasks = [] self.num_calls = 0 self.max_nodes = 0 self.min_num_calls = 0 self.num_workers = self.num_workers self.processes = [] self.task_queue = Queue() self.result_queue = Queue()
def test_start_workers(self): workers = 2 work_queue = Queue() done_queue = Queue() for repetition in range(self.test_repetitions): work_queue.put(repetition) tournament = axelrod.Tournament( name=self.test_name, players=self.players, game=self.game, turns=200, repetitions=self.test_repetitions) tournament._start_workers(workers, work_queue, done_queue) stops = 0 while stops < workers: payoffs = done_queue.get() if payoffs == 'STOP': stops += 1 self.assertEqual(stops, workers)
def test_close_does_calls_do_task(): def f(q): q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def __init__(self) -> None: super().__init__() self.queue = Queue() self.handle = MemoryProviderHandle(self.queue) self.logger = logging.getLogger("openwpm") self.cache1: DefaultDict[ VisitId, DefaultDict[TableName, List[Dict[str, Any]]] ] = defaultdict(lambda: defaultdict(list)) """The cache for entries before they are finalized""" self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list) """For all entries that have been finalized but not yet flushed out to the queue""" self.signal_list: List[Event] = []
def __init__( self, structured_storage: StructuredStorageProvider, unstructured_storage: Optional[UnstructuredStorageProvider], ) -> None: self.listener_address: Optional[Tuple[str, int]] = None self.listener_process: Optional[Process] = None self.status_queue = Queue() self.completion_queue = Queue() self.shutdown_queue = Queue() self._last_status = None self._last_status_received: Optional[float] = None self.logger = logging.getLogger("openwpm") self.storage_controller = StorageController( structured_storage, unstructured_storage, status_queue=self.status_queue, completion_queue=self.completion_queue, shutdown_queue=self.shutdown_queue, )
def test_do_tasks_twice(): def f(q): q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.do_tasks() assert (q.qsize() == 1) m.add_tasks(f, [(q, )]) m.do_tasks() assert (q.qsize() == 2)
def mp_process(self,nprocs,func,*args): images=args[0] # def worker(imgs,i,chunksize, out_q,func,*args): # """ The worker function, invoked in a process. 'images' is a # list of images to span the process upon. The results are placed in # a dictionary that's pushed to a queue. # """ # outdict = {} # for imn in range(len(imgs)-1): # print(i*chunksize+imn) # outdict[i*chunksize+imn] = func(imgs[imn],imgs[imn+1],*args[1:],i*chunksize+imn) # out_q.put(outdict) # Each process will get 'chunksize' nums and a queue to put his out # dict into out_q = Queue() chunksize = int(math.ceil((len(images)-1) / float(nprocs))) procs = [] print("Chunks of size:",chunksize) for i in range(nprocs): if i == nprocs-1: p = Process( target=worker, args=(images[chunksize * i:len(images)-1],i,chunksize,out_q,func,*args)) procs.append(p) p.start() self.loading.progress2['value']+=chunksize self.update() else: p = Process( target=worker, args=(images[chunksize * i:chunksize * (i + 1)+1],i,chunksize,out_q,func,*args)) procs.append(p) p.start() self.loading.progress2['value']+=chunksize self.update() # Collect all results into a single result dict. We know how many dicts # with results to expect. resultdict = {} for i in range(nprocs): resultdict.update(out_q.get()) # Wait for all worker processes to finish for p in procs: p.join() results=[] for j in range(len(resultdict)): results.append(resultdict[j]) return results
def solve(self, problem, X, Y): ''' Solve the multi-objective problem by multiple scalarized single-objective solvers. Parameters ---------- problem: mobo.surrogate_problem.SurrogateProblem The surrogate problem to be solved. X: np.array Current design variables. Y: np.array Current performance values. Returns ------- solution: dict A dictionary containing information of the solution.\n - solution['x']: Proposed design samples. - solution['y']: Performance of proposed design samples. ''' # initialize population sampling = self._get_sampling(X, Y) if not isinstance(sampling, np.ndarray): sampling = sampling.do(problem, self.pop_size) # generate scalarization weights weights = np.random.random((self.pop_size, Y.shape[1])) weights /= np.expand_dims(np.sum(weights, axis=1), 1) # optimization xs, ys = [], [] queue = Queue() n_active_process = 0 for i, x0 in enumerate(sampling): Process(target=optimization, args=(problem, x0, weights[i], queue)).start() n_active_process += 1 if n_active_process >= self.n_process: x, y = queue.get() xs.append(x) ys.append(y) n_active_process -= 1 # gather result for _ in range(n_active_process): x, y = queue.get() xs.append(x) ys.append(y) # construct solution self.solution = {'x': np.array(xs), 'y': np.array(ys)} return self.solution
def __init__(self, filament, **kwargs): try: log_path = os.path.join(os.path.expanduser('~'), '.fibratus', 'fibratus.log') FileHandler(log_path, mode='w+').push_application() except PermissionError: IO.write_console("ERROR - Unable to open log file for writing due to permission error") sys.exit(0) self.logger = Logger(Fibratus.__name__) self._config = YamlConfig() self.logger.info('Starting fibratus...') self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode()) self.kcontroller = KTraceController() self.ktrace_props = KTraceProps() self.ktrace_props.enable_kflags() self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME enum_handles = kwargs.pop('enum_handles', True) self.handle_repository = HandleRepository() self._handles = [] # query for handles on the # start of the kernel trace if enum_handles: self.logger.info('Enumerating system handles...') self._handles = self.handle_repository.query_handles() self.logger.info('%s handles found' % len(self._handles)) self.handle_repository.free_buffers() self.thread_registry = ThreadRegistry(self.handle_repository, self._handles) self.kevent = KEvent(self.thread_registry) self.keventq = Queue() self._adapter_classes = dict(smtp=SmtpAdapter, amqp=AmqpAdapter) self._output_adapters = self._construct_adapters() if filament: filament.keventq = self.keventq filament.logger = log_path filament.setup_adapters(self._output_adapters) self._filament = filament self.fsio = FsIO(self.kevent, self._handles) self.hive_parser = HiveParser(self.kevent, self.thread_registry) self.tcpip_parser = TcpIpParser(self.kevent) self.dll_repository = DllRepository(self.kevent) self.output_kevents = {} self.filters_count = 0
def _launch_aggregators(self): """ Launches the various data aggregators, which serialize data from all processes. * DataAggregator - sqlite database for crawl data * LevelDBAggregator - leveldb database for javascript files """ # DataAggregator self.aggregator_status_queue = Queue() self.data_aggregator = Process(target=DataAggregator.DataAggregator, args=(self.manager_params, self.aggregator_status_queue)) self.data_aggregator.daemon = True self.data_aggregator.start() self.manager_params['aggregator_address'] = self.aggregator_status_queue.get() # socket location: (address, port) # LevelDB Aggregator if self.ldb_enabled: self.ldb_status_queue = Queue() self.ldb_aggregator = Process(target=LevelDBAggregator.LevelDBAggregator, args=(self.manager_params, self.ldb_status_queue)) self.ldb_aggregator.daemon = True self.ldb_aggregator.start() self.manager_params['ldb_address'] = self.ldb_status_queue.get() # socket location: (address, port)
def test_queue(): q = Queue() m = Multiprocess() def f(q): q.push(1) m.add_tasks(f, [(q, )]) m.do_tasks() m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def run(self): """Starts the run of the tests""" results = [] worker_list = [] to_worker = Queue() from_worker = Queue() verbose = self.cl_args.verbose failfast = self.cl_args.failfast workers = int(not self.cl_args.parallel) or self.cl_args.workers for suite in self.suites: to_worker.put(suite) for _ in range(workers): to_worker.put(None) start = time.time() # A second try catch is needed here because queues can cause locking # when they go out of scope, especially when termination signals used try: for _ in range(workers): proc = Consumer(to_worker, from_worker, verbose, failfast) worker_list.append(proc) proc.start() for _ in self.suites: results.append(self.log_result(from_worker.get())) end = time.time() tests_run, errors, failures = self.compile_results( run_time=end - start, datagen_time=start - self.datagen_start, results=results) except KeyboardInterrupt: print_exception("Runner", "run", "Keyboard Interrupt, exiting...") os.killpg(0, 9) return bool(sum([errors, failures, not tests_run]))
def run_parallely_with_progress_bar( items, func, msgfunc, accumulator=NoOpAcc(), title=''): PROC_COUNT = 5 total = len(items) task_queue = Queue() done_queue = Queue() def pb_updater(inq, results_q): pb = ProgressBar(title) for i in range(total): msg, result = results_q.get() accumulator.add(result) pb.update(percent=((i+1)*100)/total, message=msg) pb.finish() accumulator.finish() # tell the workers to stop for i in range(PROC_COUNT): inq.put('STOP') def worker(inq, outq): for item in iter(inq.get, 'STOP'): result = func(item) outq.put((msgfunc(item), result)) for i in range(PROC_COUNT): Process(target=worker, args=(task_queue, done_queue)).start() updater = Process(target=pb_updater, args=(task_queue, done_queue)) updater.start() for item in items: task_queue.put(item)
def test_worker(self): tournament = axelrod.Tournament( name=self.test_name, players=self.players, game=self.game, turns=200, repetitions=self.test_repetitions) work_queue = Queue() for repetition in range(self.test_repetitions): work_queue.put(repetition) work_queue.put('STOP') done_queue = Queue() tournament._worker(work_queue, done_queue) for r in range(self.test_repetitions): new_matches = done_queue.get() self.assertEqual(len(new_matches), 15) for index_pair, match in new_matches.items(): self.assertIsInstance(index_pair, tuple) self.assertIsInstance(match, list) queue_stop = done_queue.get() self.assertEqual(queue_stop, 'STOP')
def __init__(self, name, data_sampler_method, cache_path=None, save_every=50): """ Initializes a DataStreamRecorder Parameters ---------- name : string User-friendly identifier for this data stream data_sampler_method : function Method to call to retrieve data """ Process.__init__(self) self._data_sampler_method = data_sampler_method self._has_set_sampler_params = False self._recording = False self._name = name self._cmds_q = Queue() self._data_qs = [Queue()] self._ok_q = None self._tokens_q = None self._save_every = save_every self._cache_path = cache_path self._saving_cache = cache_path is not None if self._saving_cache: self._save_path = os.path.join(cache_path, self.name) if not os.path.exists(self._save_path): os.makedirs(self._save_path) self._start_data_segment = 0 self._cur_data_segment = 0 self._saving_ps = []