def __init__(self, manager_params, browser_params):
self.manager_params = manager_params
self.browser_params = browser_params
self.listener_address = None
self.listener_process = None
self.status_queue = Queue()
self.completion_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received = None
self.logger = logging.getLogger('openwpm')
def thread_samples(adj, mSamp):
if (mSamp <= 1):
lst = []
temp_que = Queue()
# Use thread to speed up processing (multithreading)
temp_thread = threading.Thread(
None,
target=make_sample,
name=("1: " + str(mSamp)),
args=[adj, mSamp, temp_que],
daemon=True,
)
# temp_thread = threading.Thread(None, target=make_sample, name=("2:" +
# str(
# max_Sample)),
# args=[adj, max_Sample, queue], )
temp_thread.start()
temp_thread.join()
time_samp_node = temp_que.get(
) # the resulting (samples,thread_time)
temp_que.task_done()
lst.append(time_samp_node)
return (lst, True)
else:
rec_val = thread_samples(adj, mSamp - 1)
temp_que = Queue()
# Use thread to speed up processing (multithreading)
temp_thread = threading.Thread(
None,
target=make_sample,
name=("1: " + str(mSamp)),
args=[adj, mSamp, temp_que],
daemon=True,
)
# temp_thread = threading.Thread(None, target=make_sample, name=("2:" +
# str(
# max_Sample)),
# args=[adj, max_Sample, queue], )
temp_thread.start()
temp_thread.join()
time_samp_node = temp_que.get(
) # the resulting (samples,thread_time)
temp_que.task_done()
# s = queue.get()
# Use thread to speed up processing (multithreading)
if (rec_val[1]):
rec_lst = rec_val[0]
rec_lst.append(time_samp_node)
ret = (rec_lst, True)
# Return the begining of a (time x sample) x boolean tuple list
return ret
def __init__(
self,
manager_params: ManagerParamsInternal,
browser_params: List[BrowserParamsInternal],
):
self.manager_params = manager_params
self.browser_params = browser_params
self.listener_address = None
self.listener_process = None
self.status_queue = Queue()
self.completion_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received = None
self.logger = logging.getLogger("openwpm")
def run(self):
setproctitle('python.DataStreamRecorder.{0}'.format(self._name))
try:
logging.debug("Starting data recording on {0}".format(self.name))
self._tokens_q.put(("return", self.name))
while True:
if not self._cmds_q.empty():
cmd = self._cmds_q.get()
if cmd[0] == 'stop':
break
elif cmd[0] == 'pause':
self._recording = False
if self._saving_cache:
self._save_cache(self._cur_data_segment)
self._cur_data_segment += 1
self._data_qs.append(Queue())
elif cmd[0] == 'reset_data_segment':
self._start_data_segment = self._cur_data_segment
elif cmd[0] == 'resume':
self._recording = True
elif cmd[0] == 'save':
self._save_data(cmd[1], cmd[2], cmd[3])
elif cmd[0] == 'params':
self._args = cmd[1]
self._kwargs = cmd[2]
if self._recording and not self._ok_q.empty():
timestamp = self._ok_q.get()
self._tokens_q.put(("take", self.name))
data = self._data_sampler_method(*self._args,
**self._kwargs)
cur_data_q = self._data_qs[self._cur_data_segment]
if self._saving_cache and cur_data_q.qsize(
) == self._save_every:
self._save_cache(self._cur_data_segment)
cur_data_q = Queue()
self._data_qs.append(cur_data_q)
self._cur_data_segment += 1
cur_data_q.put((timestamp, data))
self._tokens_q.put(("return", self.name))
except KeyboardInterrupt:
logging.debug("Shutting down data streamer on {0}".format(
self.name))
sys.exit(0)
def __init__(self, num_cores):
self.num_cores = num_cores
self.names = set()
self.global_queue = Queue()
self.pool = []
self.channels = []
self.__terminated = False
atexit.register(self.__del__)
queues = [Queue() for _ in range(num_cores)]
for i in range(num_cores):
pipe_master, pipe_slave = Pipe()
state = Value('i', 0)
process = MRServer(i, queues, pipe_slave, state, self.global_queue)
self.channels.append(Channel(queues[i], pipe_master, state))
self.pool.append(process)
process.start()
def test_multiprocess_func_does_something():
q = Queue()
multiprocess(lambda q, x: q.push(x), [(q, 1), (q, 2)])
arr = []
while not q.empty():
arr.append(q.pop())
assert (set([1, 2]) == set(arr))
def _solve(self, X, Y, batch_size):
'''
Solve the multi-objective problem by multiple scalarized single-objective solvers.
'''
# generate scalarization weights
weights = np.random.random((batch_size, self.problem.n_obj))
weights /= np.expand_dims(np.sum(weights, axis=1), 1)
# initial solutions
X = np.vstack([X, lhs(X.shape[1], batch_size)])
F = self.problem.evaluate(X, return_values_of=['F'])
# optimization
xs, ys = [], []
queue = Queue()
n_active_process = 0
for i in range(batch_size):
x0 = X[np.argmin(augmented_tchebicheff(F, weights[i]))]
Process(target=optimization,
args=(self.problem, x0, weights[i], queue)).start()
n_active_process += 1
if n_active_process >= self.n_process:
x, y = queue.get()
xs.append(x)
ys.append(y)
n_active_process -= 1
# gather result
for _ in range(n_active_process):
x, y = queue.get()
xs.append(x)
ys.append(y)
return np.array(xs), np.array(ys)
def fit(self, X, Y):
assert not self.fit_done
assert len(X) == len(Y)
possible_labels = list(set(y_val for y in Y for y_val in y))
job_labels = np.array_split(possible_labels, self.n_jobs)
with Manager() as manager:
X_proxy = manager.list(X)
Y_proxy = manager.list(Y)
output_queue = Queue()
processes = [
Process(target=sequential_execute,
args=(output_queue, get_binary_clf_from_multilabel, [{
'X':
X_proxy,
'Y':
Y_proxy,
'label':
lbl,
'return_label':
True
} for lbl in job])) for job in job_labels
]
[p.start() for p in processes]
results = [output_queue.get()
for lbl in possible_labels] # needs to be flattened
[p.join() for p in processes]
self.classifiers = dict(results)
self.fit_done = True
def _launch_aggregators(self):
"""Launch the necessary data aggregators"""
if self.manager_params["output_format"] == "local":
self.data_aggregator = SqliteAggregator.SqliteAggregator(
self.manager_params, self.browser_params)
elif self.manager_params["output_format"] == "s3":
self.data_aggregator = S3Aggregator.S3Aggregator(
self.manager_params, self.browser_params)
else:
raise Exception("Unrecognized output format: %s" %
self.manager_params["output_format"])
self.data_aggregator.launch()
self.manager_params[
'aggregator_address'] = self.data_aggregator.listener_address
# open connection to aggregator for saving crawl details
self.sock = clientsocket(serialization='dill')
self.sock.connect(*self.manager_params['aggregator_address'])
# TODO refactor ldb aggregator to use new base classes
if self.ldb_enabled:
self.ldb_status_queue = Queue()
self.ldb_aggregator = Process(
target=LevelDBAggregator.LevelDBAggregator,
args=(self.manager_params, self.ldb_status_queue))
self.ldb_aggregator.daemon = True
self.ldb_aggregator.start()
# socket location: (address, port)
self.manager_params['ldb_address'] = self.ldb_status_queue.get()
def main(args):
resultQ = Queue()
plugins_folder = PLUGINS_FOLDER
if args.vipgo:
plugins_folder = os.path.join(
VIPGO_PATH, 'src/wp-content/plugins/newscorpau-plugins')
plugin_names = list(
filter(lambda x: os.path.isdir(os.path.join(plugins_folder, x)),
os.listdir(plugins_folder)))
if args.plugins:
assert (set(args.plugins).issubset(set(plugin_names)))
plugin_names = args.plugins
print(
f'Setting branch "{args.branch}" for {len(plugin_names)} plugins in "{plugins_folder}"...\n'
)
print(sorted(plugin_names))
multiprocess(worker, [(
plugin_name,
plugins_folder,
resultQ,
args.branch,
args.force,
) for plugin_name in plugin_names])
results = flush_results(resultQ)
print('done')
print('RESULTS\n' + '=' * 64)
print('\n'.join(results))
def classic_gen_time_Sample2(max_Samp, sample_time_lst, queue):
if max_Samp <= 0:
queue.put(sample_time_lst)
return sample_time_lst
else:
temp_que = Queue()
# Use thread to speed up processing (multithreading)
temp_thread = threading.Thread(
None,
target=make_sample,
name=("2: " + str(max_Samp)),
args=[adj, max_Samp, temp_que],
daemon=True,
)
temp_thread.start()
s = temp_que.get() # the resulting (samples,thread_time)
temp_thread.join()
# s = queue.get()
total_n = s[0]
print("2", total_n, max_Samp)
sample_time_lst.append((total_n, max_Samp))
ret = classic_gen_time_Sample2(max_Samp - 1, sample_time_lst, queue)
queue.put(ret)
return ret
def _launch_loggingserver(self):
""" sets up logging server """
self.logging_status_queue = Queue()
loggingserver = Process(target=MPLogger.loggingserver,
args=(self.manager_params['log_file'], self.logging_status_queue, ))
loggingserver.daemon = True
loggingserver.start()
return loggingserver
def __init__(self,
difficulty=1,
pool_size=4,
queue_size=16
): # NOTE: `queue_size` must be larger than 2x `pool_size`!!!
self._args_queue = Queue(queue_size)
# pipes for data dispatching
self._args_pipes = [Pipe() for _ in range(pool_size)]
self._return_queue = Queue(queue_size)
self._envs = [
self._factory_env(difficulty=difficulty) for _ in range(pool_size)
]
self._service_process = Process(target=self._service_job,
args=(self._envs, self._args_pipes,
self._args_queue,
self._return_queue))
self._service_process.start()
print('[LOG] environment service started!')
def launch(self, listener_process_runner):
"""Launch the aggregator listener process"""
self.status_queue = Queue()
self.listener_process = Process(target=listener_process_runner,
args=(self.manager_params,
self.status_queue))
self.listener_process.daemon = True
self.listener_process.start()
self.listener_address = self.status_queue.get()
def test_lambda_pickling(): f = lambda q: q.push(1) q = Queue() m = Multiprocess() m.add_tasks(f, [(q, )]) m.do_tasks() m.close() assert (q.qsize() == 1) assert (q.pop() == 1) assert (q.qsize() == 0)
def __init__(self, num_workers):
self.num_workers = num_workers
self.problem = None
self.total_tasks = 0
self.total_groups = 0
self.max_group_size = 0
self.tasks = []
self.num_calls = 0
self.max_nodes = 0
self.min_num_calls = 0
self.num_workers = self.num_workers
self.processes = []
self.task_queue = Queue()
self.result_queue = Queue()
def test_start_workers(self):
workers = 2
work_queue = Queue()
done_queue = Queue()
for repetition in range(self.test_repetitions):
work_queue.put(repetition)
tournament = axelrod.Tournament(
name=self.test_name,
players=self.players,
game=self.game,
turns=200,
repetitions=self.test_repetitions)
tournament._start_workers(workers, work_queue, done_queue)
stops = 0
while stops < workers:
payoffs = done_queue.get()
if payoffs == 'STOP':
stops += 1
self.assertEqual(stops, workers)
def test_close_does_calls_do_task():
def f(q):
q.push(1)
q = Queue()
m = Multiprocess()
m.add_tasks(f, [(q, )])
m.close()
assert (q.qsize() == 1)
assert (q.pop() == 1)
assert (q.qsize() == 0)
def __init__(self) -> None:
super().__init__()
self.queue = Queue()
self.handle = MemoryProviderHandle(self.queue)
self.logger = logging.getLogger("openwpm")
self.cache1: DefaultDict[
VisitId, DefaultDict[TableName, List[Dict[str, Any]]]
] = defaultdict(lambda: defaultdict(list))
"""The cache for entries before they are finalized"""
self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list)
"""For all entries that have been finalized but not yet flushed out to the queue"""
self.signal_list: List[Event] = []
def __init__(
self,
structured_storage: StructuredStorageProvider,
unstructured_storage: Optional[UnstructuredStorageProvider],
) -> None:
self.listener_address: Optional[Tuple[str, int]] = None
self.listener_process: Optional[Process] = None
self.status_queue = Queue()
self.completion_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received: Optional[float] = None
self.logger = logging.getLogger("openwpm")
self.storage_controller = StorageController(
structured_storage,
unstructured_storage,
status_queue=self.status_queue,
completion_queue=self.completion_queue,
shutdown_queue=self.shutdown_queue,
)
def test_do_tasks_twice():
def f(q):
q.push(1)
q = Queue()
m = Multiprocess()
m.add_tasks(f, [(q, )])
m.do_tasks()
assert (q.qsize() == 1)
m.add_tasks(f, [(q, )])
m.do_tasks()
assert (q.qsize() == 2)
def mp_process(self,nprocs,func,*args):
images=args[0]
# def worker(imgs,i,chunksize, out_q,func,*args):
# """ The worker function, invoked in a process. 'images' is a
# list of images to span the process upon. The results are placed in
# a dictionary that's pushed to a queue.
# """
# outdict = {}
# for imn in range(len(imgs)-1):
# print(i*chunksize+imn)
# outdict[i*chunksize+imn] = func(imgs[imn],imgs[imn+1],*args[1:],i*chunksize+imn)
# out_q.put(outdict)
# Each process will get 'chunksize' nums and a queue to put his out
# dict into
out_q = Queue()
chunksize = int(math.ceil((len(images)-1) / float(nprocs)))
procs = []
print("Chunks of size:",chunksize)
for i in range(nprocs):
if i == nprocs-1:
p = Process(
target=worker,
args=(images[chunksize * i:len(images)-1],i,chunksize,out_q,func,*args))
procs.append(p)
p.start()
self.loading.progress2['value']+=chunksize
self.update()
else:
p = Process(
target=worker,
args=(images[chunksize * i:chunksize * (i + 1)+1],i,chunksize,out_q,func,*args))
procs.append(p)
p.start()
self.loading.progress2['value']+=chunksize
self.update()
# Collect all results into a single result dict. We know how many dicts
# with results to expect.
resultdict = {}
for i in range(nprocs):
resultdict.update(out_q.get())
# Wait for all worker processes to finish
for p in procs:
p.join()
results=[]
for j in range(len(resultdict)):
results.append(resultdict[j])
return results
def solve(self, problem, X, Y):
'''
Solve the multi-objective problem by multiple scalarized single-objective solvers.
Parameters
----------
problem: mobo.surrogate_problem.SurrogateProblem
The surrogate problem to be solved.
X: np.array
Current design variables.
Y: np.array
Current performance values.
Returns
-------
solution: dict
A dictionary containing information of the solution.\n
- solution['x']: Proposed design samples.
- solution['y']: Performance of proposed design samples.
'''
# initialize population
sampling = self._get_sampling(X, Y)
if not isinstance(sampling, np.ndarray):
sampling = sampling.do(problem, self.pop_size)
# generate scalarization weights
weights = np.random.random((self.pop_size, Y.shape[1]))
weights /= np.expand_dims(np.sum(weights, axis=1), 1)
# optimization
xs, ys = [], []
queue = Queue()
n_active_process = 0
for i, x0 in enumerate(sampling):
Process(target=optimization,
args=(problem, x0, weights[i], queue)).start()
n_active_process += 1
if n_active_process >= self.n_process:
x, y = queue.get()
xs.append(x)
ys.append(y)
n_active_process -= 1
# gather result
for _ in range(n_active_process):
x, y = queue.get()
xs.append(x)
ys.append(y)
# construct solution
self.solution = {'x': np.array(xs), 'y': np.array(ys)}
return self.solution
def __init__(self, filament, **kwargs):
try:
log_path = os.path.join(os.path.expanduser('~'), '.fibratus', 'fibratus.log')
FileHandler(log_path, mode='w+').push_application()
except PermissionError:
IO.write_console("ERROR - Unable to open log file for writing due to permission error")
sys.exit(0)
self.logger = Logger(Fibratus.__name__)
self._config = YamlConfig()
self.logger.info('Starting fibratus...')
self.kevt_streamc = KEventStreamCollector(etw.KERNEL_LOGGER_NAME.encode())
self.kcontroller = KTraceController()
self.ktrace_props = KTraceProps()
self.ktrace_props.enable_kflags()
self.ktrace_props.logger_name = etw.KERNEL_LOGGER_NAME
enum_handles = kwargs.pop('enum_handles', True)
self.handle_repository = HandleRepository()
self._handles = []
# query for handles on the
# start of the kernel trace
if enum_handles:
self.logger.info('Enumerating system handles...')
self._handles = self.handle_repository.query_handles()
self.logger.info('%s handles found' % len(self._handles))
self.handle_repository.free_buffers()
self.thread_registry = ThreadRegistry(self.handle_repository, self._handles)
self.kevent = KEvent(self.thread_registry)
self.keventq = Queue()
self._adapter_classes = dict(smtp=SmtpAdapter, amqp=AmqpAdapter)
self._output_adapters = self._construct_adapters()
if filament:
filament.keventq = self.keventq
filament.logger = log_path
filament.setup_adapters(self._output_adapters)
self._filament = filament
self.fsio = FsIO(self.kevent, self._handles)
self.hive_parser = HiveParser(self.kevent, self.thread_registry)
self.tcpip_parser = TcpIpParser(self.kevent)
self.dll_repository = DllRepository(self.kevent)
self.output_kevents = {}
self.filters_count = 0
def _launch_aggregators(self):
"""
Launches the various data aggregators, which serialize data from all processes.
* DataAggregator - sqlite database for crawl data
* LevelDBAggregator - leveldb database for javascript files
"""
# DataAggregator
self.aggregator_status_queue = Queue()
self.data_aggregator = Process(target=DataAggregator.DataAggregator,
args=(self.manager_params, self.aggregator_status_queue))
self.data_aggregator.daemon = True
self.data_aggregator.start()
self.manager_params['aggregator_address'] = self.aggregator_status_queue.get() # socket location: (address, port)
# LevelDB Aggregator
if self.ldb_enabled:
self.ldb_status_queue = Queue()
self.ldb_aggregator = Process(target=LevelDBAggregator.LevelDBAggregator,
args=(self.manager_params, self.ldb_status_queue))
self.ldb_aggregator.daemon = True
self.ldb_aggregator.start()
self.manager_params['ldb_address'] = self.ldb_status_queue.get() # socket location: (address, port)
def test_queue():
q = Queue()
m = Multiprocess()
def f(q):
q.push(1)
m.add_tasks(f, [(q, )])
m.do_tasks()
m.close()
assert (q.qsize() == 1)
assert (q.pop() == 1)
assert (q.qsize() == 0)
def run(self):
"""Starts the run of the tests"""
results = []
worker_list = []
to_worker = Queue()
from_worker = Queue()
verbose = self.cl_args.verbose
failfast = self.cl_args.failfast
workers = int(not self.cl_args.parallel) or self.cl_args.workers
for suite in self.suites:
to_worker.put(suite)
for _ in range(workers):
to_worker.put(None)
start = time.time()
# A second try catch is needed here because queues can cause locking
# when they go out of scope, especially when termination signals used
try:
for _ in range(workers):
proc = Consumer(to_worker, from_worker, verbose, failfast)
worker_list.append(proc)
proc.start()
for _ in self.suites:
results.append(self.log_result(from_worker.get()))
end = time.time()
tests_run, errors, failures = self.compile_results(
run_time=end - start,
datagen_time=start - self.datagen_start,
results=results)
except KeyboardInterrupt:
print_exception("Runner", "run", "Keyboard Interrupt, exiting...")
os.killpg(0, 9)
return bool(sum([errors, failures, not tests_run]))
def run_parallely_with_progress_bar(
items,
func,
msgfunc,
accumulator=NoOpAcc(),
title=''):
PROC_COUNT = 5
total = len(items)
task_queue = Queue()
done_queue = Queue()
def pb_updater(inq, results_q):
pb = ProgressBar(title)
for i in range(total):
msg, result = results_q.get()
accumulator.add(result)
pb.update(percent=((i+1)*100)/total, message=msg)
pb.finish()
accumulator.finish()
# tell the workers to stop
for i in range(PROC_COUNT):
inq.put('STOP')
def worker(inq, outq):
for item in iter(inq.get, 'STOP'):
result = func(item)
outq.put((msgfunc(item), result))
for i in range(PROC_COUNT):
Process(target=worker, args=(task_queue, done_queue)).start()
updater = Process(target=pb_updater, args=(task_queue, done_queue))
updater.start()
for item in items:
task_queue.put(item)
def test_worker(self):
tournament = axelrod.Tournament(
name=self.test_name,
players=self.players,
game=self.game,
turns=200,
repetitions=self.test_repetitions)
work_queue = Queue()
for repetition in range(self.test_repetitions):
work_queue.put(repetition)
work_queue.put('STOP')
done_queue = Queue()
tournament._worker(work_queue, done_queue)
for r in range(self.test_repetitions):
new_matches = done_queue.get()
self.assertEqual(len(new_matches), 15)
for index_pair, match in new_matches.items():
self.assertIsInstance(index_pair, tuple)
self.assertIsInstance(match, list)
queue_stop = done_queue.get()
self.assertEqual(queue_stop, 'STOP')
def __init__(self,
name,
data_sampler_method,
cache_path=None,
save_every=50):
""" Initializes a DataStreamRecorder
Parameters
----------
name : string
User-friendly identifier for this data stream
data_sampler_method : function
Method to call to retrieve data
"""
Process.__init__(self)
self._data_sampler_method = data_sampler_method
self._has_set_sampler_params = False
self._recording = False
self._name = name
self._cmds_q = Queue()
self._data_qs = [Queue()]
self._ok_q = None
self._tokens_q = None
self._save_every = save_every
self._cache_path = cache_path
self._saving_cache = cache_path is not None
if self._saving_cache:
self._save_path = os.path.join(cache_path, self.name)
if not os.path.exists(self._save_path):
os.makedirs(self._save_path)
self._start_data_segment = 0
self._cur_data_segment = 0
self._saving_ps = []
