def set_affinity(affinities): """ Sets process affinity to configured processor cores. """ affinities_n = len(affinities) if affinities_n > 0: my_pid = os.getpid() try: get_lock(my_pid) # Read a process id for each processor affinity. pids = [0] * affinities_n if os.path.exists(AFFINITY_FILE): with open(AFFINITY_FILE, 'r') as f: for i,line in enumerate(f): if i < affinities_n: pids[i] = int(line.strip()) # Find free affinity. i = find_affinity(pids, my_pid) if i >= 0: os.sched_setaffinity(0, affinities[i]) #print("SET AFFINITY: {}".format(str(affinities[i]))) # Update processor affinity list. with open(AFFINITY_FILE, 'w') as f: f.write("\n".join([str(i) for i in pids])) finally: release_lock(my_pid)
def set_cpu_affinity(affinity_list): """Set CPU affinity to CPUs listed (numbered 0...n-1)""" if hasattr(os, 'sched_setaffinity'): os.sched_setaffinity(0, affinity_list) else: import psutil p = psutil.Process() if hasattr(p, 'cpu_affinity'): p.cpu_affinity(affinity_list)
def __wake_cpus(self, cpus): # Requires Python 3.3+. This will "tickle" each CPU to force it to # update its idle counters. if hasattr(os, 'sched_setaffinity'): pid = self.__gettid() save_affinity = os.sched_getaffinity(pid) for idx in cpus: os.sched_setaffinity(pid, [idx]) os.sched_getaffinity(pid) os.sched_setaffinity(pid, save_affinity)
def server(queue, args): if args.server_cpu_affinity >= 0: os.sched_setaffinity(0, [args.server_cpu_affinity]) ucp.init() if args.object_type == "numpy": import numpy as np elif args.object_type == "cupy": import cupy as np np.cuda.runtime.setDevice(args.server_dev) else: import cupy as np import rmm rmm.reinitialize( pool_allocator=True, managed_memory=False, initial_pool_size=args.rmm_init_pool_size, devices=[args.server_dev], ) np.cuda.runtime.setDevice(args.server_dev) np.cuda.set_allocator(rmm.rmm_cupy_allocator) async def run(): async def server_handler(ep): msg_recv_list = [] if not args.reuse_alloc: for _ in range(args.n_iter): msg_recv_list.append(np.zeros(args.n_bytes, dtype="u1")) else: t = np.zeros(args.n_bytes, dtype="u1") for _ in range(args.n_iter): msg_recv_list.append(t) assert msg_recv_list[0].nbytes == args.n_bytes for i in range(args.n_iter): await ep.recv(msg_recv_list[i]) await ep.send(msg_recv_list[i]) await ep.close() lf.close() lf = ucp.create_listener(server_handler, args.port) queue.put(lf.port) while not lf.closed(): await asyncio.sleep(0.5) loop = asyncio.get_event_loop() loop.run_until_complete(run()) loop.close()
def worker(wnum, input_queue, output_queue): os.sched_setaffinity(0, [wnum]) while True: try: idx, value = input_queue.get(block=False) if value == 'STOP': break output_queue.put((idx, functor(value))) except: pass os.sched_yield()
def run(self): import os all_cores = set(range(mp.cpu_count())) os.sched_setaffinity(0, all_cores) #optimizer = funcspec_to_callable(self.opt_class)(params,RemoteEvaluatorFactory**self.opt_args) def get_max_evaluators(): optimizer = self.opt_spec.opt_cls(self.pspace, None, **self.opt_spec.opt_args) return optimizer.get_max_evaluators() eval_f = RemoteEvaluatorFactory(self.task_out, self.obj_in, get_max_evaluators()) queues = dict(control_to_opt=Queue(), opt_to_control=Queue()) def run_opt(opt_class, opt_args, queues): opt = opt_class(self.pspace, eval_f, **opt_args) opt.set_threaded(queues) opt.run() tt = threading.Thread( target=run_opt, args=[self.opt_spec.opt_cls, self.opt_spec.opt_args, queues]) tt.start() status = Message('launching') new_status = True prev_time = time.time() while status.subject != 'complete': time_now = time.time() if time_now - prev_time > self.report_freq: if new_status: print('\n', status) new_status = False else: sys.stdout.write('.') sys.stdout.flush() prev_time = time_now time.sleep(0.1) queues['control_to_opt'].put('request_update') try: status = queues['opt_to_control'].get_nowait() new_status = True except Empty: pass print(status) self.finished.set()
def _set_affinity(self): if Config().get_config("affinity.enable") is True: try: core_num = Config().get_config("affinity.core_num") cpu_count = psutil.cpu_count() if core_num <= 0 or cpu_count < core_num: mask = 1 else: mask = 2**core_num - 1 os.sched_setaffinity(os.getpid(), mask) except Exception as e: Logger().error("set affinity error!", exc_info=e)
def set_cpu_affinity(mask): pid = os.getpid() if sys.platform == 'linux': print("Get CPU affinity", os.sched_getaffinity(pid)) os.sched_setaffinity(pid, mask) print("Get CPU affinity", os.sched_getaffinity(pid)) else: import psutil p = psutil.Process(pid) print("Get CPU affintiy", p.cpu_affinity) p.cpu_affinity = [0] print("Get CPU affintiy", p.cpu_affinity)
def set_cpu_affinity(self, core_list): proclist_str = f"[{','.join([str(i) for i in core_list])}]" os.environ["OMP_NUM_THREADS"] = str(len(core_list)) os.environ["OMP_SCHEDULE"] = "STATIC" os.environ["OMP_PROC_BIND"] = "CLOSE" # KMP_AFFINITY works on intel openmp (intel tensorlow, intel pytorch/ipex) # GOMP_CPU_AFFINITY works on gomp (stock pytorch) # os.sched_setaffinity works on other threads (stock tensorflow) os.environ[ "KMP_AFFINITY"] = f"verbose,granularity=fine,proclist={proclist_str},explicit" os.environ["GOMP_CPU_AFFINITY"] = proclist_str os.sched_setaffinity(0, set(core_list))
def run(self): try: # No sched_setaffinity in Windows, we handle this in MSMPI if os.name is not 'nt': all_cores = set(range(mp.cpu_count())) os.sched_setaffinity(0,all_cores) self.objectivef = self.build_objectives() self.buffer_manager.rebuild_buffers() while(True): msg = self.node_to_objective_q.get() if msg.subject == 'terminate': return if msg.subject == 'evaluate': catchment_id,buffer_id,params,data_index,translate = msg.content indata = self.buffer_manager.map_buffer(buffer_id,data_index) #for s in self.schema['model_outputs']: # _ = indata[s].T lumped = dict(params=params) for s,mapping in self.model_outputs.items(): cur_data = indata[s] if translate: cur_data = cur_data.T for out_var,agg_method in mapping.items(): if agg_method == 'flat': lumped[out_var] = cur_data else: if agg_method == 'mean': weights = self.mean_weights elif agg_method == 'volume': weights = self.vol_weights lumped[out_var] = aggregate(cur_data[self.eval_index],weights[catchment_id]) result = self.objectivef[catchment_id].evaluate(lumped) self.objective_to_node_q.put(Message('catch_result',catchment=catchment_id,result=result)) except KeyboardInterrupt: pass except Exception as e: self.objective_to_node_q.put(Message('exception',exception=e,traceback=get_traceback()))
def _cpu_affinity(self): # sched_setaffinity() was added to Python 3.3 has_sched_setaffinity = hasattr(os, 'sched_setaffinity') if not has_sched_setaffinity: if psutil is not None: proc = psutil.Process() psutil_has_cpu_affinity = hasattr(proc, 'cpu_affinity') else: psutil_has_cpu_affinity = False cpus = self.args.affinity if not cpus: stream = self._stream() # --affinity option is not set: detect isolated CPUs cpus = _get_isolated_cpus() if not cpus: # no isolated CPUs or unable to get the isolated CPUs return if not has_sched_setaffinity and not psutil_has_cpu_affinity: # unable to pin CPUs print( "WARNING: unable to pin worker processes to " "isolated CPUs, CPU affinity not available", file=stream) print("Use Python 3.3 or newer, or install psutil dependency", file=stream) return if self.args.verbose: print("Pin process to isolated CPUs: %s" % perf._format_cpu_list(cpus), file=stream) self.args.affinity = perf._format_cpu_list(cpus) else: cpus = _parse_cpu_list(cpus) if self.args.verbose: print("Pin process to CPUs: %s" % perf._format_cpu_list(cpus), file=self._stream()) if has_sched_setaffinity: os.sched_setaffinity(0, cpus) elif psutil_has_cpu_affinity: proc = psutil.Process() proc.cpu_affinity(cpus) else: print("ERROR: CPU affinity not available.", file=sys.stderr) print("Use Python 3.3 or newer, or install psutil dependency", file=stream) sys.exit(1)
def run(self): import os all_cores = set(range(mp.cpu_count())) os.sched_setaffinity(0, all_cores) self.buffer_manager.rebuild_buffers() #self.model = nodes.funcspec_to_callable(self.model) ''' Build the extent map ''' ''' Initialise our model/graphrunner Need to do a first run of fixed parameter ''' model_keys = self.model.get_input_keys() self.exe_graph = self.build_static_graph(model_keys) self.model_runner = self.model.get_runner( self.exe_graph.get_dataspecs(True), shared=False) while (True): task = self.node_to_worker_q.get() if task.subject == 'terminate': return for i in range(task['njobs']): #get_buffers() #results = run_graph(task_params) buf_id, data = self.buffer_manager.get_buffer( index=self.data_index) #for k,v in data.items(): # v[...] = i * buf_id job = task['jobs'][i] pdict = flat_params_to_dict(job['params'], self.pspace) self.exe_graph.set_parameters(pdict) iresults = self.exe_graph.get_data_flat( None, 0) # +++ refactor get_data_prepack mresults = self.model_runner.run_from_mapping( iresults, self.timesteps, self.cell_count) for k in self.model_outputs: data[k][...] = mresults[k] self.worker_to_node_q.put(buf_id)
def set_socket_affinity(gpu_id, nproc_per_node, cores): """ The process is assigned with all available physical CPU cores from the CPU socket connected to the GPU with a given id. Args: gpu_id: index of a GPU nproc_per_node: number of processes per node cores: 'all_logical' or 'single_logical' """ grouped_socket_affinities = get_grouped_socket_affinities(nproc_per_node) ungrouped_affinities = ungroup_affinities(grouped_socket_affinities, cores) os.sched_setaffinity(0, ungrouped_affinities[gpu_id])
def _experiment(f, param_names, param_values, esn_attributes=[], affinity=0, runs=10): os.sched_setaffinity(0, {affinity}) results = [] for experiment in param_values: _params = {pn: pv for pn, pv in zip(param_names, experiment)} print(_params) for i in range(runs): result, attr = f(_params, esn_attributes) results.append([result] + list(_params.values()) + attr) return results
def set_affinity(pids, cores): """ Sets PIDs' core affinity Parameters: pids: PIDs to set core affinity for cores: cores to set to """ for pid in pids: try: os.sched_setaffinity(pid, cores) except OSError: log.error("Failed to set {} PID affinity".format(pid))
def run(self): os.sched_setaffinity(self.pid, self.affinity) affinity = os.sched_getaffinity(self.pid) log.info('Process {}, affinity proc list:{}'.format(self.pid, affinity)) import mxnet as mx ctx = mx.cpu() # load mxnet model mod = self.load_model() ds = self.load_dataset() # warmup self.warmup(mod, ds) # Inform master process to continue till sample_list was issue self.lock.acquire() self.init_counter.value += 1 self.lock.release() sample_list = self.ds_queue.get() ds.load_query_samples(sample_list) # Inform master process to continue loadgen startTest self.lock.acquire() self.init_counter.value += 1 self.lock.release() while True: next_task = self.task_queue.get() if next_task == 'DONE': log.info('Exiting {}-pid:{}'.format(self.name, self.pid)) ds.unload_query_samples(sample_list) self.task_queue.task_done() break processed_results = [] query_id_list = next_task.query_id_list sample_index_list = next_task.sample_index_list batch_size = len(sample_index_list) data_np, label_np = ds.get_samples(sample_index_list) data = mx.nd.array(data_np, dtype=data_np.dtype).as_in_context(ctx) label = mx.nd.array(label_np, dtype=label_np.dtype).as_in_context(ctx) batch = mx.io.DataBatch([data], [label]) mod.forward(batch, is_train=False) out = mod.get_outputs() processed_results = self.post_proc(results=out, ids=None, expected=label_np) result = Output(query_id_list, processed_results, self.post_proc.good, self.post_proc.total, take_accuracy=self.args.accuracy) self.result_queue.put(result) self.task_queue.task_done()
def Main(): host = '192.168.1.69' port = 12349 execCore = 0 instCycleCount = 0 core = [] try: opts, args = getopt.getopt(sys.argv[1:], "hic:e:s:p:", [ "help", "instcyclecount", "core", "execcore", "serverIPaddress", "serverPort" ]) except getopt.GetoptError as err: print(str(err)) usage() sys.exit(2) for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() if o in ("-c", "--core"): core.append(int(a)) os.sched_setaffinity(0, core) if o in ("-e", "--execcore"): execCore = int(a) if o in ("-i", "--instcyclecount"): instCycleCount = 1 if o in ("-s", "--serverIPaddress"): host = a if o in ("-p", "--serverPort"): port = int(a) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((host, port)) while True: message = '' data = s.recv(1024) if data.decode('ascii') == 'end': break if data: message = execObject(data.decode('ascii'), execCore, instCycleCount) if len(message) > 0: s.sendall(message.encode('ascii')) s.close()
def set_affinity(rank): affinity_map = { 0: list(range(48, 64)) + list(range(176, 192)), 1: list(range(48, 64)) + list(range(176, 192)), 2: list(range(16, 32)) + list(range(144, 160)), 3: list(range(16, 32)) + list(range(144, 160)), 4: list(range(112, 128)) + list(range(240, 256)), 5: list(range(112, 128)) + list(range(240, 256)), 6: list(range(80, 96)) + list(range(208, 224)), 7: list(range(80, 96)) + list(range(208, 224)) } my_affinity = affinity_map[rank] os.sched_setaffinity(0, my_affinity)
def fork_and_run(max_jobs: int, client_func: Callable[[int, int, int, int], None], task_size: int, min_sz: int = 32) -> Iterable[Tuple[int, bytes]]: max_pipe_sz = min(1024**2, int(open("/proc/sys/fs/pipe-max-size").read())) buff_size = max_pipe_sz // 2 max_jobs = min(max_jobs, multiprocessing.cpu_count()) rec_per_job = max(task_size // max_jobs, min_sz) offsets: List[Tuple[int, int]] = [] start = 0 while start < task_size: end = min(task_size, start + rec_per_job) if end + rec_per_job > task_size: end = task_size offsets.append((start, end)) start = end active_pipes = 0 sel = selectors.DefaultSelector() for idx, (start, stop) in enumerate(offsets): read_fd, write_fd = os.pipe() fcntl.fcntl(write_fd, F_SETPIPE_SZ, max_pipe_sz) if os.fork() == 0: try: time.sleep(0.0) os.close(read_fd) os.sched_setaffinity(0, [idx]) client_func(write_fd, start, stop - start, buff_size) finally: os.close(write_fd) os._exit(0) else: os.close(write_fd) flag = fcntl.fcntl(read_fd, fcntl.F_GETFD) fcntl.fcntl(read_fd, fcntl.F_SETFL, flag | os.O_NONBLOCK) active_pipes += 1 sel.register(read_fd, selectors.EVENT_READ) while active_pipes: for key, mask in sel.select(): assert mask == selectors.EVENT_READ data = os.read(key.fileobj, max_pipe_sz) if len(data) == 0: active_pipes -= 1 sel.unregister(key.fileobj) os.close(key.fileobj) else: yield key.fileobj, data
def _cpu_affinity(self): # sched_setaffinity() was added to Python 3.3 has_sched_setaffinity = hasattr(os, 'sched_setaffinity') if not has_sched_setaffinity: if psutil is not None: proc = psutil.Process() psutil_has_cpu_affinity = hasattr(proc, 'cpu_affinity') else: psutil_has_cpu_affinity = False cpus = self.args.affinity if not cpus: stream = self._stream() # --affinity option is not set: detect isolated CPUs cpus = _get_isolated_cpus() if not cpus: # no isolated CPUs or unable to get the isolated CPUs return if not has_sched_setaffinity and not psutil_has_cpu_affinity: # unable to pin CPUs print("WARNING: unable to pin worker processes to " "isolated CPUs, CPU affinity not available", file=stream) print("Use Python 3.3 or newer, or install psutil dependency", file=stream) return if self.args.verbose: print("Pin process to isolated CPUs: %s" % perf._format_cpu_list(cpus), file=stream) self.args.affinity = perf._format_cpu_list(cpus) else: cpus = _parse_cpu_list(cpus) if self.args.verbose: print("Pin process to CPUs: %s" % perf._format_cpu_list(cpus), file=self._stream()) if has_sched_setaffinity: os.sched_setaffinity(0, cpus) elif psutil_has_cpu_affinity: proc = psutil.Process() proc.cpu_affinity(cpus) else: print("ERROR: CPU affinity not available.", file=sys.stderr) print("Use Python 3.3 or newer, or install psutil dependency", file=stream) sys.exit(1)
def pids_set(self, pids): """ Set Pool's PIDs Parameters: pids: Pool's PIDs """ old_pids = self.pids_get() # change core affinity for PIDs not assigned to any pool removed_pids = [pid for pid in old_pids if pid not in pids] # skip pids assigned to any pool e.g.: apps moved to other pool for pool_id in Pool.pools: if pool_id == self.pool: continue removed_pids = \ [pid for pid in removed_pids if pid not in Pool.pools[pool_id]['pids']] # remove pid form old pool if old_pids: for pool_id in Pool.pools: if pool_id == self.pool: continue pids_moved_from_other_pool = \ [pid for pid in Pool.pools[pool_id]['pids'] if pid in pids] if pids_moved_from_other_pool: log.debug("PIDs moved from other pools {}".\ format(pids_moved_from_other_pool)) # update other pool PIDs Pool.pools[pool_id]['pids'] = \ [pid for pid in Pool.pools[pool_id]['pids'] if pid not in pids] Pool.pools[self.pool]['pids'] = pids # set affinity of removed pids to default if removed_pids: log.debug("PIDs to be set to core affinity to 'Default' CPUs {}".\ format(removed_pids)) # get cores for Default Pool #0 cores = common.CONFIG_STORE.get_pool_attr('cores', 0) for pid in removed_pids: try: os.sched_setaffinity(pid, cores) except OSError: pass
def run(self): os.sched_setaffinity(self.pid, [self.cpu]) time.sleep(0.01) for _ in range(repeat): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.connect(('127.0.0.1', 8888)) sock.sendall(message) remaining = payload_sz data = [] while remaining > 0: data.append(sock.recv(payload_sz)) remaining -= len(data[-1]) sock.shutdown(socket.SHUT_RDWR)
def main(input_conn: connection.Connection, output_conn: connection.Connection, serialized_flags: str, app: Any, iterations: Optional[int] = None, pinned_cpus: Optional[Iterable[Any]] = None): """Runs the code for the receiver worker subprocess. Intended to be called with the multiprocessing.Process stdlib function. Args: input_conn: A connection object created with multiprocessing.Pipe to read data from the main process. output_conn: A connection object created with multiprocessing.Pipe to write data to the main process. serialized_flags: Flags from the main process serialized with flags.FLAGS.flags_into_string. app: Main process' app instance. iterations: Optional. The number of times the main loop will be run. If left unset, it will run forever (or until terminated by the main process). pinned_cpus: Optional. An iterable of CPU IDs to be passed to os.sched_setaffinity if set. """ if pinned_cpus is not None: os.sched_setaffinity(0, pinned_cpus) FLAGS(serialized_flags.splitlines(), known_only=True) client = app.get_client_class().from_flags() try: communicator = worker_utils.Communicator(input_conn, output_conn) communicator.greet() times_iterable = itertools.repeat(0) if iterations is None else range( iterations) for _ in times_iterable: communicator.await_from_main(protocol.Consume, protocol.AckConsume()) try: response = client.pull_message() pull_timestamp = time.time_ns() client.acknowledge_received_message(response) ack_timestamp = time.time_ns() except Exception as e: # pylint: disable=broad-except communicator.send( protocol.ReceptionReport(receive_error=repr(e))) else: communicator.send( protocol.ReceptionReport(receive_timestamp=pull_timestamp, ack_timestamp=ack_timestamp)) finally: client.close()
def prefork(self, max_workers: int = None, bindcore: bool = False): ''' bindcore binds each process to a cpu core specifily which helps reduce cache miss ,but limits the flexibility of kernel's tasks scheduling. which place you fork matters. if you fork after register ,then all process shares a unique overflow control lock , by contray if you fork before it , each process will have its own lock which means you have max_workers times maximum overflow. ''' self._child_list = list() self._isparent = True cpu_core_nums = cpu_count() if max_workers == None: max_workers = cpu_core_nums - 1 if bindcore == None: bindcore = False if (max_workers + 1) < cpu_core_nums else True try: _ = os.sched_setaffinity # currentrly not avilable on pypy except: bindcore = False if bindcore: self._bindcount = Value('i', 0) for num in range(max_workers): pid = os.fork() if pid < 0: raise OSError(f"Creating fork process failed.") elif pid == 0: self._isparent = False break else: self._child_list.append(pid) self._pid = os.getpid() if bindcore: with self._pool_lock: if self._bindcount.value < ( (max_workers + 1) // cpu_core_nums) * cpu_core_nums: os.sched_setaffinity(0, (self._bindcount.value, )) self._bindcount.value = (self._bindcount.value + 1) % cpu_core_nums
def set_affinity(gpu_id=None): r"""Set GPU affinity Args: gpu_id (int): Which gpu device. """ if gpu_id is None: gpu_id = int(os.getenv('LOCAL_RANK', 0)) dev = device(gpu_id) os.sched_setaffinity(0, dev.getCpuAffinity()) # list of ints # representing the logical cores this process is now affinitied with return os.sched_getaffinity(0)
def _set_affinity(self): if Config().get_config("affinity.enable") is True: try: core_num = Config().get_config("affinity.core_num") cpu_count = psutil.cpu_count() if core_num <= 0 or cpu_count < core_num: mask = range(1) Logger().warning( "Config item affinity.core_num invalid, use defaut (1)" ) else: mask = range(core_num) os.sched_setaffinity(os.getpid(), mask) except Exception as e: Logger().error("set affinity error!", exc_info=e)
def run_nose(args): pid, tests = args cpu_set = list(os.sched_getaffinity(0)) if len(cpu_set) > 4: offset = (pid * 4) % len(cpu_set) os.sched_setaffinity(0, set(cpu_set[offset:offset + 4])) xunitfile = "nosetests-{0:02}.xml".format(pid) print(pid, ": Running nose with", len(tests), "tests...", xunitfile) selected_tests = [x['selector'] for x in tests] call = ['nosetests', '-v', '--with-xunit', '--xunit-file=' + xunitfile ] + selected_tests subprocess.call(call)
def annotate_worker(aff, input_queue): me = threading.current_thread().name if aff: print(f"{me} starting with affinity {aff}") os.sched_setaffinity(0, aff) while True: print(f"{me} waiting") item = input_queue.get() if item is None: print(me, " got none") break print(f"{me} got {item}") id, sra, md, out_dir = item start = time.time() cmd = ["p3x-create-sars-gto", f"{out_dir}/{sra}.fasta", f"{out_dir}/{sra}.json", f"{out_dir}/{sra}.raw.gto"]; print(cmd, file=sys.stderr) subprocess.run(cmd, stdout=open(f"{out_dir}/annotate.stdout", "w"), stderr=open(f"{out_dir}/annotate.stderr", "w")) cmd = ["p3x-annotate-vigor4", "-i", f"{out_dir}/{sra}.raw.gto", "-o", f"{out_dir}/{sra}.gto"]; print(cmd, file=sys.stderr) subprocess.run(cmd, cwd=out_dir, stdout=open(f"{out_dir}/annotate.stdout", "a"), stderr=open(f"{out_dir}/annotate.stderr", "a")) end = time.time() elapsed = end - start md["annotation_elapsed"] = elapsed with open(f"{out_dir}/meta.json", "w") as f: json.dump(md, f, indent=2) with open(f"{out_dir}/RUNTIME_ANNO", "w") as f: print(f"{start}\t{end}\t{elapsed}", file=f) input_queue.task_done()
def main(): parser = get_parser() args = parser.parse_args() # SET CONFIGURATION config = load_config(args) # SET TRAINING PROCESS AFFINITY if 'affcores' in args: os.sched_setaffinity(0, args.affcores) elif 'affsockets' in args: cores = get_cores(args.affsockets, config['cpuconfig']) os.sched_setaffinity(0, cores) ## TRAIN train(config)
def set_affinity(pids, cores): """ Sets PIDs' core affinity Parameters: pids: PIDs to set core affinity for cores: cores to set to """ # set core affinity for each PID, # even if operation fails for one PID, continue with other PIDs for pid in pids: try: os.sched_setaffinity(pid, cores) except OSError: log.error("Failed to set {} PID affinity".format(pid))
def sig_test_init(test_attr): pcnt, ni, nc, nt, ng, ngap = test_attr pid_list = [] affinity_mask = {i for i in range(0, nc)} process_list.clear() # context.log_level = 'error' for i in range(pcnt): pid_list.append(run_program(ni)) for pid in pid_list: os.sched_setaffinity(pid[0], affinity_mask) os.sched_setaffinity(pid[1], affinity_mask) return pid_list
def _start_process(self, command, is_first=True): command_array = shlex.split(command) if not self.collect_output: self.proc = subprocess.Popen(command_array) elif is_first: self.proc = subprocess.Popen(command_array, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: self.proc = subprocess.Popen(command_array, stdin=self.proc.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if self.affinity: os.sched_setaffinity(self.GetPid(), self.affinity) return self.proc
def set_single_affinity(gpu_id): """ The process is assigned with the first available logical CPU core from the list of all CPU cores from the CPU socket connected to the GPU with a given id. Args: gpu_id: index of a GPU """ dev = Device(gpu_id) affinity = dev.get_cpu_affinity() # exclude unavailable cores available_cores = os.sched_getaffinity(0) affinity = list(set(affinity) & available_cores) os.sched_setaffinity(0, affinity[:1])
def initProcess(self): sd._initialize() os.sched_setaffinity(0,{1,2}) self.bf = Beamformer.get_beamformer() self.bfQueue = Beamformer.BFQueue(param_blocksize, self.bf, 0) self.input_dev = None for dev_idx, dev_dict in enumerate(sd.query_devices()): if "USBStreamer" in dev_dict['name']: self.input_dev = dev_idx if "default" in dev_dict['name']: self.output_dev = dev_idx #self.output_dev = 28 if self.input_dev is None: logging.error("Could not find USBStreamer, not beamforming") return False return True
def set_cpu_affinity(cpus): # Python 3.3 or newer? if hasattr(os, 'sched_setaffinity'): os.sched_setaffinity(0, cpus) return True try: import psutil except ImportError: return proc = psutil.Process() if not hasattr(proc, 'cpu_affinity'): return proc.cpu_affinity(cpus) return True
def test_set_affinity_and_check(self): os.sched_setaffinity(0, [0, 1]) s_ = os.sched_getaffinity(0) self.assertTrue(0 in s_) self.assertTrue(1 in s_) os.sched_setaffinity(0, [0]) s_ = os.sched_getaffinity(0) self.assertTrue(0 in s_) self.assertFalse(1 in s_) for i_ in xrange(affinity.NO_OF_CPU): os.sched_setaffinity(0, [i_]) s_ = os.sched_getaffinity(0) self.assertTrue(i_ in s_) os.sched_setaffinity(0, [0, 1]) self.assertEqual([0, 1], os.sched_getaffinity(0))
#! /usr/bin/python3 import os nb_cpu = os.sysconf(os.sysconf_names['SC_NPROCESSORS_ONLN']) while True: for i in range(0, nb_cpu): cpuset = [i] os.sched_setaffinity(0, cpuset) for j in range(0,100000000): pass
def test_demo(self): for i_ in xrange(affinity.NO_OF_CPU): os.sched_setaffinity(0,[i_]) for j_ in range(2500,6000): j_ ** j_