def getNeighbors(self, peer): semaphore = Semaphore() semaphore.acquire() neighbors = self.__layout[peer.getId()].getNeighbors() semaphore.release() return neighbors
def _spawnmain( testid: TestId, exts: List[str], # pyre-fixme[11]: Annotation `Semaphore` is not defined as a type. sem: multiprocessing.Semaphore, resultqueue: multiprocessing.Queue, ): """run a test and report progress back intended to be spawned via multiprocessing.Process. """ hasmismatch = False def mismatchcb(mismatch: Mismatch): nonlocal hasmismatch hasmismatch = True mismatch.testname = testid.name resultqueue.put(mismatch) result = TestResult(testid=testid) try: runtest(testid, exts, mismatchcb) except TestNotFoundError as e: result.exc = e except Exception as e: result.exc = e result.tb = traceback.format_exc(limit=-1) finally: if result.exc is None and hasmismatch: result.exc = MismatchError("output mismatch") resultqueue.put(result) resultqueue.close() sem.release()
class Budget(object): """Device budget (for lack of a better term) Details ------- Keeps a semaphore for the total number of resources available, and a device-specific atomic counter of available slots. """ def __init__(self, devices, n_per_device): self.devices, self.n_per_device = devices, n_per_device self.total = Semaphore(len(devices) * n_per_device) self.alloc = Array("i", len(devices) * [n_per_device]) def acquire(self): self.total.acquire() with self.alloc.get_lock(): # get the largest counter index = max(range(len(self.alloc)), key=self.alloc.__getitem__) # assert self.alloc[index] > 0 # acquire the index and decrease the counter self.alloc[index] -= 1 return index def release(self, index): with self.alloc.get_lock(): self.alloc[index] += 1 self.total.release() def __getitem__(self, index): return self.devices[index]
def setCurrentSimulationTime(self, currentSimulationTime): semaphore = Semaphore() semaphore.acquire() self.__currentSimulationTime = currentSimulationTime semaphore.release() return self.__currentSimulationTime
def _readtimestepsbond(self): # added on 2018-12-15 stepatomfiles = {} self._mkdir(self.trajatom_dir) with Pool(self.nproc, maxtasksperchild=10000) as pool: semaphore = Semaphore(360) results = pool.imap_unordered( self.bonddetector.readatombondtype, self._produce( semaphore, enumerate( zip(self.lineiter(self.bonddetector), self.erroriter() ) if self.errorfilename is not None else self. lineiter(self.bonddetector)), (self.errorfilename is not None)), 100) nstep = 0 for d, step in tqdm(results, desc="Read trajectory", unit="timestep"): for bondtypebytes, atomids in d.items(): bondtype = self._bondtype(bondtypebytes) if bondtype not in self.atombondtype: self.atombondtype.append(bondtype) stepatomfiles[bondtype] = open( os.path.join(self.trajatom_dir, f'stepatom.{bondtype}'), 'wb') stepatomfiles[bondtype].write( self.listtobytes([step, atomids])) semaphore.release() nstep += 1 pool.close() self._nstep = nstep for stepatomfile in stepatomfiles.values(): stepatomfile.close() pool.join()
def _img_processor(self, sh_img_arr: Array, sem1: Semaphore, sem2: Semaphore, ovl_arr: Array) -> None: """ Process images as needed. :param sh_img_arr: The array containing the frame to work with. :param sem1: The entrance lock. :param sem2: The exit lock. :param ovl_arr: The array containing the overlay work with. :return None: """ img_dim = (EDIT_HEIGHT, self._cur_arr_shape[1], self._cur_arr_shape[2]) img_size = int(EDIT_HEIGHT * img_dim[1] * img_dim[2]) img_arr = frombuffer(sh_img_arr.get_obj(), count=img_size, dtype=DTYPE).reshape(img_dim) while self._process_imgs: sem1.acquire() if self._use_overlay: img_pil = Image.fromarray(img_arr) draw = ImageDraw.Draw(img_pil) draw.text(OVL_POS, text=ovl_arr.value.decode(), font=OVL_FONT, fill=OVL_CLR) processed_img = asarray(img_pil) copyto(img_arr, processed_img) sem2.release()
def start(self, test_q: JoinableQueue, result_q: Queue) -> None: """ Start all worker processes :return: this object """ local_test_q = JoinableQueue() self._node_manager = Node.Manager(as_main=True, port=self.__class__._node_port) start_sem = Semaphore(self._max_simultaneous_connections) # This will be used to throttle the number of connections made when makeing distributed call to get # node-level and global-level fixtures; otherwise multiproceissing can hang on these calls if # overwhelmed fixture_sem = Semaphore(self._max_simultaneous_connections) for index in range(self._num_processes): proc = WorkerSession.start( index, self._host, self._port, start_sem, fixture_sem, local_test_q, result_q, self._node_port, ) self._worker_procs.append(proc) start_sem.release() self._test_q_process = Process(target=self._process_test_q, args=(test_q, local_test_q)) self._test_q_process.start()
def getPeerID(self, peerId): semaphore = Semaphore() semaphore.acquire() peer = self.__layout[peerId] semaphore.release() return peer
class Msg(object): """ TODO: Not documenting this class because it may go away. """ def __init__(self, size): self.s_e = Semaphore(1) self.s_f = Semaphore(0) self.s_buf = Array(ct.c_ubyte, size) def send(self, func): self.s_e.acquire() self.s_buf.acquire() send_result = func(self.s_buf._obj) self.s_buf.release() self.s_f.release() return send_result def recv(self, func): self.s_f.acquire() self.s_buf.acquire() recv_result = func(self.s_buf._obj) self.s_buf.release() self.s_e.release() return recv_result
def _build_single_scenario_proc(clean: bool, allow_offset_map: bool, scenario: str, semaphore: Semaphore): semaphore.acquire() try: _build_single_scenario(clean, allow_offset_map, scenario) finally: semaphore.release()
def generator(self, *args, **kwargs): """ This function warp generator to ParaWrapper's generator which is capable of multi-processing Once the generator function was settled, we can send worker with the task then work with full-load until meet the buff_size limit The worker's job is to feed the list and keep it contains more than <buff_size> batches """ # Initialization semaphores and numbering buff_count = Semaphore(value=0) target_remain = Semaphore(value=self.buff_size) number = str(self.gen_num) self.gen_num += 1 # Initializing list self.batch_list[number] = self.manager.list() # Assign work and send worker gen = self.datagen.generator(*args, **kwargs) worker = Process(target=self.task, args=(gen, number, target_remain, buff_count)) worker.start() while True: buff_count.acquire(block=True) ret = self.batch_list[number].pop() target_remain.release() yield ret
class Thread_Pool_Manager(object): def __init__(self, thread_num=cpu_count()): self.thread_num = thread_num print(thread_num) self.work_queue = JoinableQueue() self.work_num = Semaphore(0) self.mutex = Lock() def start_threads(self): for i in range(self.thread_num): thread = Process(target=self.do_job) thread.daemon = True # set thread as daemon thread.start() def do_job(self): global Numbers while True: # print(1) self.work_num.acquire() with self.mutex: print(1, self.work_queue.qsize()) thread_job = self.work_queue.get() print(0, self.work_queue.qsize()) thread_job.do_job(self.work_queue, self.work_num) print(self.work_queue.qsize()) self.work_queue.task_done() def join(self): self.work_queue.join() def add_job(self, job): self.work_queue.put(job) self.work_num.release()
def relay(semaphore: mp.Semaphore, queue: mp.Queue, output_lock: mp.Lock, bmsg: bytes, addr: tuple, relay_dict, recv_time: datetime): semaphore.acquire() bmsg = bytearray(bmsg) header = DNSHeader(bmsg[:12]) header.aa = 1 bmsg = header.bmsg + bmsg[12:] assert header.qdcount == 1 question = DNSQuestion(bmsg, offset=12) with output_lock: cprint(f'[{recv_time}][recv query {bytes_to_int(bmsg[:2])}]: {bmsg} from {addr}', fore='green', style='reverse') cprint_header(header, fore='green') cprint_question(question, fore='green') if question.qname in relay_dict: if relay_dict[question.qname] == '0.0.0.0': header.rcode = 3 answer = header.bmsg + bmsg[12:] mode = 'intercept ' elif question.qtype == 1: answer = fake_bmsg(bmsg, relay_dict[question.qname]) mode = 'local resolve ' else: answer = forward(bmsg) if answer is None: return mode = 'relay msg ' else: answer = forward(bmsg) mode = 'relay msg ' queue.put((answer, addr, recv_time, mode)) semaphore.release()
class EmbeddingWorker(Process): def __init__(self, queue, X, y, transformation_method, embedding_args): super().__init__() self.pause_lock = Semaphore(value=True) # lock is free self.embedding_args = embedding_args self.X = X self.y = y self.transformation_method = transformation_method self.queue = queue def callback(self, command, iteration, payload): # pausing acquires pause_lock and the following code only runs if # pause_lock is free with self.pause_lock: self.queue.put((command, iteration, payload)) def run(self): self.transformation_method(self.X, self.y, self.embedding_args, self.callback) def pause(self): self.pause_lock.acquire() def resume(self): self.pause_lock.release() def is_paused(self): return not self.pause_lock.get_value()
class Msg(object): """ Data structure encapsulating a message. """ def __init__(self, size): self.s_e = Semaphore(1) self.s_f = Semaphore(0) self.s_buf = Array(ct.c_ubyte, size) def send(self, func): self.s_e.acquire() self.s_buf.acquire() send_result = func(self.s_buf._obj) self.s_buf.release() self.s_f.release() return send_result def recv(self, func): self.s_f.acquire() self.s_buf.acquire() recv_result = func(self.s_buf._obj) self.s_buf.release() self.s_e.release() return recv_result
def _get_all(embeddings, data_sequence, start_iteration, ent_type, w_size, batch_size, processes, evalutation_semaphore=None): # The embed semaphore makes sure that the EmbedWithContext will not over produce results in relation # to the LookUpBySurfaceAndContext creation embed_semaphore = Semaphore(100) for it, link_result in \ enumerate( EmbedWithContext.run(embeddings, data_sequence, ent_type, w_size, batch_size, processes, embed_semaphore, start_iteration=start_iteration)): try: if evalutation_semaphore is not None: evalutation_semaphore.acquire(timeout=10) yield LookUpBySurfaceAndContext(link_result) except Exception as ex: print(type(ex)) print("Error: ", link_result) raise if it % batch_size == 0: embed_semaphore.release()
class DbWriter(Process): def __init__(self, queue, stop_flag): super(DbWriter, self).__init__() self.worker_control = Semaphore(MAX_WRITER_WORKERS) self.result_queue = queue self.stop_flag = stop_flag def run(self): print(" *** DB Writer online") while True and self.stop_flag.value != 1: if self.worker_control.acquire(False): task = self.result_queue.get() if task: try: worker = WriterWorker(task, self.worker_control) worker.start() except Exception as err: print(err) print("Invalid task %s" % task) self.worker_control.release() else: self.worker_control.release() time.sleep(0.3) print("stop flag: %s" % self.stop_flag.value)
def _writecoulumbmatrix(self, trajatomfilename, fc): self.dstep = {} with open( os.path.join(self.trajatom_dir, f"stepatom.{trajatomfilename}"), 'rb') as f: for line in f: s = self.bytestolist(line) self.dstep[s[0]] = s[1] n_atoms = sum(map(len, self.dstep.values())) if n_atoms > self.n_clusters: # undersampling max_counter = Counter() stepatom = np.zeros((n_atoms, 2), dtype=int) feedvector = np.zeros((n_atoms, 0)) vector_elements = defaultdict(list) with Pool(self.nproc, maxtasksperchild=10000) as pool: semaphore = Semaphore(360) results = pool.imap_unordered( self._writestepmatrix, self._produce(semaphore, enumerate(self.lineiter(self.crddetector)), None), 100) j = 0 for result in tqdm(results, desc=trajatomfilename, total=self._nstep, unit="timestep"): for stepatoma, vector, symbols_counter in result: stepatom[j] = stepatoma for element in (symbols_counter - max_counter).elements(): vector_elements[element].append( feedvector.shape[1]) feedvector = np.pad( feedvector, ((0, 0), (0, 1)), 'constant', constant_values=(0, self._coulumbdiag[element])) feedvector[ j, sum( map(lambda x: vector_elements[x[0]][:x[1]], symbols_counter.items()), [])] = vector max_counter |= symbols_counter j += 1 semaphore.release() logging.info( f"Max counter of {trajatomfilename} is {max_counter}") pool.close() choosedindexs = self._clusterdatas(np.sort(feedvector), n_clusters=self.n_clusters, n_each=self.n_each) pool.join() else: stepatom = np.array([[u, vv] for u, v in self.dstep.items() for vv in v]) choosedindexs = range(n_atoms) fc.write(self.listtobytes(stepatom[choosedindexs])) self._nstructure += len(choosedindexs)
def countNeighbors(self, peer): semaphore = Semaphore() semaphore.acquire() count = peer.countNeighbors() semaphore.release() return count
def getNeighborIt(self, peer): semaphore = Semaphore() semaphore.acquire() neighbors = [] for neighbor in self.__layout[peer.getId()].getNeighbors(): neighbors.append(neighbor.getTargetPeer()) neighborIt = neighbors.__iter__() semaphore.release() return neighborIt
class Iterator(object): """Provides an generic multithreaded iterator over the dataset.""" def __init__(self, dataset, batch_size, batchifier, pool, shuffle= False, use_padding = False, no_semaphore= 20): print("----------------- Iterator",batch_size) # Filtered games games = dataset.get_data() # print("games = {}".format(games)) # print("dataset = {} ".format(dataset)) # exit() games = batchifier.filter(games) if shuffle: random.shuffle(games) self.n_examples = len(games) self.batch_size = batch_size self.n_batches = int(math.ceil(1. * self.n_examples / self.batch_size)) batch = split_batch(games, batch_size, use_padding) print("++++ Iterator | n_examples = {},batch_size={},n_batches={}".format(self.n_examples,self.batch_size,self.n_batches)) print("shape({},{})".format(len(batch),len(batch[0]))) # no proc # self.it = (batchifier.apply(b )for b in batch) # Multi_proc self.semaphores = Semaphore(no_semaphore) it_batch = sem_iterator(l=batch, sem=self.semaphores) self.process_iterator = pool.imap(batchifier.apply, it_batch) def __len__(self): return self.n_batches def __iter__(self): return self def __next__(self): self.semaphores.release() return self.process_iterator.next() #return self.it.__next__() # trick for python 2.X def next(self): return self.__next__()
def iter_batches(self): if self.parallel > 0: with Pool(self.parallel) as pool: semaphore = Semaphore(self.parallel * 10) for batch in pool.imap( self.full_construct, self.batch_reader.read_batches(semaphore)): yield batch semaphore.release() else: for batch in self.batch_reader.read_batches(): yield self.full_construct(batch)
def addPeer(self, peer): if self.__layout.has_key(peer.getPID()): return False semaphore = Semaphore() semaphore.acquire() self.__layout[peer.getPID()] = peer semaphore.release() NetworkLogger().resgiterLoggingInfo("Add peer %s in Layout Network "%(peer.getPID())) return self.__layout.has_key(peer.getPID())
class Runner: def __init__(self): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGUSR1, signal.SIG_IGN) # Create the pool without special signal handling self.pool = Pool(processes=NUM_WORKERS) self.worker_semaphore = Semaphore(NUM_WORKERS) self.has_received_termination_signal = False for sig in termination_signals: signal.signal(sig, self.handle_termination_signal) self.units_of_work = itertools.cycle(range(100)) logging.info("Main PID {0}".format(os.getpid())) def do_work(self): while True: if self.has_received_termination_signal: logging.info( "Stopping loop because termination signal received") return self.acquire_worker_semaphore() work = (next(self.units_of_work), ) logging.info("Sending work {0}".format(work)) self.pool.apply_async( worker_function, work, callback=self.release_worker_semaphore, error_callback=self.release_worker_semaphore, ) time.sleep(2) def start(self): self.do_work() self.pool.close() self.pool.join() def handle_termination_signal(self, signum, frame): my_pid = os.getpid() logging.info("PID {0} received termination signal {1!r}".format( my_pid, signum)) self.has_received_termination_signal = True def acquire_worker_semaphore(self): self.worker_semaphore.acquire() def release_worker_semaphore(self, exception=None): self.worker_semaphore.release()
class Contractor(Process): def __init__(self, images, stop_flag, input_queue, result_queue, api_key, secret_key, time_worker_run): super(Contractor, self).__init__() self.worker_control = Semaphore(MAX_WORKERS) self.images = images self.stop_flag = stop_flag self.input_queue = input_queue self.result_queue = result_queue self.api_key = api_key self.secret_key = secret_key self.time_worker_run = time_worker_run def run(self): print(" ++ Contractor online ++ \n") while True: temp_value = None temp_value = self.stop_flag.value #print(">> %s temp value"% temp_value) if temp_value != 1: if self.worker_control.acquire(False): try: task = self.input_queue.get(False) if task: worker = Worker(self.worker_control, task, self.input_queue, self.api_key, self.secret_key, self.stop_flag, self.result_queue, self.images, self.time_worker_run) worker.start() else: self.worker_control.release() except: self.worker_control.release() else: break time.sleep(0.2) print("contractor done")
def _printmoleculeSMILESname(self): mname = [] with open(self.moleculefilename, 'w') as fm, open(self.moleculetemp2filename, 'rb') as ft, Pool(self.nproc, maxtasksperchild=1000) as pool: semaphore = Semaphore(360) results = pool.imap(self._calmoleculeSMILESname, self._produce(semaphore, ft, ()), 10) for index, (name, atoms, bonds) in enumerate(results): self._loggingprocessing(index) mname.append(name) print(name, ",".join([str(x) for x in atoms]), ";".join( [",".join([str(y) for y in x]) for x in bonds]), file=fm) semaphore.release() self._mname = mname
def run_shard(q: Queue, sem: Semaphore, num, total): global _MP_QUEUE _MP_QUEUE = q sys.argv.append("--internal-num-shards=" + str(total)) sys.argv.append("--internal-shard=" + str(num)) # sys.argv.append("--pretend") print("shard", num, sys.argv) try: libcxx_main() print("Job", num, "completed") except Exception as e: print("Job", num, "failed!!", e) sem.release()
class Barrier: def __init__(self, n): self.n = n self.counter = SharedCounter(0) self.barrier = Semaphore(0) def wait(self): with self.counter.lock: self.counter.val.value += 1 if self.counter.val.value == self.n: self.barrier.release() self.barrier.acquire() self.barrier.release()
class Barrier: def __init__(self, n): self.n = n self.counter = SharedCounter(0) self.barrier = Semaphore(0) def wait(self): with self.counter.lock: self.counter.val.value += 1 if self.counter.val.value == self.n: self.barrier.release() self.barrier.acquire() self.barrier.release()
def _start_token_generation(sema: mp.Semaphore, tokens_per_second: int, stop_event: threading.Event): """ Releases number of tokens_per_second tokens per second on the Semaphore """ logging.debug('token generation thread started') while not stop_event.is_set(): time.sleep(2 / tokens_per_second) try: sema.release() sema.release() except ValueError: continue logging.debug('token generation thread stoopped')
class WorkQueue(object): def __init__(self): self.request_rfd, self.request_wfd = os.pipe() self.response_rfd, self.response_wfd = os.pipe() self.response_reader = ResponseReader(self.response_rfd) self.request_sem = Semaphore() self.response_sem = Semaphore() def submit_request(self, id, address, head, body): try: ip_str, port = address ipa, ipb, ipc, ipd = map(int, ip_str.split(".")) except: port = ipa = ipb = ipc = ipd = 0 os.write( self.request_wfd, REQUEST_HEADER.pack(id, ipa, ipb, ipc, ipd, port, len(head), len(body)) + head + body ) def get_request(self): self.request_sem.acquire() header = "" bytes_to_read = REQUEST_HEADER.size while bytes_to_read: header += os.read(self.request_rfd, bytes_to_read) bytes_to_read = REQUEST_HEADER.size - len(header) id, ipa, ipb, ipc, ipd, port, head_len, body_len = REQUEST_HEADER.unpack(header) head = StringIO() bytes_to_read = head_len while bytes_to_read: head.write(os.read(self.request_rfd, bytes_to_read)) bytes_to_read = head_len - head.tell() body = StringIO() bytes_to_read = body_len while bytes_to_read: body.write(os.read(self.request_rfd, bytes_to_read)) bytes_to_read = body_len - body.tell() self.request_sem.release() return id, (".".join(map(str, [ipa, ipb, ipc, ipd])), port), head.getvalue(), body.getvalue() def submit_response(self, id, response): self.response_sem.acquire() response_output = response.output() keep_alive = "\x01" if response.headers.get("Connection") == "Keep-Alive" else "\x00" os.write(self.response_wfd, RESPONSE_HEADER.pack(id, len(response_output)) + response_output + keep_alive) self.response_sem.release() def get_response(self): return self.response_reader.read()
def generate_ripe_request_tokens(sema: mp.Semaphore, limit: int, finish_event: threading.Event): """ Generates RIPE_REQUESTS_PER_SECOND tokens on the Semaphore """ logger.debug('generate thread started') while not finish_event.is_set(): time.sleep(2 / limit) try: sema.release() sema.release() except ValueError: continue logger.debug('generate thread stoopped')
class ForkingWorker(BaseWorker): def __init__(self, num_processes=1): # Set up sync primitives, to communicate with the spawned children self._semaphore = Semaphore(num_processes) self._slots = Array('i', [0] * num_processes) def spawn_child(self): """Forks and executes the job.""" self._semaphore.acquire() # responsible for the blocking # Select an empty slot from self._slots (the first 0 value is picked) # The implementation guarantees there will always be at least one empty slot for slot, value in enumerate(self._slots): if value == 0: break # The usual hardcore forking action child_pid = os.fork() if child_pid == 0: # Within child # Disable signal handlers signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) random.seed() try: self.fake_work() finally: # This is the new stuff. Remember, we're in the child process # currently. When all work is done here, free up the current # slot (by writing a 0 in the slot position). This # communicates to the parent that the current child has died # (so can safely be forgotten about). self._slots[slot] = 0 self._semaphore.release() os._exit(0) else: # Within parent, keep track of the new child by writing its PID # into the first free slot index. self._slots[slot] = child_pid def wait_for_children(self): for child_pid in self._slots: if child_pid != 0: os.waitpid(child_pid, 0) def get_id(self): return os.getpid()
class Iterator(object): """Provides an generic multithreaded iterator over the dataset.""" def __init__(self, dataset, batch_size, batchifier, pool, shuffle=False, use_padding=False, no_semaphore=20): # Filtered games games = dataset.get_data() games = batchifier.filter(games) games = batchifier.split(games) if shuffle: random.shuffle(games) self.batch_size = batch_size self.n_batches = int(math.ceil(1. * len(games) / self.batch_size)) if use_padding: self.n_examples = self.n_batches * self.batch_size else: self.n_examples = len(games) batch = split_batch(games, batch_size, use_padding) # no proc # self.it = (batchifier.apply(b) for b in batch) # Multi_proc self.semaphores = Semaphore(no_semaphore) it_batch = sem_iterator(l=batch, sem=self.semaphores) self.process_iterator = pool.imap(batchifier.apply, it_batch) def __len__(self): return self.n_batches def __iter__(self): return self def __next__(self): self.semaphores.release() return self.process_iterator.next() # return self.it.__next__() # trick for python 2.X def next(self): return self.__next__()
class Pool(object): """Create a pool of object with the given class_type, args, and kwargs. The get method will return any objects currently unused from the pool. The put method will put back the object, marking it as unused for future use. Putting blocking=True will wait until an object is available, while blocking=False will return immediately if there is no object available """ def __init__(self, class_type, n, blocking=True, callback=lambda x: x, args=(), kwargs={}): self.class_type = class_type self.items = [] self.used = Array('i', [0] * n) self.lock = Lock() self.count = Semaphore(n) self.n = n self.blocking = blocking try: for i in range(n): my_item = self.class_type(*args, **kwargs) callback(my_item) self.items.append(my_item) except Exception as exc: raise type(exc)('%s: %s' % (self.class_type, exc.message)) def get(self): self.lock.acquire(self.blocking) try: if not self.count.acquire(self.blocking): return None for i, my_item in enumerate(self.items): if self.used[i] == 0: self.used[i] = 1 return my_item finally: self.lock.release() def put(self, item): self.lock.acquire() try: for i, my_item in enumerate(self.items): if my_item == item: self.used[i] = 0 self.count.release() return True return False finally: self.lock.release()
class CheckableSem: def __init__(self): self._sem = Semaphore() self._num_permits = 1 def acquire(self): self._sem.acquire() self._num_permits -= 1 def release(self): self._sem.release() self._num_permits += 1 def almost_perfect_get_num_permits(self): return self._num_permits
def _printatomroute(self, atomeach): with open(self.atomroutefilename, 'w') as f, Pool(self.nproc, maxtasksperchild=1000) as pool: allmoleculeroute = [] semaphore = Semaphore(360) results = pool.imap(self._getatomroute, self._produce( semaphore, enumerate(zip(atomeach, self._atomtype), start=1), ()), 10) for index, route in enumerate(results): self._loggingprocessing(index) moleculeroute, routestr = route print(routestr, file=f) for mroute in moleculeroute: if not mroute in allmoleculeroute: allmoleculeroute.append(mroute) semaphore.release() return allmoleculeroute
def definyPeerTrading(self): value =0; peerAux ="" semaphore = Semaphore() semaphore.acquire() for peer,trust in self.__peersTrading.iteritems(): if trust >= value: value = trust peerAux = peer semaphore.release() return (peerAux,value)
async def server_work(reader, writer): ''' Handle main communication with C#. ''' while True: # print("wait for server work") # Wait for an instruction from C# and act upon. # TODO: handle disconnects (they might throw some exceptions around read). recv = await reader.readline() # print(recv) try: data = recv.decode('utf-8').split() except (UnicodeDecodeError, AttributeError): print(f"Invalid data received from C#: {recv}") continue if data[0] == "start": num_threads = int(data[1]) model_choice = int(data[2]) image_size = int(data[3]) model_path = data[4] visualization_type = int(data[5]) video_path = data[6] frame_rate = int(data[7]) layers_to_see = [] layers_to_see.append("pred") for layers in range(8, len(data)): layers_to_see.append(data[layers]) # Start the cam worker. image_queue = Queue(num_threads) semaphore = Semaphore(0) Process(target=camera_worker, args=[ image_queue, semaphore, image_size, visualization_type, video_path, frame_rate ]).start() # Make the initial prediction. image_model, layers_indices = init(model_choice, layers_to_see, model_path) semaphore.release() image = image_queue.get() # Start the prediction workers. for index in range(num_threads): Process(target=worker_thread, args=(index + 1, image_queue, semaphore, model_choice, layers_to_see, model_path)).start() await predict_and_write(0, image, image_model, layers_indices, writer)
def run_mp(nproc, **arg): pool = Pool(nproc, maxtasksperchild=1000) semaphore = Semaphore(nproc * 150) try: results = multiopen(pool=pool, semaphore=semaphore, **arg) for item in results: yield item semaphore.release() except: logging.exception("run_mp failed") pool.terminate() raise else: pool.close() finally: pool.join()
def removeNeighbor(self, source, target): if (not self.__layout.has_key(source.getId())) and (not self.__layout.has_key(target.getId())) : return False if source.hasNeighbor(target): return False if target.hasNeighbor(source): return False semaphore = Semaphore() semaphore.acquire() flag = source.removeNeighbor(target) target.removeNeighbor(source) semaphore.release() return flag
def start_multiprocess(self, csv_file_name): self.get_download_task_from_csv(csv_file_name) semaphores = Semaphore(self._thread_number) for video_url in self._video_url: semaphores.acquire() process = multiprocessing.Process( target=self.run_downloader, args=(semaphores, video_url, self._MAX_VIDEO_DURATION_DIFF)) process.daemon = True process.start() # semaphores.release() for _ in range(0, self._thread_number): semaphores.acquire() semaphores.release()
def _calhmm(self): with open(self.originfilename, 'wb') if self.getoriginfile or not self.runHMM else Placeholder() as fo, open(self.hmmfilename, 'wb') if self.runHMM else Placeholder() as fh, open(self.moleculetempfilename, 'rb') as ft, open(self.moleculetemp2filename, 'wb') as ft2, Pool(self.nproc, maxtasksperchild=1000) as pool: semaphore = Semaphore(360) results = pool.imap_unordered( self._getoriginandhmm, self._produce(semaphore, ft, ()), 10) for index, (originsignal, hmmsignal, mlist) in enumerate(results): self._loggingprocessing(index) if 1 in hmmsignal or self.printfiltersignal or not self.runHMM: if self.getoriginfile: fo.write(self._compress( "".join([str(i) for i in originsignal.tolist()]))) if self.runHMM: fh.write(self._compress( "".join([str(i) for i in hmmsignal.tolist()]))) ft2.write(self._compress(mlist.strip())) semaphore.release()
def main(): gps_n = Semaphore(0) gps_s = Semaphore(1) gps_coords_stack = Manager().list() gps = GPS(gps_coords_stack, gps_n, gps_s) gps.start() # Get the first position z = gps.getPosition() dt = 0.05 range_std = 5. # Means meters # Instantiate the filter filterk = ExtendedKalmanFilter(2, 1, 0) # 1 type of value of position, but in 2 dimensions. sensor provides position in (x,y) so use 2 # Insert first position filterk.x = array(z) # Pretty sure this sets up the taylor series filterk.F = eye(2) + array([[0,1], [0,0]])*dt # Sets the uncertainty filterk.R = np.diag([range_std**2]) # Trains it using white noise? filterk.Q[0:2, 0:2] = Q_discrete_white_noise(2, dt=dt, var=0.1) filterk.Q[2, 2] = 0.1 # Covariance matrix filterk.P *= 50 for i in range(10): # Pull a value from the GPS stack gps_n.acquire() gps_s.acquire() result = gps_coords_stack.pop() gps_s.release() # Put new z value in filterk.predict_update(array(result), HJacobian_at, hx) #this maaaaay need to be formatted differently, otherwise just put the longitude and lattitude as an array [x,y] # Get the predicted value np.append(xs, filterk.x) print(filterk.x)
class EventMasterProcess(SatoriProcess): def __init__(self): super(EventMasterProcess, self).__init__('event master') self.sem = Semaphore(0) def do_run(self): listener = Listener(address=(settings.EVENT_HOST, settings.EVENT_PORT)) master = Master(mapper=TrivialMapper()) master.listen(listener) self.sem.release() master.run() def start(self, *args, **kwargs): super(EventMasterProcess, self).start(*args, **kwargs) while True: if self.sem.acquire(False): return if not self.is_alive(): raise RuntimeError('Event master failed to start') sleep(0)
def removePeer(self, peer): flag = True if not self.__layout.has_key(peer.getPID()): return False semaphore = Semaphore() semaphore.acquire() ''' pode travar pois estou chamando um sema dentro do outro? ''' del self.__layout[peer.getPID()] flag = not self.__layout.has_key(peer.getPID()) semaphore.release() return flag
def connect(self, priority): sem = Semaphore() sem.acquire() if self.getPeer().isConnected(): return network = self.getPeer().getNetwork() node = None if netowork.countNodes() > 0: idx = randint(0, network.countNodes() - 1) graph = network.getGraph() node = graph.keys()[idx] network.addNode(self.getPeer()) if node: network.createConnection(self.getPeer(), node) self.getPeer().connected() # randon time for disconnect disconnectionTime = randint(3600, 28800) self.getPeer().setDisconnectionTime(disconnectionTime) sem.release()
class Barrier(object): def __init__(self, total = 2): self.waiting = 0 self.total = total self.waitSem = Semaphore() self.waitSem.acquire() self.mutex = Semaphore() def sync(self): self.mutex.acquire() if self.waiting == self.total - 1: self.waitSem.release() else: self.waiting += 1 self.mutex.release() self.waitSem.acquire() self.waiting -= 1 if self.waiting ==0: self.mutex.release() else: self.waitSem.release()
class TestLongFS3(TestLPBase): def MOCKED_DB_mod_file_dicom_exec(self, f): orig_isfile = os.path.isfile def mock_isfile(s, *args, **kwargs): return os.path.isfile(os.path.join(self.db_dir, "test_db")) if "stats.db" in s else orig_isfile(s) with patch("os.path.isfile", side_effect=mock_isfile), patch("pysqlcipher.dbapi2.connect", lambda s, *args, **kwargs: self.db_conn if "stats.db" in s else self.jobdb_conn): f() def call_START(self, mock_pid_ptid_mapping=MagicMock(has_key=lambda *args, **kwargs: False)): self.Locks = [Lock() for _ in range(0, 3)] self.NL_L = Semaphore(2) self.free_threads = Semaphore(5) mock_proc_inter_instance = MagicMock(free_threads=self.free_threads, Locks=self.Locks, NL_L=self.NL_L, pid_ptid_mapping=MagicMock( has_key=lambda *args, **kwargs: False) ) with patch("processing_interface.ProcessingInterface.Instance", side_effect=lambda *args, **kwargs: mock_proc_inter_instance): LP = LongProcess.LongProcess(self.test_dicoms_dirs, [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs]) self.__test_Locks() LP._LongProcess__START() self.__test_Locks() print LP._Process__success assert(LP._Process__success == "SUCCESS") assert(LP._Process__state == STATES.LongFS1) return LP def __test_Locks(self): assert(any([L.acquire(False) for L in self.Locks])) assert(self.NL_L.acquire(False) and self.NL_L.acquire(False) and not self.NL_L.acquire(False)) for L in self.Locks: L.release() self.NL_L.release() self.NL_L.release() for _ in range(0, 5): assert(self.free_threads.acquire(False)) assert(not self.free_threads.acquire(False)) for _ in range(0, 5): self.free_threads.release() def vanilla_f(self, mock_make_call, mock__print_log, message1, message2, state1, state2): with patch("FS.FreeSurfer.make_call", side_effect=mock_make_call), patch("FS.FreeSurfer._FreeSurfer__print_log", side_effect=mock__print_log), patch("path_constants.jobs", self.jobs), patch("path_constants_webinterface.jobs", self.jobs): LP = self.call_START() LP._LongProcess__LongFS3() assert(LP._Process__state == state2) assert(LP._Process__success == message2) self.__test_Locks() def test_LongFS3_vanilla(self): def mock__print_log(*args): try: args[-1].write(args[-2]) except: print args def mock_make_call(*args): return 0 f = lambda: self.vanilla_f( mock_make_call, mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) assert(len(running) == 3) assert(running[0][0] == "0810300310") def test_LongFS3_join(self): def mock__print_log(*args): args[-1].write(args[-2]) def mock_make_call(*args): # time.sleep(5) return 0 class MockProcess(multiprocessing.Process): count = [] def join(self, *args, **kwargs): self.count += [1] print "COUNT ", self.count if(len(self.count) == 2): raise RuntimeError("Mock join error") super(MockProcess, self).join(*args, **kwargs) def f(): with patch("multiprocessing.Process", MockProcess): self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS", "Error during FreeSurfer processing: Mock join error", STATES.LongFS1, STATES.QUITTING) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) print running assert(len(running) == 3) def test_LongFS3_start(self): def mock__print_log(*args): args[-1].write(args[-2]) def mock_make_call(*args): time.sleep(5) return 0 class MockProcess(multiprocessing.Process): count = [] def start(self, *args, **kwargs): self.count += [1] print self.count if(len(self.count) == 2): raise RuntimeError("Mock start error") super(MockProcess, self).start(*args, **kwargs) def f(): with patch("multiprocessing.Process", MockProcess): self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS", "Error during FreeSurfer processing: Mock start error", STATES.LongFS1, STATES.QUITTING) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) print running assert(len(running) == 3) def test_LongFS3_make_call_error(self): def mock__print_log(*args): args[-1].write(args[-2]) def mock_make_call(*args): raise RuntimeError("Mock make_call error") def f(): self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS", "Error during FreeSurfer processing: Mock make_call error.", STATES.LongFS1, STATES.QUITTING) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) print running assert(len(running) == 3) def test_LongFS3_mock_file_write(self): def mock__print_log(*args): args[-1].write(args[-2]) def mock_make_call(*args): return 0 orig_mod_file = aux_func.read_write_files.mod_file def mock_mod_file(*args, **kwargs): add = True try: tmp = args[2]([], 1) add = len(tmp) == 1 except: add = False if(add): print "%s %s %s" % (add, args[1][0], [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][0]) if(args[1][0] == [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][1] and add): raise RuntimeError("Mock mod_file error") orig_mod_file(*args, **kwargs) def f(): with patch("aux_func.read_write_files.mod_file", mock_mod_file): self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) print running assert(len(running) == 3) def test_LongFS3_mock_file_remove(self): def mock__print_log(*args): args[-1].write(args[-2]) def mock_make_call(*args): return 0 orig_mod_file = aux_func.read_write_files.mod_file def mock_mod_file(*args, **kwargs): add = True try: tmp = args[2]([], 1) add = len(tmp) != 1 except: add = False if(add): print "%s %s %s" % (add, args[1][0], [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][0]) if(args[1][0] == [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs][1] and add): raise RuntimeError("Mock mod_file error") orig_mod_file(*args, **kwargs) def f(): with patch("aux_func.read_write_files.mod_file", mock_mod_file): self.vanilla_f(mock_make_call, mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.GET_FS_ERROR3) self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) print self.jobs print running assert(len(running) != 1)
def process(self, input_filepath, msi_loci, config): self.__reset() # Generate dictionary read counts for loci counts = {} loci = [] for locus in msi_loci: counts[locus.locus()] = {} loci.append(locus) # Generate input and output queues and (mutex) semaphores # for each. queue_out = Queue() queue_in = Queue() queue_full = BoundedSemaphore(100) full = Semaphore(0) empty = BoundedSemaphore(40) mutex_out = Semaphore(1) mutex_in = Semaphore(1) # Set amount of consumer threads; minimum one. consumer_threads = config['threads'] - 1 if consumer_threads < 1: consumer_threads = 1 # Create producer thread; currently only using single thread # since I/O is more of the limiter than CPU bound processes. self.__producer = Process(target=self.extract_reads, args=( input_filepath, msi_loci, full, empty, mutex_out, queue_in, consumer_threads)) self.__producer.start() # Spawn the set amount of threads/processes if self.debug_output: tprint('Main> Generating {0} analyzer process(es).'.format(consumer_threads)) for i in range(0, consumer_threads): p = Process(target=self.read_analyzer, args=( queue_in, queue_out, full, empty, mutex_in, mutex_out, queue_full)) self.__consumers.append(p) self.__consumers[-1].start() # Iterate through the loci, fetching any reads and pushing them to # the pool of threads, collecting the output as they process it. query_delay = 0.050 # In seconds loop_counter = 0 proc_check_interval = 100 while (not queue_out.empty() or self.has_live_threads()): # Sleep for the set amount of time so the queue isn't constantly # getting hammered with queries time.sleep(query_delay) loop_counter += 1 if loop_counter % proc_check_interval is 0: # Time to check that the consumers # didn't die while the producer is still producing mutex_out.acquire() self.status_check(queue_out.qsize()) mutex_out.release() while not queue_out.empty(): # There is data on the queue to be processed; # the return from the queue should be a tuple # with (locus, repeat_count) mutex_out.acquire() result = queue_out.get() locus = result[0] repeat_count = result[1] if repeat_count >= 0: if locus not in counts: counts[locus] = {} if repeat_count not in counts[locus]: counts[locus][repeat_count] = 0 counts[locus][repeat_count] += 1 mutex_out.release() queue_full.release() if not self.has_live_threads(): # All processes should have terminated. if self.debug_output: tprint('Main> All processes complete.') break # end while loop return counts
class SatoriMasterProcess(SatoriProcess): def __init__(self, is_daemon): super(SatoriMasterProcess, self).__init__('master') self.name = 'master' self.is_daemon = is_daemon self.sem = Semaphore(0) def do_handle_signal(self, signum, frame): for process in reversed(self.started): logging.info('Terminating %s', process.name) process.terminate() # wait for each child so it can deinitialize while other processes (like event master) still exist process.join(5) process.kill() def do_run(self): logger = logging.getLogger() logger.setLevel(logging.DEBUG) for handler in logger.handlers: logger.removeHandler(handler) formatter = logging.Formatter("%(asctime)s - %(name)s - %(process)d - %(levelname)s - %(message)s") ensuredirs(os.path.dirname(settings.LOG_FILE)) file_handler = logging.FileHandler(settings.LOG_FILE) file_handler.setFormatter(formatter) logger.addHandler(file_handler) console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) logger.addHandler(console_handler) ensuredirs(os.path.dirname(settings.PID_FILE)) fp = open(settings.PID_FILE, 'a+') try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: fp.close() raise RuntimeError('Satori already running') fp.seek(0) fp.truncate() fp.write(str(os.getpid())) fp.flush() os.chdir('/') os.umask(0) os.setsid() logging.info('Loading ARS interface...') import satori.core.api logging.info('ARS interface loaded.') if self.is_daemon: logger.removeHandler(console_handler) os.close(sys.stdin.fileno()) os.open('/dev/null', os.O_RDWR) os.close(sys.stdout.fileno()) os.open('/dev/null', os.O_RDWR) os.close(sys.stderr.fileno()) os.open('/dev/null', os.O_RDWR) self.sem.release() from satori.core.management.processes import EventMasterProcess, DbevNotifierProcess, ThriftServerProcess, TwistedHttpServerProcess, UwsgiHttpServerProcess, DebugQueueProcess, CheckingMasterProcess, PrintingMasterProcess to_start = [ EventMasterProcess(), DebugQueueProcess(), DbevNotifierProcess(), CheckingMasterProcess(), PrintingMasterProcess(), ThriftServerProcess(), # choose one of the two following: TwistedHttpServerProcess(), #UwsgiHttpServerProcess(), ] self.started = [] for process in to_start: process.start() self.started.append(process) for process in reversed(self.started): process.join() fp.close() os.remove(settings.PID_FILE) def start(self, *args, **kwargs): super(SatoriMasterProcess, self).start(*args, **kwargs) while True: if self.sem.acquire(False): return if not self.is_alive(): raise RuntimeError('Satori master failed to start') sleep(0) def stop(self): fp = open(settings.PID_FILE, 'a+') try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: fp.seek(0) pid = int(fp.read()) fp.close() try: os.kill(pid, SIGINT) except OSError: pass else: fp.close() print 'Satori not running'
class TestQUITTING(TestLPBase): def MOCKED_DB_mod_file_dicom_exec(self, f): orig_isfile = os.path.isfile def mock_isfile(s, *args, **kwargs): return orig_isfile(os.path.join(self.test_installation, "stats.db")) if "stats.db" in s else orig_isfile(s) with patch("os.path.isfile", side_effect=mock_isfile), patch("path_constants.installation", self.test_installation): f() def call_START(self, mock_pid_ptid_mapping=MagicMock(has_key=lambda *args, **kwargs: False)): self.Locks = [Lock() for _ in range(0, 3)] self.NL_L = Semaphore(2) self.free_threads = Semaphore(5) mock_proc_inter_instance = MagicMock(free_threads=self.free_threads, Locks=self.Locks, NL_L=self.NL_L, pid_ptid_mapping=MagicMock( has_key=lambda *args, **kwargs: False) ) with patch("processing_interface.ProcessingInterface.Instance", side_effect=lambda *args, **kwargs: mock_proc_inter_instance): LP = LongProcess.LongProcess(self.test_dicoms_dirs, [os.path.split(dicom_dir)[-1] for dicom_dir in self.test_dicoms_dirs]) self.__test_Locks() LP._LongProcess__START() self.__test_Locks() assert(LP._Process__success == "SUCCESS") assert(LP._Process__state == STATES.LongFS1) return LP def __test_Locks(self): assert(any([L.acquire(False) for L in self.Locks])) assert(self.NL_L.acquire(False) and self.NL_L.acquire(False) and not self.NL_L.acquire(False)) for L in self.Locks: L.release() self.NL_L.release() self.NL_L.release() for _ in range(0, 5): assert(self.free_threads.acquire(False)) assert(not self.free_threads.acquire(False)) for _ in range(0, 5): self.free_threads.release() def vanilla_f(self, mock__print_log, message1, message2, state1, state2): with patch("path_constants.jobs", self.jobs), patch("path_constants_webinterface.jobs", self.jobs): LP = self.call_START() LP._LongProcess__QUITTING() print LP._Process__success assert(LP._Process__state == state2) assert(LP._Process__success == message2) self.__test_Locks() def test_QUITTING_vanilla(self): def mock__print_log(*args): try: args[-1].write(args[-2]) except: print args f = lambda: self.vanilla_f( mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.DONE) with patch("path_constants.fs_subjectsdir", self.test_fs_sub),\ patch("path_constants_webinterface.fs_subjectsdir", self.test_fs_sub), \ patch("path_constants.fs_dir", FS_DIR): self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) assert(len(running) == 0) with open(os.path.join(self.jobs, "done"), "r") as f: done = pickle.load(f) assert(len(done) == 3) for i in range(0, len(TEST_FS_RES)): assert(done[i][0] == str(2911752899 + i)) for i in range(1, len(TEST_FS_RES) + 1): assert(not os.path.isdir(os.path.join( self.test_fs_sub, (TEMPLATE_NAME + str(i) + ".long." + TEMPLATE_NAME)))) def test_QUITTING_db(self): def mock__print_log(*args): try: args[-1].write(args[-2]) except: print args f = lambda: self.vanilla_f( mock__print_log, "SUCCESS", "SUCCESS", STATES.LongFS1, STATES.DONE) with\ patch("path_constants.fs_subjectsdir", self.test_fs_sub),\ patch("path_constants_webinterface.fs_subjectsdir", self.test_fs_sub),\ patch("path_constants.fs_dir", FS_DIR): self.MOCKED_DB_mod_file_dicom_exec(f) with open(os.path.join(self.jobs, "running"), "r") as f: running = pickle.load(f) assert(len(running) == 0) with open(os.path.join(self.jobs, "done"), "r") as f: done = pickle.load(f) assert(len(done) == 3) for i in range(0, len(TEST_FS_RES)): assert(done[i][0] == str(2911752899 + i)) for i in range(1, len(TEST_FS_RES) + 1): assert(not os.path.isdir(os.path.join( self.test_fs_sub, (TEMPLATE_NAME + str(i) + ".long." + TEMPLATE_NAME)))) with sqlite3.connect(os.path.join(self.test_installation, "stats.db")) as conn: cur = conn.cursor() cur.execute("PRAGMA key='f$$pm->>>'") ret = cur.execute( "select ptid from errors group by ptid").fetchall() assert(len(ret) == 3)
def countPeers(self): semaphore = Semaphore() semaphore.acquire() tamPeers = len(self.__layout) semaphore.release() return tamPeers
class DownloadManager: """Class to start and track status of download processes.""" def __init__(self, api): self.api = api self.config = self.api.session.config self.download_progress = dict() self.download_progress['active'] = dict() self.download_progress['completed'] = list() self.download_progress['failed'] = list() self.pool = Pool(CORES_TO_USE, initializer=initialize_download, maxtasksperchild=1) self.sema = Semaphore() def start_dl(self, broadcast): """Adds a download task to the multiprocessing pool""" print("[{0}] Adding Download: {1}".format(current_datetimestring(), broadcast.title)) self.pool.apply_async(Download(broadcast).start, (), callback=self._callback_dispatcher) self.sema.acquire() self.active_downloads[broadcast.id] = broadcast self.sema.release() def review_broadcast_status(self, broadcast, download_ok): """Starts download of broadcast replay if not already gotten; or, resumes interrupted live download. Print status to console. """ old_title = broadcast.title broadcast.lock_name = False broadcast.update_info() if broadcast.isreplay and broadcast.replay_downloaded: return None elif download_ok and broadcast.islive: broadcast.dl_failures += 1 failure_message = None if broadcast.dl_failures > MAX_DOWNLOAD_ATTEMPTS: if broadcast.islive and broadcast.available: broadcast.wait_for_replay = True broadcast.dl_failures = 0 print("[{0}] Too many live resume attempts, waiting for replay: {1} {2}".format( current_datetimestring(), old_title, failure_message)) else: failure_message = "\n\tExceeded maximum download attempts " if broadcast.failure_reason is not None: failure_message += "with the following error:\n\t" + \ str(broadcast.failure_reason) elif not (broadcast.islive or broadcast.isreplay or broadcast.dl_failures == 0): failure_message = "\n\tBroadcast no longer available." elif broadcast.dl_failures > 0: print("[{0}] Resuming download (Attempt {1} of {2}): {3}".format( current_datetimestring(), broadcast.dl_failures, MAX_DOWNLOAD_ATTEMPTS, broadcast.title)) elif broadcast.isreplay and not broadcast.replay_downloaded: print("[{0}] Downloading replay of: " "{1}".format(current_datetimestring(), broadcast.title)) else: return None if failure_message is not None: print("[{0}] Failed: {1} {2}".format(current_datetimestring(), old_title, failure_message)) self.sema.acquire() self.failed_downloads.append((current_datetimestring(), broadcast)) self.sema.release() else: self.start_dl(broadcast) def _callback_dispatcher(self, results): """Unpacks callback argument and passes to appropriate cleanup method""" download_ok, broadcast = results self.sema.acquire() del self.active_downloads[broadcast.id] self.sema.release() if download_ok: print("[{0}] Completed: {1}".format(current_datetimestring(), broadcast.title)) self.sema.acquire() self.completed_downloads.append((current_datetimestring(), broadcast)) self.sema.release() else: broadcast.dl_failures += 1 self.review_broadcast_status(broadcast, download_ok) @property def status(self): """Retrieve status string for printing to console""" self.sema.acquire() active = len(self.active_downloads) complete = len(self.completed_downloads) failed = len(self.failed_downloads) self.sema.release() cur_status = "{0} active downloads, {1} completed downloads, " \ "{2} failed downloads".format(active, complete, failed) return "[{0}] {1}".format(current_datetimestring(), cur_status) @property def currently_downloading(self): """Returns list of the broadcast.title property of all active broadcast downloads""" self.sema.acquire() _ = [broadcast.title for _, broadcast in self.active_downloads.items()] self.sema.release() return _ @property def active_downloads(self): """Return dictionary of active downloads""" return self.download_progress['active'] @property def completed_downloads(self): """Return list of completed downloads""" return self.download_progress['completed'] @property def failed_downloads(self): """Return list of failed downloads""" return self.download_progress['failed']
def getCurrentSimulationTime(self): semaphore = Semaphore() semaphore.acquire() time = self.__currentSimulationTime semaphore.release() return time
class ForkingWorker(BaseWorker): ## # Overridden from BaseWorker def __init__(self, num_processes=1): # Set up sync primitives, to communicate with the spawned children self.num_processes = num_processes # This semaphore is used as a "worker pool guard" to keep the number # of spawned workers in the pool to the specified maximum (and block # the .spawn_child() call after that) self._semaphore = Semaphore(num_processes) # This array of integers represents a slot per worker and holds the # actual pids (process ids) of the worker's children. Initially, the # array-of-pids is all zeroes. When a new child is spawned, the pid # is written into the slot. WHen a child finishes, it resets its own # slot to 0 again, effectively freeing up the slot (and allowing new # children to be spawned). self._pids = Array('i', [0] * num_processes) # This array of integers also represents a slot per worker and also # holds the actual pids of the worker's children. The difference with # _pids, however, is that this array's slots don't get reset # immediately when the children end. In order for Unix subprocesses # to actually disappear from the process list (and freeing up the # memory), they need to be waitpid()'ed for by the parent process. # When each new child is spawned, it waitpid()'s for the (finished) # child that was previously in that slot before it claims the new # slot. This mainly avoids ever-growing process lists and slowly # growing the memory footprint. self._waitfor = Array('i', [0] * num_processes) # This array of booleans represent workers that are in their idle # state (i.e. they are waiting for work). During this time, it is # safe to terminate them when the user requests so. Once they start # processing work, they flip their idle state and won't be terminated # while they're still doing work. self._idle = Array('b', [False] * num_processes) def get_ident(self): return os.getpid() def spawn_child(self): """Forks and executes the job.""" # Responsible for the blocking, may be interrupted by SIGINT or # SIGTERM, the worker's main loop will catch it with Interruptable(): self._semaphore.acquire() self._fork() def wait_for_children(self): """ Wait for children to finish their execution. This function should block until all children are finished. Must be interruptable by another press of Ctrl+C, which kicks off forceful termination. """ # As soon as we can acquire all slots, we're done executing with Interruptable(): for pid in self._pids: if pid != 0: print 'waiting for pid %d to finish gracefully...' % (pid,) waitpid(pid) def terminate_idle_children(self): for slot, idle in enumerate(self._idle): pid = self._pids[slot] if idle: print '==> Killing idle pid {}'.format(pid) kill(pid, signal.SIGKILL) #os.waitpid(pid, 0) # necessary? else: print '==> Waiting for pid {} (still busy)'.format(pid) def kill_children(self): """ Force-kill all children. This function should block until all children are terminated. """ # As soon as we can acquire all slots, we're done executing for pid in self._pids: if pid != 0: print 'killing pid %d...' % (pid,) kill(pid, signal.SIGKILL) self.wait_for_children() ## # Helper methods (specific to forking workers) def _fork(self): # noqa slot = self._claim_slot() # The usual hardcore forking action child_pid = os.fork() if child_pid == 0: random.seed() # Within child try: def _mark_busy(slot): def _inner(): self._idle[slot] = False return _inner self._idle[slot] = True self.main_child(_mark_busy(slot)) finally: # Remember, we're in the child process currently. When all # work is done here, free up the current slot (by writing # a 0 in the slot position). This communicates to the parent # that the current child has died (so can safely be forgotten # about). self._pids[slot] = 0 self._semaphore.release() os._exit(0) else: # Within parent, keep track of the new child by writing its PID # into the first free slot index. self._pids[slot] = child_pid self._waitfor[slot] = child_pid def _claim_slot(self): slot = self._find_empty_slot() self._wait_for_previous_worker(slot) return slot def _find_empty_slot(self): # Select an empty slot from self._pids (the first 0 value is picked) # The implementation guarantees there will always be at least one empty slot for slot, value in enumerate(self._pids): if value == 0: return slot raise RuntimeError('This should never happen.') def _wait_for_previous_worker(self, slot): if self._waitfor[slot] > 0: os.waitpid(self._waitfor[slot], 0) self._waitfor[slot] = 0
class Scheduler(): def __init__(self): self.ready_list = [] self.last_run = None;\ self.semafore=Semaphore(1); # Add a process to the run list def add_process(self, process): #set the index as the last element of the list at the begining #RACE condition here!! ''' Say a process with priority 1 comes into the ready_list with 5 elements The ready_list priorities are as shown ready_list_priorities={10,8,8,6,4} the index=len(self.ready_list) line executes and gets the value 5. This means index =5 then say before the next line is executed, another process is added to the ready_list, its priority is 3 then ready_list_priorities={10,8,8,6,4,3} but index is still 5 so new process will be inserted here rather than at the end of the list so ready_list_priorities={10,8,8,6,4,1,3} this is incorrect we must lock this method and only allow one process to access it at a time maybe with a semaphore ''' self.semafore.acquire() index=len(self.ready_list) #find the first priority that is less than the priority of the process for i in range (len(self.ready_list)-1,-1,-1): if self.ready_list[i].priority<process.priority: index = i #add the item there self.ready_list.insert(index, process) self.semafore.release() return def remove_process(self, process): #do more shit to ensure that all things remain the way they are self.ready_list.remove(process) # Selects the process with the best priority. # If more than one have the same priority these are selected in round-robin fashion. def select_process(self): #return none if the process list is empty if len(self.ready_list)==0: return None #otherwise check to see if the lastrun process process is the same as the current running process if self.last_run == self.ready_list[0]: #make sure the list doesnt contain only one process if len(self.ready_list)>1: #check to see that here are no more processes with the same or higher priority level #because the ready_list is already sorted, we only need to check the next element and check that it is not #of equal or higher priority if(self.last_run.priority<=self.ready_list[1].priority): #if it is, remove this process from the readylist and put it back in #This has the effect of placing it behind all the processes with the same priority level as this process self.remove_process(self.last_run) self.add_process(self.last_run) #return the process that is at the head of the queue and move it to the tail of the queue self.last_run = self.ready_list[0] return self.ready_list[0] # Suspends the currently running process by sending it a STOP signal. @staticmethod def suspend(process): os.kill(process.pid, signal.SIGSTOP) # Resumes a process by sending it a CONT signal. @staticmethod def resume(process): if process.pid: # if the process has a pid it has started os.kill(process.pid, signal.SIGCONT) else: process.run() def run(self): current_process = None while True: #print('length of ready_list:', len(self.ready_list)) next_process = self.select_process() if next_process == None: # no more processes controller_write.write('terminate\n') sys.exit() if next_process != current_process: if current_process: self.suspend(current_process) current_process = next_process self.resume(current_process) time.sleep(1) # need to remove dead processes from the list try: current_process_finished = ( os.waitpid(current_process.pid, os.WNOHANG) != (0, 0) ) except ChildProcessError: current_process_finished = True if current_process_finished: print('remove process', current_process.pid, 'from ready list') self.remove_process(current_process) current_process = None
def getPeers(self): semaphore = Semaphore() semaphore.acquire() peers = self.__layout.itervalues() semaphore.release() return peers
class URLGather(object): def __init__(self, *args, **kwargs): self.url = kwargs.get("url") if not self.url: raise Exception("No URL to gather") self.max_depth = kwargs.get("depth", 1) self.workers = kwargs.get("workers", 1) self.max_errors = kwargs.get("acceptable_errors", None) self.out = kwargs.get("out", "/tmp/") if not self.out.endswith("/"): self.out += "/" self.out += "url_gather/" if not os.path.exists(self.out): os.makedirs(self.out) self.collector_file = kwargs.get("collector_file") self.collector_class = kwargs.get("collector_class") self._load_collector() self._gathered_urls = set() # initiate multiprocessing resources self._pool = Pool(self.workers) self._semaphore = Semaphore(self.workers) self._manager = Manager() self._url_children = self._manager.dict() self._url_errors = self._manager.dict() self._url_events = {} def _load_collector(self): if self.collector_file: if os.path.isfile(self.collector_file): if self.collector_class: # TODO load custom collector with open(self.collector_file, "r") as custom_code: import_code(custom_code.read(), "custom_collector", 1) m = __import__("custom_collector") self.collector = m.__getattribute__(self.collector_class) else: raise Exception("Undefined custom collector class name.") else: raise Exception("Custom collector file %s not found." % self.collector_file) else: self.collector = DefaultCollector def run(self): self._gather_url(self.url) self._pool.close() self._pool.join() if self._exceed_max_errors(): error_log = "%serror.log" % self.out with open(error_log, "w+") as f: f.write(self._url_errors) print "REACHED MAX ERRORS. SEE %s FILE TO MORE DETAILS" % error_log def task_done(self, result): self._semaphore.release() def _exceed_max_errors(self): return self.max_errors is not None and self.max_errors >= 0 and len(self._url_errors) > self.max_errors def _wait_children_in_thread(self, url): while url not in self._url_children: time.sleep(0.1) event = self._url_events.pop(url) event.set() def _wait_for_children(self, url): event = Event() self._url_events[url] = event t = Thread(target=self._wait_children_in_thread, args=(url,)) t.start() event.wait() def _gather_url(self, url, current_depth=0): if self._exceed_max_errors() or url in self._gathered_urls: return self._gathered_urls.add(url) gather_children = current_depth < self.max_depth self._semaphore.acquire() if self._exceed_max_errors(): return self._pool.apply_async( run_in_child, (url, self.collector, self._url_children, gather_children, self.out, self._url_errors), callback=self.task_done, ) # gather children links if gather_children: self._wait_for_children(url) if self._url_children[url]: for child in self._url_children[url]: self._gather_url(child, current_depth + 1)
class Barrier: """Simple reusable semaphore barrier. Python 2.6 doesn't have multiprocessing barriers so we implement this. See http://greenteapress.com/semaphores/downey08semaphores.pdf, p. 41. """ def __init__(self, n, timeout=None): self.n = n self.to = timeout self.count = Value('i', 0) self.mutex = Semaphore(1) self.turnstile1 = Semaphore(0) self.turnstile2 = Semaphore(1) def wait(self): if not self.mutex.acquire(timeout=self.to): raise BarrierTimeoutError() self.count.value += 1 if self.count.value == self.n: if not self.turnstile2.acquire(timeout=self.to): raise BarrierTimeoutError() self.turnstile1.release() self.mutex.release() if not self.turnstile1.acquire(timeout=self.to): raise BarrierTimeoutError() self.turnstile1.release() if not self.mutex.acquire(timeout=self.to): raise BarrierTimeoutError() self.count.value -= 1 if self.count.value == 0: if not self.turnstile1.acquire(timeout=self.to): raise BarrierTimeoutError() self.turnstile2.release() self.mutex.release() if not self.turnstile2.acquire(timeout=self.to): raise BarrierTimeoutError() self.turnstile2.release()