def compute(self, split): import subprocess devnull = open(os.devnull, 'w') p = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.quiet and devnull or sys.stderr, shell=self.shell) def read(stdin): try: it = self.prev.iterator(split) # fetch the first item for first in it: break else: return if isinstance(first, str) and first.endswith('\n'): stdin.write(first) stdin.writelines(it) else: stdin.write("%s\n"%first) stdin.writelines(itertools.imap(lambda x:"%s\n"%x, it)) finally: stdin.close() devnull.close() spawn(read, p.stdin) return p.stdout
def start_instances_cb(request_id, response, exception): if exception is not None: log.error("start exception: " + str(exception)) if "Rate Limit Exceeded" in str(exception): retry_list.append(request_id) elif "was not found" in str(exception): util.spawn(f"/apps/slurm/scripts/resume.py {request_id}")
def initialize(cls, is_master): FileBroadcast.initialize(is_master) def run(): sock = env.ctx.socket(zmq.REP) sock.setsockopt(zmq.LINGER, 0) port = sock.bind_to_random_port("tcp://0.0.0.0") cls.master_addr = 'tcp://%s:%d' % (cls.host, port) logger.debug("TreeBroadcast tracker started at %s", cls.master_addr) while True: uuid = sock.recv_pyobj() guide = cls.guides.get(uuid, '') if not guide: logger.warning("broadcast %s is not registered", uuid) sock.send_pyobj(guide) sock.close() logger.debug("TreeBroadcast tracker stopped") if is_master: spawn(run) while cls.master_addr is None: time.sleep(0.01) env.register('TreeBroadcastTrackerAddr', cls.master_addr) else: cls.master_addr = env.get('TreeBroadcastTrackerAddr') logger.debug("TreeBroadcast initialized")
def startServer(self): def run(): sock = env.ctx.socket(zmq.REP) sock.setsockopt(zmq.LINGER, 0) port = sock.bind_to_random_port("tcp://0.0.0.0") self.serverAddr = 'tcp://%s:%d' % (self.host,port) logger.debug("server started at %s", self.serverAddr) while True: msg = sock.recv_pyobj() if msg == SourceInfo.StopBroadcast: sock.send_pyobj(0) break id = msg sock.send_pyobj(id < len(self.blocks) and self.blocks[id] or None) sock.close() logger.debug("stop TreeBroadcast server %s", self.serverAddr) # clear cache self.cache.put(self.uuid, None) self.blocks = [] self.value = None spawn(run) while self.serverAddr is None: time.sleep(0.01) #logger.debug("server started...") self.listOfSources = [SourceInfo(self.serverAddr, self.total_blocks, self.total_bytes, self.block_size)]
def startGuide(self): def run(): sock = env.ctx.socket(zmq.REP) port = sock.bind_to_random_port("tcp://0.0.0.0") self.guide_addr = "tcp://%s:%d" % (self.host, port) logger.debug("guide start at %s", self.guide_addr) sources = {} while True: msg = sock.recv_pyobj() if msg == SourceInfo.StopBroadcast: sock.send_pyobj(0) break addr = msg # use the first one to recover if addr in sources: ssi = self.listOfSources[0] sock.send_pyobj(ssi) continue while True: ssi = self.selectSuitableSource(addr) if not ssi: self.MaxDegree += 1 ssi = self.selectSuitableSource(addr) if ssi: if not self.check_activity(ssi): ssi.failed = True else: break logger.debug("sending selected sourceinfo %s", ssi.addr) sock.send_pyobj(ssi) o = SourceInfo(addr, self.total_blocks, self.total_bytes, self.block_size) logger.debug("Adding possible new source to listOfSource: %s", o) sources[addr] = o self.listOfSources.append(o) sources[addr].parents.append(ssi) sock.close() logger.debug("Sending stop notification to %d servers ...", len(self.listOfSources)) for source_info in self.listOfSources: self.stopServer(source_info.addr) self.listOfSources = [] self.unregisterValue(self.uuid) spawn(run) # wait for guide to start while self.guide_addr is None: time.sleep(0.01) self.registerValue(self.uuid, self.guide_addr) logger.debug("guide started...")
def start_logger(self, output): sock = env.ctx.socket(zmq.PULL) port = sock.bind_to_random_port("tcp://0.0.0.0") def collect_log(): while True: line = sock.recv() output.write(line) spawn(collect_log) host = socket.gethostname() addr = "tcp://%s:%d" % (host, port) logger.debug("log collecter start at %s", addr) return addr
def start(self): spawn(self.run) while self.addr is None: time.sleep(0.01)
def __init__(self, nthreads): self.nthreads = nthreads self.requests = Queue.Queue() self.results = Queue.Queue(1) for i in range(nthreads): spawn(self._worker_thread)