def compute(self, split):
import subprocess
devnull = open(os.devnull, 'w')
p = subprocess.Popen(self.command, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=self.quiet and devnull or sys.stderr,
shell=self.shell)
def read(stdin):
try:
it = self.prev.iterator(split)
# fetch the first item
for first in it:
break
else:
return
if isinstance(first, str) and first.endswith('\n'):
stdin.write(first)
stdin.writelines(it)
else:
stdin.write("%s\n"%first)
stdin.writelines(itertools.imap(lambda x:"%s\n"%x, it))
finally:
stdin.close()
devnull.close()
spawn(read, p.stdin)
return p.stdout
def start_instances_cb(request_id, response, exception):
if exception is not None:
log.error("start exception: " + str(exception))
if "Rate Limit Exceeded" in str(exception):
retry_list.append(request_id)
elif "was not found" in str(exception):
util.spawn(f"/apps/slurm/scripts/resume.py {request_id}")
def initialize(cls, is_master):
FileBroadcast.initialize(is_master)
def run():
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
cls.master_addr = 'tcp://%s:%d' % (cls.host, port)
logger.debug("TreeBroadcast tracker started at %s",
cls.master_addr)
while True:
uuid = sock.recv_pyobj()
guide = cls.guides.get(uuid, '')
if not guide:
logger.warning("broadcast %s is not registered", uuid)
sock.send_pyobj(guide)
sock.close()
logger.debug("TreeBroadcast tracker stopped")
if is_master:
spawn(run)
while cls.master_addr is None:
time.sleep(0.01)
env.register('TreeBroadcastTrackerAddr', cls.master_addr)
else:
cls.master_addr = env.get('TreeBroadcastTrackerAddr')
logger.debug("TreeBroadcast initialized")
def startServer(self):
def run():
sock = env.ctx.socket(zmq.REP)
sock.setsockopt(zmq.LINGER, 0)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.serverAddr = 'tcp://%s:%d' % (self.host,port)
logger.debug("server started at %s", self.serverAddr)
while True:
msg = sock.recv_pyobj()
if msg == SourceInfo.StopBroadcast:
sock.send_pyobj(0)
break
id = msg
sock.send_pyobj(id < len(self.blocks) and self.blocks[id] or None)
sock.close()
logger.debug("stop TreeBroadcast server %s", self.serverAddr)
# clear cache
self.cache.put(self.uuid, None)
self.blocks = []
self.value = None
spawn(run)
while self.serverAddr is None:
time.sleep(0.01)
#logger.debug("server started...")
self.listOfSources = [SourceInfo(self.serverAddr,
self.total_blocks, self.total_bytes,
self.block_size)]
def startGuide(self):
def run():
sock = env.ctx.socket(zmq.REP)
port = sock.bind_to_random_port("tcp://0.0.0.0")
self.guide_addr = "tcp://%s:%d" % (self.host, port)
logger.debug("guide start at %s", self.guide_addr)
sources = {}
while True:
msg = sock.recv_pyobj()
if msg == SourceInfo.StopBroadcast:
sock.send_pyobj(0)
break
addr = msg
# use the first one to recover
if addr in sources:
ssi = self.listOfSources[0]
sock.send_pyobj(ssi)
continue
while True:
ssi = self.selectSuitableSource(addr)
if not ssi:
self.MaxDegree += 1
ssi = self.selectSuitableSource(addr)
if ssi:
if not self.check_activity(ssi):
ssi.failed = True
else:
break
logger.debug("sending selected sourceinfo %s", ssi.addr)
sock.send_pyobj(ssi)
o = SourceInfo(addr, self.total_blocks,
self.total_bytes, self.block_size)
logger.debug("Adding possible new source to listOfSource: %s",
o)
sources[addr] = o
self.listOfSources.append(o)
sources[addr].parents.append(ssi)
sock.close()
logger.debug("Sending stop notification to %d servers ...", len(self.listOfSources))
for source_info in self.listOfSources:
self.stopServer(source_info.addr)
self.listOfSources = []
self.unregisterValue(self.uuid)
spawn(run)
# wait for guide to start
while self.guide_addr is None:
time.sleep(0.01)
self.registerValue(self.uuid, self.guide_addr)
logger.debug("guide started...")
def start_logger(self, output):
sock = env.ctx.socket(zmq.PULL)
port = sock.bind_to_random_port("tcp://0.0.0.0")
def collect_log():
while True:
line = sock.recv()
output.write(line)
spawn(collect_log)
host = socket.gethostname()
addr = "tcp://%s:%d" % (host, port)
logger.debug("log collecter start at %s", addr)
return addr
def start(self):
spawn(self.run)
while self.addr is None:
time.sleep(0.01)
def __init__(self, nthreads):
self.nthreads = nthreads
self.requests = Queue.Queue()
self.results = Queue.Queue(1)
for i in range(nthreads):
spawn(self._worker_thread)
