def connect(self, address):
connection = tcp_client.connect(self, address)
if connection:
msg = "eserver:%d:%s:%f\n"%(self.port,self.hostname, self.emark)
pylib_io.verbout("Initial msg: "+msg+" send to "+str(address))
connection.write(msg)
return connection
def proc_read(self):
"""
Try to read a result from the connection. Return a list of
(results or empty string), with empty string indicating a
broken connection.
"""
self.last_activity = time.time()
tmp = self.connection.read()
ret = list()
for i in tmp:
if i== "":
ret.append("")
else:
if not self.initialized():
tmp = i.split(":")
try:
slave_port = int(tmp[1])
slave_name = tmp[2]
slave_emark = float(tmp[3])
self.name = slave_name
self.emark = slave_emark
pylib_io.verbout("Slave "+str(self.connection)+" initialized")
except (IndexError, ValueError):
pass
else:
try:
res = xresult(i)
ret.append(res)
del self.open_jobs[res.key()]
except (IndexError, KeyError):
pass
return ret
def announce_self(
self,
addr,
):
msg = "eserver:%d:%s:%f\n" % (self.port, self.hostname, self.emark)
pylib_io.verbout("Announcing: " + msg + " to " + str(addr))
self.udpout.sendto(msg, 0, addr)
def proc_read(self):
"""
Try to read a result from the connection. Return a list of
(results or empty string), with empty string indicating a
broken connection.
"""
self.last_activity = time.time()
tmp = self.connection.read()
ret = list()
for i in tmp:
if i == "":
ret.append("")
else:
if not self.initialized():
tmp = i.split(":")
try:
slave_port = int(tmp[1])
slave_name = tmp[2]
slave_emark = float(tmp[3])
self.name = slave_name
self.emark = slave_emark
pylib_io.verbout("Slave " + str(self.connection) +
" initialized")
except (IndexError, ValueError):
pass
else:
try:
res = xresult(i)
ret.append(res)
del self.open_jobs[res.key()]
except (IndexError, KeyError):
pass
return ret
def handle_announce(self):
(data, sender) = self.rec_sock.recvfrom(1024)
slave_addr, port = sender
if slave_addr in self.slaves:
return
if not announce_matcher.match(data):
return
slave_name = "<unknown>"
slave_emark = -1.0
tmp = data.split(":")
slave_port = int(tmp[1])
if port < 1000 or port > 65535:
return
try:
slave_name = tmp[2]
slave_emark = float(tmp[3])
except (IndexError, ValueError):
pass
try:
connection = self.client.connect((slave_addr, slave_port))
pylib_io.verbout("New slave: " + str(connection))
slave = eslave(connection, slave_addr, slave_name, slave_emark)
self.slaves[slave_addr] = slave
except (socket.timeout, socket.error):
pass
def connect(self, address):
connection = tcp_client.connect(self, address)
if connection:
msg = "eserver:%d:%s:%f\n"%(self.port,self.hostname, self.emark)
pylib_io.verbout("Initial msg: "+msg+" send to "+str(address))
connection.write(msg)
return connection
def handle_announce(self):
(data,sender) = self.rec_sock.recvfrom(1024)
slave_addr, port = sender
if slave_addr in self.slaves:
return
if not announce_matcher.match(data):
return
slave_name = "<unknown>"
slave_emark = -1.0
tmp = data.split(":")
slave_port = int(tmp[1])
if port<1000 or port > 65535:
return
try:
slave_name = tmp[2]
slave_emark = float(tmp[3])
except (IndexError, ValueError):
pass
try:
connection = self.client.connect((slave_addr, slave_port))
pylib_io.verbout("New slave: "+str(connection))
slave = eslave(connection, slave_addr, slave_name, slave_emark)
self.slaves[slave_addr] = slave
except (socket.timeout, socket.error):
pass
def check_and_report(self):
res = self.runner.get_result()
if res != None:
self.connection.write(str(res)+"\n")
pylib_io.verbout("Job"+str(self)+"done: "+str(res))
return True
else:
return False
def check_and_report(self):
res = self.runner.get_result()
if res != None:
self.connection.write(str(res)+"\n")
pylib_io.verbout("Job"+str(self)+"done: "+str(res))
return True
else:
return False
def sync(self):
"""
Safe the protocol to the associated disk file (if any and if
necessary).
"""
if self.filename and not self.synced:
pylib_io.verbout("Syncing " + self.filename)
fp = pylib_io.flexopen(self.filename, "w")
fp.write(self.__str__())
pylib_io.flexclose(fp)
self.synced = True
def sync(self):
"""
Safe the protocol to the associated disk file (if any and if
necessary).
"""
if self.filename and not self.synced:
pylib_io.verbout("Syncing "+self.filename)
fp = pylib_io.flexopen(self.filename, "w")
fp.write(self.__str__())
pylib_io.flexclose(fp)
self.synced = True
def handle_connect(self, listener):
"""
Handle a slave trying to connect.
"""
connection = listener.accept()
if not connection:
return
slave_addr = connection.peer_adr()
pylib_io.verbout("New pre-slave: "+str(connection))
slave = eslave(connection, slave_addr)
self.slaves[slave_addr] = slave
def handle_connect(self, listener):
"""
Handle a slave trying to connect.
"""
connection = listener.accept()
if not connection:
return
slave_addr = connection.peer_adr()
pylib_io.verbout("New pre-slave: " + str(connection))
slave = eslave(connection, slave_addr)
self.slaves[slave_addr] = slave
def activate_strat(self):
"""
Activate a strategy. This will fail with IndexError if no
open strategy is available either in full or in string form.
"""
name, specdir, protdir = self.strats.pop(0)
strat = pylib_eprot.estrat_task(name, self.auto_sync)
try:
strat.parse(specdir, protdir)
self.jobqueue.extend(strat.generate_jobs())
self.processing[strat.name] = strat
except IOError, problem:
pylib_io.verbout("Cannot parse "+name+": "+str(problem)+"\n");
def activate_strat(self):
"""
Activate a strategy. This will fail with IndexError if no
open strategy is available either in full or in string form.
"""
name, specdir, protdir = self.strats.pop(0)
strat = pylib_eprot.estrat_task(name, self.auto_sync)
try:
strat.parse(specdir, protdir)
self.jobqueue.extend(strat.generate_jobs())
self.processing[strat.name] = strat
except IOError, problem:
pylib_io.verbout("Cannot parse " + name + ": " + str(problem) +
"\n")
def __init__(self, port):
self.udpout = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
inst = None
while port < 65536:
try:
self.port = port
self.listen_sock.bind(("", port))
self.listen_sock.listen(5)
pylib_io.verbout("TCP Server listening on port "+str(self.port))
return
except socket.error, inst:
port = port+1
def __init__(self, port):
self.udpout = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
inst = None
while port < 65536:
try:
self.port = port
self.listen_sock.bind(("", port))
self.listen_sock.listen(5)
pylib_io.verbout("TCP Server listening on port "+str(self.port))
return
except socket.error, inst:
port = port+1
def process(self):
announce_timer = pylib_generic.timer(0)
check_load_timer = pylib_generic.timer(0)
load_ok = True
while True:
if announce_timer.expired():
if self.config.mode == "announce":
self.announce()
elif self.config.mode == "connect":
self.connect()
announce_timer.set(10.0)
if check_load_timer.expired():
load_ok = self.check_load()
check_load_timer.set(30.0)
ractive = [self.server]+self.running+\
[i for i in self.connections if i.readable()]
wactive = [i for i in self.connections if i.sendable()]
ready = select.select(ractive, wactive, [], 1)
for i in ready[0]:
if i == self.server:
new_conn = self.server.accept(pylib_tcp.jobend_re)
pylib_io.verbout("New connection " + str(new_conn))
self.connections.append(new_conn)
if i in self.connections:
commands = i.read()
for command in commands:
if command == "":
pylib_io.verbout("Connection " + str(i) +
" terminated")
self.connections.remove(i)
else:
self.dispatch(i, command)
if i in self.running:
if i.check_and_report():
self.running.remove(i)
for i in ready[1]:
i.send()
if len(self.running) < self.max_jobs() and load_ok:
self.run_job()
def dispatch(self, connection, command):
cl = command.split("\n")
cl = pylib_io.clean_list(cl)
pylib_io.verbout(cl[0])
if cl:
if cl[0] == "run":
self.create_job(connection, cl)
elif cl[0] == "ls":
self.list_state(connection)
elif cl[0] == "restart":
pylib_io.closerange(sys.stderr.fileno()+1, 10000)
os.execv(sys.argv[0], sys.argv)
elif cl[0] == "version":
connection.write(service+" "+version+"\n")
else:
connection.write("Error: Unkown command "+cl[0]+".\n")
def dispatch(self, connection, command):
cl = command.split("\n")
cl = pylib_io.clean_list(cl)
pylib_io.verbout(cl[0])
if cl:
if cl[0] == "run":
self.create_job(connection, cl)
elif cl[0] == "ls":
self.list_state(connection)
elif cl[0] == "restart":
pylib_io.closerange(sys.stderr.fileno()+1, 10000)
os.execv(sys.argv[0], sys.argv)
elif cl[0] == "version":
connection.write(service+" "+version+"\n")
else:
connection.write("Error: Unkown command "+cl[0]+".\n")
def process(self):
announce_timer = pylib_generic.timer(0)
check_load_timer = pylib_generic.timer(0)
load_ok = True
while True:
if announce_timer.expired():
if self.config.mode == "announce":
self.announce()
elif self.config.mode == "connect":
self.connect()
announce_timer.set(10.0)
if check_load_timer.expired():
load_ok = self.check_load()
check_load_timer.set(30.0)
ractive = [self.server]+self.running+\
[i for i in self.connections if i.readable()]
wactive = [i for i in self.connections if i.sendable()]
ready = select.select(ractive, wactive, [], 1)
for i in ready[0]:
if i == self.server:
new_conn = self.server.accept(pylib_tcp.jobend_re)
pylib_io.verbout("New connection "+str(new_conn))
self.connections.append(new_conn)
if i in self.connections:
commands = i.read()
for command in commands:
if command == "":
pylib_io.verbout("Connection "+str(i)+" terminated")
self.connections.remove(i)
else:
self.dispatch(i, command)
if i in self.running:
if i.check_and_report():
self.running.remove(i)
for i in ready[1]:
i.send()
if len(self.running) < self.max_jobs() and load_ok:
self.run_job()
def eserver_get_reply(address, command):
"""
Connect to an eserver, issue a command, and return the first
complete reply (or None) if the connection fails.
"""
try:
conn = pylib_tcp.tcp_client().connect(address)
except (socket.error, socket.timeout):
pylib_io.verbout("No running server found.")
return None
conn.write(command)
while conn.sendable():
conn.send()
res = []
while not res:
res = conn.read()
conn.close()
return res[0]
def eserver_get_reply(address, command):
"""
Connect to an eserver, issue a command, and return the first
complete reply (or None) if the connection fails.
"""
try:
conn = pylib_tcp.tcp_client().connect(address)
except (socket.error, socket.timeout):
pylib_io.verbout("No running server found.")
return None
conn.write(command)
while conn.sendable():
conn.send()
res = []
while not res:
res = conn.read()
conn.close()
return res[0]
def process_complete_jobs(decoder, stratset, job_db, resdir = ""):
"""
Process all jobs in resdir. Store the results in stratset, move
processed complete files to donedir (if this argument is given).
"""
os.chdir(resdir)
names = os.listdir(".")
for job in names:
if ejobname_re.match(job):
pylib_io.verbout("Parsing "+job);
res = decode_results(decoder, stratset, job)
if res:
pylib_io.verbout("Found "+job+" complete");
job_db.del_entry(job)
joberr = job[:-4]+".err"
try:
os.unlink(job)
os.unlink(joberr)
except OSError:
pass
def create_job(self, connection, command_list):
job = waiting_job(connection, command_list)
pylib_io.verbout("Creating job"+str(job))
self.jobs.append(job)
def announce_self(self, addr, ):
msg = "eserver:%d:%s:%f\n"%(self.port,self.hostname, self.emark)
pylib_io.verbout("Announcing: "+msg+" to "+str(addr))
self.udpout.sendto(msg ,0, addr)
def create_job(self, connection, command_list):
job = waiting_job(connection, command_list)
pylib_io.verbout("Creating job"+str(job))
self.jobs.append(job)
def add_job(self, job):
self.open_jobs[job.key()] = job
pylib_io.verbout("Adding:"+str(job))
self.connection.write(str(job))
def sneak_strat(self, strat):
self.strats.sneak_strat(strat, self.config.specdir, self.config.protdir)
pylib_io.verbout("New job (%d open):"%(len(self.strats.strats),)+str(strat))
def add_job(self, job):
self.open_jobs[job.key()] = job
pylib_io.verbout("Adding:" + str(job))
self.connection.write(str(job))
def sneak_strat(self, strat):
self.strats.sneak_strat(strat, self.config.specdir,
self.config.protdir)
pylib_io.verbout("New job (%d open):" % (len(self.strats.strats), ) +
str(strat))
