class Processes (object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - 0b111111
# '0b111111111111111111000000' (around a minute, 63 seconds)
_dispatch = {}
def __init__ (self, reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
from exabgp.configuration.environment import environment
self.highres = environment.settings().api.highres
def clean (self):
self._process = {}
self._encoder = {}
self._broken = []
self._respawning = {}
def _terminate (self, process):
self.logger.processes("Terminating process %s" % process)
try:
self._process[process].terminate()
except OSError:
# the process is most likely already dead
pass
self._process[process].wait()
del self._process[process]
def terminate (self):
for process in list(self._process):
if not self.silence:
try:
self.write(process,self._encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" % process)
self.clean()
def _start (self, process):
try:
if process in self._process:
self.logger.processes("process already running")
return
if process not in self.reactor.configuration.processes:
self.logger.processes("Can not start process, no configuration for it (anymore ?)")
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ['TERM'] = 'dumb'
run = self.reactor.configuration.processes[process].get('run','')
if run:
api = self.reactor.configuration.processes[process]['encoder']
self._encoder[process] = Response.Text(text_version) if api == 'text' else Response.JSON(json_version,self.highres)
self._process[process] = subprocess.Popen(
run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][around_now] > self.respawn_number:
self.logger.processes(
"Too many respawn for %s (%d) terminating program" % (process,self.respawn_number),
'critical'
)
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
except (subprocess.CalledProcessError,OSError,ValueError),exc:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(exc))
class Processes (object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - pow(2,6) + 1 # '0b111111111111111111000000' (around a minute, 63 seconds)
_dispatch = {}
# names = {
# Message.ID.NOTIFICATION : 'neighbor-changes',
# Message.ID.OPEN : 'receive-opens',
# Message.ID.KEEPALIVE : 'receive-keepalives',
# Message.ID.UPDATE : 'receive-updates',
# Message.ID.ROUTE_REFRESH : 'receive-refresh',
# Message.ID.OPERATIONAL : 'receive-operational',
# }
def __init__ (self,reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
from exabgp.configuration.environment import environment
self.highres = environment.settings().api.highres
def clean (self):
self._process = {}
self._api = {}
self._api_encoder = {}
self._neighbor_process = {}
self._broken = []
self._respawning = {}
def _terminate (self,process):
self.logger.processes("Terminating process %s" % process)
try:
self._process[process].terminate()
except OSError:
# the process is most likely already dead
pass
self._process[process].wait()
del self._process[process]
def terminate (self):
for process in list(self._process):
if not self.silence:
try:
self.write(process,self._api_encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" % process)
pass
self.clean()
def _start (self,process):
try:
if process in self._process:
self.logger.processes("process already running")
return
if not process in self.reactor.configuration.process:
self.logger.processes("Can not start process, no configuration for it (anymore ?)")
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ['TERM']='dumb'
run = self.reactor.configuration.process[process].get('run','')
if run:
api = self.reactor.configuration.process[process]['encoder']
self._api_encoder[process] = JSON('3.4.0',self.highres) if api == 'json' else Text('3.3.2')
self._process[process] = subprocess.Popen(run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][around_now] > self.respawn_number:
self.logger.processes("Too many respawn for %s (%d) terminating program" % (process,self.respawn_number),'critical')
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
neighbor = self.reactor.configuration.process[process]['neighbor']
self._neighbor_process.setdefault(neighbor,[]).append(process)
except (subprocess.CalledProcessError,OSError,ValueError),e:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(e))
class Processes (object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - pow(2,6) + 1 # '0b111111111111111111000000' (around a minute, 63 seconds)
_dispatch = {}
# names = {
# Message.ID.NOTIFICATION : 'neighbor-changes',
# Message.ID.OPEN : 'receive-opens',
# Message.ID.KEEPALIVE : 'receive-keepalives',
# Message.ID.UPDATE : 'receive-updates',
# Message.ID.ROUTE_REFRESH : 'receive-refresh',
# Message.ID.OPERATIONAL : 'receive-operational',
# }
def __init__ (self,reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
from exabgp.configuration.environment import environment
self.highres = environment.settings().api.highres
def clean (self):
self._process = {}
self._encoder = {}
self._events= {}
self._neighbor_process = {}
self._broken = []
self._respawning = {}
def _terminate (self,process):
self.logger.processes("Terminating process %s" % process)
try:
self._process[process].terminate()
except OSError:
# the process is most likely already dead
pass
self._process[process].wait()
del self._process[process]
def terminate (self):
for process in list(self._process):
if not self.silence:
try:
self.write(process,self._encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" % process)
pass
self.clean()
def _start (self,process):
events = self.reactor.configuration.process[process]
for event,present in events.iteritems():
if event in ('run','encoder'):
continue
if present:
self._events.setdefault(process,[]).append(event)
try:
if process in self._process:
self.logger.processes("process already running")
return
if not process in self.reactor.configuration.process:
self.logger.processes("Can not start process, no configuration for it (anymore ?)")
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ['TERM']='dumb'
run = self.reactor.configuration.process[process].get('run','')
if run:
api = self.reactor.configuration.process[process]['encoder']
self._encoder[process] = JSON('3.4.6',self.highres) if api == 'json' else Text('3.3.2')
self._process[process] = subprocess.Popen(run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][around_now] > self.respawn_number:
self.logger.processes("Too many respawn for %s (%d) terminating program" % (process,self.respawn_number),'critical')
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
neighbor = self.reactor.configuration.process[process]['neighbor']
self._neighbor_process.setdefault(neighbor,[]).append(process)
except (subprocess.CalledProcessError,OSError,ValueError),e:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(e))
class Processes(object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - 0b111111
# '0b111111111111111111000000' (around a minute, 63 seconds)
_dispatch = {}
def __init__(self, reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
def clean(self):
self._process = {}
self._encoder = {}
self._broken = []
self._respawning = {}
def _terminate(self, process):
self.logger.processes("Terminating process %s" % process)
try:
self._process[process].terminate()
except OSError:
# the process is most likely already dead
pass
self._process[process].wait()
del self._process[process]
def terminate(self):
for process in list(self._process):
if not self.silence:
try:
self.write(process, self._encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" %
process)
self.clean()
def _start(self, process):
try:
if process in self._process:
self.logger.processes("process already running")
return
if process not in self.reactor.configuration.processes:
self.logger.processes(
"Can not start process, no configuration for it (anymore ?)"
)
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ['TERM'] = 'dumb'
run = self.reactor.configuration.processes[process].get('run', '')
if run:
api = self.reactor.configuration.processes[process]['encoder']
self._encoder[process] = Response.Text(
text_version) if api == 'text' else Response.JSON(
json_version)
self._process[process] = subprocess.Popen(
run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(),
fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][
around_now] > self.respawn_number:
self.logger.processes(
"Too many respawn for %s (%d) terminating program"
% (process, self.respawn_number), 'critical')
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
except (subprocess.CalledProcessError, OSError, ValueError), exc:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(exc))
class Processes(object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - 0b111111
# '0b111111111111111111000000' (around a minute, 63 seconds)
_dispatch = {}
def __init__(self, reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
def clean(self):
self._process = {}
self._encoder = {}
self._broken = []
self._respawning = {}
def _terminate(self, process):
self.logger.processes("Terminating process %s" % process)
try:
self._process[process].terminate()
except OSError:
# the process is most likely already dead
pass
self._process[process].wait()
del self._process[process]
def terminate(self):
for process in list(self._process):
if not self.silence:
try:
self.write(process, self._encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" %
process)
self.clean()
def _start(self, process):
try:
if process in self._process:
self.logger.processes("process already running")
return
if process not in self.reactor.configuration.processes:
self.logger.processes(
"Can not start process, no configuration for it (anymore ?)"
)
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ['TERM'] = 'dumb'
run = self.reactor.configuration.processes[process].get('run', '')
if run:
api = self.reactor.configuration.processes[process]['encoder']
self._encoder[process] = Response.Text(
text_version) if api == 'text' else Response.JSON(
json_version)
self._process[process] = subprocess.Popen(
run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(),
fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][
around_now] > self.respawn_number:
self.logger.processes(
"Too many respawn for %s (%d) terminating program"
% (process, self.respawn_number), 'critical')
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
except (subprocess.CalledProcessError, OSError, ValueError) as exc:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(exc))
def start(self, restart=False):
for process in self.reactor.configuration.processes:
if restart and process in list(self._process):
self._terminate(process)
self._start(process)
for process in list(self._process):
if process not in self.reactor.configuration.processes:
self._terminate(process)
def broken(self, neighbor):
if self._broken:
for process in self.reactor.configuration.processes:
if process in self._broken:
return True
return False
def fds(self):
return [self._process[process].stdout for process in self._process]
def received(self):
consumed_data = False
buffered = {}
for process in list(self._process):
try:
proc = self._process[process]
# proc.poll returns None if the process is still fine
# -[signal], like -15, if the process was terminated
if proc.poll() is not None and self.reactor.respawn:
raise ValueError('child died')
r, _, _ = select.select([
proc.stdout,
], [], [], 0)
if r:
try:
while True:
# Calling next() on Linux and OSX works perfectly well
# but not on OpenBSD where it always raise StopIteration
# and only readline() works
raw = buffered.get(process,
'') + proc.stdout.readline()
if not raw.endswith('\n'):
buffered[process] = raw
continue
buffered[process] = ''
line = raw.rstrip()
consumed_data = True
self.logger.processes(
"Command from process %s : %s " %
(process, line))
if raw == '':
raise IOError('Child process died')
yield (process, formated(line))
except IOError as exc:
if not exc.errno or exc.errno in error.fatal:
# if the program exists we can get an IOError with errno code zero !
self.logger.processes(
"Issue with the process, terminating it and restarting it"
)
self._terminate(process)
self._start(process)
elif exc.errno in error.block:
# we often see errno.EINTR: call interrupted and
# we most likely have data, we will try to read them a the next loop iteration
pass
else:
self.logger.processes(
"unexpected errno received from forked process (%s)"
% errstr(exc))
except StopIteration:
if not consumed_data:
self.logger.processes(
"The process died, trying to respawn it")
self._terminate(process)
self._start(process)
except (subprocess.CalledProcessError, OSError, ValueError):
self.logger.processes(
"Issue with the process, terminating it and restarting it")
self._terminate(process)
self._start(process)
def write(self, process, string, neighbor=None):
# XXX: FIXME: This is potentially blocking
while True:
try:
self._process[process].stdin.write('%s\n' % string)
except IOError as exc:
self._broken.append(process)
if exc.errno == errno.EPIPE:
self._broken.append(process)
self.logger.processes(
"Issue while sending data to our helper program")
raise ProcessError()
else:
# Could it have been caused by a signal ? What to do.
self.logger.processes(
"Error received while SENDING data to helper program, retrying (%s)"
% errstr(exc))
continue
break
try:
self._process[process].stdin.flush()
except IOError as exc:
# AFAIK, the buffer should be flushed at the next attempt.
self.logger.processes(
"Error received while FLUSHING data to helper program, retrying (%s)"
% errstr(exc))
return True
def _notify(self, neighbor, event):
for process in neighbor.api[event]:
yield process
# do not do anything if silenced
# no-self-argument
def silenced(function):
def closure(self, *args):
if self.silence:
return
return function(self, *args)
return closure
# invalid-name
@silenced
def up(self, neighbor):
for process in self._notify(neighbor, 'neighbor-changes'):
self.write(process, self._encoder[process].up(neighbor), neighbor)
@silenced
def connected(self, neighbor):
for process in self._notify(neighbor, 'neighbor-changes'):
self.write(process, self._encoder[process].connected(neighbor),
neighbor)
@silenced
def down(self, neighbor, reason):
for process in self._notify(neighbor, 'neighbor-changes'):
self.write(process, self._encoder[process].down(neighbor, reason),
neighbor)
@silenced
def packets(self, neighbor, direction, category, header, body):
for process in self._notify(neighbor, '%s-packets' % direction):
self.write(
process,
self._encoder[process].packets(neighbor, direction, category,
header, body), neighbor)
@silenced
def notification(self, neighbor, direction, code, subcode, data, header,
body):
for process in self._notify(neighbor, 'neighbor-changes'):
self.write(
process,
self._encoder[process].notification(neighbor, direction, code,
subcode, data, header,
body), neighbor)
@silenced
def message(self, message_id, neighbor, direction, message, header, *body):
self._dispatch[message_id](self, neighbor, direction, message, header,
*body)
# registering message functions
# no-self-argument
def register_process(message_id, storage=_dispatch):
def closure(function):
def wrap(*args):
function(*args)
storage[message_id] = wrap
return wrap
return closure
# notifications are handled in the loop as they use different arguments
@register_process(Message.CODE.OPEN)
def _open(self, peer, direction, message, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.OPEN.SHORT)):
self.write(
process, self._encoder[process].open(peer, direction, message,
header, body), peer)
@register_process(Message.CODE.UPDATE)
def _update(self, peer, direction, update, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.UPDATE.SHORT)):
self.write(
process, self._encoder[process].update(peer, direction, update,
header, body), peer)
@register_process(Message.CODE.NOTIFICATION)
def _notification(self, peer, direction, message, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.NOTIFICATION.SHORT)):
self.write(
process,
self._encoder[process].notification(peer, direction, message,
header, body), peer)
# unused-argument, must keep the API
@register_process(Message.CODE.KEEPALIVE)
def _keepalive(self, peer, direction, keepalive, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.KEEPALIVE.SHORT)):
self.write(
process,
self._encoder[process].keepalive(peer, direction, header,
body), peer)
@register_process(Message.CODE.ROUTE_REFRESH)
def _refresh(self, peer, direction, refresh, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.ROUTE_REFRESH.SHORT)):
self.write(
process,
self._encoder[process].refresh(peer, direction, refresh,
header, body), peer)
@register_process(Message.CODE.OPERATIONAL)
def _operational(self, peer, direction, operational, header, body):
for process in self._notify(
peer, '%s-%s' % (direction, Message.CODE.OPERATIONAL.SHORT)):
self.write(
process,
self._encoder[process].operational(peer, direction,
operational.category,
operational, header, body),
peer)
class Processes(object):
# how many time can a process can respawn in the time interval
respawn_number = 5
respawn_timemask = 0xFFFFFF - pow(2, 6) + 1 # '0b111111111111111111000000' (around a minute, 63 seconds)
def __init__(self, reactor):
self.logger = Logger()
self.reactor = reactor
self.clean()
self.silence = False
def clean(self):
self._process = {}
self._api = {}
self._api_encoder = {}
self._neighbor_process = {}
self._broken = []
self._respawning = {}
def _terminate(self, process):
self.logger.processes("Terminating process %s" % process)
self._process[process].terminate()
self._process[process].wait()
del self._process[process]
def terminate(self):
for process in list(self._process):
if not self.silence:
try:
self.write(process, self._api_encoder[process].shutdown())
except ProcessError:
pass
self.silence = True
time.sleep(0.1)
for process in list(self._process):
try:
self._terminate(process)
except OSError:
# we most likely received a SIGTERM signal and our child is already dead
self.logger.processes("child process %s was already dead" % process)
pass
self.clean()
def _start(self, process):
try:
if process in self._process:
self.logger.processes("process already running")
return
if not process in self.reactor.configuration.process:
self.logger.processes("Can not start process, no configuration for it (anymore ?)")
return
# Prevent some weird termcap data to be created at the start of the PIPE
# \x1b[?1034h (no-eol) (esc)
os.environ["TERM"] = "dumb"
run = self.reactor.configuration.process[process].get("run", "")
if run:
api = self.reactor.configuration.process[process]["encoder"]
self._api_encoder[process] = JSON("2.0") if api == "json" else Text("1.0")
self._process[process] = subprocess.Popen(
run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=preexec_helper
# This flags exists for python 2.7.3 in the documentation but on on my MAC
# creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
)
fcntl.fcntl(self._process[process].stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
self.logger.processes("Forked process %s" % process)
around_now = int(time.time()) & self.respawn_timemask
if process in self._respawning:
if around_now in self._respawning[process]:
self._respawning[process][around_now] += 1
# we are respawning too fast
if self._respawning[process][around_now] > self.respawn_number:
self.logger.processes(
"Too many respawn for %s (%d) terminating program" % (process, self.respawn_number),
"critical",
)
raise ProcessError()
else:
# reset long time since last respawn
self._respawning[process] = {around_now: 1}
else:
# record respawing
self._respawning[process] = {around_now: 1}
neighbor = self.reactor.configuration.process[process]["neighbor"]
self._neighbor_process.setdefault(neighbor, []).append(process)
except (subprocess.CalledProcessError, OSError, ValueError), e:
self._broken.append(process)
self.logger.processes("Could not start process %s" % process)
self.logger.processes("reason: %s" % str(e))
