def connected (self): if not self.serving: return try: for sock in self._sockets: try: io, _ = sock.accept() except socket.error as exc: if exc.errno in error.block: continue raise AcceptError('could not accept a new connection (%s)' % errstr(exc)) try: if sock.family == socket.AF_INET: local_ip = io.getpeername()[0] # local_ip,local_port remote_ip = io.getsockname()[0] # remote_ip,remote_port elif sock.family == socket.AF_INET6: local_ip = io.getpeername()[0] # local_ip,local_port,local_flow,local_scope remote_ip = io.getsockname()[0] # remote_ip,remote_port,remote_flow,remote_scope else: raise AcceptError('unexpected address family (%d)' % sock.family) fam = self._family_AFI_map[sock.family] yield Incoming(fam,remote_ip,local_ip,io) except socket.error as exc: raise AcceptError('could not setup a new connection (%s)' % errstr(exc)) except NetworkError as exc: self.logger.network(str(exc),'critical')
def connected(self): if not self.serving: return try: for sock in self._sockets: try: io, _ = sock.accept() except socket.error, exc: if exc.errno in error.block: continue raise AcceptError( 'could not accept a new connection (%s)' % errstr(exc)) try: if sock.family == socket.AF_INET: local_ip = io.getpeername()[0] # local_ip,local_port remote_ip = io.getsockname()[ 0] # remote_ip,remote_port elif sock.family == socket.AF_INET6: local_ip = io.getpeername()[ 0] # local_ip,local_port,local_flow,local_scope remote_ip = io.getsockname( )[0] # remote_ip,remote_port,remote_flow,remote_scope else: raise AcceptError('unexpected address family (%d)' % sock.family) fam = self._family_AFI_map[sock.family] yield Incoming(fam, remote_ip, local_ip, io) except socket.error, exc: raise AcceptError('could not setup a new connection (%s)' % errstr(exc))
def _reader (self, number): # The function must not be called if it does not return with no data with a smaller size as parameter if not self.io: self.close() raise NotConnected('Trying to read on a closed TCP connection') if number == 0: yield b'' return while not self.reading(): yield b'' data = b'' reported = '' while True: try: while True: if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error on purpose') read = self.io.recv(number) if not read: self.close() self.logger.wire("%s %s lost TCP session with peer" % (self.name(),self.peer),source=self.session()) raise LostConnection('the TCP connection was closed by the remote end') data += read number -= len(read) if not number: self.logger.wire( LazyFormat( "%s %-32s RECEIVED " % ( self.name(), '%s / %s' % (self.local,self.peer) ), read ), source=self.session() ) yield data return yield b'' except socket.timeout as exc: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer),source=self.session()) raise TooSlowError('Timeout while reading data from the network (%s)' % errstr(exc)) except socket.error as exc: if exc.args[0] in error.block: message = "%s %s blocking io problem mid-way through reading a message %s, trying to complete" % (self.name(),self.peer,errstr(exc)) if message != reported: reported = message self.logger.wire(message,'debug',self.session()) yield b'' elif exc.args[0] in error.fatal: self.close() raise LostConnection('issue reading on the socket: %s' % errstr(exc)) # what error could it be ! else: self.logger.wire("%s %s undefined error reading on socket" % (self.name(),self.peer),source=self.session()) raise NetworkError('Problem while reading data from the network (%s)' % errstr(exc))
def write(self, process, string, neighbor=None): if string is None: return True # XXX: FIXME: This is potentially blocking while True: try: self._process[process].stdin.write(bytes_ascii('%s\n' % string)) except IOError as exc: self._broken.append(process) if exc.errno == errno.EPIPE: self._broken.append(process) self.logger.debug( 'issue while sending data to our helper program', 'process') raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.debug( 'error received while sending data to helper program, retrying (%s)' % errstr(exc), 'process') continue break try: self._process[process].stdin.flush() except IOError as exc: # AFAIK, the buffer should be flushed at the next attempt. self.logger.debug( 'error received while FLUSHING data to helper program, retrying (%s)' % errstr(exc), 'process') return True
def _reader (self, number): # The function must not be called if it does not return with no data with a smaller size as parameter if not self.io: self.close() raise NotConnected('Trying to read on a closed TCP conncetion') if number == 0: yield '' return while not self.reading(): yield '' data = '' reported = '' while True: try: while True: if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error in purpose') read = self.io.recv(number) if not read: self.close() self.logger.wire("%s %s lost TCP session with peer" % (self.name(),self.peer)) raise LostConnection('the TCP connection was closed by the remote end') data += read number -= len(read) if not number: self.logger.wire( LazyFormat( "%s %-32s RECEIVED " % ( self.name(), '%s / %s' % (self.local,self.peer) ), read ) ) yield data return yield '' except socket.timeout,exc: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer)) raise TooSlowError('Timeout while reading data from the network (%s)' % errstr(exc)) except socket.error,exc: if exc.args[0] in error.block: message = "%s %s blocking io problem mid-way through reading a message %s, trying to complete" % (self.name(),self.peer,errstr(exc)) if message != reported: reported = message self.logger.wire(message,'debug') yield '' elif exc.args[0] in error.fatal: self.close() raise LostConnection('issue reading on the socket: %s' % errstr(exc)) # what error could it be ! else: self.logger.wire("%s %s undefined error reading on socket" % (self.name(),self.peer)) raise NetworkError('Problem while reading data from the network (%s)' % errstr(exc))
def _reader (self,number): # The function must not be called if it does not return with no data with a smaller size as parameter if not self.io: self.close() raise NotConnected('Trying to read on a close TCP conncetion') if number == 0: yield '' return # XXX: one of the socket option is to recover the size of the buffer # XXX: we could use it to not have to put together the string with multiple reads # XXX: and get rid of the self.read_timeout option while not self.reading(): yield '' data = '' reported = '' while True: try: while True: if self._reading is None: self._reading = time.time() elif time.time() > self._reading + self.read_timeout: self.close() self.logger.wire("%s %s peer is too slow (we were told there was data on the socket but we can not read up to what should be there)" % (self.name(),self.peer)) raise TooSlowError('Waited to read for data on a socket for more than %d second(s)' % self.read_timeout) if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error in purpose') read = self.io.recv(number) if not read: self.close() self.logger.wire("%s %s lost TCP session with peer" % (self.name(),self.peer)) raise LostConnection('the TCP connection was closed by the remote end') data += read number -= len(read) if not number: self.logger.wire(LazyFormat("%s %-32s RECEIVED " % (self.name(),'%s / %s' % (self.local,self.peer)),od,read)) self._reading = None yield data return except socket.timeout,e: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer)) raise TooSlowError('Timeout while reading data from the network (%s)' % errstr(e)) except socket.error,e: if e.args[0] in error.block: message = "%s %s blocking io problem mid-way through reading a message %s, trying to complete" % (self.name(),self.peer,errstr(e)) if message != reported: reported = message self.logger.wire(message,'debug') elif e.args[0] in error.fatal: self.close() raise LostConnection('issue reading on the socket: %s' % errstr(e)) # what error could it be ! else: self.logger.wire("%s %s undefined error reading on socket" % (self.name(),self.peer)) raise NetworkError('Problem while reading data from the network (%s)' % errstr(e))
def writer(self, data): if not self.io: # XXX: FIXME: Make sure it does not hold the cleanup during the closing of the peering session yield True return while not self.writing(): yield False self.logger.debug(LazyFormat('sending TCP payload', data), self.session()) # The first while is here to setup the try/catch block once as it is very expensive while True: try: while True: if self.defensive and random.randint(0, 2): raise socket.error(errno.EAGAIN, 'raising network error on purpose') # we can not use sendall as in case of network buffer filling # it does raise and does not let you know how much was sent number = self.io.send(data) if not number: self.close() self.logger.warning( '%s %s lost TCP connection with peer' % (self.name(), self.peer), self.session()) raise LostConnection('lost the TCP connection') data = data[number:] if not data: yield True return yield False except socket.error as exc: if exc.args[0] in error.block: self.logger.debug( '%s %s blocking io problem mid-way through writing a message %s, trying to complete' % (self.name(), self.peer, errstr(exc)), self.session()) yield False elif exc.errno == errno.EPIPE: # The TCP connection is gone. self.close() raise NetworkError('Broken TCP connection') elif exc.args[0] in error.fatal: self.close() self.logger.critical( '%s %s problem sending message (%s)' % (self.name(), self.peer, errstr(exc)), self.session()) raise NetworkError( 'Problem while writing data to the network (%s)' % errstr(exc)) # what error could it be ! else: self.logger.critical( '%s %s undefined error writing on socket' % (self.name(), self.peer), self.session()) yield False
def _reader (self,number): # The function must not be called if it does not return with no data with a smaller size as parameter if not self.io: self.close() raise NotConnected('Trying to read on a close TCP conncetion') if number == 0: yield '' return # XXX: one of the socket option is to recover the size of the buffer # XXX: we could use it to not have to put together the string with multiple reads # XXX: and get rid of the self.read_timeout option while not self.reading(): yield '' data = '' while True: try: while True: if self._reading is None: self._reading = time.time() elif time.time() > self._reading + self.read_timeout: self.close() self.logger.wire("%s %s peer is too slow (we were told there was data on the socket but we can not read up to what should be there)" % (self.name(),self.peer)) raise TooSlowError('Waited to read for data on a socket for more than %d second(s)' % self.read_timeout) if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error in purpose') read = self.io.recv(number) if not read: self.close() self.logger.wire("%s %s lost TCP session with peer" % (self.name(),self.peer)) raise LostConnection('the TCP connection was closed by the remote end') data += read number -= len(read) if not number: self.logger.wire(LazyFormat("%s %-32s RECEIVED " % (self.name(),'%s / %s' % (self.local,self.peer)),od,read)) self._reading = None yield data return except socket.timeout,e: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer)) raise TooSlowError('Timeout while reading data from the network (%s)' % errstr(e)) except socket.error,e: if e.args[0] in error.block: self.logger.wire("%s %s blocking io problem mid-way through reading a message %s, trying to complete" % (self.name(),self.peer,errstr(e)),'debug') elif e.args[0] in error.fatal: self.close() raise LostConnection('issue reading on the socket: %s' % errstr(e)) # what error could it be ! else: self.logger.wire("%s %s undefined error reading on socket" % (self.name(),self.peer)) raise NetworkError('Problem while reading data from the network (%s)' % errstr(e))
def writer (self,data): if not self.io: # XXX: FIXME: Make sure it does not hold the cleanup during the closing of the peering session yield True return if not self.writing(): yield False return self.logger.wire(LazyFormat("%s %-32s SENDING " % (self.name(),'%s / %s' % (self.local,self.peer)),od,data)) # The first while is here to setup the try/catch block once as it is very expensive while True: try: while True: if self._writing is None: self._writing = time.time() elif time.time() > self._writing + self.read_timeout: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer)) raise TooSlowError('Waited to write for data on a socket for more than %d second(s)' % self.read_timeout) if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error in purpose') # we can not use sendall as in case of network buffer filling # it does raise and does not let you know how much was sent nb = self.io.send(data) if not nb: self.close() self.logger.wire("%s %s lost TCP connection with peer" % (self.name(),self.peer)) raise LostConnection('lost the TCP connection') data = data[nb:] if not data: self._writing = None yield True return yield False except socket.error,e: if e.args[0] in error.block: self.logger.wire("%s %s blocking io problem mid-way through writing a message %s, trying to complete" % (self.name(),self.peer,errstr(e)),'debug') yield False elif e.errno == errno.EPIPE: # The TCP connection is gone. self.close() raise NetworkError('Broken TCP connection') elif e.args[0] in error.fatal: self.close() self.logger.wire("%s %s problem sending message (%s)" % (self.name(),self.peer,errstr(e))) raise NetworkError('Problem while writing data to the network (%s)' % errstr(e)) # what error could it be ! else: self.logger.wire("%s %s undefined error writing on socket" % (self.name(),self.peer)) yield False
def writer (self,data): if not self.io: # XXX: FIXME: Make sure it does not hold the cleanup during the closing of the peering session yield True return if not self.writing(): yield False return self.logger.wire(LazyFormat("%s %-32s SENDING " % (self.name(),'%s / %s' % (self.local,self.peer)),od,data)) # The first while is here to setup the try/catch block once as it is very expensive while True: try: while True: if self._writing is None: self._writing = time.time() elif time.time() > self._writing + self.read_timeout: self.close() self.logger.wire("%s %s peer is too slow" % (self.name(),self.peer)) raise TooSlowError('Waited to write for data on a socket for more than %d second(s)' % self.read_timeout) if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error in purpose') # we can not use sendall as in case of network buffer filling # it does raise and does not let you know how much was sent nb = self.io.send(data) if not nb: self.close() self.logger.wire("%s %s lost TCP connection with peer" % (self.name(),self.peer)) raise LostConnection('lost the TCP connection') data = data[nb:] if not data: self._writing = None yield True return yield False except socket.error,e: if e.args[0] in error.block: self.logger.wire("%s %s blocking io problem mid-way through writing a message %s, trying to complete" % (self.name(),self.peer,errstr(e)),'debug') elif e.errno == errno.EPIPE: # The TCP connection is gone. self.close() raise NetworkError('Broken TCP connection') elif e.args[0] in error.fatal: self.close() self.logger.wire("%s %s problem sending message (%s)" % (self.name(),self.peer,errstr(e))) raise NetworkError('Problem while writing data to the network (%s)' % errstr(e)) # what error could it be ! else: self.logger.wire("%s %s undefined error writing on socket" % (self.name(),self.peer)) yield False
def writer (self, data): if not self.io: # XXX: FIXME: Make sure it does not hold the cleanup during the closing of the peering session yield True return while not self.writing(): yield False self.logger.debug(LazyFormat('sending TCP payload',data),self.session()) # The first while is here to setup the try/catch block once as it is very expensive while True: try: while True: if self.defensive and random.randint(0,2): raise socket.error(errno.EAGAIN,'raising network error on purpose') # we can not use sendall as in case of network buffer filling # it does raise and does not let you know how much was sent number = self.io.send(data) if not number: self.close() self.logger.warning('%s %s lost TCP connection with peer' % (self.name(),self.peer),self.session()) raise LostConnection('lost the TCP connection') data = data[number:] if not data: yield True return yield False except socket.error as exc: if exc.args[0] in error.block: self.logger.debug( '%s %s blocking io problem mid-way through writing a message %s, trying to complete' % ( self.name(), self.peer, errstr(exc) ), self.session() ) yield False elif exc.errno == errno.EPIPE: # The TCP connection is gone. self.close() raise NetworkError('Broken TCP connection') elif exc.args[0] in error.fatal: self.close() self.logger.critical('%s %s problem sending message (%s)' % (self.name(),self.peer,errstr(exc)),self.session()) raise NetworkError('Problem while writing data to the network (%s)' % errstr(exc)) # what error could it be ! else: self.logger.critical('%s %s undefined error writing on socket' % (self.name(),self.peer),self.session()) yield False
def connect(io, ip, port, afi, md5): try: if afi == AFI.ipv4: io.connect((ip, port)) if afi == AFI.ipv6: io.connect((ip, port, 0, 0)) except socket.error as exc: if exc.errno == errno.EINPROGRESS: return if md5: raise NotConnected( 'Could not connect to peer %s:%d, check your MD5 password (%s)' % (ip, port, errstr(exc))) raise NotConnected('Could not connect to peer %s:%d (%s)' % (ip, port, errstr(exc)))
def received (self): for process in list(self._process): try: proc = self._process[process] r,_,_ = select.select([proc.stdout,],[],[],0) if r: try: for line in proc.stdout: line = line.rstrip() if line: self.logger.processes("Command from process %s : %s " % (process,line)) yield (process,formated(line)) else: self.logger.processes("The process died, trying to respawn it") self._terminate(process) self._start(process) break except IOError,e: if e.errno == errno.EINTR: # call interrupted pass # we most likely have data, we will try to read them a the next loop iteration elif e.errno != errno.EAGAIN: # no more data self.logger.processes("unexpected errno received from forked process (%s)" % errstr(e)) except (subprocess.CalledProcessError,OSError,ValueError): self.logger.processes("Issue with the process, terminating it and restarting it") self._terminate(process) self._start(process)
def received(self): consumed_data = False for process in list(self._process): try: proc = self._process[process] poll = proc.poll() # proc.poll returns None if the process is still fine # -[signal], like -15, if the process was terminated if poll is not None: self._handle_problem(process) return r, _, _ = select.select([ proc.stdout, ], [], [], 0) if not r: continue try: # Calling next() on Linux and OSX works perfectly well # but not on OpenBSD where it always raise StopIteration # and only readline() works buf = str_ascii(proc.stdout.read(16384)) if buf == '' and poll is not None: # if proc.poll() is None then # process is fine, we received an empty line because # we're doing .readline() on a non-blocking pipe and # the process maybe has nothing to send yet self._handle_problem(process) continue raw = self._buffer.get(process, '') + buf while '\n' in raw: line, raw = raw.split('\n', 1) line = line.rstrip() consumed_data = True self.logger.debug( 'command from process %s : %s ' % (process, line), 'process') yield (process, formated(line)) self._buffer[process] = raw except IOError as exc: if not exc.errno or exc.errno in error.fatal: # if the program exits we can get an IOError with errno code zero ! self._handle_problem(process) elif exc.errno in error.block: # we often see errno.EINTR: call interrupted and # we most likely have data, we will try to read them a the next loop iteration pass else: self.logger.debug( 'unexpected errno received from forked process (%s)' % errstr(exc), 'process') except StopIteration: if not consumed_data: self._handle_problem(process) except (subprocess.CalledProcessError, OSError, ValueError): self._handle_problem(process)
def TTL (io, ip, ttl): # None (ttl-security unset) or zero (maximum TTL) is the same thing if ttl: try: io.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl) except socket.error as exc: raise TTLError('This OS does not support IP_TTL (ttl-security) for %s (%s)' % (ip,errstr(exc)))
def connected(self): if not self.serving: return try: for sock, (host, _) in self._sockets.items(): try: io, _ = sock.accept() if sock.family == socket.AF_INET: local_ip, local_port = io.getpeername() remote_ip, remote_port = io.getsockname() elif sock.family == socket.AF_INET6: local_ip, local_port, local_flow, local_scope = io.getpeername( ) remote_ip, remote_port, remote_flow, remote_scope = io.getsockname( ) else: raise AcceptError('unexpected address family (%d)' % sock.family) fam = self._family_AFI_map[sock.family] yield Incoming(fam, remote_ip, local_ip, io) break except socket.error, exc: if exc.errno in error.block: continue raise AcceptError( 'could not accept a new connection (%s)' % errstr(exc)) except NetworkError, exc: self.logger.network(str(exc), 'critical') raise exc
def connected(self): if not self.serving: return try: for sock, (host, _) in self._sockets.items(): try: io, _ = sock.accept() if sock.family == socket.AF_INET: local_ip, local_port = io.getpeername() remote_ip, remote_port = io.getsockname() elif sock.family == socket.AF_INET6: local_ip, local_port, local_flow, local_scope = io.getpeername() remote_ip, remote_port, remote_flow, remote_scope = io.getsockname() else: raise AcceptError("unexpected address family (%d)" % sock.family) fam = self._family_AFI_map[sock.family] yield Incoming(fam, remote_ip, local_ip, io) break except socket.error, e: if e.errno in error.block: continue raise AcceptError("could not accept a new connection (%s)" % errstr(e)) except NetworkError, e: self.logger.network(str(e), "critical") raise e
def received(self): lines = [] for process in list(self._process): try: proc = self._process[process] r, _, _ = select.select([ proc.stdout, ], [], [], 0) if r: try: line = proc.stdout.readline().rstrip() if line: self.logger.processes( "Command from process %s : %s " % (process, line)) lines.append((process, line)) else: self.logger.processes( "The process died, trying to respawn it") self._terminate(process) self._start(process) except IOError, e: if e.errno == errno.EINTR: # call interrupted pass # we most likely have data, we will try to read them a the next loop iteration elif e.errno != errno.EAGAIN: # no more data self.logger.processes( "unexpected errno received from forked process (%s)" % errstr(e)) except (subprocess.CalledProcessError, OSError, ValueError): self.logger.processes( "Issue with the process, terminating it and restarting it") self._terminate(process) self._start(process) return lines
def _bind (self,ip,port): try: if isipv6(ip): s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP) try: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except (socket.error,AttributeError): pass s.bind((ip,port,0,0)) elif isipv4(ip): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) try: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except (socket.error,AttributeError): pass s.bind((ip,port)) else: return None s.setblocking(0) ##s.settimeout(0.0) s.listen(self._backlog) return s except socket.error, e: if e.args[0] == errno.EADDRINUSE: raise BindingError('could not listen on %s:%d, the port already in use by another application' % (ip,self._port)) elif e.args[0] == errno.EADDRNOTAVAIL: raise BindingError('could not listen on %s:%d, this is an invalid address' % (ip,self._port)) else: raise BindingError('could not listen on %s:%d (%s)' % (ip,self._port,errstr(e)))
def received (self): for process in list(self._process): try: proc = self._process[process] # proc.poll returns None if the process is still fine # -[signal], like -15, if the process was terminated if proc.poll() is not None and self.reactor.respawn: raise ValueError('child died') r,_,_ = select.select([proc.stdout,],[],[],0) if r: try: line = proc.stdout.next().rstrip() if line: self.logger.processes("Command from process %s : %s " % (process,line)) yield (process,formated(line)) else: self.logger.processes("The process died, trying to respawn it") self._terminate(process) self._start(process) break except IOError,e: if not e.errno or e.errno in error.fatal: # if the program exists we can get an IOError with errno code zero ! self.logger.processes("Issue with the process' PIPE, terminating it and restarting it") self._terminate(process) self._start(process) elif e.errno in error.block: # we often see errno.EINTR: call interrupted and # we most likely have data, we will try to read them a the next loop iteration pass else: self.logger.processes("unexpected errno received from forked process (%s)" % errstr(e)) except StopIteration: pass
def write(self, process, string, peer=None): failure = 0 while True: try: self._process[process].stdin.write('%s\n' % string) self._process[process].stdin.flush() except IOError, exc: failure += 1 if failure >= 5: self.logger.processes( "Too many attempt to send data to helper program, aborting" ) raise ProcessError() if exc.errno in error.block: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while sending data to helper program, retrying (%s)" % errstr(exc)) continue if exc.errno == errno.EPIPE: self.logger.processes( "Issue while sending data to our helper program, it left us restarting it" ) self._terminate(process) self._start(process) continue self.logger.processes( "Fatal error on writing to PIPE, ignoring as it can not recovered" ) raise ProcessError() break
def TTL (io, ip, ttl): # None (ttl-security unset) or zero (maximum TTL) is the same thing if ttl: try: io.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl) except socket.error,exc: raise TTLError('This OS does not support IP_TTL (ttl-security) for %s (%s)' % (ip,errstr(exc)))
def TTLv6(io, ip, ttl): if ttl: try: io.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_UNICAST_HOPS, ttl) except socket.error as exc: raise TTLError( 'This OS does not support unicast_hops (ttl-security) for %s (%s)' % (ip, errstr(exc)))
def received(self): consumed_data = False buffered = {} for process in list(self._process): try: proc = self._process[process] # proc.poll returns None if the process is still fine # -[signal], like -15, if the process was terminated if proc.poll() is not None and self.reactor.respawn: raise ValueError('child died') r, _, _ = select.select([ proc.stdout, ], [], [], 0) if r: try: while True: # Calling next() on Linux and OSX works perfectly well # but not on OpenBSD where it always raise StopIteration # and only readline() works raw = buffered.get(process, '') + proc.stdout.readline() if not raw.endswith('\n'): buffered[process] = raw continue buffered[process] = '' line = raw.rstrip() consumed_data = True self.logger.processes( "Command from process %s : %s " % (process, line)) if raw == '': raise IOError('Child process died') yield (process, formated(line)) except IOError, exc: if not exc.errno or exc.errno in error.fatal: # if the program exists we can get an IOError with errno code zero ! self.logger.processes( "Issue with the process, terminating it and restarting it" ) self._terminate(process) self._start(process) elif exc.errno in error.block: # we often see errno.EINTR: call interrupted and # we most likely have data, we will try to read them a the next loop iteration pass else: self.logger.processes( "unexpected errno received from forked process (%s)" % errstr(exc)) except StopIteration: if not consumed_data: self.logger.processes( "The process died, trying to respawn it") self._terminate(process) self._start(process)
def MD5(io, ip, port, md5): if md5: os = platform.system() if os == 'FreeBSD': if md5 != 'kernel': raise MD5Error( 'FreeBSD requires that you set your MD5 key via ipsec.conf.\n' 'Something like:\n' 'flush;\n' 'add <local ip> <peer ip> tcp 0x1000 -A tcp-md5 "password";' ) try: TCP_MD5SIG = 0x10 io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, 1) except socket.error: raise MD5Error( 'FreeBSD requires that you rebuild your kernel to enable TCP MD5 Signatures:\n' 'options IPSEC\n' 'options TCP_SIGNATURE\n' 'device crypto\n') elif os == 'Linux': try: # __kernel_sockaddr_storage n_af = IP.toaf(ip) n_addr = IP.pton(ip) n_port = socket.htons(port) # pack 'x' is padding, so we want the struct # Do not use '!' for the pack, the network (big) endian switch in # struct.pack is fighting against inet_pton and htons (note the n) if IP.toafi(ip) == AFI.ipv4: # SS_MAXSIZE is 128 but addr_family, port and ipaddr (8 bytes total) are written independently of the padding SS_MAXSIZE_PADDING = 128 - calcsize('HH4s') # 8 sockaddr = pack('HH4s%dx' % SS_MAXSIZE_PADDING, socket.AF_INET, n_port, n_addr) else: SS_MAXSIZE_PADDING = 128 - calcsize('HI16sI') # 28 SIN6_FLOWINFO = 0 SIN6_SCOPE_ID = 0 sockaddr = pack('HHI16sI%dx' % SS_MAXSIZE_PADDING, n_af, n_port, SIN6_FLOWINFO, n_addr, SIN6_SCOPE_ID) TCP_MD5SIG_MAXKEYLEN = 80 key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, len(md5), md5) TCP_MD5SIG = 14 io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) except socket.error, exc: raise MD5Error( 'This linux machine does not support TCP_MD5SIG, you can not use MD5 (%s)' % errstr(exc)) else: raise MD5Error('ExaBGP has no MD5 support for %s' % os)
def received (self): consumed_data = False for process in list(self._process): try: proc = self._process[process] poll = proc.poll() # proc.poll returns None if the process is still fine # -[signal], like -15, if the process was terminated if poll is not None: self._handle_problem(process) return r,_,_ = select.select([proc.stdout,],[],[],0) if not r: continue try: # Calling next() on Linux and OSX works perfectly well # but not on OpenBSD where it always raise StopIteration # and only readline() works buf = str_ascii(proc.stdout.read(16384)) if buf == '' and poll is not None: # if proc.poll() is None then # process is fine, we received an empty line because # we're doing .readline() on a non-blocking pipe and # the process maybe has nothing to send yet self._handle_problem(process) continue raw = self._buffer.get(process,'') + buf while '\n' in raw: line,raw = raw.split('\n',1) line = line.rstrip() consumed_data = True self.logger.debug('command from process %s : %s ' % (process,line),'process') yield (process,formated(line)) self._buffer[process] = raw except IOError as exc: if not exc.errno or exc.errno in error.fatal: # if the program exits we can get an IOError with errno code zero ! self._handle_problem(process) elif exc.errno in error.block: # we often see errno.EINTR: call interrupted and # we most likely have data, we will try to read them a the next loop iteration pass else: self.logger.debug('unexpected errno received from forked process (%s)' % errstr(exc),'process') except StopIteration: if not consumed_data: self._handle_problem(process) except (subprocess.CalledProcessError,OSError,ValueError): self._handle_problem(process)
def connect (io, ip, port, afi, md5): try: if afi == AFI.ipv4: io.connect((ip,port)) if afi == AFI.ipv6: io.connect((ip,port,0,0)) except socket.error as exc: if exc.errno == errno.EINPROGRESS: return if md5: raise NotConnected('Could not connect to peer %s:%d, check your MD5 password (%s)' % (ip,port,errstr(exc))) raise NotConnected('Could not connect to peer %s:%d (%s)' % (ip,port,errstr(exc)))
def __init__ (self, afi, peer, local, io): Connection.__init__(self,afi,peer,local) self.logger.wire("Connection from %s" % self.peer) try: self.io = io async(self.io,peer) nagle(self.io,peer) except NetworkError,exc: self.close() raise NotConnected(errstr(exc))
def __init__(self, afi, peer, local, io): Connection.__init__(self, afi, peer, local) self.logger.debug('connection from %s' % self.peer, 'network') try: self.io = io async (self.io, self.peer) nagle(self.io, self.peer) self.success() except NetworkError as exc: self.close() raise NotConnected(errstr(exc))
def __init__ (self, afi, peer, local, io): Connection.__init__(self,afi,peer,local) self.logger.debug('connection from %s' % self.peer,'network') try: self.io = io asynchronous(self.io, self.peer) nagle(self.io,self.peer) self.success() except NetworkError as exc: self.close() raise NotConnected(errstr(exc))
def MD5(io, ip, port, afi, md5): if md5: os = platform.system() if os == 'FreeBSD': if md5 != 'kernel': raise MD5Error( 'FreeBSD requires that you set your MD5 key via ipsec.conf.\n' 'Something like:\n' 'flush;\n' 'add <local ip> <peer ip> tcp 0x1000 -A tcp-md5 "password";' ) try: TCP_MD5SIG = 0x10 io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, 1) except socket.error, e: raise MD5Error( 'FreeBSD requires that you rebuild your kernel to enable TCP MD5 Signatures:\n' 'options IPSEC\n' 'options TCP_SIGNATURE\n' 'device crypto\n') elif os == 'Linux': try: TCP_MD5SIG = 14 TCP_MD5SIG_MAXKEYLEN = 80 n_port = socket.htons(port) if afi == AFI.ipv4: SS_PADSIZE = 120 n_addr = socket.inet_pton(socket.AF_INET, ip) tcp_md5sig = 'HH4s%dx2xH4x%ds' % (SS_PADSIZE, TCP_MD5SIG_MAXKEYLEN) md5sig = struct.pack(tcp_md5sig, socket.AF_INET, n_port, n_addr, len(md5), md5) if afi == AFI.ipv6: SS_PADSIZE = 100 SIN6_FLOWINFO = 0 SIN6_SCOPE_ID = 0 n_addr = socket.inet_pton(socket.AF_INET6, ip) tcp_md5sig = 'HHI16sI%dx2xH4x%ds' % (SS_PADSIZE, TCP_MD5SIG_MAXKEYLEN) md5sig = struct.pack(tcp_md5sig, socket.AF_INET6, n_port, SIN6_FLOWINFO, n_addr, SIN6_SCOPE_ID, len(md5), md5) io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, md5sig) except socket.error, e: raise MD5Error( 'This linux machine does not support TCP_MD5SIG, you can not use MD5 (%s)' % errstr(e))
def received (self): consumed_data = False buffered = {} for process in list(self._process): try: proc = self._process[process] # proc.poll returns None if the process is still fine # -[signal], like -15, if the process was terminated if proc.poll() is not None and self.reactor.respawn: raise ValueError('child died') r,_,_ = select.select([proc.stdout,],[],[],0) if r: try: while True: # Calling next() on Linux and OSX works perfectly well # but not on OpenBSD where it always raise StopIteration # and only readline() works raw = buffered.get(process,'') + proc.stdout.readline() if not raw.endswith('\n'): buffered[process] = raw continue buffered[process] = '' line = raw.rstrip() consumed_data = True self.logger.processes("Command from process %s : %s " % (process,line)) if raw == '': raise IOError('Child process died') yield (process,formated(line)) except IOError,exc: if not exc.errno or exc.errno in error.fatal: # if the program exists we can get an IOError with errno code zero ! self.logger.processes("Issue with the process, terminating it and restarting it") self._terminate(process) self._start(process) elif exc.errno in error.block: # we often see errno.EINTR: call interrupted and # we most likely have data, we will try to read them a the next loop iteration pass else: self.logger.processes("unexpected errno received from forked process (%s)" % errstr(exc)) except StopIteration: if not consumed_data: self.logger.processes("The process died, trying to respawn it") self._terminate(process) self._start(process)
def write(self, process, string): while True: try: self._process[process].stdin.write('%s\r\n' % string) except IOError, e: self._broken.append(process) if e.errno == errno.EPIPE: self._broken.append(process) self.logger.processes( "Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while SENDING data to helper program, retrying (%s)" % errstr(e)) continue break
def connected (self): if not self.serving: return try: for sock,(host,_) in self._sockets.items(): try: io, _ = sock.accept() local_ip,local_port = io.getpeername() remote_ip,remote_port = io.getsockname() yield Incoming(AFI.ipv4,remote_ip,local_ip,io) break except socket.error, e: if e.errno in error.block: continue raise AcceptError('could not accept a new connection (%s)' % errstr(e)) except NetworkError,e: self.logger.network(str(e),'critical') raise e
def write(self, process, string, neighbor=None): # XXX: FIXME: This is potentially blocking while True: try: self._process[process].stdin.write('%s\n' % string) except IOError, exc: self._broken.append(process) if exc.errno == errno.EPIPE: self._broken.append(process) self.logger.processes( "Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while SENDING data to helper program, retrying (%s)" % errstr(exc)) continue break
def asynchronous(io, ip): try: io.setblocking(0) except socket.error as exc: raise AsyncError('could not set socket non-blocking for %s (%s)' % (ip,errstr(exc)))
self._broken.append(process) if e.errno == errno.EPIPE: self._broken.append(process) self.logger.processes("Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes("Error received while SENDING data to helper program, retrying (%s)" % errstr(e)) continue break try: self._process[process].stdin.flush() except IOError,e: # AFAIK, the buffer should be flushed at the next attempt. self.logger.processes("Error received while FLUSHING data to helper program, retrying (%s)" % errstr(e)) return True def _notify (self,peer,event): neighbor = peer.neighbor.peer_address for process in self._neighbor_process.get(neighbor,[]): if process in self._process: yield process for process in self._neighbor_process.get('*',[]): if process in self._process: yield process def reset (self,peer): if self.silence: return for process in self._notify(peer,'*'):
raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while SENDING data to helper program, retrying (%s)" % errstr(e)) continue break try: self._process[process].stdin.flush() except IOError, e: # AFAIK, the buffer should be flushed at the next attempt. self.logger.processes( "Error received while FLUSHING data to helper program, retrying (%s)" % errstr(e)) return True def _notify(self, neighbor, event): for process in self._neighbor_process.get(neighbor, []): if process in self._process: yield process for process in self._neighbor_process.get('*', []): if process in self._process: yield process def up(self, neighbor): if self.silence: return for process in self._notify(neighbor, 'neighbor-changes'): self.write(process, self._api_encoder[process].up(neighbor))
def _write (self, process, string, neighbor=None): if string is None: return True # XXX: FIXME: This is potentially blocking while True: try: self._process[process].stdin.write(bytes_ascii('%s\n' % string)) except IOError as exc: self._broken.append(process) if exc.errno == errno.EPIPE: self._broken.append(process) self.logger.debug('issue while sending data to our helper program','process') raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.debug('error received while sending data to helper program, retrying (%s)' % errstr(exc),'process') continue break try: self._process[process].stdin.flush() except IOError as exc: # AFAIK, the buffer should be flushed at the next attempt. self.logger.debug('error received while FLUSHING data to helper program, retrying (%s)' % errstr(exc),'process') return True
def MD5(io, ip, port, md5, md5_base64): platform_os = platform.system() if platform_os == 'FreeBSD': if md5: if md5 != 'kernel': raise MD5Error( 'FreeBSD requires that you set your MD5 key via ipsec.conf.\n' 'Something like:\n' 'flush;\n' 'add <local ip> <peer ip> tcp 0x1000 -A tcp-md5 "password";' ) try: TCP_MD5SIG = 0x10 io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, 1) except socket.error: raise MD5Error( 'FreeBSD requires that you rebuild your kernel to enable TCP MD5 Signatures:\n' 'options IPSEC\n' 'options TCP_SIGNATURE\n' 'device crypto\n') elif platform_os == 'Linux': try: md5_bytes = None if md5: if md5_base64 is True: try: md5_bytes = base64.b64decode(md5) except TypeError: raise MD5Error("Failed to decode base 64 encoded PSK") elif md5_base64 is None and not re.match('.*[^a-f0-9].*', md5): # auto options = [md5 + '==', md5 + '=', md5] for md5 in options: try: md5_bytes = base64.b64decode(md5) break except TypeError: pass # __kernel_sockaddr_storage n_af = IP.toaf(ip) n_addr = IP.pton(ip) n_port = socket.htons(port) # pack 'x' is padding, so we want the struct # Do not use '!' for the pack, the network (big) endian switch in # struct.pack is fighting against inet_pton and htons (note the n) if IP.toafi(ip) == AFI.ipv4: # SS_MAXSIZE is 128 but addr_family, port and ipaddr (8 bytes total) are written independently of the padding SS_MAXSIZE_PADDING = 128 - calcsize('HH4s') # 8 sockaddr = pack('HH4s%dx' % SS_MAXSIZE_PADDING, socket.AF_INET, n_port, n_addr) else: SS_MAXSIZE_PADDING = 128 - calcsize('HI16sI') # 28 SIN6_FLOWINFO = 0 SIN6_SCOPE_ID = 0 sockaddr = pack('HHI16sI%dx' % SS_MAXSIZE_PADDING, n_af, n_port, SIN6_FLOWINFO, n_addr, SIN6_SCOPE_ID) TCP_MD5SIG_MAXKEYLEN = 80 TCP_MD5SIG = 14 if md5_bytes: key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, len(md5_bytes), md5_bytes) io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) elif md5: md5_bytes = bytes(md5, 'ascii') key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, len(md5_bytes), md5_bytes) io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) # else: # key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, 0, b'') # io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) except socket.error as exc: if exc.errno != errno.ENOENT: raise MD5Error( 'This linux machine does not support TCP_MD5SIG, you can not use MD5 (%s)' % errstr(exc)) elif md5: raise MD5Error('ExaBGP has no MD5 support for %s' % platform_os)
def async (io, ip): try: io.setblocking(0) except socket.error as exc: raise AsyncError('could not set socket non-blocking for %s (%s)' % (ip, errstr(exc)))
raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while SENDING data to helper program, retrying (%s)" % errstr(exc)) continue break try: self._process[process].stdin.flush() except IOError, exc: # AFAIK, the buffer should be flushed at the next attempt. self.logger.processes( "Error received while FLUSHING data to helper program, retrying (%s)" % errstr(exc)) return True def _notify(self, neighbor, event): for process in neighbor.api[event]: yield process # do not do anything if silenced # no-self-argument def silenced(function): def closure(self, *args): if self.silence: return return function(self, *args)
if ttl: try: io.setsockopt(socket.IPPROTO_IP, socket.IP_MINTTL, ttl) except socket.error, exc: raise TTLError( 'This OS does not support IP_MINTTL (ttl-security) for %s (%s)' % (ip, errstr(exc))) except AttributeError: pass try: io.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl) except socket.error, exc: raise TTLError( 'This OS does not support IP_MINTTL or IP_TTL (ttl-security) for %s (%s)' % (ip, errstr(exc))) def async (io, ip): try: io.setblocking(0) except socket.error, exc: raise AsyncError('could not set socket non-blocking for %s (%s)' % (ip, errstr(exc))) def ready(io): logger = Logger() warned = False start = time.time()
self._broken.append(process) if exc.errno == errno.EPIPE: self._broken.append(process) self.logger.processes("Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes("Error received while SENDING data to helper program, retrying (%s)" % errstr(exc)) continue break try: self._process[process].stdin.flush() except IOError,exc: # AFAIK, the buffer should be flushed at the next attempt. self.logger.processes("Error received while FLUSHING data to helper program, retrying (%s)" % errstr(exc)) return True def _notify (self, neighbor, event): for process in neighbor.api[event]: yield process # do not do anything if silenced # no-self-argument def silenced (function): def closure (self, *args): if self.silence: return return function(self,*args)
def TTLv6 (io, ip, ttl): if ttl: try: io.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_UNICAST_HOPS, ttl) except socket.error as exc: raise TTLError('This OS does not support unicast_hops (ttl-security) for %s (%s)' % (ip,errstr(exc)))
def write(self, process, string): while True: try: self._process[process].stdin.write("%s\n" % string) except IOError, e: self._broken.append(process) if e.errno == errno.EPIPE: self._broken.append(process) self.logger.processes("Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes( "Error received while SENDING data to helper program, retrying (%s)" % errstr(e) ) continue break
def write (self, process, string, peer=None): failure = 0 while True: try: self._process[process].stdin.write('%s\n' % string) self._process[process].stdin.flush() except IOError,exc: failure += 1 if failure >= 5: self.logger.processes("Too many attempt to send data to helper program, aborting") raise ProcessError() if exc.errno in error.block: # Could it have been caused by a signal ? What to do. self.logger.processes("Error received while sending data to helper program, retrying (%s)" % errstr(exc)) continue if exc.errno == errno.EPIPE: self.logger.processes("Issue while sending data to our helper program, it left us restarting it") self._terminate(process) self._start(process) continue self.logger.processes("Fatal error on writing to PIPE, ignoring as it can not recovered") raise ProcessError() break
def MIN_TTL (io, ip, ttl): # None (ttl-security unset) or zero (maximum TTL) is the same thing if ttl: try: io.setsockopt(socket.IPPROTO_IP, socket.IP_MINTTL, ttl) except socket.error,exc: raise TTLError('This OS does not support IP_MINTTL (ttl-security) for %s (%s)' % (ip,errstr(exc))) except AttributeError: pass try: io.setsockopt(socket.IPPROTO_IP, socket.IP_TTL, ttl) except socket.error,exc: raise TTLError('This OS does not support IP_MINTTL or IP_TTL (ttl-security) for %s (%s)' % (ip,errstr(exc))) def async (io, ip): try: io.setblocking(0) except socket.error,exc: raise AsyncError('could not set socket non-blocking for %s (%s)' % (ip,errstr(exc))) def ready (io): logger = Logger() warned = False start = time.time() while True:
def write (self, process, string, neighbor=None): # XXX: FIXME: This is potentially blocking while True: try: self._process[process].stdin.write('%s\n' % string) except IOError,exc: self._broken.append(process) if exc.errno == errno.EPIPE: self._broken.append(process) self.logger.processes("Issue while sending data to our helper program") raise ProcessError() else: # Could it have been caused by a signal ? What to do. self.logger.processes("Error received while SENDING data to helper program, retrying (%s)" % errstr(exc)) continue break
def MD5 (io, ip, port, md5, md5_base64): platform_os = platform.system() if platform_os == 'FreeBSD': if md5: if md5 != 'kernel': raise MD5Error( 'FreeBSD requires that you set your MD5 key via ipsec.conf.\n' 'Something like:\n' 'flush;\n' 'add <local ip> <peer ip> tcp 0x1000 -A tcp-md5 "password";' ) try: TCP_MD5SIG = 0x10 io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, 1) except socket.error: raise MD5Error( 'FreeBSD requires that you rebuild your kernel to enable TCP MD5 Signatures:\n' 'options IPSEC\n' 'options TCP_SIGNATURE\n' 'device crypto\n' ) elif platform_os == 'Linux': try: if md5: md5_bytes = None if md5_base64 is True: try: md5_bytes = base64.b64decode(md5) except TypeError: raise MD5Error("Failed to decode base 64 encoded PSK") elif md5_base64 is None and not re.match('.*[^a-f0-9].*', md5): # auto options = [md5+'==', md5+'=', md5] for md5 in options: try: md5_bytes = base64.b64decode(md5) break except TypeError: pass # __kernel_sockaddr_storage n_af = IP.toaf(ip) n_addr = IP.pton(ip) n_port = socket.htons(port) # pack 'x' is padding, so we want the struct # Do not use '!' for the pack, the network (big) endian switch in # struct.pack is fighting against inet_pton and htons (note the n) if IP.toafi(ip) == AFI.ipv4: # SS_MAXSIZE is 128 but addr_family, port and ipaddr (8 bytes total) are written independently of the padding SS_MAXSIZE_PADDING = 128 - calcsize('HH4s') # 8 sockaddr = pack('HH4s%dx' % SS_MAXSIZE_PADDING, socket.AF_INET, n_port, n_addr) else: SS_MAXSIZE_PADDING = 128 - calcsize('HI16sI') # 28 SIN6_FLOWINFO = 0 SIN6_SCOPE_ID = 0 sockaddr = pack('HHI16sI%dx' % SS_MAXSIZE_PADDING, n_af, n_port, SIN6_FLOWINFO, n_addr, SIN6_SCOPE_ID) TCP_MD5SIG_MAXKEYLEN = 80 TCP_MD5SIG = 14 if md5_bytes: key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, len(md5_bytes), md5_bytes) io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) elif md5: md5_bytes = bytes_ascii(md5) key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, len(md5_bytes), md5_bytes) io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) # else: # key = pack('2xH4x%ds' % TCP_MD5SIG_MAXKEYLEN, 0, b'') # io.setsockopt(socket.IPPROTO_TCP, TCP_MD5SIG, sockaddr + key) except socket.error as exc: if exc.errno != errno.ENOENT: raise MD5Error('This linux machine does not support TCP_MD5SIG, you can not use MD5 (%s)' % errstr(exc)) elif md5: raise MD5Error('ExaBGP has no MD5 support for %s' % platform_os)