def connect(self):
super(OutboundEventSocket, self).connect()
# Starts event handler for this client/session.
self.start_event_handler()
# Sends connect and sets timeout while connecting.
timer = Timeout(self.transport.get_connect_timeout())
timer.start()
try:
connect_response = self._protocol_send("connect")
if not connect_response.is_success():
self.disconnect()
raise ConnectError("Error while connecting")
except Timeout:
self.disconnect()
raise ConnectError("Timeout connecting")
finally:
timer.cancel()
# Sets channel and channel unique id from this event
self._channel = connect_response
self._uuid = connect_response.get_header("Unique-ID")
# Set connected flag to True
self.connected = True
# Sets event filter or raises ConnectError
if self._filter:
if self._is_eventjson:
filter_response = self.eventjson(self._filter)
else:
filter_response = self.eventplain(self._filter)
if not filter_response.is_success():
self.disconnect()
raise ConnectError("Event filter failure")
def connect(self):
super(OutboundEventSocket, self).connect()
# Starts event handler for this client/session.
self.start_event_handler()
# Sends connect and sets timeout while connecting.
timer = Timeout(self.transport.get_connect_timeout())
timer.start()
try:
connect_response = self._protocol_send("connect")
if not connect_response.is_success():
raise ConnectError("Error while connecting")
except Timeout:
raise ConnectError("Timeout connecting")
finally:
timer.cancel()
# Sets channel and channel unique id from this event
self._channel = connect_response
self._uuid = connect_response.get_header("Unique-ID")
# Set connected flag to True
self.connected = True
# Sets event filter or raises ConnectError
if self._filter:
if self._is_eventjson:
self.trace("using eventjson")
filter_response = self.eventjson(self._filter)
else:
self.trace("using eventplain")
filter_response = self.eventplain(self._filter)
if not filter_response.is_success():
raise ConnectError("Event filter failure")
def _handle_request(self, listener_name, sock, addr):
client = TFileObjectTransport(sock.makefile())
itrans = self.tfactory.getTransport(client)
otrans = self.tfactory.getTransport(client)
iprot = self.pfactory.getProtocol(itrans)
oprot = self.pfactory.getProtocol(otrans)
try:
while True:
(name, type, seqid) = iprot.readMessageBegin()
request_start = time.time()
try:
timeout_con = Timeout(self.cfg.timeout, Timeout)
timeout_con.start()
if name not in self.wsgi._processMap:
iprot.skip(TType.STRUCT)
iprot.readMessageEnd()
x = TApplicationException(
TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name))
oprot.writeMessageBegin(
name, TMessageType.EXCEPTION, seqid)
x.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
raise ThriftFuncNotFound
else:
self.wsgi._processMap[name](self.wsgi, seqid, iprot, oprot)
except ThriftFuncNotFound, ex:
self.log.error("Unknown function %s" % (name))
self.log.access(
addr, name, "FUNC_NOT_FOUND", time.time() - request_start)
except Timeout, ex:
self.log.error("A greenlet process timeout.")
self.log.access(
addr, name, "TIMEOUT", time.time() - request_start)
def run_node_shutdown(self):
results = []
failed_results = []
shutdown_count = math.floor(len(self.running_nodes)*self.shutdown_fraction)
self.failed_nodes = set()
while len(self.shutdown_nodes) + len(self.failed_nodes) < shutdown_count:
address = random.choice(self.running_nodes)
timeout = Timeout(self.shutdown_timeout, TestTimeout())
timeout.start()
try:
TestClient(address).shutdown()
results.append(TextTestResult(self.get_node_name(address), 'Shutdown signal sent.'))
self.running_nodes.remove(address)
self.shutdown_nodes.append(address)
l.info('Shutdown node ' + address)
except TestTimeout:
self.failed_nodes.add(address)
failed_results.append(TextTestResult(self.get_node_name(address), 'Node timed-out when signaled to shutdown.'))
l.exception('Shutdown node ' + address)
except KeyValueError, error:
self.failed_nodes.add(address)
failed_results.append(TextTestResult(self.get_node_name(address), 'Node returned error when signaled to shutdown. ' + str(error)))
l.exception('Shutdown node ' + address)
except Exception, error:
self.failed_nodes.add(address)
failed_results.append(TextTestResult(self.get_node_name(address), 'Node does not want to shutdown. ' + str(error)))
l.exception('Shutdown node ' + address)
def __call__(self, *args):
timer = Timeout(self.timeout)
timer.start()
try:
result = self.function(*args)
finally:
timer.cancel()
return result
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Timeout):
bad_clusters.append(cluster)
finally:
t.cancel()
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Timeout):
pass
finally:
t.cancel()
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instancesall.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Timeout):
bad_clusters.append(cluster)
finally:
t.cancel()
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Timeout):
pass
finally:
t.cancel()
def close(self):
"""
Send a *bye* message to the :class:`AnnounceServer` and disconnect.
"""
timeout = Timeout(1)
timeout.start()
try:
self._announce.bye(self)
except Timeout:
pass
finally:
timeout.cancel()
self._announce.close()
self._mailbox.stop()
[gevent.kill(g) for g in (self._greenlet, self._greenlet_ann_sub) if g]
def _wait_auth_request(self):
'''
Waits until auth/request event is received.
'''
# Sets timeout to wait for auth/request
timer = Timeout(self.transport.get_connect_timeout())
timer.start()
try:
# When auth/request is received,
# _auth_request method will wake up async result
# so we will just wait this event here.
return self._wait_auth_event.get()
except Timeout:
raise ConnectError("Timeout waiting auth/request")
finally:
timer.cancel()
def _wait_auth_request(self):
'''
Waits until auth/request event is received.
'''
# Sets timeout to wait for auth/request
timer = Timeout(self.transport.get_connect_timeout())
timer.start()
try:
# When auth/request is received,
# _authRequest method in EventSocket will push this event to queue
# so we will just wait this event here.
return self._response_queue.get()
except Timeout:
raise ConnectError("Timeout waiting auth/request")
finally:
timer.cancel()
def _run_queue_item(self, queue_item):
temp_file = os.path.join(self.downloader._tmp_directory,
queue_item.filename)
try:
self._logger.debug("begin download")
timeout = Timeout(self.downloader.timeout)
timeout.start()
try:
self._download_file(temp_file, queue_item)
finally:
timeout.cancel()
self._unpack_file(temp_file, queue_item)
except NotFound:
self._handle_notfound(queue_item)
except Timeout:
self._handle_timeout(queue_item)
except BadFile as e:
abort = self._handle_badfile(queue_item)
if abort:
raise # thereby killing the whole download
except (gevent.socket.error, ftplib.Error):
self._handle_ioerror(queue_item)
except (greenlet.GreenletExit, KeyboardInterrupt, SystemExit):
raise
else:
self._server_sleep_backoff = 0
# unfortunately gevent doesn't have JoinablePriorityQueues
self.downloader._file_complete()
finally:
try:
os.unlink(temp_file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def _run_queue_item(self, queue_item):
temp_file = os.path.join(self.downloader._tmp_directory,
queue_item.filename)
try:
self._logger.debug("begin download")
timeout = Timeout(self.downloader.timeout)
timeout.start()
try:
self._download_file(temp_file, queue_item)
finally:
timeout.cancel()
self._unpack_file(temp_file, queue_item)
except NotFound:
self._handle_notfound(queue_item)
except Timeout:
self._handle_timeout(queue_item)
except BadFile as e:
abort = self._handle_badfile(queue_item)
if abort:
raise # thereby killing the whole download
except (gevent.socket.error, ftplib.Error):
self._handle_ioerror(queue_item)
except (greenlet.GreenletExit, KeyboardInterrupt, SystemExit):
raise
else:
self._server_sleep_backoff = 0
# unfortunately gevent doesn't have JoinablePriorityQueues
self.downloader._file_complete()
finally:
try:
os.unlink(temp_file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def test_down_nodes(self):
results = []
for node in self.shutdown_nodes:
timeout = Timeout(self.shutdown_timeout)
timeout.start()
try:
TestClient(node).put('test', self.get_value('test'))
results.append(TextTestResult(self.get_node_name(node), 'Node still responding to requests.'))
except:
pass
finally:
timeout.cancel()
if len(results) > 0:
self.results.append(ClusteredTestResult(
map(self.get_node_name, self.shutdown_nodes),
results
))
else:
self.results.append(TextTestResult(
'Check shutdown nodes',
'All shutdown nodes not responding to requests.'
))
def _handle_request(self, listener_name, sock, addr):
client = TFileObjectTransport(sock.makefile())
itrans = self.tfactory.getTransport(client)
otrans = self.tfactory.getTransport(client)
iprot = self.pfactory.getProtocol(itrans)
oprot = self.pfactory.getProtocol(otrans)
try:
while True:
(name, type, seqid) = iprot.readMessageBegin()
request_start = time.time()
try:
timeout_con = Timeout(self.cfg.timeout, Timeout)
timeout_con.start()
if name not in self.wsgi._processMap:
iprot.skip(TType.STRUCT)
iprot.readMessageEnd()
x = TApplicationException(
TApplicationException.UNKNOWN_METHOD, "Unknown function %s" % (name))
oprot.writeMessageBegin(
name, TMessageType.EXCEPTION, seqid)
x.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
raise ThriftFuncNotFound
else:
self.wsgi._processMap[name](self.wsgi, seqid, iprot, oprot)
except ThriftFuncNotFound, ex:
self.log.error("Unknown function %s" % (name))
self.log.access(
addr, name, "FUNC_NOT_FOUND", time.time() - request_start)
break
except Timeout, ex:
self.log.error("A greenlet process timeout.")
self.log.access(
addr, name, "TIMEOUT", time.time() - request_start)
break
filedesc:
@author: serg
"""
from gevent.hub import Waiter, get_hub
from gevent import core
import time
from gevent.timeout import Timeout
import gevent
from timeout import TimeoutException
from gevent import monkey
monkey.patch_all()
seconds = 3
timeout = Timeout(3)
timeout.start()
# timeout = Timeout.start_new(seconds)
try:
print 'sleep'
# gevent.sleep(10)
time.sleep(10)
# exception will be raised here, after *seconds* passed since start() call
finally:
timeout.cancel()
# Timeout(0.1).start()
# time.sleep(1)
# # gevent.sleep(0.2)
# print 'finish'
class SAPRouterServerHandler(Loggeable, SAPNIServerHandler):
@property
def hostname(self):
return self.config.get("hostname", "sapnw702")
@property
def release(self):
return self.config.get("release", 721)
@property
def router_version(self):
return self.config.get("router_version", 40)
@property
def router_version_patch(self):
return self.config.get("router_version_patch", 4)
@property
def info_password(self):
return self.config.get("info_password", None)
@property
def external_admin(self):
return self.config.get("external_admin", False)
@property
def timeout(self):
return self.config.get("timeout", 5)
@property
def route_table_filename(self):
return self.config.get("route_table_filename", "saprouttab")
@property
def route_table_working_directory(self):
return self.config.get("route_table_working_directory", "/usr/sap/router")
@property
def time_started(self):
return self.server.time_started
@property
def pid(self):
return self.server.pid
@property
def parent_pid(self):
return self.config.get("parent_pid", 0)
@property
def parent_port(self):
return self.config.get("parent_pid", 0)
def __init__(self, request, client_address, server):
"""Initialization"""
self.config = server.config
client_ip, client_port = client_address
server_ip, server_port = server.server_address
self.session = server.session_manager.get_session("saprouter",
client_ip,
client_port,
server_ip,
server_port)
SAPNIServerHandler.__init__(self, request, client_address, server)
def setup(self):
"""Add the client to the current client lists"""
SAPNIServerHandler.setup(self)
self.server.clients_count += 1
self.server.clients[self.client_address].id = self.server.clients_count
self.server.clients[self.client_address].address = self.client_address[0]
self.server.clients[self.client_address].connected_on = datetime.today()
def finish(self):
"""Closes the connection and deletes the client from the clients list"""
self.close()
SAPNIServerHandler.finish(self)
def handle(self):
"""Handle data from the client. Treat timeouts inside the handle method"""
# Set the timeout
self._timeout = Timeout(self.timeout)
self._timeout.start()
# Try to handle the request
try:
while not self.closed.is_set():
# Check if the current client was already routed to a target service
if self.server.clients[self.client_address].routed:
# In that case we should treat the request as routed,
# and do not handle further packets from this client
self.handle_routed()
break
else:
# Otherwise, we should expect for a route request within the timeout
# defined. Receive and store the packet
self.packet = self.request.recv()
# Pass the control to the handle_data function
self.handle_data()
except Timeout as t:
# If this is another timeout, raise it so another block can
# catch it
if t is not self._timeout:
raise t
self.handle_timeout()
finally:
self._timeout.cancel()
def handle_data(self):
"""Handles a received packet"""
self.session.add_event("Received packet", request=str(self.packet))
if SAPRouter not in self.packet or not router_is_known_type(self.packet):
self.logger.debug("Invalid packet sent to SAPRouter")
router = self.packet[SAPRouter]
if router_is_route(router):
return self.handle_route(router)
elif router_is_control(router):
return self.handle_control(router)
elif router_is_admin(router):
return self.handle_admin(router)
def handle_routed(self):
"""Handles a packet for an already routed client."""
self.logger.debug("Handling routed message")
# We create a new raw StreamSocket for passing it to the virtual
# service. The SAP router service should take care of the NI layer
# and perform the reassembling, keep alive, etc.
stream_socket = StreamSocket(self.request.ins)
# Now handle the virtual service, from now on the virtual service
# would take care of this client
self.server.clients[self.client_address].target_service.handle_virtual(stream_socket,
self.client_address)
def handle_route(self, pkt):
"""Handles route messages"""
self.logger.debug("Handling route message")
# Perform some checks on the route request
if self.check_route(pkt):
# Route the request
self.route_request(pkt)
def check_route(self, pkt):
"""Checks if a route request is valid.
"""
# Check the route NI version
if pkt.route_ni_version > self.router_version:
self.logger.debug("Route request version greater")
# TODO: Check if we need to return an error
# Cehck the number of routes
if len(pkt.route_string) <= 0:
self.logger.debug("Invalid number of routes in route request")
# TODO: Check if we need to return an error
# Check the number of entries
if pkt.route_entries < 2 and pkt.route_entries != len(pkt.route_string):
self.logger.debug("Invalid number of entries in route request")
# TODO: Check if we need to return an error
# Check the number of remaining entries
if pkt.route_rest_nodes >= pkt.route_entries:
self.logger.debug("Invalid route rest nodes number")
# TODO: Check if we need to return an error
# Check the offset value against the length
if pkt.route_offset >= pkt.route_length:
self.logger.debug("Invalid route string offset")
# TODO: Check if we need to return an error
# Check the offset value against the remaining hops
actual_offset = sum([len(x) for x in pkt.route_string[:pkt.route_rest_nodes]])
if pkt.route_offset != actual_offset:
self.logger.debug("Invalid route string offset")
# TODO: Check if we need to return an error
# Check that the first hop is the SAP Router
first_hop = pkt.route_string[0]
if first_hop.hostname != self.server.listener_address or \
first_hop.port != self.server.listener_port:
self.logger.debug("Invalid first hop in route string")
# TODO: Check if we need to return an error
return True
def route_request(self, pkt):
"""Perform a lookup on the route table and routes the packet accordingly
if allowed.
"""
route_string = pkt.route_string[pkt.route_rest_nodes]
(action, talk_mode, password) = self.server.route_table.lookup_target(route_string.hostname,
int(route_string.port))
if action == RouteTable.ROUTE_DENY:
self.logger.debug("Route to %s:%s denied" % (route_string.hostname,
route_string.port))
self.return_error(return_code=-94,
error="%s: route permission denied (%s to %s, %s)" % (self.hostname,
self.server.listener_address,
route_string.hostname,
route_string.port))
return
elif talk_mode != RouteTable.MODE_ANY and talk_mode != pkt.route_talk_mode:
self.logger.debug("Talk mode (%d) to %s:%s denied" % (pkt.route_talk_mode,
route_string.hostname,
route_string.port))
self.return_error(return_code=2) # TODO: Return the proper error
return
elif action == RouteTable.ROUTE_ALLOW:
if password:
if password == route_string.password:
self.logger.debug("Valid password for route to %s:%s" % (route_string.hostname,
route_string.port))
self.session.add_event("Route request allowed, valid password", data={"target_host": route_string.hostname,
"target_port": route_string.port,
"password": route_string.password},
request=str(pkt))
else:
self.logger.debug("Invalid password for route to %s:%s" % (route_string.hostname,
route_string.port))
self.session.add_event("Route request allowed, invalid password", data={"target_host": route_string.hostname,
"target_port": route_string.port,
"password": route_string.password},
request=str(pkt))
self.return_error(return_code=3) # TODO: Return the proper error
return
else:
self.logger.debug("Route request allowed to %s:%s" % (route_string.hostname,
route_string.port))
self.session.add_event("Route request allowed", data={"target": route_string.hostname,
"port": route_string.port,
"password": route_string.password},
request=str(pkt))
# The route is accepted, now look the service for the target address/port
# and register it as routed
service = self.server.service_manager.find_service_by_address(route_string.hostname,
int(route_string.port))
# If the service wasn't found, we should return a timeout message,
# meaning that the SAP Router tried to connect to the target service
# but it didn't responded
if service is None:
self.logger.debug("Target service %s:%s not available" % (route_string.hostname,
route_string.port))
self.session.add_event("Target service not available", data={"target": route_string.hostname,
"port": route_string.port,
"password": route_string.password},
request=str(pkt))
else:
self.logger.debug("Target service %s:%s found, registering and routing" % (route_string.hostname,
route_string.port))
# First cancel the timeout as a valid route was specified
self._timeout.cancel()
# Register the current client as routed and set the target
# address, port and service
self.server.clients[self.client_address].routed = True
self.server.clients[self.client_address].connected = True
self.server.clients[self.client_address].target_service = service
self.server.clients[self.client_address].talk_mode = pkt.route_talk_mode
self.server.clients[self.client_address].partner = route_string.hostname
self.server.clients[self.client_address].service = int(route_string.port)
# Send a PONG message to notify the client the route was accepted
self.request.send(SAPRouter(type=SAPRouter.SAPROUTER_PONG))
def handle_control(self, pkt):
"""Handles control messages"""
opcode_str = router_control_opcodes[pkt.opcode] if pkt.opcode in router_control_opcodes else "unknown"
self.logger.debug("Handling control message, opcode %d (%s)",
pkt.opcode, opcode_str)
# Version request
if pkt.opcode == 1:
self.logger.debug("Received version request (client version %d)", pkt.version)
self.server.clients[self.client_address].ni_version = pkt.version
self.request.send(SAPRouter(type=SAPRouter.SAPROUTER_CONTROL,
version=self.router_version,
opcode=2,
return_code=-13))
else:
self.logger.debug("Unhandled opcode %d (%s)",
pkt.opcode, opcode_str)
return self.return_error(return_code=-13,
error="invalid client version",
detail="NiBufIProcMsg: unknown opcode 3 received")
def handle_admin(self, pkt):
"""Handles admin messages"""
self.logger.debug("Handling admin message, command %d (%s)",
pkt.adm_command,
router_adm_commands[pkt.adm_command])
if not self.external_admin:
self.logger.debug("External administration disabled")
return self.return_error(return_code=-94,
error="Admin from remote denied")
# Information request
if pkt.adm_command == 2:
self.logger.debug("Received information request (password %s)", pkt.adm_password)
# If a password was specified but doesn't match, return error
if self.info_password and self.info_password != pkt.adm_password.strip("\x00"):
self.session.add_event("Information request invalid password", data=pkt.adm_password, request=str(self.packet))
return self.return_error(return_code=-94,
error="route denied")
else:
self.session.add_event("Information request valid password", data=pkt.adm_password, request=str(self.packet))
return self.return_info()
# Trace connection request
if pkt.adm_command == 12:
self.logger.debug("Received trace connection request (# clients: %s)", pkt.adm_client_count)
for client_id in pkt.adm_client_ids:
for client in self.server.clients:
if self.server.clients[client].id == client_id:
self.server.clients[client].traced = True
return
self.logger.debug("Unhandled command %d (%s)",
pkt.adm_command,
router_adm_commands[pkt.adm_command])
def handle_timeout(self):
"""Handles timeout"""
self.logger.debug("Timed out client")
self.return_error(return_code=-5,
error="connection timed out",
detail="RTPENDLIST::timeoutPend: no route received within %ds (CONNECTED)" % self.timeout)
def return_info(self):
"""Returns an information request response"""
self.logger.debug("Returning information request")
info_clients = []
for client in list(self.server.clients.values()):
info_client = SAPRouterInfoClient(id=client.id)
info_client.address = client.address
if client.routed:
info_client.partner = client.partner
info_client.service = client.service
info_client.connected_on = unix_time(client.connected_on)
info_client.flag_traced = client.traced
info_client.flag_routed = client.routed
info_client.flag_connected = client.connected
info_clients.append(info_client)
info_clients = "".join([str(client) for client in info_clients])
info_pkt = Raw(info_clients)
self.request.send(info_pkt)
self.session.add_event("Returned information request", response=str(info_pkt))
__, server_port = self.server.server_address
info_pkt = SAPRouterInfoServer(pid=self.pid,
ppid=self.parent_pid,
started_on=unix_time(self.time_started),
port=server_port,
pport=self.parent_port)
hexdump(info_pkt)
self.request.send(info_pkt)
self.session.add_event("Returned information request", data={"packet": "info_packet"},
response=str(info_pkt))
info_pkt = Raw("Total no. of clients: %d\x00" % len(self.server.clients))
self.request.send(info_pkt)
self.session.add_event("Returned information request", data={"packet": "total_no_clients"},
response=str(info_pkt))
info_pkt = Raw("Working directory : %s\x00" % self.route_table_working_directory)
self.request.send(info_pkt)
self.session.add_event("Returned information request", data={"packet": "working_directory"},
response=str(info_pkt))
info_pkt = Raw("Routtab : %s\x00" % self.route_table_filename)
self.request.send(info_pkt)
self.session.add_event("Returned information request", data={"packet": "routtab"},
response=str(info_pkt))
self.request.close()
def return_error(self, **options):
"""Returns an error response"""
self.logger.debug("Returning error code %d (%s)", options.get("return_code"),
router_return_codes[options.get("return_code")])
error_text = SAPRouterError(release=str(self.release),
version=str(self.router_version),
error_time=datetime.now().strftime(SAPRouterError.time_format),
location="SAPRouter %d.%d on '%s'" % (self.router_version,
self.router_version_patch,
self.hostname))
for field in list(options.keys()):
setattr(error_text, field, options[field])
error_pkt = SAPRouter(type=SAPRouter.SAPROUTER_ERROR,
version=self.router_version,
opcode=0,
return_code=options.get("return_code"),
err_text_value=error_text)
self.request.send(error_pkt)
self.session.add_event("Returned error",
data={"return_code": options.get("return_code"),
"error_msg": router_return_codes[options.get("return_code")]},
response=str(error_pkt))