def serve_forever(self, server):
try:
if os.path.exists(self.path):
os.unlink(self.path)
self.sockfd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sockfd.bind(self.path)
os.chmod(self.path, self.permissions)
self.sockfd.listen(10)
except OSError as err:
self.logger.error('Cannot start socket server: {0}'.format(
str(err)))
return
while True:
try:
fd, addr = self.sockfd.accept()
except OSError as err:
if err.errno == errno.ECONNABORTED:
break
self.logger.error('accept() failed: {0}'.format(str(err)))
break
handler = self.UnixSocketHandler(self, fd, addr)
handler.conn = server.on_connection(handler)
spawn_thread(handler.handle_connection)
self.sockfd.close()
def connect(self, url, parent, **kwargs):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.parent = parent
if not self.parent:
raise RuntimeError('ClientTransportSock can be only created inside of a class')
timeout = kwargs.get('timeout', 30)
if url.path:
self.path = url.path
try:
while True:
try:
self.sock.connect(self.path)
self.connected = True
debug_log('Connected to {0}', self.path)
break
except (socket.error, OSError) as err:
if timeout:
timeout -= 1
time.sleep(1)
continue
else:
self.close()
debug_log('Socket connection exception: {0}', err)
raise
except KeyboardInterrupt:
self.close()
self.sock.close()
raise
spawn_thread(self.recv)
def serve_forever(self, server):
try:
if os.path.exists(self.path):
os.unlink(self.path)
self.sockfd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sockfd.bind(self.path)
os.chmod(self.path, self.permissions)
self.sockfd.listen(10)
except OSError as err:
self.logger.error('Cannot start socket server: {0}'.format(str(err)))
return
while True:
try:
fd, addr = self.sockfd.accept()
except OSError as err:
if err.errno == errno.ECONNABORTED:
break
self.logger.error('accept() failed: {0}'.format(str(err)))
break
handler = self.UnixSocketHandler(self, fd, addr)
handler.conn = server.on_connection(handler)
spawn_thread(handler.handle_connection)
self.sockfd.close()
def connect(self, url, parent, **kwargs):
self.parent = parent
if 'fobj' in kwargs:
self.fobj = kwargs.pop('fobj')
else:
self.fd = int(url.hostname)
self.fobj = os.fdopen(self.fd, 'w+b', 0)
spawn_thread(self.recv)
def connect(self, url, parent, **kwargs):
self.parent = parent
if 'fobj' in kwargs:
self.fobj = kwargs.pop('fobj')
else:
self.fd = int(url.hostname)
self.fobj = os.fdopen(self.fd, 'w+b', 0)
spawn_thread(self.recv)
def call_task_async(self, name, *args, timeout=3600, callback=None):
def wait_on_complete(tid):
self.call_sync('task.wait', tid, timeout=timeout)
callback(self.call_sync('task.status', tid))
tid = self.call_sync('task.submit', name, list(args))
if callback:
spawn_thread(wait_on_complete, tid)
return tid
def call_task_async(self, name, *args, timeout=3600, callback=None):
def wait_on_complete(tid):
self.call_sync('task.wait', tid, timeout=timeout)
callback(self.call_sync('task.status', tid))
tid = self.call_sync('task.submit', name, list(args))
if callback:
spawn_thread(wait_on_complete, tid)
return tid
def on_rpc_call(self, id, data):
if self.rpc is None:
self.send_error(id, errno.EINVAL, 'Server functionality is not supported')
return
if not isinstance(self.rpc, rpc.RpcContext):
self.send_error(id, errno.EINVAL, 'Incompatible context')
return
if 'method' not in data or 'args' not in data:
self.send_error(id, errno.EINVAL, 'Malformed request')
return
def run_async(id, args):
try:
result = self.rpc.dispatch_call(
args['method'],
args['args'],
sender=self,
streaming=self.streaming
)
except rpc.RpcException as err:
self.trace('RPC error: id={0} code={0} message={1} extra={2}'.format(
id,
err.code,
err.message,
err.extra
))
self.send_error(id, err.code, err.message)
else:
if isinstance(result, rpc.RpcStreamingResponse):
self.pending_iterators[id] = PendingIterator(result)
try:
first, seqno = self.pending_iterators[id].advance()
self.trace('RPC response fragment: id={0} seqno={1} result={2}'.format(id, seqno, first))
self.send_fragment(id, seqno, first)
except StopIteration as stp:
self.trace('RPC response end: id={0}'.format(id))
self.send_end(id, stp.args[0])
del self.pending_iterators[id]
return
else:
self.trace('RPC response: id={0} result={1}'.format(id, result))
self.send_response(id, result)
self.trace('RPC call: id={0} method={1} args={2}'.format(id, data['method'], data['args']))
spawn_thread(run_async, id, data, threadpool=True)
return
def connect(self, url, parent, **kwargs):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.fd = self.sock.makefile('rwb')
self.parent = parent
if not self.parent:
raise RuntimeError('ClientTransportSock can be only created inside of a class')
timeout = kwargs.get('timeout', 30)
if url.path:
self.path = url.path
while True:
try:
self.sock.connect(self.path)
debug_log('Connected to {0}', self.path)
break
except (socket.error, OSError) as err:
if timeout:
timeout -= 1
time.sleep(1)
continue
else:
self.close()
debug_log('Socket connection exception: {0}', err)
raise
recv_t = spawn_thread(target=self.recv, daemon=True)
recv_t.start()
def __process_events(self):
while True:
name, args = self.event_queue.get()
if not name:
return
with self.event_distribution_lock:
for h in self.event_handlers.get(name, []):
if getattr(h, 'sync', False):
with h.lock:
with contextlib.suppress(BaseException):
h(args)
else:
spawn_thread(h, args, threadpool=True)
if self.event_callback:
with contextlib.suppress(BaseException):
self.event_callback(name, args)
def connect(self, url, parent, **kwargs):
self.parent = parent
s = None
for item in socket.getaddrinfo(url.hostname, url.port, type=socket.SOCK_STREAM, proto=socket.IPPROTO_TCP):
af, type, proto, canonname, sockaddr = item
try:
s = socket.socket(af, type, proto)
s.connect(sockaddr)
except socket.error:
continue
else:
break
if s is None:
raise RuntimeError('Cannot connect to {0}'.format(url.hostname))
self.fobj = s.makefile('rwb')
self.parent.on_open()
spawn_thread(self.recv)
def serve_forever(self, server):
self.sockfd = socket.socket(self.af, socket.SOCK_STREAM, socket.IPPROTO_TCP)
self.sockfd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sockfd.bind((self.hostname, self.port))
self.sockfd.listen(10)
while True:
try:
fd, addr = self.sockfd.accept()
except OSError as err:
if err.errno == errno.ECONNABORTED:
break
self.logger.error('accept() failed: {0}'.format(str(err)))
break
handler = self.TCPSocketHandler(self, fd, addr)
handler.conn = server.on_connection(handler)
spawn_thread(handler.handle_connection)
self.sockfd.close()
def on_rpc_continue(self, id, data):
seqno = data
self.trace('RPC continuation: id={0} seqno={1}'.format(id, seqno))
if id not in self.pending_iterators:
self.trace('RPC pending call {0} not found'.format(id))
self.send_error(id, errno.ENOENT, 'Invalid call')
return
def run_async(id):
it = self.pending_iterators[id]
try:
fragment = it.request_chunk(seqno)
self.trace(
'RPC response fragment: id={0} seqno={1} result={2}'.
format(id, seqno, fragment))
self.send_fragment(id, seqno, fragment)
except StopIteration as stp:
self.trace('RPC response end: id={0}'.format(id))
self.send_end(id, seqno)
if not it.view:
with self.request_lock:
self.pending_iterators.pop(id, None)
if id in self.requests:
del self.requests[id]
self.send_close(id)
return
except rpc.RpcException as err:
self.trace(
'RPC error: id={0} code={0} message={1} extra={2}'.format(
id, err.code, err.message, err.extra))
with self.request_lock:
self.pending_iterators.pop(id, None)
if id in self.requests:
self.send_error(id, err.code, err.message, err.extra)
del self.requests[id]
spawn_thread(run_async, id, threadpool=True)
def connect(self, url, parent, **kwargs):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.parent = parent
self.terminated = False
if not self.parent:
raise RuntimeError(
'ClientTransportUnix can be only created inside of a class')
timeout = kwargs.get('timeout', 30)
if url.path:
self.path = url.path
try:
while True:
try:
self.sock.connect(self.path)
self.connected = True
debug_log('Connected to {0}', self.path)
break
except (socket.error, OSError) as err:
if err.errno == errno.EPERM:
raise
if timeout:
timeout -= 1
time.sleep(1)
continue
else:
with contextlib.suppress(OSError):
self.sock.close()
debug_log('Socket connection exception: {0}', err)
raise
except KeyboardInterrupt:
self.sock.close()
raise
self.parent.on_open()
spawn_thread(self.recv)
def serve_forever(self, server):
self.sockfd = socket.socket(self.af, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
self.sockfd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sockfd.bind((self.hostname, self.port))
self.sockfd.listen(10)
while True:
try:
fd, addr = self.sockfd.accept()
except OSError as err:
if err.errno == errno.ECONNABORTED:
break
self.logger.error('accept() failed: {0}'.format(str(err)))
break
handler = self.TCPSocketHandler(self, fd, addr)
handler.conn = server.on_connection(handler)
spawn_thread(handler.handle_connection)
self.sockfd.close()
def connect(self, url, parent, **kwargs):
self.parent = parent
s = None
for item in socket.getaddrinfo(url.hostname,
url.port,
type=socket.SOCK_STREAM,
proto=socket.IPPROTO_TCP):
af, type, proto, canonname, sockaddr = item
try:
s = socket.socket(af, type, proto)
s.connect(sockaddr)
except socket.error:
continue
else:
break
if s is None:
raise RuntimeError('Cannot connect to {0}'.format(url.hostname))
self.fobj = s.makefile('rwb')
self.parent.on_open()
spawn_thread(self.recv)
def serve_forever(self, server):
try:
if os.path.exists(self.path):
os.unlink(self.path)
self.sockfd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sockfd.bind(self.path)
os.chmod(self.path, 0o775)
self.sockfd.listen(self.backlog)
except OSError as err:
self.logger.error('Cannot start socket server: {0}'.format(str(err)))
return
while True:
try:
fd, addr = self.sockfd.accept()
except OSError as err:
self.logger.error('accept() failed: {0}'.format(str(err)))
continue
handler = self.UnixSocketHandler(self, fd, addr)
handler.conn = server.on_connection(handler)
spawn_thread(handler.handle_connection, threadpool=True)
def connect(self, url, **kwargs):
self.parse_url(url)
if not self.scheme:
self.scheme = kwargs.get('scheme',"ws")
else:
if 'scheme' in kwargs:
raise ValueError('Connection scheme cannot be delared in both url and arguments.')
if self.scheme is "http":
self.scheme = "ws"
builder = ClientTransportBuilder()
self.transport = builder.create(self.scheme)
self.transport.connect(self.parsed_url, self, **kwargs)
debug_log('Connection opened, local address {0}', self.transport.address)
if self.use_bursts:
self.event_thread = spawn_thread(target=self.__event_emitter, args=())
self.event_thread.start()
def on_rpc_call(self, id, data):
if self.rpc is None:
self.send_error(id, errno.EINVAL,
'Server functionality is not supported')
return
if not isinstance(self.rpc, rpc.RpcContext):
self.send_error(id, errno.EINVAL, 'Incompatible context')
return
if 'method' not in data or 'args' not in data:
self.send_error(id, errno.EINVAL, 'Malformed request')
return
if self.call_queue_limit and len(
self.requests) >= self.call_queue_limit:
self.send_error(id, errno.EBUSY,
'Number of simultaneous requests exceeded')
return
def run_async(id, args):
try:
result = self.rpc.dispatch_call(args['method'],
args['args'],
sender=self,
streaming=self.streaming)
except rpc.RpcException as err:
self.trace(
'RPC error: id={0} code={1} message={2} extra={3}'.format(
id, err.code, err.message, err.extra))
with self.request_lock:
if id in self.requests:
del self.requests[id]
self.send_error(id, err.code, err.message, err.extra)
else:
if isinstance(result, rpc.RpcStreamingResponse):
it = PendingIterator(result, args.get('view', False))
self.pending_iterators[id] = it
try:
first, seqno = it.advance()
self.trace(
'RPC response fragment: id={0} seqno={1} result={2}'
.format(id, seqno, first))
self.send_fragment(id, seqno, first)
except StopIteration as stp:
self.trace('RPC response end: id={0}'.format(id))
self.send_end(id, stp.args[0])
if not it.view:
with self.request_lock:
self.pending_iterators.pop(id, None)
if id in self.requests:
del self.requests[id]
self.send_close(id)
return
else:
with self.request_lock:
if id in self.requests:
del self.requests[id]
self.trace(
'RPC response: id={0} result={1}'.format(
id, result))
self.send_response(id, result)
self.trace('RPC call: id={0} method={1} args={2} view={3}'.format(
id, data['method'], data['args'], data.get('view', False)))
with self.request_lock:
self.requests[id] = spawn_thread(run_async,
id,
data,
threadpool=True)
def on_open(self):
self.event_thread = spawn_thread(self.__process_events)
def connect(self, url, parent, **kwargs):
self.url = url
self.parent = parent
self.username = url.username
self.port = url.port
if not self.parent:
raise RuntimeError('ClientTransportSSH can be only created inside of a class')
if url.hostname:
self.hostname = url.hostname
elif url.netloc:
self.hostname = url.netloc
if '@' in self.hostname:
temp, self.hostname = self.hostname.split('@')
elif url.path:
self.hostname = url.path
if not self.username:
self.username = kwargs.get('username', None)
else:
if 'username' in kwargs:
raise ValueError('Username cannot be delared in both url and arguments.')
if not self.username:
raise ValueError('Username is not declared.')
if not self.hostname:
self.hostname = kwargs.get('hostname', None)
else:
if 'hostname' in kwargs:
raise ValueError('Hostname cannot be delared in both url and arguments.')
if not self.hostname:
raise ValueError('Hostname is not declared.')
if not self.port:
self.port = kwargs.get('port', 22)
else:
if 'port' in kwargs:
raise ValueError('Port cannot be delared in both url and arguments.')
self.password = kwargs.get('password', None)
self.pkey = kwargs.get('pkey', None)
self.key_filename = kwargs.get('key_filename', None)
if not self.pkey and not self.password and not self.key_filename:
raise ValueError('No password, key_filename nor pkey for authentication declared.')
self.host_key_file = kwargs.get('host_key_file', None)
self.timeout = kwargs.get('timeout', 30)
debug_log('Trying to connect to {0}', self.hostname)
try:
self.ssh = paramiko.SSHClient()
logging.getLogger("paramiko").setLevel(logging.WARNING)
if self.host_key_file:
self.look_for_keys = False
try:
self.ssh.load_host_keys(self.host_key_file)
except IOError:
debug_log('Cannot read host key file: {0}. SSH transport is closing.', self.host_key_file)
self.close()
raise
else:
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(
self.hostname,
port=self.port,
username=self.username,
password=self.password,
pkey=self.pkey,
look_for_keys=self.look_for_keys,
key_filename=self.key_filename,
timeout=self.timeout
)
self.connected = True
debug_log('Connected to {0}', self.hostname)
except paramiko.AuthenticationException as err:
debug_log('Authentication exception: {0}', err)
raise
except paramiko.BadHostKeyException as err:
debug_log('Bad host key exception: {0}', err)
raise
except paramiko.SSHException as err:
debug_log('SSH exception: {0}', err)
raise
except socket.error as err:
debug_log('Socket exception: {0}', err)
raise
self.stdin, self.stdout, self.stderr = self.ssh.exec_command(
"sh /usr/local/libexec/dispatcher/ssh_transport_catcher",
bufsize=0
)
self.channel = self.ssh.get_transport().open_session()
self.parent.on_open()
spawn_thread(self.recv)
def on_events_event_burst(self, id, data):
for i in data['events']:
spawn_thread(self.__process_event, i['name'], i['args'], threadpool=True)
def on_events_event(self, id, data):
spawn_thread(self.__process_event, data['name'], data['args'], threadpool=True)
def decode(self, msg, fds):
self.__replace_fds(msg, fds)
if 'namespace' not in msg:
self.error_callback(ClientError.INVALID_JSON_RESPONSE)
return
if 'name' not in msg:
self.error_callback(ClientError.INVALID_JSON_RESPONSE)
return
if msg['namespace'] == 'events' and msg['name'] == 'event':
args = msg['args']
t = spawn_thread(target=self.__process_event, args=(args['name'], args['args']))
t.start()
return
if msg['namespace'] == 'events' and msg['name'] == 'event_burst':
args = msg['args']
for i in args['events']:
t = spawn_thread(target=self.__process_event, args=(i['name'], i['args']))
t.start()
return
if msg['namespace'] == 'events' and msg['name'] == 'logout':
self.error_callback(ClientError.LOGOUT)
return
if msg['namespace'] == 'rpc':
if msg['name'] == 'call':
if self.rpc is None:
self.__send_error(msg['id'], errno.EINVAL, 'Server functionality is not supported')
return
if 'args' not in msg:
self.__send_error(msg['id'], errno.EINVAL, 'Malformed request')
return
args = msg['args']
if 'method' not in args or 'args' not in args:
self.__send_error(msg['id'], errno.EINVAL, 'Malformed request')
return
def run_async(msg, args):
try:
result = self.rpc.dispatch_call(args['method'], args['args'], sender=self)
except rpc.RpcException as err:
self.__send_error(msg['id'], err.code, err.message)
else:
self.__send_response(msg['id'], result)
t = spawn_thread(target=run_async, args=(msg, args))
t.start()
return
if msg['name'] == 'response':
if msg['id'] in self.pending_calls.keys():
call = self.pending_calls[msg['id']]
call.result = msg['args']
call.completed.set()
if call.callback is not None:
call.callback(msg['args'])
del self.pending_calls[str(call.id)]
else:
if self.error_callback is not None:
self.error_callback(ClientError.SPURIOUS_RPC_RESPONSE, msg['id'])
if msg['name'] == 'error':
if msg['id'] in self.pending_calls.keys():
call = self.pending_calls[msg['id']]
call.result = None
call.error = msg['args']
call.completed.set()
if call.callback is not None:
call.callback(rpc.RpcException(obj=call.error))
del self.pending_calls[str(call.id)]
if self.error_callback is not None:
self.error_callback(ClientError.RPC_CALL_ERROR)
def connect(self, url, parent, **kwargs):
self.url = url
self.parent = parent
self.username = url.username
self.port = url.port
if not self.parent:
raise RuntimeError(
'ClientTransportSSH can be only created inside of a class')
if url.hostname:
self.hostname = url.hostname
elif url.netloc:
self.hostname = url.netloc
if '@' in self.hostname:
temp, self.hostname = self.hostname.split('@')
elif url.path:
self.hostname = url.path
if not self.username:
self.username = kwargs.get('username', None)
else:
if 'username' in kwargs:
raise ValueError(
'Username cannot be delared in both url and arguments.')
if not self.username:
raise ValueError('Username is not declared.')
if not self.hostname:
self.hostname = kwargs.get('hostname', None)
else:
if 'hostname' in kwargs:
raise ValueError(
'Hostname cannot be delared in both url and arguments.')
if not self.hostname:
raise ValueError('Hostname is not declared.')
if not self.port:
self.port = kwargs.get('port', 22)
else:
if 'port' in kwargs:
raise ValueError(
'Port cannot be delared in both url and arguments.')
self.password = kwargs.get('password', None)
self.pkey = kwargs.get('pkey', None)
self.key_filename = kwargs.get('key_filename', None)
if not self.pkey and not self.password and not self.key_filename:
raise ValueError(
'No password, key_filename nor pkey for authentication declared.'
)
self.host_key_file = kwargs.get('host_key_file', None)
self.timeout = kwargs.get('timeout', 30)
debug_log('Trying to connect to {0}', self.hostname)
try:
self.ssh = paramiko.SSHClient()
logging.getLogger("paramiko").setLevel(logging.WARNING)
if self.host_key_file:
self.look_for_keys = False
try:
self.ssh.load_host_keys(self.host_key_file)
except IOError:
debug_log(
'Cannot read host key file: {0}. SSH transport is closing.',
self.host_key_file)
self.close()
raise
else:
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.hostname,
port=self.port,
username=self.username,
password=self.password,
pkey=self.pkey,
look_for_keys=self.look_for_keys,
key_filename=self.key_filename,
timeout=self.timeout)
self.connected = True
debug_log('Connected to {0}', self.hostname)
except paramiko.AuthenticationException as err:
debug_log('Authentication exception: {0}', err)
raise
except paramiko.BadHostKeyException as err:
debug_log('Bad host key exception: {0}', err)
raise
except paramiko.SSHException as err:
debug_log('SSH exception: {0}', err)
raise
except socket.error as err:
debug_log('Socket exception: {0}', err)
raise
self.stdin, self.stdout, self.stderr = self.ssh.exec_command(
"sh /usr/local/libexec/dispatcher/ssh_transport_catcher",
bufsize=0)
self.channel = self.ssh.get_transport().open_session()
self.parent.on_open()
spawn_thread(self.recv)