Ejemplo n.º 1
0
def sessionhdl(connection, authname, skipauth=False):
    # For now, trying to test the console stuff, so let's just do n4.
    authenticated = False
    authdata = None
    cfm = None
    if skipauth:
        authenticated = True
        cfm = configmanager.ConfigManager(tenant=None, username=authname)
    elif authname:
        authdata = auth.authorize(authname, element=None)
        if authdata is not None:
            cfm = authdata[1]
            authenticated = True
    send_data(connection, "Confluent -- v0 --")
    while not authenticated:  # prompt for name and passphrase
        send_data(connection, {'authpassed': 0})
        response = tlvdata.recv(connection)
        authname = response['username']
        passphrase = response['password']
        # note(jbjohnso): here, we need to authenticate, but not
        # authorize a user.  When authorization starts understanding
        # element path, that authorization will need to be called
        # per request the user makes
        authdata = auth.check_user_passphrase(authname, passphrase)
        if authdata is None:
            auditlog.log(
                {'operation': 'connect', 'user': authname, 'allowed': False})
        else:
            authenticated = True
            cfm = authdata[1]
    send_data(connection, {'authpassed': 1})
    request = tlvdata.recv(connection)
    while request is not None:
        try:
            process_request(
                connection, request, cfm, authdata, authname, skipauth)
        except exc.ConfluentException as e:
            if ((not isinstance(e, exc.LockedCredentials)) and
                    e.apierrorcode == 500):
                tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {'errorcode': e.apierrorcode,
                                   'error': e.apierrorstr,
                                   'detail': e.get_error_body()})
            send_data(connection, {'_requestdone': 1})
        except SystemExit:
            sys.exit(0)
        except:
            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {'errorcode': 500,
                                      'error': 'Unexpected error'})
            send_data(connection, {'_requestdone': 1})
        request = tlvdata.recv(connection)
Ejemplo n.º 2
0
 def authenticate(self, username, password):
     tlvdata.send(self.connection, {
         'username': username,
         'password': password
     })
     authdata = tlvdata.recv(self.connection)
     if authdata['authpassed'] == 1:
         self.authenticated = True
Ejemplo n.º 3
0
def send_request(operation, path, server, parameters=None):
    """This function iterates over all the responses
    received from the server.

    :param operation:  The operation to request, retrieve, update, delete,
                       create, start, stop
    :param path: The URI path to the resource to operate on
    :param server: The socket to send data over
    :param parameters:  Parameters if any to send along with the request
    """
    payload = {'operation': operation, 'path': path}
    if parameters is not None:
        payload['parameters'] = parameters
    tlvdata.send(server, payload)
    result = tlvdata.recv(server)
    while '_requestdone' not in result:
        yield result
        result = tlvdata.recv(server)
Ejemplo n.º 4
0
def start_term(authname, cfm, connection, params, path, authdata, skipauth):
    elems = path.split('/')
    if len(elems) < 4 or elems[1] != 'nodes':
        raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
    node = elems[2]
    ccons = ClientConsole(connection)
    skipreplay = False
    if params and 'skipreplay' in params and params['skipreplay']:
        skipreplay = True
    if elems[3] == "console":
        consession = consoleserver.ConsoleSession(node=node,
                                                  configmanager=cfm,
                                                  username=authname,
                                                  datacallback=ccons.sendall,
                                                  skipreplay=skipreplay)
    elif len(elems) >= 6 and elems[3:5] == ['shell', 'sessions']:
        if len(elems) == 7:
            sessionid = elems[5]
        else:
            sessionid = None
        consession = shellserver.ShellSession(node=node,
                                              configmanager=cfm,
                                              username=authname,
                                              datacallback=ccons.sendall,
                                              skipreplay=skipreplay,
                                              sessionid=sessionid)

    else:
        raise exc.InvalidArgumentException('Invalid path {0}'.format(path))

    if consession is None:
        raise Exception("TODO")
    send_data(connection, {'started': 1})
    ccons.startsending()
    bufferage = consession.get_buffer_age()
    if bufferage is not False:
        send_data(connection, {'bufferage': bufferage})
    while consession is not None:
        data = tlvdata.recv(connection)
        if type(data) == dict:
            if data['operation'] == 'stop':
                consession.destroy()
                return
            elif data['operation'] == 'break':
                consession.send_break()
                continue
            elif data['operation'] == 'reopen':
                consession.reopen()
                continue
            else:
                process_request(connection, data, cfm, authdata, authname,
                                skipauth)
                continue
        if not data:
            consession.destroy()
            return
        consession.write(data)
Ejemplo n.º 5
0
def send_request(operation, path, server, parameters=None):
    """This function iterates over all the responses
    received from the server.

    :param operation:  The operation to request, retrieve, update, delete,
                       create, start, stop
    :param path: The URI path to the resource to operate on
    :param server: The socket to send data over
    :param parameters:  Parameters if any to send along with the request
    """
    payload = {'operation': operation, 'path': path}
    if parameters is not None:
        payload['parameters'] = parameters
    tlvdata.send(server, payload)
    result = tlvdata.recv(server)
    while '_requestdone' not in result:
        yield result
        result = tlvdata.recv(server)
Ejemplo n.º 6
0
def try_assimilate(drone):
    try:
        remote = connect_to_collective(None, drone)
    except socket.error:
        # Oh well, unable to connect, hopefully the rest will be
        # in order
        return
    tlvdata.send(
        remote, {
            'collective': {
                'operation': 'assimilate',
                'name': get_myname(),
                'txcount': cfm._txcount
            }
        })
    tlvdata.recv(remote)  # the banner
    tlvdata.recv(remote)  # authpassed... 0..
    answer = tlvdata.recv(remote)
    if answer and 'error' in answer:
        connect_to_leader(None, None, leader=remote.getpeername()[0])
Ejemplo n.º 7
0
def start_term(authname, cfm, connection, params, path):
    elems = path.split('/')
    if len(elems) < 4 or elems[1] != 'nodes':
        raise exc.InvalidArgumentException('Invalid path {0}'.format(path))
    node = elems[2]
    ccons = ClientConsole(connection)
    skipreplay = False
    if params and 'skipreplay' in params and params['skipreplay']:
        skipreplay = True
    if elems[3] == "console":
        consession = consoleserver.ConsoleSession(
            node=node, configmanager=cfm, username=authname,
            datacallback=ccons.sendall, skipreplay=skipreplay)
    elif len(elems) >= 6 and elems[3:5] == ['shell', 'sessions']:
        if len(elems) == 7:
            sessionid = elems[5]
        else:
            sessionid = None
        consession = shellserver.ShellSession(
            node=node, configmanager=cfm, username=authname,
            datacallback=ccons.sendall, skipreplay=skipreplay,
            sessionid=sessionid)

    else:
        raise exc.InvalidArgumentException('Invalid path {0}'.format(path))

    if consession is None:
        raise Exception("TODO")
    send_data(connection, {'started': 1})
    ccons.startsending()
    bufferage = consession.get_buffer_age()
    if bufferage is not False:
        send_data(connection, {'bufferage': bufferage})
    while consession is not None:
        data = tlvdata.recv(connection)
        if type(data) == dict:
            if data['operation'] == 'stop':
                consession.destroy()
                return
            elif data['operation'] == 'break':
                consession.send_break()
                continue
            elif data['operation'] == 'reopen':
                consession.reopen()
                continue
            else:
                raise Exception("TODO")
        if not data:
            consession.destroy()
            return
        consession.write(data)
Ejemplo n.º 8
0
 def __init__(self, server=None):
     self.connection = None
     if server is None:
         if 'CONFLUENT_HOST' in os.environ:
             self.serverloc = os.environ['CONFLUENT_HOST']
         else:
             self.serverloc = '/var/run/confluent/api.sock'
     else:
         self.serverloc = server
     if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
         self._connect_unix()
     else:
         self._connect_tls()
     tlvdata.recv(self.connection)
     authdata = tlvdata.recv(self.connection)
     if authdata['authpassed'] == 1:
         self.authenticated = True
     else:
         self.authenticated = False
     if not self.authenticated and 'CONFLUENT_USER' in os.environ:
         username = os.environ['CONFLUENT_USER']
         passphrase = os.environ['CONFLUENT_PASSPHRASE']
         self.authenticate(username, passphrase)
Ejemplo n.º 9
0
 def __init__(self, server=None):
     self.connection = None
     if server is None:
         if 'CONFLUENT_HOST' in os.environ:
             self.serverloc = os.environ['CONFLUENT_HOST']
         else:
             self.serverloc = '/var/run/confluent/api.sock'
     else:
         self.serverloc = server
     if os.path.isabs(self.serverloc) and os.path.exists(self.serverloc):
         self._connect_unix()
     else:
         self._connect_tls()
     tlvdata.recv(self.connection)
     authdata = tlvdata.recv(self.connection)
     if authdata['authpassed'] == 1:
         self.authenticated = True
     else:
         self.authenticated = False
     if not self.authenticated and 'CONFLUENT_USER' in os.environ:
         username = os.environ['CONFLUENT_USER']
         passphrase = os.environ['CONFLUENT_PASSPHRASE']
         self.authenticate(username, passphrase)
Ejemplo n.º 10
0
 def attachsession(self, session):
     self.clisession = session
     self.data_handler = session.data_handler
     termreq = {
         'proxyconsole': {
             'name': self.myname,
             'user': self.user,
             'tenant': self.cfm.tenant,
             'node': self.node,
             'skipreplay': self.skipreplay,
             'width': self.initsize[0],
             'height': self.initsize[1],
             #TODO(jjohnson2): declare myself as a proxy,
             #facilitate redirect rather than relay on manager change
         },
     }
     try:
         remote = socket.create_connection(
             (self.managerinfo['address'], 13001))
         remote = ssl.wrap_socket(remote,
                                  cert_reqs=ssl.CERT_NONE,
                                  keyfile='/etc/confluent/privkey.pem',
                                  certfile='/etc/confluent/srvcert.pem')
         if not util.cert_matches(self.managerinfo['fingerprint'],
                                  remote.getpeercert(binary_form=True)):
             raise Exception('Invalid peer certificate')
     except Exception:
         eventlet.sleep(3)
         if self.clisession:
             self.clisession.detach()
         self.detachsession(None)
         return
     tlvdata.recv(remote)
     tlvdata.recv(remote)
     tlvdata.send(remote, termreq)
     self.remote = remote
     eventlet.spawn(self.relay_data)
Ejemplo n.º 11
0
def term_interact(authdata, authname, ccons, cfm, connection, consession,
                  skipauth):
    send_data(connection, {'started': 1})
    ccons.startsending()
    bufferage = consession.get_buffer_age()
    if bufferage is not False:
        send_data(connection, {'bufferage': bufferage})
    while consession is not None:
        data = tlvdata.recv(connection)
        if type(data) == dict:
            if data['operation'] == 'stop':
                consession.destroy()
                break
            elif data['operation'] == 'break':
                consession.send_break()
                continue
            elif data['operation'] == 'reopen':
                consession.reopen()
                continue
            elif data['operation'] == 'pause':
                ccons.xmit = False
                continue
            elif data['operation'] == 'resume':
                ccons.xmit = True
                continue
            elif data['operation'] == 'resize':
                consession.resize(width=data['width'], height=data['height'])
                continue
            else:
                try:
                    process_request(connection, data, cfm, authdata, authname,
                                    skipauth)
                except Exception as e:
                    tracelog.log(traceback.format_exc(),
                                 ltype=log.DataTypes.event,
                                 event=log.Events.stacktrace)
                    send_data(connection, {
                        'errorcode': 500,
                        'error': 'Unexpected error - ' + str(e)
                    })
                    send_data(connection, {'_requestdone': 1})
                continue
        if not data:
            consession.destroy()
            break
        consession.write(data)
Ejemplo n.º 12
0
def handle_connection(connection, cert, request, local=False):
    global currentleader
    global retrythread
    operation = request['operation']
    if cert:
        cert = crypto.dump_certificate(crypto.FILETYPE_ASN1, cert)
    else:
        if not local:
            return
        if operation in ('show', 'delete'):
            if not list(cfm.list_collective()):
                tlvdata.send(
                    connection, {
                        'collective': {
                            'error':
                            'Collective mode not '
                            'enabled on this '
                            'system'
                        }
                    })
                return
            if follower:
                linfo = cfm.get_collective_member_by_address(currentleader)
                remote = socket.create_connection((currentleader, 13001))
                remote = ssl.wrap_socket(remote,
                                         cert_reqs=ssl.CERT_NONE,
                                         keyfile='/etc/confluent/privkey.pem',
                                         certfile='/etc/confluent/srvcert.pem')
                cert = remote.getpeercert(binary_form=True)
                if not (linfo
                        and util.cert_matches(linfo['fingerprint'], cert)):
                    remote.close()
                    tlvdata.send(connection, {
                        'error':
                        'Invalid certificate, '
                        'redo invitation process'
                    })
                    connection.close()
                    return
                tlvdata.recv(remote)  # ignore banner
                tlvdata.recv(remote)  # ignore authpassed: 0
                tlvdata.send(remote, {
                    'collective': {
                        'operation': 'getinfo',
                        'name': get_myname()
                    }
                })
                collinfo = tlvdata.recv(remote)
            else:
                collinfo = {}
                populate_collinfo(collinfo)
            try:
                cfm.check_quorum()
                collinfo['quorum'] = True
            except exc.DegradedCollective:
                collinfo['quorum'] = False
            if operation == 'show':
                tlvdata.send(connection, {'collective': collinfo})
            elif operation == 'delete':
                todelete = request['member']
                if (todelete == collinfo['leader']
                        or todelete in collinfo['active']):
                    tlvdata.send(
                        connection, {
                            'collective': {
                                'error':
                                '{0} is still active, stop the confluent service to remove it'
                                .format(todelete)
                            }
                        })
                    return
                if todelete not in collinfo['offline']:
                    tlvdata.send(
                        connection, {
                            'collective': {
                                'error':
                                '{0} is not a recognized collective member'.
                                format(todelete)
                            }
                        })
                    return
                cfm.del_collective_member(todelete)
                tlvdata.send(
                    connection, {
                        'collective': {
                            'status':
                            'Successfully deleted {0}'.format(todelete)
                        }
                    })
                connection.close()
            return
        if 'invite' == operation:
            try:
                cfm.check_quorum()
            except exc.DegradedCollective:
                tlvdata.send(connection, {
                    'collective': {
                        'error': 'Collective does not have quorum'
                    }
                })
                return
            #TODO(jjohnson2): Cannot do the invitation if not the head node, the certificate hand-carrying
            #can't work in such a case.
            name = request['name']
            invitation = invites.create_server_invitation(name)
            tlvdata.send(connection,
                         {'collective': {
                             'invitation': invitation
                         }})
            connection.close()
        if 'join' == operation:
            invitation = request['invitation']
            try:
                invitation = base64.b64decode(invitation)
                name, invitation = invitation.split(b'@', 1)
                name = util.stringify(name)
            except Exception:
                tlvdata.send(
                    connection,
                    {'collective': {
                        'status': 'Invalid token format'
                    }})
                connection.close()
                return
            host = request['server']
            try:
                remote = socket.create_connection((host, 13001))
                # This isn't what it looks like.  We do CERT_NONE to disable
                # openssl verification, but then use the invitation as a
                # shared secret to validate the certs as part of the join
                # operation
                remote = ssl.wrap_socket(remote,
                                         cert_reqs=ssl.CERT_NONE,
                                         keyfile='/etc/confluent/privkey.pem',
                                         certfile='/etc/confluent/srvcert.pem')
            except Exception:
                tlvdata.send(
                    connection, {
                        'collective': {
                            'status': 'Failed to connect to {0}'.format(host)
                        }
                    })
                connection.close()
                return
            mycert = util.get_certificate_from_file(
                '/etc/confluent/srvcert.pem')
            cert = remote.getpeercert(binary_form=True)
            proof = base64.b64encode(
                invites.create_client_proof(invitation, mycert, cert))
            tlvdata.recv(remote)  # ignore banner
            tlvdata.recv(remote)  # ignore authpassed: 0
            tlvdata.send(remote, {
                'collective': {
                    'operation': 'enroll',
                    'name': name,
                    'hmac': proof
                }
            })
            rsp = tlvdata.recv(remote)
            if 'error' in rsp:
                tlvdata.send(connection,
                             {'collective': {
                                 'status': rsp['error']
                             }})
                connection.close()
                return
            proof = rsp['collective']['approval']
            proof = base64.b64decode(proof)
            j = invites.check_server_proof(invitation, mycert, cert, proof)
            if not j:
                remote.close()
                tlvdata.send(connection,
                             {'collective': {
                                 'status': 'Bad server token'
                             }})
                connection.close()
                return
            tlvdata.send(connection, {'collective': {'status': 'Success'}})
            connection.close()
            currentleader = rsp['collective']['leader']
            f = open('/etc/confluent/cfg/myname', 'w')
            f.write(name)
            f.close()
            log.log({
                'info': 'Connecting to collective due to join',
                'subsystem': 'collective'
            })
            eventlet.spawn_n(connect_to_leader,
                             rsp['collective']['fingerprint'], name)
    if 'enroll' == operation:
        #TODO(jjohnson2): error appropriately when asked to enroll, but the master is elsewhere
        mycert = util.get_certificate_from_file('/etc/confluent/srvcert.pem')
        proof = base64.b64decode(request['hmac'])
        myrsp = invites.check_client_proof(request['name'], mycert, cert,
                                           proof)
        if not myrsp:
            tlvdata.send(connection, {'error': 'Invalid token'})
            connection.close()
            return
        myrsp = base64.b64encode(myrsp)
        fprint = util.get_fingerprint(cert)
        myfprint = util.get_fingerprint(mycert)
        cfm.add_collective_member(get_myname(),
                                  connection.getsockname()[0], myfprint)
        cfm.add_collective_member(request['name'],
                                  connection.getpeername()[0], fprint)
        myleader = get_leader(connection)
        ldrfprint = cfm.get_collective_member_by_address(
            myleader)['fingerprint']
        tlvdata.send(
            connection, {
                'collective': {
                    'approval': myrsp,
                    'fingerprint': ldrfprint,
                    'leader': get_leader(connection)
                }
            })
    if 'assimilate' == operation:
        drone = request['name']
        droneinfo = cfm.get_collective_member(drone)
        if not droneinfo:
            tlvdata.send(
                connection,
                {'error': 'Unrecognized leader, '
                 'redo invitation process'})
            return
        if not util.cert_matches(droneinfo['fingerprint'], cert):
            tlvdata.send(
                connection,
                {'error': 'Invalid certificate, '
                 'redo invitation process'})
            return
        if request['txcount'] < cfm._txcount:
            tlvdata.send(
                connection, {
                    'error': 'Refusing to be assimilated by inferior'
                    'transaction count',
                    'txcount': cfm._txcount,
                })
            return
        if connecting.active:
            # don't try to connect while actively already trying to connect
            tlvdata.send(connection, {'status': 0})
            connection.close()
            return
        if (currentleader == connection.getpeername()[0] and follower
                and not follower.dead):
            # if we are happily following this leader already, don't stir
            # the pot
            tlvdata.send(connection, {'status': 0})
            connection.close()
            return
        log.log({
            'info': 'Connecting in response to assimilation',
            'subsystem': 'collective'
        })
        eventlet.spawn_n(connect_to_leader,
                         None,
                         None,
                         leader=connection.getpeername()[0])
        tlvdata.send(connection, {'status': 0})
        connection.close()
    if 'getinfo' == operation:
        drone = request['name']
        droneinfo = cfm.get_collective_member(drone)
        if not (droneinfo
                and util.cert_matches(droneinfo['fingerprint'], cert)):
            tlvdata.send(
                connection,
                {'error': 'Invalid certificate, '
                 'redo invitation process'})
            connection.close()
            return
        collinfo = {}
        populate_collinfo(collinfo)
        tlvdata.send(connection, collinfo)
    if 'connect' == operation:
        drone = request['name']
        droneinfo = cfm.get_collective_member(drone)
        if not (droneinfo
                and util.cert_matches(droneinfo['fingerprint'], cert)):
            tlvdata.send(
                connection,
                {'error': 'Invalid certificate, '
                 'redo invitation process'})
            connection.close()
            return
        myself = connection.getsockname()[0]
        if connecting.active:
            tlvdata.send(connection, {
                'error': 'Connecting right now',
                'backoff': True
            })
            connection.close()
            return
        if myself != get_leader(connection):
            tlvdata.send(
                connection, {
                    'error': 'Cannot assimilate, our leader is '
                    'in another castle',
                    'leader': currentleader
                })
            connection.close()
            return
        if request['txcount'] > cfm._txcount:
            retire_as_leader()
            tlvdata.send(
                connection, {
                    'error': 'Client has higher tranasaction count, '
                    'should assimilate me, connecting..',
                    'txcount': cfm._txcount
                })
            log.log({
                'info': 'Connecting to leader due to superior '
                'transaction count',
                'subsystem': collective
            })
            eventlet.spawn_n(connect_to_leader, None, None,
                             connection.getpeername()[0])
            connection.close()
            return
        if retrythread:
            retrythread.cancel()
            retrythread = None
        with leader_init:
            cfm.update_collective_address(request['name'],
                                          connection.getpeername()[0])
            tlvdata.send(connection, cfm._dump_keys(None, False))
            tlvdata.send(connection, cfm._cfgstore['collective'])
            tlvdata.send(connection, {})  # cfm.get_globals())
            cfgdata = cfm.ConfigManager(None)._dump_to_json()
            tlvdata.send(connection, {
                'txcount': cfm._txcount,
                'dbsize': len(cfgdata)
            })
            connection.sendall(cfgdata)
        #tlvdata.send(connection, {'tenants': 0}) # skip the tenants for now,
        # so far unused anyway
        if not cfm.relay_slaved_requests(drone, connection):
            if not retrythread:  # start a recovery if everyone else seems
                # to have disappeared
                retrythread = eventlet.spawn_after(30 + random.random(),
                                                   start_collective)
Ejemplo n.º 13
0
def sessionhdl(connection, authname, skipauth=False):
    # For now, trying to test the console stuff, so let's just do n4.
    authenticated = False
    authdata = None
    cfm = None
    if skipauth:
        authenticated = True
        cfm = configmanager.ConfigManager(tenant=None, username=authname)
    elif authname:
        authdata = auth.authorize(authname, element=None)
        if authdata is not None:
            cfm = authdata[1]
            authenticated = True
    send_data(connection, "Confluent -- v0 --")
    while not authenticated:  # prompt for name and passphrase
        send_data(connection, {'authpassed': 0})
        response = tlvdata.recv(connection)
        authname = response['username']
        passphrase = response['password']
        # note(jbjohnso): here, we need to authenticate, but not
        # authorize a user.  When authorization starts understanding
        # element path, that authorization will need to be called
        # per request the user makes
        authdata = auth.check_user_passphrase(authname, passphrase)
        if authdata is None:
            auditlog.log({
                'operation': 'connect',
                'user': authname,
                'allowed': False
            })
        else:
            authenticated = True
            cfm = authdata[1]
    send_data(connection, {'authpassed': 1})
    request = tlvdata.recv(connection)
    while request is not None:
        try:
            process_request(connection, request, cfm, authdata, authname,
                            skipauth)
        except exc.ConfluentException as e:
            if ((not isinstance(e, exc.LockedCredentials))
                    and e.apierrorcode == 500):
                tracelog.log(traceback.format_exc(),
                             ltype=log.DataTypes.event,
                             event=log.Events.stacktrace)
            send_data(
                connection, {
                    'errorcode': e.apierrorcode,
                    'error': e.apierrorstr,
                    'detail': e.get_error_body()
                })
            send_data(connection, {'_requestdone': 1})
        except SystemExit:
            sys.exit(0)
        except:
            tracelog.log(traceback.format_exc(),
                         ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {
                'errorcode': 500,
                'error': 'Unexpected error'
            })
            send_data(connection, {'_requestdone': 1})
        request = tlvdata.recv(connection)
Ejemplo n.º 14
0
def dispatch_request(nodes, manager, element, configmanager, inputdata,
                     operation, isnoderange):
    a = configmanager.get_collective_member(manager)
    try:
        remote = socket.create_connection((a['address'], 13001))
        remote.settimeout(180)
        remote = ssl.wrap_socket(remote,
                                 cert_reqs=ssl.CERT_NONE,
                                 keyfile='/etc/confluent/privkey.pem',
                                 certfile='/etc/confluent/srvcert.pem')
    except Exception:
        for node in nodes:
            if a:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} is unreachable'.format(a['name']))
            else:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    '"{0}" is not recognized as a collective member'.format(
                        manager))

        return
    if not util.cert_matches(a['fingerprint'],
                             remote.getpeercert(binary_form=True)):
        raise Exception("Invalid certificate on peer")
    banner = tlvdata.recv(remote)
    vers = banner.split()[2]
    if vers == b'v0':
        pvers = 2
    elif vers == b'v1':
        pvers = 4
    if sys.version_info[0] < 3:
        pvers = 2
    tlvdata.recv(remote)
    myname = collective.get_myname()
    dreq = b'\x01\x03' + msgpack.packb(
        {
            'name': myname,
            'nodes': list(nodes),
            'path': element,
            'tenant': configmanager.tenant,
            'operation': operation,
            'inputdata': inputdata,
            'isnoderange': isnoderange
        },
        use_bin_type=False)
    tlvdata.send(remote, {'dispatch': {'name': myname, 'length': len(dreq)}})
    remote.sendall(dreq)
    while True:
        try:
            rlen = remote.recv(8)
        except Exception:
            for node in nodes:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} went unreachable'.format(a['name']))
            return
        while len(rlen) < 8:
            try:
                nlen = remote.recv(8 - len(rlen))
            except Exception:
                nlen = 0
            if not nlen:
                for node in nodes:
                    yield msg.ConfluentResourceUnavailable(
                        node, 'Collective member {0} went unreachable'.format(
                            a['name']))
                return
            rlen += nlen
        rlen = struct.unpack('!Q', rlen)[0]
        if rlen == 0:
            break
        try:
            rsp = remote.recv(rlen)
        except Exception:
            for node in nodes:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} went unreachable'.format(a['name']))
            return
        while len(rsp) < rlen:
            try:
                nrsp = remote.recv(rlen - len(rsp))
            except Exception:
                nrsp = 0
            if not nrsp:
                for node in nodes:
                    yield msg.ConfluentResourceUnavailable(
                        node, 'Collective member {0} went unreachable'.format(
                            a['name']))
                return
            rsp += nrsp
        try:
            rsp = msg.msg_deserialize(rsp)
        except Exception:
            rsp = exc.deserialize_exc(rsp)
        if isinstance(rsp, Exception):
            raise rsp
        if not rsp:
            raise Exception(
                'Error in cross-collective serialize/deserialze, see remote logs'
            )
        yield rsp
Ejemplo n.º 15
0
def dispatch_request(nodes, manager, element, configmanager, inputdata,
                     operation):
    a = configmanager.get_collective_member(manager)
    try:
        remote = socket.create_connection((a['address'], 13001))
        remote.settimeout(90)
        remote = ssl.wrap_socket(remote,
                                 cert_reqs=ssl.CERT_NONE,
                                 keyfile='/etc/confluent/privkey.pem',
                                 certfile='/etc/confluent/srvcert.pem')
    except Exception:
        for node in nodes:
            if a:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} is unreachable'.format(a['name']))
            else:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    '"{0}" is not recognized as a collective member'.format(
                        manager))

        return
    if not util.cert_matches(a['fingerprint'],
                             remote.getpeercert(binary_form=True)):
        raise Exception("Invalid certificate on peer")
    tlvdata.recv(remote)
    tlvdata.recv(remote)
    myname = collective.get_myname()
    dreq = pickle.dumps({
        'name': myname,
        'nodes': list(nodes),
        'path': element,
        'tenant': configmanager.tenant,
        'operation': operation,
        'inputdata': inputdata
    })
    tlvdata.send(remote, {'dispatch': {'name': myname, 'length': len(dreq)}})
    remote.sendall(dreq)
    while True:
        try:
            rlen = remote.recv(8)
        except Exception:
            for node in nodes:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} went unreachable'.format(a['name']))
            return
        while len(rlen) < 8:
            try:
                nlen = remote.recv(8 - len(rlen))
            except Exception:
                nlen = 0
            if not nlen:
                for node in nodes:
                    yield msg.ConfluentResourceUnavailable(
                        node, 'Collective member {0} went unreachable'.format(
                            a['name']))
                return
            rlen += nlen
        rlen = struct.unpack('!Q', rlen)[0]
        if rlen == 0:
            break
        try:
            rsp = remote.recv(rlen)
        except Exception:
            for node in nodes:
                yield msg.ConfluentResourceUnavailable(
                    node,
                    'Collective member {0} went unreachable'.format(a['name']))
            return
        while len(rsp) < rlen:
            try:
                nrsp = remote.recv(rlen - len(rsp))
            except Exception:
                nrsp = 0
            if not nrsp:
                for node in nodes:
                    yield msg.ConfluentResourceUnavailable(
                        node, 'Collective member {0} went unreachable'.format(
                            a['name']))
                return
            rsp += nrsp
        rsp = pickle.loads(rsp)
        if isinstance(rsp, Exception):
            raise rsp
        yield rsp
Ejemplo n.º 16
0
 def authenticate(self, username, password):
     tlvdata.send(self.connection,
                  {'username': username, 'password': password})
     authdata = tlvdata.recv(self.connection)
     if authdata['authpassed'] == 1:
         self.authenticated = True
Ejemplo n.º 17
0
def sessionhdl(connection, authname, skipauth=False):
    # For now, trying to test the console stuff, so let's just do n4.
    authenticated = False
    authdata = None
    cfm = None
    if skipauth:
        authenticated = True
        cfm = configmanager.ConfigManager(tenant=None)
    elif authname:
        authdata = auth.authorize(authname, element=None)
        if authdata is not None:
            cfm = authdata[1]
            authenticated = True
    send_data(connection, "Confluent -- v0 --")
    while not authenticated:  # prompt for name and passphrase
        send_data(connection, {'authpassed': 0})
        response = tlvdata.recv(connection)
        authname = response['username']
        passphrase = response['password']
        # note(jbjohnso): here, we need to authenticate, but not
        # authorize a user.  When authorization starts understanding
        # element path, that authorization will need to be called
        # per request the user makes
        authdata = auth.check_user_passphrase(authname, passphrase)
        if authdata is None:
            auditlog.log(
                {'operation': 'connect', 'user': authname, 'allowed': False})
        else:
            authenticated = True
            cfm = authdata[1]
    send_data(connection, {'authpassed': 1})
    request = tlvdata.recv(connection)
    while request is not None:
        try:
            process_request(
                connection, request, cfm, authdata, authname, skipauth)
        except exc.ForbiddenRequest:
            send_data(connection, {'errorcode': 403,
                                      'error': 'Forbidden'})
            send_data(connection, {'_requestdone': 1})
        except exc.TargetEndpointBadCredentials:
            send_data(connection, {'errorcode': 502,
                                      'error': 'Bad Credentials'})
            send_data(connection, {'_requestdone': 1})
        except exc.TargetEndpointUnreachable as tu:
            send_data(connection, {'errorcode': 504,
                                      'error': 'Unreachable Target - ' + str(
                                          tu)})
            send_data(connection, {'_requestdone': 1})
        except exc.NotImplementedException:
            send_data(connection, {'errorcode': 501,
                                      'error': 'Not Implemented'})
            send_data(connection, {'_requestdone': 1})
        except exc.NotFoundException as nfe:
            send_data(connection, {'errorcode': 404,
                                      'error': str(nfe)})
            send_data(connection, {'_requestdone': 1})
        except exc.InvalidArgumentException as iae:
            send_data(connection, {'errorcode': 400,
                                      'error': 'Bad Request - ' + str(iae)})
            send_data(connection, {'_requestdone': 1})
        except exc.LockedCredentials as lockedcred:
            send_data(connection, {'errorcode': 500,
                                      'error': 'Locked Credential Store'})
            send_data(connection, {'_requestdone': 1})
        except SystemExit:
            sys.exit(0)
        except:
            tracelog.log(traceback.format_exc(), ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {'errorcode': 500,
                                      'error': 'Unexpected error'})
            send_data(connection, {'_requestdone': 1})
        request = tlvdata.recv(connection)
Ejemplo n.º 18
0
def process_request(connection, request, cfm, authdata, authname, skipauth):
    if not isinstance(request, dict):
        raise ValueError
    operation = request['operation']
    path = request['path']
    params = request.get('parameters', None)
    hdlr = None
    if not skipauth:
        authdata = auth.authorize(authdata[2], path, authdata[3], operation)
        auditmsg = {
            'operation': operation,
            'user': authdata[2],
            'target': path,
        }
        if authdata[3] is not None:
            auditmsg['tenant'] = authdata[3]
        if authdata is None:
            auditmsg['allowed'] = False
            auditlog.log(auditmsg)
            raise exc.ForbiddenRequest()
        auditmsg['allowed'] = True
        auditlog.log(auditmsg)
    try:
        if operation == 'start':
            elems = path.split('/')
            if elems[3] != "console":
                raise exc.InvalidArgumentException()
            node = elems[2]
            ccons = ClientConsole(connection)
            skipreplay = False
            if params and 'skipreplay' in params and params['skipreplay']:
                skipreplay = True
            consession = consoleserver.ConsoleSession(
                node=node, configmanager=cfm, username=authname,
                datacallback=ccons.sendall, skipreplay=skipreplay)
            if consession is None:
                raise Exception("TODO")
            send_data(connection, {'started': 1})
            ccons.startsending()
            bufferage = consession.get_buffer_age()
            if bufferage is not False:
                send_data(connection, {'bufferage': bufferage})
            while consession is not None:
                data = tlvdata.recv(connection)
                if type(data) == dict:
                    if data['operation'] == 'stop':
                        consession.destroy()
                        return
                    elif data['operation'] == 'break':
                        consession.send_break()
                        continue
                    elif data['operation'] == 'reopen':
                        consession.reopen()
                        continue
                    else:
                        raise Exception("TODO")
                if not data:
                    consession.destroy()
                    return
                consession.write(data)
        elif operation == 'shutdown':
            configmanager.ConfigManager.shutdown()
        else:
            hdlr = pluginapi.handle_path(path, operation, cfm, params)
    except exc.NotFoundException as e:
        send_data(connection, {"errorcode": 404,
                                  "error": "Target not found - " + str(e)})
        send_data(connection, {"_requestdone": 1})
    except exc.InvalidArgumentException as e:
        send_data(connection, {"errorcode": 400,
                                  "error": "Bad Request - " + str(e)})
        send_data(connection, {"_requestdone": 1})
    send_response(hdlr, connection)
    return
Ejemplo n.º 19
0
 def relay_data(self):
     data = tlvdata.recv(self.remote)
     while data:
         self.data_handler(data)
         data = tlvdata.recv(self.remote)
Ejemplo n.º 20
0
def connect_to_leader(cert=None, name=None, leader=None):
    global currentleader
    global follower
    if leader is None:
        leader = currentleader
    log.log({
        'info': 'Attempting connection to leader {0}'.format(leader),
        'subsystem': 'collective'
    })
    try:
        remote = connect_to_collective(cert, leader)
    except socket.error as e:
        log.log({
            'error':
            'Collective connection attempt to {0} failed: {1}'
            ''.format(leader, str(e)),
            'subsystem':
            'collective'
        })
        return False
    with connecting:
        with cfm._initlock:
            banner = tlvdata.recv(remote)  # the banner
            vers = banner.split()[2]
            if vers != b'v2':
                raise Exception(
                    'This instance only supports protocol 2, synchronize versions between collective members'
                )
            tlvdata.recv(remote)  # authpassed... 0..
            if name is None:
                name = get_myname()
            tlvdata.send(
                remote, {
                    'collective': {
                        'operation': 'connect',
                        'name': name,
                        'txcount': cfm._txcount
                    }
                })
            keydata = tlvdata.recv(remote)
            if not keydata:
                return False
            if 'error' in keydata:
                if 'backoff' in keydata:
                    log.log({
                        'info':
                        'Collective initialization in progress on '
                        '{0}'.format(leader),
                        'subsystem':
                        'collective'
                    })
                    return False
                if 'leader' in keydata:
                    log.log({
                        'info':
                        'Prospective leader {0} has redirected this '
                        'member to {1}'.format(leader, keydata['leader']),
                        'subsystem':
                        'collective'
                    })
                    ldrc = cfm.get_collective_member_by_address(
                        keydata['leader'])
                    if ldrc and ldrc['name'] == name:
                        raise Exception("Redirected to self")
                    return connect_to_leader(name=name,
                                             leader=keydata['leader'])
                if 'txcount' in keydata:
                    log.log({
                        'info':
                        'Prospective leader {0} has inferior '
                        'transaction count, becoming leader'
                        ''.format(leader),
                        'subsystem':
                        'collective',
                        'subsystem':
                        'collective'
                    })
                    return become_leader(remote)
                return False
                follower.kill()
                cfm.stop_following()
                follower = None
            if follower:
                follower.kill()
                cfm.stop_following()
                follower = None
            log.log({
                'info': 'Following leader {0}'.format(leader),
                'subsystem': 'collective'
            })
            colldata = tlvdata.recv(remote)
            # the protocol transmits global data, but for now we ignore it
            globaldata = tlvdata.recv(remote)
            dbi = tlvdata.recv(remote)
            dbsize = dbi['dbsize']
            dbjson = b''
            while (len(dbjson) < dbsize):
                ndata = remote.recv(dbsize - len(dbjson))
                if not ndata:
                    try:
                        remote.close()
                    except Exception:
                        pass
                    raise Exception("Error doing initial DB transfer")
                dbjson += ndata
            cfm.clear_configuration()
            try:
                cfm._restore_keys(keydata, None, sync=False)
                for c in colldata:
                    cfm._true_add_collective_member(c,
                                                    colldata[c]['address'],
                                                    colldata[c]['fingerprint'],
                                                    sync=False)
                #for globvar in globaldata:
                #    cfm.set_global(globvar, globaldata[globvar], False)
                cfm._txcount = dbi.get('txcount', 0)
                cfm.ConfigManager(tenant=None)._load_from_json(dbjson,
                                                               sync=False)
                cfm.commit_clear()
            except Exception:
                cfm.stop_following()
                cfm.rollback_clear()
                raise
            currentleader = leader
        #spawn this as a thread...
        follower = eventlet.spawn(follow_leader, remote, leader)
    return True
Ejemplo n.º 21
0
def sessionhdl(connection, authname, skipauth=False):
    # For now, trying to test the console stuff, so let's just do n4.
    authenticated = False
    authdata = None
    cfm = None
    if skipauth:
        authenticated = True
        cfm = configmanager.ConfigManager(tenant=None, username=authname)
    elif authname:
        authdata = auth.authorize(authname, element=None)
        if authdata is not None:
            cfm = authdata[1]
            authenticated = True
    send_data(connection, "Confluent -- v0 --")
    while not authenticated:  # prompt for name and passphrase
        send_data(connection, {'authpassed': 0})
        response = tlvdata.recv(connection)
        authname = response['username']
        passphrase = response['password']
        # note(jbjohnso): here, we need to authenticate, but not
        # authorize a user.  When authorization starts understanding
        # element path, that authorization will need to be called
        # per request the user makes
        authdata = auth.check_user_passphrase(authname, passphrase)
        if authdata is None:
            auditlog.log({
                'operation': 'connect',
                'user': authname,
                'allowed': False
            })
        else:
            authenticated = True
            cfm = authdata[1]
    send_data(connection, {'authpassed': 1})
    request = tlvdata.recv(connection)
    while request is not None:
        try:
            process_request(connection, request, cfm, authdata, authname,
                            skipauth)
        except exc.ForbiddenRequest:
            send_data(connection, {'errorcode': 403, 'error': 'Forbidden'})
            send_data(connection, {'_requestdone': 1})
        except exc.TargetEndpointBadCredentials:
            send_data(connection, {
                'errorcode': 502,
                'error': 'Bad Credentials'
            })
            send_data(connection, {'_requestdone': 1})
        except exc.TargetEndpointUnreachable as tu:
            send_data(connection, {
                'errorcode': 504,
                'error': 'Unreachable Target - ' + str(tu)
            })
            send_data(connection, {'_requestdone': 1})
        except exc.NotImplementedException:
            send_data(connection, {
                'errorcode': 501,
                'error': 'Not Implemented'
            })
            send_data(connection, {'_requestdone': 1})
        except exc.NotFoundException as nfe:
            send_data(connection, {'errorcode': 404, 'error': str(nfe)})
            send_data(connection, {'_requestdone': 1})
        except exc.InvalidArgumentException as iae:
            send_data(connection, {
                'errorcode': 400,
                'error': 'Bad Request - ' + str(iae)
            })
            send_data(connection, {'_requestdone': 1})
        except exc.LockedCredentials as lockedcred:
            send_data(connection, {
                'errorcode': 500,
                'error': 'Locked Credential Store'
            })
            send_data(connection, {'_requestdone': 1})
        except exc.ConfluentException as e:
            if e.apierrorcode == 500:
                tracelog.log(traceback.format_exc(),
                             ltype=log.DataTypes.event,
                             event=log.Events.stacktrace)
            send_data(
                connection, {
                    'errorcode': e.apierrorcode,
                    'error': e.apierrorstr,
                    'detail': e.get_error_body()
                })
            send_data(connection, {'_requestdone': 1})
        except SystemExit:
            sys.exit(0)
        except:
            tracelog.log(traceback.format_exc(),
                         ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {
                'errorcode': 500,
                'error': 'Unexpected error'
            })
            send_data(connection, {'_requestdone': 1})
        request = tlvdata.recv(connection)
Ejemplo n.º 22
0
def sessionhdl(connection, authname, skipauth=False, cert=None):
    # For now, trying to test the console stuff, so let's just do n4.
    authenticated = False
    authdata = None
    cfm = None
    if skipauth:
        authenticated = True
        cfm = configmanager.ConfigManager(tenant=None, username=authname)
    elif authname:
        authdata = auth.authorize(authname, element=None)
        if authdata is not None:
            cfm = authdata[1]
            authenticated = True
    send_data(connection, "Confluent -- v0 --")
    while not authenticated:  # prompt for name and passphrase
        send_data(connection, {'authpassed': 0})
        response = tlvdata.recv(connection)
        if 'collective' in response:
            return collective.handle_connection(connection, cert,
                                                response['collective'])
        if 'dispatch' in response:
            dreq = tlvdata.recvall(connection, response['dispatch']['length'])
            return pluginapi.handle_dispatch(connection, cert, dreq,
                                             response['dispatch']['name'])
        if 'proxyconsole' in response:
            return start_proxy_term(connection, cert, response['proxyconsole'])
        authname = response['username']
        passphrase = response['password']
        # note(jbjohnso): here, we need to authenticate, but not
        # authorize a user.  When authorization starts understanding
        # element path, that authorization will need to be called
        # per request the user makes
        authdata = auth.check_user_passphrase(authname, passphrase)
        if authdata is None:
            auditlog.log({
                'operation': 'connect',
                'user': authname,
                'allowed': False
            })
        else:
            authenticated = True
            cfm = authdata[1]
    send_data(connection, {'authpassed': 1})
    request = tlvdata.recv(connection)
    if 'collective' in request and skipauth:
        if not libssl:
            tlvdata.send(
                connection, {
                    'collective': {
                        'error':
                        'Server either does not have '
                        'python-pyopenssl installed or has an '
                        'incorrect version installed '
                        '(e.g. pyOpenSSL would need to be '
                        'replaced with python-pyopenssl)'
                    }
                })
            return
        return collective.handle_connection(connection,
                                            None,
                                            request['collective'],
                                            local=True)
    while request is not None:
        try:
            process_request(connection, request, cfm, authdata, authname,
                            skipauth)
        except exc.ConfluentException as e:
            if ((not isinstance(e, exc.LockedCredentials))
                    and e.apierrorcode == 500):
                tracelog.log(traceback.format_exc(),
                             ltype=log.DataTypes.event,
                             event=log.Events.stacktrace)
            send_data(
                connection, {
                    'errorcode': e.apierrorcode,
                    'error': e.apierrorstr,
                    'detail': e.get_error_body()
                })
            send_data(connection, {'_requestdone': 1})
        except SystemExit:
            sys.exit(0)
        except:
            tracelog.log(traceback.format_exc(),
                         ltype=log.DataTypes.event,
                         event=log.Events.stacktrace)
            send_data(connection, {
                'errorcode': 500,
                'error': 'Unexpected error'
            })
            send_data(connection, {'_requestdone': 1})
        request = tlvdata.recv(connection)
Ejemplo n.º 23
0
def connect_to_leader(cert=None, name=None, leader=None):
    global currentleader
    global cfginitlock
    global follower
    if cfginitlock is None:
        cfginitlock = threading.RLock()
    if leader is None:
        leader = currentleader
    try:
        remote = connect_to_collective(cert, leader)
    except socket.error:
        return False
    with connecting:
        with cfginitlock:
            tlvdata.recv(remote)  # the banner
            tlvdata.recv(remote)  # authpassed... 0..
            if name is None:
                name = get_myname()
            tlvdata.send(
                remote, {
                    'collective': {
                        'operation': 'connect',
                        'name': name,
                        'txcount': cfm._txcount
                    }
                })
            keydata = tlvdata.recv(remote)
            if not keydata:
                return False
            if 'error' in keydata:
                if 'backoff' in keydata:
                    eventlet.spawn_after(random.random(), connect_to_leader,
                                         cert, name, leader)
                    return True
                if 'leader' in keydata:
                    ldrc = cfm.get_collective_member_by_address(
                        keydata['leader'])
                    if ldrc and ldrc['name'] == name:
                        raise Exception("Redirected to self")
                    return connect_to_leader(name=name,
                                             leader=keydata['leader'])
                if 'txcount' in keydata:
                    return become_leader(remote)
                print(keydata['error'])
                return False
            if follower is not None:
                follower.kill()
                cfm.stop_following()
                follower = None
            colldata = tlvdata.recv(remote)
            globaldata = tlvdata.recv(remote)
            dbi = tlvdata.recv(remote)
            dbsize = dbi['dbsize']
            dbjson = ''
            while (len(dbjson) < dbsize):
                ndata = remote.recv(dbsize - len(dbjson))
                if not ndata:
                    try:
                        remote.close()
                    except Exception:
                        pass
                    raise Exception("Error doing initial DB transfer")
                dbjson += ndata
            cfm.clear_configuration()
            try:
                cfm._restore_keys(keydata, None, sync=False)
                for c in colldata:
                    cfm._true_add_collective_member(c,
                                                    colldata[c]['address'],
                                                    colldata[c]['fingerprint'],
                                                    sync=False)
                for globvar in globaldata:
                    cfm.set_global(globvar, globaldata[globvar], False)
                cfm._txcount = dbi.get('txcount', 0)
                cfm.ConfigManager(tenant=None)._load_from_json(dbjson,
                                                               sync=False)
                cfm.commit_clear()
            except Exception:
                cfm.stop_following()
                cfm.rollback_clear()
                raise
            currentleader = leader
        #spawn this as a thread...
        follower = eventlet.spawn(follow_leader, remote)
    return True