Ejemplo n.º 1
0
class FileWrapRemote(FileWrapBase):
    def __init__(self, uri, type=None, username='******', password=''):
        super(FileWrapRemote, self).__init__(uri, type)
        self.client = Client()
        self.username = username
        self.password = password

    def readdir(self):
        if not self.is_dir:
            raise NotADirectoryError
        self.client.connect('ws://'+self.hostname)
        self.client.login_user(self.username, self.password)
        for e in self.client.call_sync('filesystem.list_dir', self.path):
            yield FileWrapRemote(
                PurePath(self.uri).joinpath(e['name']), type=e['type'], username=self.username, password=self.password)

    def _map_type(self, val):
        self._freenas_mappings = {
            'DIRECTORY': FileType.dir,
            'FILE': FileType.file,
        }
        return self._freenas_mappings[val]

    def _get_type(self):
        #self.client.connect('ws://'+self.hostname)
        #self.client.login_user(self.username, self.password)
        """ TODO """
        return self._map_type('DIRECTORY')

    def _get_parent(self):
        return FileWrapRemote(PurePath(self.uri).parent.as_posix(), username=self.username, password=self.password)
Ejemplo n.º 2
0
def get_replication_client(dispatcher, remote):
    host = dispatcher.call_sync(
        'peer.query',
        [('address', '=', remote), ('type', '=', 'replication')],
        {'single': True}
    )
    if not host:
        raise TaskException(errno.ENOENT, 'There are no known keys to connect to {0}'.format(remote))

    with open('/etc/replication/key') as f:
        pkey = RSAKey.from_private_key(f)

    credentials = host['credentials']

    try:
        client = Client()
        with tempfile.NamedTemporaryFile('w') as host_key_file:
            host_key_file.write(credentials['hostkey'])
            host_key_file.flush()
            client.connect(
                'ws+ssh://replication@{0}'.format(remote),
                port=credentials['port'],
                host_key_file=host_key_file.name,
                pkey=pkey
            )
        client.login_service('replicator')
        return client

    except (AuthenticationException, SSHException):
        raise TaskException(errno.EAUTH, 'Cannot connect to {0}'.format(remote))
    except (OSError, ConnectionRefusedError):
        raise TaskException(errno.ECONNREFUSED, 'Cannot connect to {0}'.format(remote))
    except IOError:
        raise TaskException(errno.EINVAL, 'Provided host key is not valid')
Ejemplo n.º 3
0
    def process_request(self, req, resp):
        # Do not require auth to access index
        if req.relative_uri == '/':
            return

        auth = req.get_header("Authorization")
        if auth is None or not auth.startswith('Basic '):
            raise falcon.HTTPUnauthorized(
                'Authorization token required',
                'Provide a Basic Authentication header',
                ['Basic realm="FreeNAS"'],
            )
        try:
            username, password = base64.b64decode(auth[6:]).decode('utf8').split(':', 1)
        except binascii.Error:
            raise falcon.HTTPUnauthorized(
                'Invalid Authorization token',
                'Provide a valid Basic Authentication header',
                ['Basic realm="FreeNAS"'],
            )

        try:
            client = Client()
            client.connect('unix:')
            client.login_user(username, password, check_password=True)
            req.context['client'] = client
        except RpcException as e:
            if e.code == errno.EACCES:
                raise falcon.HTTPUnauthorized(
                    'Invalid credentials',
                    'Verify your credentials and try again.',
                    ['Basic realm="FreeNAS"'],
                )
            raise falcon.HTTPUnauthorized('Unknown authentication error', str(e), ['Basic realm="FreeNAS"'])
Ejemplo n.º 4
0
class SyslogProvider(Provider):
    def initialize(self, context):
        self.client = Client()
        self.client.connect('unix:///var/run/logd.sock')

    @generator
    def query(self, filter=None, params=None):
        return self.client.call_sync('logd.logging.query', filter, params)
Ejemplo n.º 5
0
    def setUp(self):
        super(BaseTestCase, self).setUp()

        assert self.context is not None
        self.ssh_client = self.context.ssh_client
        self.client = Client()
        self.client.connect('ws://{0}'.format(self.context.hostname))
        self.client.login_user(self.context.username, self.context.password)
        load_schema_definitions(self.client)
Ejemplo n.º 6
0
    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()
Ejemplo n.º 7
0
def main(name, *args):
    connection = Client()
    connection.connect('127.0.0.1')
    connection.login_service('ups')

    connection.emit_event('service.ups.signal', {
        'name': name,
        'type': os.environ['NOTIFYTYPE'],
    })

    connection.disconnect()
Ejemplo n.º 8
0
 def setUp(self):
     try:
         self.conn = Client()
         self.conn.event_callback = self.on_event
         self.conn.connect(os.getenv('TESTHOST', '127.0.0.1'))
         self.conn.login_user(os.getenv('TESTUSER', 'root'),
                              os.getenv('TESTPWD', ''),
                              timeout=self.task_timeout)
         self.conn.subscribe_events('*')
     except:
         raise
Ejemplo n.º 9
0
class Context(object):
    def __init__(self, *args, **kwargs):
        self.client = None
        self.entity_subscribers = {}

    def start_entity_subscribers(self):
        for i in ENTITY_SUBSCRIBERS:
            if i in self.entity_subscribers:
                self.entity_subscribers[i].stop()
                del self.entity_subscribers[i]

            e = EntitySubscriber(self.client, i)
            e.start()
            self.entity_subscribers[i] = e

    def wait_entity_subscribers(self):
        for i in self.entity_subscribers.values():
            i.wait_ready()

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('collectd_{0}'.format(PLUGIN_NAME))
                # enable streaming responses as they are needed but entitysubscriber for
                # reliable performace and such
                self.client.call_sync('management.enable_features',
                                      ['streaming_responses'])
                self.start_entity_subscribers()
                self.wait_entity_subscribers()
                return
            except (OSError, RpcException) as err:
                collectd.warning(
                    "{0} collectd plugin could not connect to server retrying in 5 seconds"
                    .format(PLUGIN_NAME))
                time.sleep(5)

    def connection_error(self, event, **kwargs):
        if event in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
            collectd.info(
                '{0} collectd plugin connection to dispatcher lost'.format(
                    PLUGIN_NAME))
            self.connect()

    def init_dispatcher(self):
        self.client = Client()
        self.client.on_error(self.connection_error)
        self.connect()

    def disk_temps(self):
        for disk in self.entity_subscribers['disk'].query(
            ('status.smart_info.temperature', '!=', None)):
            yield (disk['name'], disk['status']['smart_info']['temperature'])
Ejemplo n.º 10
0
def main(*args):
    connection = Client()
    connection.connect('127.0.0.1')
    connection.login_service('smtp')

    parser = argparse.ArgumentParser(description='Process email')
    parser.add_argument('-i', dest='strip_leading_dot', action='store_false',
                        default=True, help='see sendmail(8) -i')
    parser.add_argument('-t', dest='parse_recipients', action='store_true',
                        default=False,
                        help='parse recipients from message')
    parser.usage = ' '.join(parser.format_usage().split(' ')[1:-1])
    parser.usage += ' [email_addr|user] ..'
    args, to_addrs = parser.parse_known_args()
    if not to_addrs and not args.parse_recipients:
        parser.exit(message=parser.format_usage())
    msg = sys.stdin.read()

    em_parser = email.parser.Parser()
    em = em_parser.parsestr(msg)
    if args.parse_recipients:
        # Strip away the comma based delimiters and whitespace.
        to_addrs = map(str.strip, em.get('To').split(','))

    if not to_addrs or not to_addrs[0]:
        to_addrs = ['root']

    margs = {}
    margs['extra_headers'] = dict(em)
    margs['extra_headers'].update({
        'X-Mailer': 'FreeNAS',
        'X-FreeNAS-Host': socket.gethostname(),
    })
    margs['subject'] = em.get('Subject')

    if em.is_multipart():
        margs['attachments'] = filter(
            lambda part: part.get_content_maintype() != 'multipart',
            em.walk()
        )
        margs['message'] = (
            'This is a MIME formatted message.  If you see '
            'this text it means that your email software '
            'does not support MIME formatted messages.')
    else:
        margs['message'] = ''.join(email.iterators.body_line_iterator(em))

    if to_addrs:
        margs['to'] = to_addrs

    connection.call_sync('mail.send', margs)
    connection.disconnect()
Ejemplo n.º 11
0
class Context(object):
    def __init__(self, *args, **kwargs):
        self.client = None
        self.entity_subscribers = {}

    def start_entity_subscribers(self):
        for i in ENTITY_SUBSCRIBERS:
            if i in self.entity_subscribers:
                self.entity_subscribers[i].stop()
                del self.entity_subscribers[i]

            e = EntitySubscriber(self.client, i)
            e.start()
            self.entity_subscribers[i] = e

    def wait_entity_subscribers(self):
        for i in self.entity_subscribers.values():
            i.wait_ready()

    def connect(self):
        while True:
            try:
                self.client.connect("unix:")
                self.client.login_service("collectd_{0}".format(PLUGIN_NAME))
                # enable streaming responses as they are needed but entitysubscriber for
                # reliable performace and such
                self.client.call_sync("management.enable_features", ["streaming_responses"])
                self.start_entity_subscribers()
                self.wait_entity_subscribers()
                return
            except (OSError, RpcException) as err:
                collectd.warning(
                    "{0} collectd plugin could not connect to server retrying in 5 seconds".format(PLUGIN_NAME)
                )
                time.sleep(5)

    def connection_error(self, event, **kwargs):
        if event in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
            collectd.info("{0} collectd plugin connection to dispatcher lost".format(PLUGIN_NAME))
            self.connect()

    def init_dispatcher(self):
        self.client = Client()
        self.client.on_error(self.connection_error)
        self.connect()

    def disk_temps(self):
        for disk in self.entity_subscribers["disk"].query(("status.smart_info.temperature", "!=", None)):
            yield (disk["name"], disk["status"]["smart_info"]["temperature"])
Ejemplo n.º 12
0
 def __init__(self):
     self.logger = logging.getLogger(self.__class__.__name__)
     self.msock = msock.client.Client()
     self.msock.on_closed = self.on_msock_close
     self.rpc_fd = -1
     self.connection_id = None
     self.jobs = []
     self.state = ConnectionState.OFFLINE
     self.config = None
     self.keepalive = None
     self.connected_at = None
     self.cv = Condition()
     self.rpc = RpcContext()
     self.client = Client()
     self.server = Server()
     self.middleware_endpoint = None
Ejemplo n.º 13
0
    def connect_keepalive(self):
        while True:
            try:
                if not self.connection_id:
                    self.connection_id = uuid.uuid4()

                self.msock.connect(SUPPORT_PROXY_ADDRESS)
                self.logger.info("Connecting to {0}".format(SUPPORT_PROXY_ADDRESS))
                self.rpc_fd = self.msock.create_channel(0)
                time.sleep(1)  # FIXME
                self.client = Client()
                self.client.connect("fd://", fobj=self.rpc_fd)
                self.client.channel_serializer = MSockChannelSerializer(self.msock)
                self.client.standalone_server = True
                self.client.enable_server()
                self.client.register_service("debug", DebugService(self))
                self.client.call_sync(
                    "server.login", str(self.connection_id), socket.gethostname(), get_version(), "none"
                )
                self.set_state(ConnectionState.CONNECTED)
            except BaseException as err:
                self.logger.warning("Failed to initiate support connection: {0}".format(err), exc_info=True)
                self.msock.disconnect()
            else:
                self.connected_at = datetime.now()
                with self.cv:
                    self.cv.wait_for(lambda: self.state in (ConnectionState.LOST, ConnectionState.OFFLINE))
                    if self.state == ConnectionState.OFFLINE:
                        return

            self.logger.warning("Support connection lost, retrying in 10 seconds")
            time.sleep(10)
Ejemplo n.º 14
0
    def test_unix_server(self):
        sockpath = os.path.join(os.getcwd(),
                                'test.{0}.sock'.format(os.getpid()))
        sockurl = 'unix://' + sockpath

        context = RpcContext()
        context.register_service('test', TestService)
        server = Server()
        server.rpc = context
        server.start(sockurl)
        threading.Thread(target=server.serve_forever, daemon=True).start()

        # Spin until server is ready
        while not os.path.exists(sockpath):
            time.sleep(0.1)

        client = Client()
        client.connect(sockurl)
        self.assertTrue(client.connected)
        self.assertEqual(client.call_sync('test.hello', 'freenas'),
                         'Hello World, freenas')

        client.disconnect()
        server.close()
        os.unlink(sockpath)
Ejemplo n.º 15
0
    def main(self):
        if len(sys.argv) != 2:
            print("Invalid number of arguments", file=sys.stderr)
            sys.exit(errno.EINVAL)

        key = sys.argv[1]
        logging.basicConfig(level=logging.DEBUG)

        self.datastore = get_default_datastore()
        self.configstore = ConfigStore(self.datastore)
        self.conn = Client()
        self.conn.connect('unix:')
        self.conn.login_service('task.{0}'.format(os.getpid()))
        self.conn.enable_server()
        self.conn.rpc.register_service_instance('taskproxy', self.service)
        self.conn.call_sync('task.checkin', key)
        setproctitle.setproctitle('task executor (idle)')

        while True:
            try:
                task = self.task.get()
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                if task['debugger']:
                    sys.path.append('/usr/local/lib/dispatcher/pydev')

                    import pydevd
                    host, port = task['debugger']
                    pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True)

                module = imp.load_source('plugin', task['filename'])
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                try:
                    self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore)
                    self.instance.configstore = self.configstore
                    self.running.set()
                    result = self.instance.run(*task['args'])
                except BaseException as err:
                    print("Task exception: {0}".format(str(err)), file=sys.stderr)
                    traceback.print_exc(file=sys.stderr)
                    self.put_status('FAILED', exception=err)
                else:
                    self.put_status('FINISHED', result=result)

            except RpcException as err:
                print("RPC failed: {0}".format(str(err)), file=sys.stderr)
                sys.exit(errno.EBADMSG)
            except socket.error as err:
                print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr)
                sys.exit(errno.ETIMEDOUT)

            if task['debugger']:
                import pydevd
                pydevd.stoptrace()

            setproctitle.setproctitle('task executor (idle)')
Ejemplo n.º 16
0
 def setUp(self):
     try:
         self.conn = Client()
         self.conn.event_callback = self.on_event
         self.conn.connect(os.getenv('TESTHOST', '127.0.0.1'))
         self.conn.login_user(os.getenv('TESTUSER', 'root'), os.getenv('TESTPWD', ''), timeout = self.task_timeout)
         self.conn.subscribe_events('*')
     except:
         raise
Ejemplo n.º 17
0
    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()
Ejemplo n.º 18
0
    def setUp(self):
        super(BaseTestCase, self).setUp()

        assert self.context is not None
        self.ssh_client = self.context.ssh_client
        self.client = Client()
        self.client.connect('ws://{0}'.format(self.context.hostname))
        self.client.login_user(self.context.username, self.context.password)
        load_schema_definitions(self.client)
Ejemplo n.º 19
0
def get_freenas_peer_client(parent, remote):
    try:
        address = socket.gethostbyname(remote)
    except socket.error as err:
        raise TaskException(err.errno, '{0} is unreachable'.format(remote))

    host = parent.dispatcher.call_sync(
        'peer.query', [
            ('or', [
                ('credentials.address', '=', remote),
                ('credentials.address', '=', address),
            ]),
            ('type', '=', 'freenas')
        ],
        {'single': True}
    )
    if not host:
        raise TaskException(errno.ENOENT, 'There are no known keys to connect to {0}'.format(remote))

    with io.StringIO() as f:
        f.write(parent.configstore.get('peer.freenas.key.private'))
        f.seek(0)
        pkey = RSAKey.from_private_key(f)

    credentials = host['credentials']

    try:
        client = Client()
        with tempfile.NamedTemporaryFile('w') as host_key_file:
            host_key_file.write(remote + ' ' + credentials['hostkey'])
            host_key_file.flush()
            client.connect(
                'ws+ssh://freenas@{0}'.format(wrap_address(remote)),
                port=credentials['port'],
                host_key_file=host_key_file.name,
                pkey=pkey
            )
        client.login_service('replicator')
        return client

    except (AuthenticationException, SSHException):
        raise TaskException(errno.EAUTH, 'Cannot connect to {0}'.format(remote))
    except OSError as err:
        raise TaskException(errno.ECONNREFUSED, 'Cannot connect to {0}: {1}'.format(remote, err))
Ejemplo n.º 20
0
def main(*args):
    connection = Client()
    connection.connect("127.0.0.1")
    connection.login_service("smtp")

    parser = argparse.ArgumentParser(description="Process email")
    parser.add_argument("-i", dest="strip_leading_dot", action="store_false", default=True, help="see sendmail(8) -i")
    parser.add_argument(
        "-t", dest="parse_recipients", action="store_true", default=False, help="parse recipients from message"
    )
    parser.usage = " ".join(parser.format_usage().split(" ")[1:-1])
    parser.usage += " [email_addr|user] .."
    args, to_addrs = parser.parse_known_args()
    if not to_addrs and not args.parse_recipients:
        parser.exit(message=parser.format_usage())
    msg = sys.stdin.read()

    em_parser = email.parser.Parser()
    em = em_parser.parsestr(msg)
    if args.parse_recipients:
        # Strip away the comma based delimiters and whitespace.
        to_addrs = map(str.strip, em.get("To").split(","))

    if not to_addrs or not to_addrs[0]:
        to_addrs = ["root"]

    margs = {}
    margs["extra_headers"] = dict(em)
    margs["extra_headers"].update({"X-Mailer": "FreeNAS", "X-FreeNAS-Host": socket.gethostname()})
    margs["subject"] = em.get("Subject")

    if em.is_multipart():
        margs["attachments"] = filter(lambda part: part.get_content_maintype() != "multipart", em.walk())
        margs["message"] = (
            "This is a MIME formatted message.  If you see "
            "this text it means that your email software "
            "does not support MIME formatted messages."
        )
    else:
        margs["message"] = "".join(email.iterators.body_line_iterator(em))

    if to_addrs:
        margs["to"] = to_addrs

    connection.call_sync("mail.send", margs)
    connection.disconnect()
Ejemplo n.º 21
0
    def connect_keepalive(self):
        while True:
            try:
                if not self.connection_id:
                    self.connection_id = uuid.uuid4()

                self.msock.connect(SUPPORT_PROXY_ADDRESS)
                self.logger.info(
                    'Connecting to {0}'.format(SUPPORT_PROXY_ADDRESS))
                self.rpc_fd = self.msock.create_channel(0)
                time.sleep(1)  # FIXME
                self.client = Client()
                self.client.connect('fd://', fobj=self.rpc_fd)
                self.client.channel_serializer = MSockChannelSerializer(
                    self.msock)
                self.client.standalone_server = True
                self.client.enable_server()
                self.client.register_service('debug', DebugService(self))
                self.client.call_sync('server.login', str(self.connection_id),
                                      socket.gethostname(), get_version(),
                                      'none')
                self.set_state(ConnectionState.CONNECTED)
            except BaseException as err:
                self.logger.warning(
                    'Failed to initiate support connection: {0}'.format(err),
                    exc_info=True)
                self.msock.disconnect()
            else:
                self.connected_at = datetime.now()
                with self.cv:
                    self.cv.wait_for(lambda: self.state in (
                        ConnectionState.LOST, ConnectionState.OFFLINE))
                    if self.state == ConnectionState.OFFLINE:
                        return

            self.logger.warning(
                'Support connection lost, retrying in 10 seconds')
            time.sleep(10)
Ejemplo n.º 22
0
 def __init__(self):
     self.logger = logging.getLogger(self.__class__.__name__)
     self.msock = msock.client.Client()
     self.msock.on_closed = self.on_msock_close
     self.rpc_fd = -1
     self.connection_id = None
     self.jobs = []
     self.state = ConnectionState.OFFLINE
     self.config = None
     self.keepalive = None
     self.connected_at = None
     self.cv = Condition()
     self.rpc = RpcContext()
     self.client = Client()
     self.server = Server()
     self.middleware_endpoint = None
Ejemplo n.º 23
0
 def __init__(self):
     self.hostname = None
     self.connection = Client()
     self.ml = None
     self.logger = logging.getLogger('cli')
     self.plugin_dirs = []
     self.task_callbacks = {}
     self.plugins = {}
     self.variables = VariableStore()
     self.root_ns = RootNamespace('')
     self.event_masks = ['*']
     self.event_divert = False
     self.event_queue = six.moves.queue.Queue()
     self.keepalive_timer = None
     self.argparse_parser = None
     config.instance = self
Ejemplo n.º 24
0
class LogdLogHandler(logging.Handler):
    def __init__(self, level=logging.NOTSET, address=None, ident=None):
        super(LogdLogHandler, self).__init__(level)
        self.address = address or 'unix:///var/run/logd.sock'
        self.ident = ident or os.path.basename(sys.executable)
        self.client = Client()
        self.client.connect(self.address)

    def emit(self, record):
        try:
            if not self.client.connected:
                self.client.connect(self.address)

            item = {
                'timestamp': datetime.utcfromtimestamp(record.created),
                'priority': PRIORITY_MAP.get(record.levelno, 'INFO'),
                'message': record.getMessage(),
                'identifier': self.ident,
                'thread': record.threadName,
                'tid': record.thread,
                'module_name': record.name,
                'source_language': 'python',
                'source_file': record.pathname,
                'source_line': record.lineno,
            }

            if record.exc_info:
                item['exception'] = ''.join(
                    traceback.format_exception(*record.exc_info))

            self.client.call_async('logd.logging.push', None, item)
        except:
            self.handleError(record)

    def close(self):
        super(LogdLogHandler, self).close()
        self.client.disconnect()
Ejemplo n.º 25
0
class LogdLogHandler(logging.Handler):
    def __init__(self, level=logging.NOTSET, address=None, ident=None):
        super(LogdLogHandler, self).__init__(level)
        self.address = address or 'unix:///var/run/logd.sock'
        self.ident = ident or os.path.basename(sys.executable)
        self.client = Client()
        self.client.connect(self.address)

    def emit(self, record):
        try:
            if not self.client.connected:
                self.client.connect(self.address)

            item = {
                'timestamp': datetime.utcfromtimestamp(record.created),
                'priority': PRIORITY_MAP.get(record.levelno, 'INFO'),
                'message': record.getMessage(),
                'identifier': self.ident,
                'thread': record.threadName,
                'tid': record.thread,
                'module_name': record.name,
                'source_language': 'python',
                'source_file': record.pathname,
                'source_line': record.lineno,
            }

            if record.exc_info:
                item['exception'] = ''.join(traceback.format_exception(*record.exc_info))

            self.client.call_async('logd.logging.push', None, item)
        except:
            self.handleError(record)

    def close(self):
        super(LogdLogHandler, self).close()
        self.client.disconnect()
Ejemplo n.º 26
0
def get_freenas_peer_client(parent, remote):
    try:
        address = socket.gethostbyname(remote)
    except socket.error as err:
        raise TaskException(err.errno, '{0} is unreachable'.format(remote))

    host = parent.dispatcher.call_sync(
        'peer.query', [
            ('or', [
                ('credentials.address', '=', remote),
                ('credentials.address', '=', address),
            ]),
            ('type', '=', 'freenas')
        ],
        {'single': True}
    )
    if not host:
        raise TaskException(errno.ENOENT, 'There are no known keys to connect to {0}'.format(remote))

    with io.StringIO() as f:
        f.write(parent.configstore.get('peer.freenas.key.private'))
        f.seek(0)
        pkey = RSAKey.from_private_key(f)

    credentials = host['credentials']

    try:
        client = Client()
        with tempfile.NamedTemporaryFile('w') as host_key_file:
            host_key_file.write(remote + ' ' + credentials['hostkey'])
            host_key_file.flush()
            client.connect(
                'ws+ssh://freenas@{0}'.format(wrap_address(remote)),
                port=credentials['port'],
                host_key_file=host_key_file.name,
                pkey=pkey
            )
        client.login_service('replicator')
        return client

    except (AuthenticationException, SSHException):
        raise TaskException(errno.EAUTH, 'Cannot connect to {0}'.format(remote))
    except OSError as err:
        raise TaskException(errno.ECONNREFUSED, 'Cannot connect to {0}: {1}'.format(remote, err))
Ejemplo n.º 27
0
class BaseTestCase(unittest.TestCase):
    def __init__(self, methodName):
        super(BaseTestCase, self).__init__(methodName)
        self.context = None

    def setUp(self):
        super(BaseTestCase, self).setUp()

        assert self.context is not None
        self.ssh_client = self.context.ssh_client
        self.client = Client()
        self.client.connect('ws://{0}'.format(self.context.hostname))
        self.client.login_user(self.context.username, self.context.password)
        load_schema_definitions(self.client)

    def tearDown(self):
        self.client.disconnect()

    def ssh_exec(self, command, output=False):
        _, stdout, stderr = self.ssh_client.exec_command(command)
        exitcode = stdout.channel.recv_exit_status()
        if output:
            return exitcode, stdout.read(), stderr.read()

        return exitcode

    def get_params_schema(self, method):
        return get_methods(self.client, method).get('params-schema')

    def get_result_schema(self, method):
        return get_methods(self.client, method).get('results-schema')

    def assertConformsToSchema(self, obj, schema, strict=False):
        errors = verify_schema(schema, obj, strict)
        if errors:
            raise AssertionError(
                'Object {0} does not match {1} schema. Errors: {2}'.format(
                    obj, schema, errors))

    def assertConformsToNamedSchema(self, obj, schema_name, strict=False):
        schema = get_schema(schema_name)
        if not schema:
            raise AssertionError('Schema {0} is unknown'.format(schema_name))
        self.assertConformsToSchema(obj, schema, strict)
Ejemplo n.º 28
0
class BaseTestCase(unittest.TestCase):
    def __init__(self, methodName):
        super(BaseTestCase, self).__init__(methodName)
        self.context = None

    def setUp(self):
        super(BaseTestCase, self).setUp()

        assert self.context is not None
        self.ssh_client = self.context.ssh_client
        self.client = Client()
        self.client.connect('ws://{0}'.format(self.context.hostname))
        self.client.login_user(self.context.username, self.context.password)
        load_schema_definitions(self.client)

    def tearDown(self):
        self.client.disconnect()

    def ssh_exec(self, command, output=False):
        _, stdout, stderr = self.ssh_client.exec_command(command)
        exitcode = stdout.channel.recv_exit_status()
        if output:
            return exitcode, stdout.read(), stderr.read()

        return exitcode

    def get_params_schema(self, method):
        return get_methods(self.client, method).get('params-schema')

    def get_result_schema(self, method):
        return get_methods(self.client, method).get('results-schema')

    def assertConformsToSchema(self, obj, schema, strict=False):
        errors = verify_schema(schema, obj, strict)
        if errors:
            raise AssertionError('Object {0} does not match {1} schema. Errors: {2}'.format(obj, schema, errors))

    def assertConformsToNamedSchema(self, obj, schema_name, strict=False):
        schema = get_schema(schema_name)
        if not schema:
            raise AssertionError('Schema {0} is unknown'.format(schema_name))
        self.assertConformsToSchema(obj, schema, strict)
Ejemplo n.º 29
0
    def test_unix_server(self):
        sockpath = os.path.join(os.getcwd(), 'test.{0}.sock'.format(os.getpid()))
        sockurl = 'unix://' + sockpath

        context = RpcContext()
        context.register_service('test', TestService)
        server = Server()
        server.rpc = context
        server.start(sockurl)
        threading.Thread(target=server.serve_forever, daemon=True).start()

        # Spin until server is ready
        while not os.path.exists(sockpath):
            time.sleep(0.1)

        client = Client()
        client.connect(sockurl)
        self.assertTrue(client.connected)
        self.assertEqual(client.call_sync('test.hello', 'freenas'), 'Hello World, freenas')

        client.disconnect()
        server.close()
        os.unlink(sockpath)
Ejemplo n.º 30
0
    def process_request(self, req, resp):
        # Do not require auth to access index
        if req.relative_uri == '/':
            return

        auth = req.get_header("Authorization")
        if auth is None or not auth.startswith('Basic '):
            raise falcon.HTTPUnauthorized(
                'Authorization token required',
                'Provide a Basic Authentication header',
                ['Basic realm="FreeNAS"'],
            )
        try:
            username, password = base64.b64decode(
                auth[6:]).decode('utf8').split(':', 1)
        except binascii.Error:
            raise falcon.HTTPUnauthorized(
                'Invalid Authorization token',
                'Provide a valid Basic Authentication header',
                ['Basic realm="FreeNAS"'],
            )

        try:
            client = Client()
            client.connect('unix:')
            client.login_user(username, password, check_password=True)
            req.context['client'] = client
        except RpcException as e:
            if e.code == errno.EACCES:
                raise falcon.HTTPUnauthorized(
                    'Invalid credentials',
                    'Verify your credentials and try again.',
                    ['Basic realm="FreeNAS"'],
                )
            raise falcon.HTTPUnauthorized('Unknown authentication error',
                                          str(e), ['Basic realm="FreeNAS"'])
Ejemplo n.º 31
0
class RESTApi(object):
    def __init__(self):
        self.logger = logging.getLogger('restd')
        self._cruds = []
        self._threads = []
        self._rpcs = {}
        self._schemas = {}
        self._used_schemas = set()
        self._services = {}
        self._tasks = {}
        self.api = falcon.API(middleware=[
            AuthMiddleware(),
            JSONTranslator(),
        ])
        self.api.add_route('/', SwaggerResource(self))

        gevent.signal(signal.SIGINT, self.die)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.dispatcher = Client()
        self.dispatcher.on_error(on_error)
        self.connect()

    def init_metadata(self):
        self._tasks = self.dispatcher.call_sync('discovery.get_tasks')
        self._schemas = self.dispatcher.call_sync('discovery.get_schema')
        for service in self.dispatcher.call_sync('discovery.get_services'):
            self._services[service] = self.dispatcher.call_sync(
                'discovery.get_methods', service)
            for method in self._services[service]:
                self._rpcs['{0}.{1}'.format(service, method['name'])] = method

    def load_plugins(self):
        pluginsdir = os.path.realpath(
            os.path.join(os.path.dirname(__file__), '..', 'plugins'))
        for i in glob.glob1(pluginsdir, "*.py"):
            try:
                loader = importlib.machinery.SourceFileLoader(
                    i.split('.')[0], os.path.join(pluginsdir, i))
                mod = loader.load_module()
            except:
                self.logger.error('Failed to load plugin %s', i, exc_info=True)
                raise
            mod._init(self)

    def connect(self):
        while True:
            try:
                self.dispatcher.connect('unix:')
                self.dispatcher.login_service('restd')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def __call__(self, environ, start_response):
        if 'HTTP_X_REAL_IP' in environ:
            environ['PATH_INFO'] = environ.get('PATH_INFO',
                                               '').replace('/api/v2.0', '', 1)
        return self.api.__call__(environ, start_response)

    def register_crud(self, klass):
        ins = klass(self, self.dispatcher)
        self._cruds.append(ins)

    def register_singleitem(self, klass):
        klass(self, self.dispatcher)

    def register_resource(self, klass):
        klass(self)

    def run(self):
        self.init_dispatcher()
        self.init_metadata()
        self.load_plugins()

        server4 = WSGIServer(('0.0.0.0', 8889),
                             self,
                             handler_class=RESTWSGIHandler)
        self._threads = [gevent.spawn(server4.serve_forever)]
        checkin()
        gevent.joinall(self._threads)

    def die(self, *args):
        gevent.killall(self._threads)
        sys.exit(0)
Ejemplo n.º 32
0
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#####################################################################

from threading import Lock
from freenas.dispatcher.client import Client
from freenas.dispatcher.rpc import RpcException

SERVICED_SOCKET = 'unix:///var/run/serviced.sock'
_client = Client()
_lock = Lock()


class ServicedException(RpcException):
    pass


def checkin():
    with _lock:
        try:
            _client.connect(SERVICED_SOCKET)
            return _client.call_sync('serviced.job.checkin')
        except RpcException as err:
            raise ServicedException(err.code, err.message, err.extra)
        finally:
Ejemplo n.º 33
0
class Context(object):
    def __init__(self):
        self.logger = logging.getLogger(self.__class__.__name__)
        self.msock = msock.client.Client()
        self.msock.on_closed = self.on_msock_close
        self.rpc_fd = -1
        self.connection_id = None
        self.jobs = []
        self.state = ConnectionState.OFFLINE
        self.config = None
        self.keepalive = None
        self.connected_at = None
        self.cv = Condition()
        self.rpc = RpcContext()
        self.client = Client()
        self.server = Server()
        self.middleware_endpoint = None

    def start(self, configpath, sockpath):
        signal.signal(signal.SIGUSR2, lambda signo, frame: self.connect())
        self.read_config(configpath)
        self.server.rpc = RpcContext()
        self.server.rpc.register_service_instance('control',
                                                  ControlService(self))
        self.server.start(sockpath)
        threading.Thread(target=self.server.serve_forever,
                         name='server thread',
                         daemon=True).start()

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect_dispatcher()

        self.middleware_endpoint = Client()
        self.middleware_endpoint.on_error(on_error)
        self.connect_dispatcher()

    def connect_dispatcher(self):
        while True:
            try:
                self.middleware_endpoint.connect('unix:')
                self.middleware_endpoint.login_service('debugd')
                self.middleware_endpoint.enable_server()
                self.middleware_endpoint.register_service(
                    'debugd.management', ControlService(self))
                self.middleware_endpoint.resume_service('debugd.management')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def read_config(self, path):
        try:
            with open(path) as f:
                self.config = json.load(f)
        except (IOError, OSError, ValueError) as err:
            self.logger.fatal('Cannot open config file: {0}'.format(str(err)))
            self.logger.fatal('Exiting.')
            sys.exit(1)

    def connect(self, discard=False):
        if discard:
            self.connection_id = None

        self.keepalive = threading.Thread(target=self.connect_keepalive,
                                          daemon=True)
        self.keepalive.start()

    def connect_keepalive(self):
        while True:
            try:
                if not self.connection_id:
                    self.connection_id = uuid.uuid4()

                self.msock.connect(SUPPORT_PROXY_ADDRESS)
                self.logger.info(
                    'Connecting to {0}'.format(SUPPORT_PROXY_ADDRESS))
                self.rpc_fd = self.msock.create_channel(0)
                time.sleep(1)  # FIXME
                self.client = Client()
                self.client.connect('fd://', fobj=self.rpc_fd)
                self.client.channel_serializer = MSockChannelSerializer(
                    self.msock)
                self.client.standalone_server = True
                self.client.enable_server()
                self.client.register_service('debug', DebugService(self))
                self.client.call_sync('server.login', str(self.connection_id),
                                      socket.gethostname(), get_version(),
                                      'none')
                self.set_state(ConnectionState.CONNECTED)
            except BaseException as err:
                self.logger.warning(
                    'Failed to initiate support connection: {0}'.format(err),
                    exc_info=True)
                self.msock.disconnect()
            else:
                self.connected_at = datetime.now()
                with self.cv:
                    self.cv.wait_for(lambda: self.state in (
                        ConnectionState.LOST, ConnectionState.OFFLINE))
                    if self.state == ConnectionState.OFFLINE:
                        return

            self.logger.warning(
                'Support connection lost, retrying in 10 seconds')
            time.sleep(10)

    def disconnect(self):
        self.connected_at = None
        self.set_state(ConnectionState.OFFLINE)
        self.client.disconnect()
        self.msock.destroy_channel(0)
        self.msock.disconnect()
        self.jobs.clear()

    def on_msock_close(self):
        self.connected_at = None
        self.set_state(ConnectionState.LOST)

    def run_job(self, job):
        self.jobs.append(job)
        job.context = self
        job.start()

    def set_state(self, state):
        with self.cv:
            self.state = state
            self.cv.notify_all()
Ejemplo n.º 34
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('etcd')
        self.root = None
        self.configfile = None
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.plugin_dirs = []
        self.renderers = {}
        self.managed_files = {}

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore(self.configfile)
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('etcd')
                self.client.enable_server()
                self.client.register_service('etcd.generation',
                                             FileGenerationService(self))
                self.client.register_service('etcd.management',
                                             ManagementService(self))
                self.client.register_service('etcd.debug', DebugService())
                self.client.resume_service('etcd.generation')
                self.client.resume_service('etcd.management')
                self.client.resume_service('etcd.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def init_renderers(self):
        for name, impl in TEMPLATE_RENDERERS.items():
            self.renderers[name] = impl(self)

    def parse_config(self, filename):
        try:
            f = open(filename, 'r')
            self.config = json.load(f)
            f.close()
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error(
                'Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['etcd']['plugin-dirs']

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for root, dirs, files in os.walk(dir):
            for name in files:
                abspath = os.path.join(root, name)
                path = os.path.relpath(abspath, dir)
                name, ext = os.path.splitext(path)

                if name in self.managed_files.keys():
                    continue

                if ext in TEMPLATE_RENDERERS.keys():
                    self.managed_files[name] = abspath
                    self.logger.info('Adding managed file %s [%s]', name, ext)

    def generate_file(self, file_path):
        if file_path not in self.managed_files.keys():
            raise RpcException(errno.ENOENT, 'No such file')

        template_path = self.managed_files[file_path]
        name, ext = os.path.splitext(template_path)
        if ext not in self.renderers.keys():
            raise RuntimeError("Can't find renderer for {0}".format(file_path))

        renderer = self.renderers[ext]
        try:
            return renderer.render_template(template_path)
        except Exception as e:
            self.logger.warn('Cannot generate file {0}: {1}'.format(
                file_path, str(e)))
            return "# FILE GENERATION FAILED: {0}\n".format(str(e))

    def emit_event(self, name, params):
        self.client.emit_event(name, params)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c',
                            metavar='CONFIG',
                            default=DEFAULT_CONFIGFILE,
                            help='Middleware config file')
        parser.add_argument('-f',
                            action='store_true',
                            default=False,
                            help='Run in foreground')
        parser.add_argument('mountpoint',
                            metavar='MOUNTPOINT',
                            default='/etc',
                            help='/etc mount point')
        args = parser.parse_args()
        configure_logging('/var/log/etcd.log', 'DEBUG')

        setproctitle.setproctitle('etcd')
        self.root = args.mountpoint
        self.configfile = args.c
        self.parse_config(args.c)
        self.scan_plugins()
        self.init_renderers()
        self.init_datastore()
        self.init_dispatcher()
        self.client.wait_forever()
Ejemplo n.º 35
0
class Main(object):
    def __init__(self):
        self.config = None
        self.client = None
        self.datastore = None
        self.configstore = None
        self.rtsock_thread = None
        self.dhcp_clients = {}
        self.dhcp_lock = threading.RLock()
        self.logger = logging.getLogger('networkd')
        self.default_interface = None
        self.cv = Condition()

    def dhclient_pid(self, interface):
        path = os.path.join('/var/run', 'dhclient.{0}.pid'.format(interface))
        if not os.path.exists(path):
            return None

        try:
            with open(path) as f:
                pid = int(f.read().strip())
                return pid
        except (IOError, ValueError):
            return None

    def dhclient_running(self, interface):
        pid = self.dhclient_pid(interface)
        if not pid:
            return False

        try:
            os.kill(pid, 0)
            return True
        except OSError:
            return False

    def configure_dhcp(self, interface, block=False, timeout=None):
        if interface in self.dhcp_clients:
            self.logger.info('Interface {0} already configured by DHCP'.format(interface))
            return True

        def bind(old_lease, lease):
            self.logger.info('{0} DHCP lease on {1} from {2}, valid for {3} seconds'.format(
                'Renewed' if old_lease else 'Acquired',
                interface,
                client.server_address,
                lease.lifetime,
                interface
            ))

            if old_lease is None or lease.client_ip != old_lease.client_ip:
                self.logger.info('Assigning IP address {0} to interface {1}'.format(lease.client_ip, interface))
                alias = lease.client_interface
                iface = netif.get_interface(interface)

                if old_lease:
                    try:
                        addr = first_or_default(lambda a: a.address == old_lease.client_ip, iface.addresses)
                        if addr:
                            iface.remove_address(addr)
                    except OSError as err:
                        self.logger.error('Cannot remove alias {0}: {1}'.format(old_lease.client_ip, err.strerror))

                try:
                    iface.add_address(netif.InterfaceAddress(netif.AddressFamily.INET, alias))
                except OSError as err:
                    self.logger.error('Cannot add alias to {0}: {1}'.format(interface, err.strerror))

            if lease.router and self.configstore.get('network.dhcp.assign_gateway'):
                try:
                    rtable = netif.RoutingTable()
                    newroute = default_route(lease.router)
                    if rtable.default_route_ipv4 != newroute:
                        if rtable.default_route_ipv4:
                            self.logger.info('DHCP default route changed from {0} to {1}'.format(
                                rtable.default_route_ipv4,
                                newroute
                            ))
                            rtable.delete(rtable.default_route_ipv4)
                            rtable.add(default_route(lease.router))
                        else:
                            self.logger.info('Adding default route via {0}'.format(lease.router))
                            rtable.add(default_route(lease.router))
                except OSError as err:
                    self.logger.error('Cannot configure default route: {0}'.format(err.strerror))

            if lease.dns_addresses and self.configstore.get('network.dhcp.assign_dns'):
                inp = []
                addrs = []
                proc = subprocess.Popen(
                    ['/sbin/resolvconf', '-a', interface],
                    stdout=subprocess.PIPE,
                    stdin=subprocess.PIPE
                )

                for i in lease.dns_addresses:
                    # Filter out bogus DNS server addresses
                    if str(i) in ('127.0.0.1', '0.0.0.0', '255.255.255.255'):
                        continue

                    inp.append('nameserver {0}'.format(i))
                    addrs.append(i)

                if lease.domain_name:
                    inp.append('search {0}'.format(lease.domain_name))

                proc.communicate('\n'.join(inp).encode('ascii'))
                proc.wait()
                self.client.emit_event('network.dns.configured', {
                    'addresses': addrs,
                })
                self.logger.info('Updated DNS configuration')
            else:
                subprocess.call(['/sbin/resolvconf', '-d', interface])
                self.client.emit_event('network.dns.configured', {
                    'addresses': [],
                })
                self.logger.info('Deleted DNS configuration')

        def reject(reason):
            self.logger.info('DHCP request rejected on {0}: {1}'.format(interface, reason))
            self.deconfigure_dhcp(interface)
            if not block:
                t = threading.Timer(60, self.configure_dhcp, args=(interface,))
                t.start()

        def unbind(lease, reason):
            reasons = {
                dhcp.client.UnbindReason.EXPIRE: 'expired',
                dhcp.client.UnbindReason.REVOKE: 'revoked'
            }

            self.logger.info('DHCP lease on {0}: {1}'.format(interface, reasons.get(reason, 'revoked')))

        def state_change(state):
            self.client.emit_event('network.interface.changed', {
                'operation': 'update',
                'ids': [interface]
            })

            self.client.emit_event('network.changed', {
                'operation': 'update'
            })

        with self.dhcp_lock:
            client = dhcp.client.Client(interface, lambda: socket.gethostname().split('.')[0])
            client.on_bind = bind
            client.on_unbind = unbind
            client.on_reject = reject
            client.on_state_change = state_change
            client.start()
            self.dhcp_clients[interface] = client

        if block:
            ret = client.wait_for_bind(timeout)
            if ret is None:
                client.stop()
                del self.dhcp_clients[interface]

            return ret is not None

        return True

    def deconfigure_dhcp(self, interface):
        with self.dhcp_lock:
            client = self.dhcp_clients[interface]
            client.release()
            client.stop()
            del self.dhcp_clients[interface]

    def renew_dhcp(self, interface):
        if interface not in self.dhcp_clients:
            raise RpcException(errno.ENXIO, 'Interface {0} is not configured for DHCP'.format(interface))

        if not self.dhcp_clients[interface].lease:
            raise RpcException(errno.ENOENT, 'Cannot renew without a lease')

        self.dhcp_clients[interface].request(renew=True, timeout=30)

    def interface_detached(self, name):
        self.logger.warn('Interface {0} detached from the system'.format(name))

    def interface_attached(self, name):
        self.logger.warn('Interface {0} attached to the system'.format(name))

    def using_dhcp_for_gateway(self):
        for i in self.datastore.query('network.interfaces'):
            if i.get('dhcp') and self.configstore.get('network.dhcp.assign_gateway'):
                return True

        return False

    def scan_interfaces(self):
        self.logger.info('Scanning available network interfaces...')
        existing = []

        # Add newly plugged NICs to DB
        for i in list(netif.list_interfaces().values()):
            existing.append(i.name)

            # We want only physical NICs
            if i.cloned:
                continue

            if i.name in ('mgmt0', 'nat0'):
                continue

            if i.name.startswith(('tap', 'brg')):
                continue

            if not self.datastore.exists('network.interfaces', ('id', '=', i.name)):
                self.logger.info('Found new interface {0} ({1})'.format(i.name, i.type.name))
                self.datastore.insert('network.interfaces', {
                    'enabled': False,
                    'id': i.name,
                    'name': None,
                    'cloned': False,
                    'type': i.type.name,
                    'dhcp': False,
                    'noipv6': False,
                    'rtadv': False,
                    'mtu': i.mtu,
                    'media': None,
                    'mediaopts': [],
                    'aliases': [],
                    'capabilities': {
                        'add': [],
                        'del': []
                    }
                })

        # Remove unplugged NICs from DB
        for i in self.datastore.query('network.interfaces', ('id', 'nin', existing), ('cloned', '=', False)):
            self.datastore.delete('network.interfaces', i['id'])

    def init_datastore(self):
        try:
            self.datastore = get_datastore(self.config)
        except DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def connect(self, resume=False):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('networkd')
                self.client.enable_server()
                self.register_schemas()
                self.client.register_service('networkd.configuration', ConfigurationService(self))
                self.client.register_service('networkd.debug', DebugService())
                if resume:
                    self.client.resume_service('networkd.configuration')
                    self.client.resume_service('networkd.debug')

                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect(resume=True)

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_routing_socket(self):
        self.rtsock_thread = RoutingSocketEventSource(self)
        self.rtsock_thread.start()

    def register_schemas(self):
        self.client.register_schema('network-aggregation-protocols', {
            'type': 'string',
            'enum': list(netif.AggregationProtocol.__members__.keys())
        })

        self.client.register_schema('network-lagg-port-flags', {
            'type': 'array',
            'items': {'$ref': 'network-lagg-port-flags-items'}
        })

        self.client.register_schema('network-lagg-port-flags-items', {
            'type': 'string',
            'enum': list(netif.LaggPortFlags.__members__.keys())
        })

        self.client.register_schema('network-interface-flags', {
            'type': 'array',
            'items': {'$ref': 'network-interface-flags-items'}
        })

        self.client.register_schema('network-interface-flags-items', {
            'type': 'string',
            'enum': list(netif.InterfaceFlags.__members__.keys())
        })

        self.client.register_schema('network-interface-capabilities', {
            'type': 'array',
            'items': {'$ref': 'network-interface-capabilities-items'}
        })

        self.client.register_schema('network-interface-capabilities-items', {
            'type': 'string',
            'enum': list(netif.InterfaceCapability.__members__.keys())
        })

        self.client.register_schema('network-interface-mediaopts', {
            'type': 'array',
            'items': {'$ref': 'network-interface-mediaopts-items'}
        })

        self.client.register_schema('network-interface-mediaopts-items', {
            'type': 'string',
            'enum': list(netif.InterfaceMediaOption.__members__.keys())
        })

        self.client.register_schema('network-interface-nd6-flag', {
            'type': 'array',
            'items': {'$ref': 'network-interface-nd6-flag-items'}
        })

        self.client.register_schema('network-interface-nd6-flag-items', {
            'type': 'string',
            'enum': list(netif.NeighborDiscoveryFlags.__members__.keys())
        })

        self.client.register_schema('network-interface-type', {
            'type': 'string',
            'enum': [
                'LOOPBACK',
                'ETHER',
                'VLAN',
                'BRIDGE',
                'LAGG'
            ]
        })

        self.client.register_schema('network-interface-dhcp-state', {
            'type': 'string',
            'enum': [
                'INIT',
                'SELECTING',
                'REQUESTING',
                'INIT_REBOOT',
                'REBOOTING',
                'BOUND',
                'RENEWING',
                'REBINDING'
            ]
        })

        self.client.register_schema('network-interface-status', {
            'type': 'object',
            'properties': {
                'name': {'type': 'string'},
                'link_state': {'$ref': 'network-interface-status-linkstate'},
                'link_address': {'type': 'string'},
                'mtu': {'type': 'integer'},
                'media_type': {'type': 'string'},
                'media_subtype': {'type': 'string'},
                'active_media_type': {'type': 'string'},
                'active_media_subtype': {'type': 'string'},
                'media_options': {'$ref': 'network-interface-mediaopts'},
                'cloned': {'type': 'boolean'},
                'capabilities': {'$ref': 'network-interface-capabilities'},
                'flags': {'$ref': 'network-interface-flags'},
                'dhcp': {
                    'type': 'object',
                    'properties': {
                        'state': {'$ref': 'network-interface-dhcp-state'},
                        'server_address': {'type': 'string'},
                        'server_name': {'type': 'string'},
                        'lease_starts_at': {'type': 'datetime'},
                        'lease_ends_at': {'type': 'datetime'}
                    }
                },
                'aliases': {
                    'type': 'array',
                    'items': {'$ref': 'network-interface-alias'}
                },
                'nd6_flags': {
                    'type': 'array',
                    'items': {'$ref': 'network-interface-nd6-flag'}
                },
                'ports': {
                    'oneOf': [
                        {'type': 'null'},
                        {
                            'type': 'array',
                            'members': {
                                'type': 'object',
                                'properties': {
                                    'name': {'type': 'string'},
                                    'flags': {'$ref': 'network-lagg-port-flags'}
                                }
                            }
                        }
                    ]
                },
                'members': {
                    'oneOf': [
                        {'type': 'null'},
                        {
                            'type': 'array',
                            'members': {'type': 'string'}
                        }
                    ]
                },
                'parent': {'type': ['string', 'null']},
                'tag': {'type': ['integer', 'null']}
            }
        })

        self.client.register_schema('network-interface-status-linkstate', {
            'type': 'string',
            'enum': list(netif.InterfaceLinkState.__members__.keys())
        })

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        args = parser.parse_args()
        configure_logging('/var/log/networkd.log', 'DEBUG')
        setproctitle.setproctitle('networkd')
        self.config = args.c
        self.init_datastore()
        self.init_dispatcher()
        self.scan_interfaces()
        self.init_routing_socket()
        self.client.resume_service('networkd.configuration')
        self.client.resume_service('networkd.debug')
        self.logger.info('Started')
        self.client.wait_forever()
Ejemplo n.º 36
0
class Context(object):
    def __init__(self):
        self.service = TaskProxyService(self)
        self.task = queue.Queue(1)
        self.datastore = None
        self.configstore = None
        self.conn = None
        self.instance = None
        self.running = Event()

    def put_status(self, state, result=None, exception=None):
        obj = {
            'status': state,
            'result': None
        }

        if result is not None:
            obj['result'] = result

        if exception is not None:
            obj['error'] = serialize_error(exception)

        self.conn.call_sync('task.put_status', obj)

    def task_progress_handler(self, args):
        if self.instance:
            self.instance.task_progress_handler(args)

    def collect_fds(self, obj):
        if isinstance(obj, dict):
            for v in obj.values():
                if isinstance(v, FileDescriptor):
                    yield v
                else:
                    yield from self.collect_fds(v)

        if isinstance(obj, (list, tuple)):
            for o in obj:
                if isinstance(o, FileDescriptor):
                    yield o
                else:
                    yield from self.collect_fds(o)

    def close_fds(self, fds):
        for i in fds:
            try:
                os.close(i.fd)
            except OSError:
                pass

    def main(self):
        if len(sys.argv) != 2:
            print("Invalid number of arguments", file=sys.stderr)
            sys.exit(errno.EINVAL)

        key = sys.argv[1]
        configure_logging(None, logging.DEBUG)

        self.datastore = get_datastore()
        self.configstore = ConfigStore(self.datastore)
        self.conn = Client()
        self.conn.connect('unix:')
        self.conn.login_service('task.{0}'.format(os.getpid()))
        self.conn.enable_server()
        self.conn.rpc.register_service_instance('taskproxy', self.service)
        self.conn.register_event_handler('task.progress', self.task_progress_handler)
        self.conn.call_sync('task.checkin', key)
        setproctitle.setproctitle('task executor (idle)')

        while True:
            try:
                task = self.task.get()
                logging.root.setLevel(self.conn.call_sync('management.get_logging_level'))
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                if task['debugger']:
                    sys.path.append('/usr/local/lib/dispatcher/pydev')

                    import pydevd
                    host, port = task['debugger']
                    pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True)

                name, _ = os.path.splitext(os.path.basename(task['filename']))
                module = load_module_from_file(name, task['filename'])
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                fds = list(self.collect_fds(task['args']))

                try:
                    self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore)
                    self.instance.configstore = self.configstore
                    self.instance.user = task['user']
                    self.instance.environment = task['environment']
                    self.running.set()
                    result = self.instance.run(*task['args'])
                except BaseException as err:
                    print("Task exception: {0}".format(str(err)), file=sys.stderr)
                    traceback.print_exc(file=sys.stderr)

                    if hasattr(self.instance, 'rollback'):
                        self.put_status('ROLLBACK')
                        try:
                            self.instance.rollback(*task['args'])
                        except BaseException as rerr:
                            print("Task exception during rollback: {0}".format(str(rerr)), file=sys.stderr)
                            traceback.print_exc(file=sys.stderr)

                    self.put_status('FAILED', exception=err)
                else:
                    self.put_status('FINISHED', result=result)
                finally:
                    self.close_fds(fds)
                    self.running.clear()

            except RpcException as err:
                print("RPC failed: {0}".format(str(err)), file=sys.stderr)
                print(traceback.format_exc(), flush=True)
                sys.exit(errno.EBADMSG)
            except socket.error as err:
                print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr)
                sys.exit(errno.ETIMEDOUT)

            if task['debugger']:
                import pydevd
                pydevd.stoptrace()

            setproctitle.setproctitle('task executor (idle)')
Ejemplo n.º 37
0
class Context(object):
    def __init__(self):
        self.logger = logging.getLogger('schedulerd')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.scheduler = None
        self.active_tasks = {}

    def init_datastore(self):
        try:
            self.datastore = get_datastore(self.config)
        except DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_scheduler(self):
        store = MongoDBJobStore(database='freenas',
                                collection='calendar_tasks',
                                client=self.datastore.client)
        self.scheduler = BackgroundScheduler(jobstores={'default': store},
                                             timezone=pytz.utc)
        self.scheduler.start()

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('schedulerd')
                self.client.enable_server()
                self.client.register_service('scheduler.management',
                                             ManagementService(self))
                self.client.register_service('scheduler.debug', DebugService())
                self.client.resume_service('scheduler.management')
                self.client.resume_service('scheduler.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def run_job(self, *args, **kwargs):
        tid = self.client.call_sync(
            'task.submit_with_env', args[0], args[1:], {
                'RUN_AS_USER': '******',
                'CALENDAR_TASK_NAME': kwargs.get('name')
            })

        self.active_tasks[kwargs['id']] = tid
        self.client.call_sync('task.wait', tid, timeout=None)
        result = self.client.call_sync('task.status', tid)
        if result['state'] != 'FINISHED':
            try:
                self.client.call_sync(
                    'alert.emit', {
                        'name':
                        'scheduler.task.failed',
                        'severity':
                        'CRITICAL',
                        'description':
                        'Task {0} has failed: {1}'.format(
                            kwargs.get('name', tid),
                            result['error']['message']),
                    })
            except RpcException as e:
                self.logger.error('Failed to emit alert', exc_info=True)

        del self.active_tasks[kwargs['id']]
        self.datastore.insert('schedulerd.runs', {
            'job_id': kwargs['id'],
            'task_id': result['id']
        })

    def emit_event(self, name, params):
        self.client.emit_event(name, params)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c',
                            metavar='CONFIG',
                            default=DEFAULT_CONFIGFILE,
                            help='Middleware config file')
        parser.add_argument('-f',
                            action='store_true',
                            default=False,
                            help='Run in foreground')
        args = parser.parse_args()
        configure_logging('/var/log/schedulerd.log', 'DEBUG')
        setproctitle.setproctitle('schedulerd')
        self.config = args.c
        self.init_datastore()
        self.init_scheduler()
        self.init_dispatcher()
        self.client.wait_forever()
Ejemplo n.º 38
0
    def run(self, peer, initial_credentials):
        hostid = self.dispatcher.call_sync('system.info.host_uuid')
        hostname = self.dispatcher.call_sync(
            'system.general.get_config')['hostname']
        remote_peer_name = hostname
        credentials = peer['credentials']
        remote = credentials.get('address')
        port = credentials.get('port', 22)
        username = initial_credentials.get('username')
        password = initial_credentials.get('password')
        auth_code = initial_credentials.get('auth_code')
        key_auth = initial_credentials.get('key_auth')

        local_ssh_config = self.dispatcher.call_sync('service.sshd.get_config')

        if self.datastore.exists('peers', ('credentials.address', '=', remote),
                                 ('type', '=', 'freenas')):
            raise TaskException(
                errno.EEXIST,
                'FreeNAS peer entry for {0} already exists'.format(remote))

        remote_client = Client()

        try:
            if auth_code:
                try:
                    remote_client.connect('ws://{0}'.format(
                        wrap_address(remote)))
                except (AuthenticationException, OSError,
                        ConnectionRefusedError):
                    raise TaskException(
                        errno.ECONNABORTED,
                        'Cannot connect to {0}:{1}'.format(remote, port))

                try:
                    remote_host_uuid, pubkey = remote_client.call_sync(
                        'peer.freenas.auth_with_code', auth_code, hostname,
                        local_ssh_config['port'])
                except RpcException as err:
                    raise TaskException(err.code, err.message)

                try:
                    self.dispatcher.call_sync('peer.freenas.put_temp_pubkey',
                                              pubkey)
                    if not self.dispatcher.test_or_wait_for_event(
                            'peer.changed',
                            lambda ar: ar['operation'] == 'create' and
                            remote_host_uuid in ar['ids'],
                            lambda: self.datastore.exists(
                                'peers', ('id', '=', remote_host_uuid)),
                            timeout=30):
                        raise TaskException(
                            errno.EAUTH,
                            'FreeNAS peer creation failed. Check connection to host {0}.'
                            .format(remote))
                finally:
                    self.dispatcher.call_sync(
                        'peer.freenas.remove_temp_pubkey', pubkey)

            else:
                try:
                    if key_auth:
                        with io.StringIO() as f:
                            f.write(
                                self.configstore.get(
                                    'peer.freenas.key.private'))
                            f.seek(0)
                            pkey = RSAKey.from_private_key(f)

                        max_tries = 50
                        while True:
                            try:
                                remote_client.connect(
                                    'ws+ssh://freenas@{0}'.format(
                                        wrap_address(remote)),
                                    pkey=pkey,
                                    port=port)
                                break
                            except AuthenticationException:
                                if max_tries:
                                    max_tries -= 1
                                    time.sleep(1)
                                else:
                                    raise
                    else:
                        remote_client.connect('ws+ssh://{0}@{1}'.format(
                            username, wrap_address(remote)),
                                              port=port,
                                              password=password)

                    remote_client.login_service('replicator')
                except (AuthenticationException, OSError,
                        ConnectionRefusedError):
                    raise TaskException(
                        errno.ECONNABORTED,
                        'Cannot connect to {0}:{1}'.format(remote, port))

                local_host_key, local_pub_key = self.dispatcher.call_sync(
                    'peer.freenas.get_ssh_keys')
                remote_host_key, remote_pub_key = remote_client.call_sync(
                    'peer.freenas.get_ssh_keys')
                ip_at_remote_side = remote_client.local_address[0]

                remote_hostname = remote_client.call_sync(
                    'system.general.get_config')['hostname']

                remote_host_key = remote_host_key.rsplit(' ', 1)[0]
                local_host_key = local_host_key.rsplit(' ', 1)[0]

                if remote_client.call_sync('peer.query',
                                           [('id', '=', hostid)]):
                    raise TaskException(
                        errno.EEXIST,
                        'Peer entry of {0} already exists at {1}'.format(
                            hostname, remote))

                peer['credentials'] = {
                    '%type': 'freenas-credentials',
                    'pubkey': remote_pub_key,
                    'hostkey': remote_host_key,
                    'port': port,
                    'address': remote_hostname
                }

                local_id = remote_client.call_sync('system.info.host_uuid')
                peer['id'] = local_id
                peer['name'] = remote_hostname
                ip = socket.gethostbyname(remote)

                created_ids = self.join_subtasks(
                    self.run_subtask('peer.freenas.create_local', peer, ip,
                                     True))

                peer['id'] = hostid
                peer['name'] = remote_peer_name
                peer['credentials'] = {
                    '%type': 'freenas-credentials',
                    'pubkey': local_pub_key,
                    'hostkey': local_host_key,
                    'port': local_ssh_config['port'],
                    'address': hostname
                }

                try:
                    call_task_and_check_state(remote_client,
                                              'peer.freenas.create_local',
                                              peer, ip_at_remote_side)
                except TaskException:
                    self.datastore.delete('peers', local_id)
                    self.dispatcher.dispatch_event('peer.changed', {
                        'operation': 'delete',
                        'ids': [local_id]
                    })
                    raise

                return created_ids[0]
        finally:
            remote_client.disconnect()
Ejemplo n.º 39
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('dscached')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.rpc = RpcContext()
        self.rpc.streaming_enabled = True
        self.rpc.streaming_burst = 16
        self.client = None
        self.server = None
        self.plugin_dirs = []
        self.plugins = {}
        self.directories = []
        self.users_cache = TTLCacheStore()
        self.groups_cache = TTLCacheStore()
        self.hosts_cache = TTLCacheStore()
        self.cache_ttl = 7200
        self.search_order = []
        self.cache_enumerations = True
        self.cache_lookups = True
        self.home_directory_root = None
        self.account_service = AccountService(self)
        self.group_service = GroupService(self)
        self.rpc.register_service_instance('dscached.account',
                                           self.account_service)
        self.rpc.register_service_instance('dscached.group',
                                           self.group_service)
        self.rpc.register_service_instance('dscached.host', HostService(self))
        self.rpc.register_service_instance('dscached.idmap',
                                           IdmapService(self))
        self.rpc.register_service_instance('dscached.management',
                                           ManagementService(self))
        self.rpc.register_service_instance('dscached.debug', DebugService())

    def get_active_directories(self):
        return list(
            filter(lambda d: d and d.state == DirectoryState.BOUND,
                   self.directories))

    def get_searched_directories(self):
        return list(
            filter(lambda d: d and d.state == DirectoryState.BOUND,
                   (self.get_directory_by_name(n)
                    for n in self.get_search_order())))

    def get_search_order(self):
        return self.search_order

    def get_directory_by_domain(self, domain_name):
        return first_or_default(lambda d: d.domain_name == domain_name,
                                self.directories)

    def get_directory_by_name(self, name):
        return first_or_default(lambda d: d.name == name, self.directories)

    def get_directory_for_id(self, uid=None, gid=None):
        if uid is not None:
            if uid == 0:
                # Special case for root user
                return first_or_default(lambda d: d.plugin_type == 'local',
                                        self.directories)

            return first_or_default(
                lambda d: d.max_uid and d.max_uid >= uid >= d.min_uid,
                self.directories)

        if gid is not None:
            if gid == 0:
                # Special case for wheel group
                return first_or_default(lambda d: d.plugin_type == 'local',
                                        self.directories)

            return first_or_default(
                lambda d: d.max_gid and d.max_gid >= gid >= d.min_gid,
                self.directories)

    def get_home_directory(self, directory, username):
        if not self.home_directory_root:
            return '/nonexistent'

        return os.path.join(self.home_directory_root,
                            f'{username}@{directory.domain_name}')

    def wait_for_etcd(self):
        self.client.test_or_wait_for_event(
            'plugin.service_resume',
            lambda args: args['name'] == 'etcd.generation',
            lambda: 'etcd.generation' in self.client.call_sync(
                'discovery.get_services'))

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore()
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_server(self, address):
        self.server = Server(self)
        self.server.rpc = self.rpc
        self.server.streaming = True
        self.server.start(address, transport_options={'permissions': 0o777})
        thread = Thread(target=self.server.serve_forever)
        thread.name = 'ServerThread'
        thread.daemon = True
        thread.start()

    def parse_config(self, filename):
        try:
            with open(filename, 'r') as f:
                self.config = json.load(f)
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error(
                'Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['dscached']['plugin-dirs']

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('dscached')
                self.client.enable_server(self.rpc)
                self.client.resume_service('dscached.account')
                self.client.resume_service('dscached.group')
                self.client.resume_service('dscached.host')
                self.client.resume_service('dscached.idmap')
                self.client.resume_service('dscached.management')
                self.client.resume_service('dscached.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for f in os.listdir(dir):
            name, ext = os.path.splitext(os.path.basename(f))
            if ext != '.py':
                continue

            try:
                plugin = load_module_from_file(name, os.path.join(dir, f))
                plugin._init(self)
            except:
                self.logger.error('Cannot initialize plugin {0}'.format(f),
                                  exc_info=True)

    def register_plugin(self, name, cls):
        self.plugins[name] = cls
        self.logger.info('Registered plugin {0} (class {1})'.format(name, cls))

    def register_schema(self, name, schema):
        self.client.register_schema(name, schema)

    def register_schemas(self):
        from freenas.dispatcher.model import context
        for name, schema in (s.__named_json_schema__()
                             for s in context.local_json_schema_objects):
            self.logger.debug(f'Registering schema: {name}')
            self.client.register_schema(name, schema)

    def init_directories(self):
        for i in self.datastore.query('directories'):
            try:
                directory = Directory(self, i)
                self.directories.append(directory)
                directory.configure()
            except:
                continue

    def load_config(self):
        self.search_order = self.configstore.get('directory.search_order')
        self.cache_ttl = self.configstore.get('directory.cache_ttl')
        self.cache_enumerations = self.configstore.get(
            'directory.cache_enumerations')
        self.cache_lookups = self.configstore.get('directory.cache_lookups')
        self.home_directory_root = self.configstore.get(
            'system.home_directory_root')

    def checkin(self):
        checkin()

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c',
                            metavar='CONFIG',
                            default=DEFAULT_CONFIGFILE,
                            help='Middleware config file')
        parser.add_argument('-s',
                            metavar='SOCKET',
                            default=DEFAULT_SOCKET_ADDRESS,
                            help='Socket address to listen on')
        args = parser.parse_args()
        configure_logging('dscached', 'DEBUG')

        setproctitle('dscached')
        self.config = args.c
        self.parse_config(self.config)
        self.init_datastore()
        self.init_dispatcher()
        self.load_config()
        self.init_server(args.s)
        self.scan_plugins()
        self.register_schemas()
        self.wait_for_etcd()
        self.init_directories()
        self.checkin()
        self.client.wait_forever()
Ejemplo n.º 40
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('etcd')
        self.root = None
        self.configfile = None
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.plugin_dirs = []
        self.renderers = {}
        self.managed_files = {}

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore(self.configfile)
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('etcd')
                self.client.enable_server()
                self.client.register_service('etcd.generation', FileGenerationService(self))
                self.client.register_service('etcd.management', ManagementService(self))
                self.client.register_service('etcd.debug', DebugService())
                self.client.resume_service('etcd.generation')
                self.client.resume_service('etcd.management')
                self.client.resume_service('etcd.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def init_renderers(self):
        for name, impl in TEMPLATE_RENDERERS.items():
            self.renderers[name] = impl(self)

    def parse_config(self, filename):
        try:
            f = open(filename, 'r')
            self.config = json.load(f)
            f.close()
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error('Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['etcd']['plugin-dirs']

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for root, dirs, files in os.walk(dir):
            for name in files:
                abspath = os.path.join(root, name)
                path = os.path.relpath(abspath, dir)
                name, ext = os.path.splitext(path)

                if name in self.managed_files.keys():
                    continue

                if ext in TEMPLATE_RENDERERS.keys():
                    self.managed_files[name] = abspath
                    self.logger.info('Adding managed file %s [%s]', name, ext)

    def generate_file(self, file_path):
        if file_path not in self.managed_files.keys():
            raise RpcException(errno.ENOENT, 'No such file')

        template_path = self.managed_files[file_path]
        name, ext = os.path.splitext(template_path)
        if ext not in self.renderers.keys():
            raise RuntimeError("Can't find renderer for {0}".format(file_path))

        renderer = self.renderers[ext]
        try:
            return renderer.render_template(template_path)
        except Exception as e:
            self.logger.warn('Cannot generate file {0}: {1}'.format(file_path, str(e)))
            return "# FILE GENERATION FAILED: {0}\n".format(str(e))

    def emit_event(self, name, params):
        self.client.emit_event(name, params)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        parser.add_argument('-f', action='store_true', default=False, help='Run in foreground')
        parser.add_argument('mountpoint', metavar='MOUNTPOINT', default='/etc', help='/etc mount point')
        args = parser.parse_args()
        configure_logging('/var/log/etcd.log', 'DEBUG')

        setproctitle.setproctitle('etcd')
        self.root = args.mountpoint
        self.configfile = args.c
        self.parse_config(args.c)
        self.scan_plugins()
        self.init_renderers()
        self.init_datastore()
        self.init_dispatcher()
        self.client.wait_forever()
Ejemplo n.º 41
0
 def __init__(self, level=logging.NOTSET, address=None, ident=None):
     super(LogdLogHandler, self).__init__(level)
     self.address = address or 'unix:///var/run/logd.sock'
     self.ident = ident or os.path.basename(sys.executable)
     self.client = Client()
     self.client.connect(self.address)
Ejemplo n.º 42
0
class Context(object):
    def __init__(self):
        self.service = TaskProxyService(self)
        self.task = queue.Queue(1)
        self.datastore = None
        self.configstore = None
        self.conn = None
        self.instance = None
        self.module_cache = {}
        self.running = Event()

    def put_status(self, state, result=None, exception=None):
        obj = {'status': state, 'result': None}

        if result is not None:
            obj['result'] = result

        if exception is not None:
            obj['error'] = serialize_error(exception)

        self.conn.call_sync('task.put_status', obj)

    def task_progress_handler(self, args):
        if self.instance:
            self.instance.task_progress_handler(args)

    def collect_fds(self, obj):
        if isinstance(obj, dict):
            for v in obj.values():
                if isinstance(v, FileDescriptor):
                    yield v
                else:
                    yield from self.collect_fds(v)

        if isinstance(obj, (list, tuple)):
            for o in obj:
                if isinstance(o, FileDescriptor):
                    yield o
                else:
                    yield from self.collect_fds(o)

    def close_fds(self, fds):
        for i in fds:
            try:
                os.close(i.fd)
            except OSError:
                pass

    def run_task_hooks(self, instance, task, type, **extra_env):
        for hook, props in task['hooks'].get(type, {}).items():
            try:
                if props['condition'] and not props['condition'](
                        *task['args']):
                    continue
            except BaseException as err:
                print(err)
                continue

            instance.join_subtasks(
                instance.run_subtask(hook, *task['args'], **extra_env))

    def main(self):
        if len(sys.argv) != 2:
            print("Invalid number of arguments", file=sys.stderr)
            sys.exit(errno.EINVAL)

        key = sys.argv[1]
        configure_logging(None, logging.DEBUG)

        self.datastore = get_datastore()
        self.configstore = ConfigStore(self.datastore)
        self.conn = Client()
        self.conn.connect('unix:')
        self.conn.login_service('task.{0}'.format(os.getpid()))
        self.conn.enable_server()
        self.conn.call_sync('management.enable_features',
                            ['streaming_responses'])
        self.conn.rpc.register_service_instance('taskproxy', self.service)
        self.conn.register_event_handler('task.progress',
                                         self.task_progress_handler)
        self.conn.call_sync('task.checkin', key)
        setproctitle.setproctitle('task executor (idle)')

        while True:
            try:
                task = self.task.get()
                logging.root.setLevel(
                    self.conn.call_sync('management.get_logging_level'))
                setproctitle.setproctitle('task executor (tid {0})'.format(
                    task['id']))

                if task['debugger']:
                    sys.path.append('/usr/local/lib/dispatcher/pydev')

                    import pydevd
                    host, port = task['debugger']
                    pydevd.settrace(host,
                                    port=port,
                                    stdoutToServer=True,
                                    stderrToServer=True)

                name, _ = os.path.splitext(os.path.basename(task['filename']))
                module = self.module_cache.get(task['filename'])
                if not module:
                    module = load_module_from_file(name, task['filename'])
                    self.module_cache[task['filename']] = module

                setproctitle.setproctitle('task executor (tid {0})'.format(
                    task['id']))
                fds = list(self.collect_fds(task['args']))

                try:
                    dispatcher = DispatcherWrapper(self.conn)
                    self.instance = getattr(module,
                                            task['class'])(dispatcher,
                                                           self.datastore)
                    self.instance.configstore = self.configstore
                    self.instance.user = task['user']
                    self.instance.environment = task['environment']
                    self.running.set()
                    self.run_task_hooks(self.instance, task, 'before')
                    result = self.instance.run(*task['args'])
                    self.run_task_hooks(self.instance,
                                        task,
                                        'after',
                                        result=result)
                except BaseException as err:
                    print("Task exception: {0}".format(str(err)),
                          file=sys.stderr)
                    traceback.print_exc(file=sys.stderr)

                    if hasattr(self.instance, 'rollback'):
                        self.put_status('ROLLBACK')
                        try:
                            self.instance.rollback(*task['args'])
                        except BaseException as rerr:
                            print("Task exception during rollback: {0}".format(
                                str(rerr)),
                                  file=sys.stderr)
                            traceback.print_exc(file=sys.stderr)

                    # Main task is already failed at this point, so ignore hook errors
                    with contextlib.suppress(RpcException):
                        self.run_task_hooks(self.instance,
                                            task,
                                            'error',
                                            error=serialize_error(err))

                    self.put_status('FAILED', exception=err)
                else:
                    self.put_status('FINISHED', result=result)
                finally:
                    self.close_fds(fds)
                    self.running.clear()

            except RpcException as err:
                print("RPC failed: {0}".format(str(err)), file=sys.stderr)
                print(traceback.format_exc(), flush=True)
                sys.exit(errno.EBADMSG)
            except socket.error as err:
                print("Cannot connect to dispatcher: {0}".format(str(err)),
                      file=sys.stderr)
                sys.exit(errno.ETIMEDOUT)

            if task['debugger']:
                import pydevd
                pydevd.stoptrace()

            setproctitle.setproctitle('task executor (idle)')
Ejemplo n.º 43
0
    def main(self):
        if len(sys.argv) != 2:
            print("Invalid number of arguments", file=sys.stderr)
            sys.exit(errno.EINVAL)

        key = sys.argv[1]
        configure_logging(None, logging.DEBUG)

        self.datastore = get_datastore()
        self.configstore = ConfigStore(self.datastore)
        self.conn = Client()
        self.conn.connect('unix:')
        self.conn.login_service('task.{0}'.format(os.getpid()))
        self.conn.enable_server()
        self.conn.rpc.register_service_instance('taskproxy', self.service)
        self.conn.register_event_handler('task.progress', self.task_progress_handler)
        self.conn.call_sync('task.checkin', key)
        setproctitle.setproctitle('task executor (idle)')

        while True:
            try:
                task = self.task.get()
                logging.root.setLevel(self.conn.call_sync('management.get_logging_level'))
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                if task['debugger']:
                    sys.path.append('/usr/local/lib/dispatcher/pydev')

                    import pydevd
                    host, port = task['debugger']
                    pydevd.settrace(host, port=port, stdoutToServer=True, stderrToServer=True)

                name, _ = os.path.splitext(os.path.basename(task['filename']))
                module = load_module_from_file(name, task['filename'])
                setproctitle.setproctitle('task executor (tid {0})'.format(task['id']))

                fds = list(self.collect_fds(task['args']))

                try:
                    self.instance = getattr(module, task['class'])(DispatcherWrapper(self.conn), self.datastore)
                    self.instance.configstore = self.configstore
                    self.instance.user = task['user']
                    self.instance.environment = task['environment']
                    self.running.set()
                    result = self.instance.run(*task['args'])
                except BaseException as err:
                    print("Task exception: {0}".format(str(err)), file=sys.stderr)
                    traceback.print_exc(file=sys.stderr)

                    if hasattr(self.instance, 'rollback'):
                        self.put_status('ROLLBACK')
                        try:
                            self.instance.rollback(*task['args'])
                        except BaseException as rerr:
                            print("Task exception during rollback: {0}".format(str(rerr)), file=sys.stderr)
                            traceback.print_exc(file=sys.stderr)

                    self.put_status('FAILED', exception=err)
                else:
                    self.put_status('FINISHED', result=result)
                finally:
                    self.close_fds(fds)
                    self.running.clear()

            except RpcException as err:
                print("RPC failed: {0}".format(str(err)), file=sys.stderr)
                print(traceback.format_exc(), flush=True)
                sys.exit(errno.EBADMSG)
            except socket.error as err:
                print("Cannot connect to dispatcher: {0}".format(str(err)), file=sys.stderr)
                sys.exit(errno.ETIMEDOUT)

            if task['debugger']:
                import pydevd
                pydevd.stoptrace()

            setproctitle.setproctitle('task executor (idle)')
Ejemplo n.º 44
0
class Main(object):
    def __init__(self):
        self.client = None
        self.datastore = None
        self.configstore = None
        self.config = None
        self.mgmt = None
        self.vm_started = Event()
        self.containers = {}
        self.tokens = {}
        self.logger = logging.getLogger('containerd')
        self.bridge_interface = None
        self.used_nmdms = []

    def init_datastore(self):
        try:
            self.datastore = get_datastore(self.config)
        except DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def allocate_nmdm(self):
        for i in range(0, 255):
            if i not in self.used_nmdms:
                self.used_nmdms.append(i)
                return i

    def release_nmdm(self, index):
        self.used_nmdms.remove(index)

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('containerd')
                self.client.enable_server()
                self.client.register_service('containerd.management', ManagementService(self))
                self.client.register_service('containerd.debug', DebugService(gevent=True, builtins={"context": self}))
                self.client.resume_service('containerd.management')
                self.client.resume_service('containerd.debug')

                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.use_bursts = True
        self.client.on_error(on_error)
        self.connect()

    def init_mgmt(self):
        self.mgmt = ManagementNetwork(self, MGMT_INTERFACE, MGMT_ADDR)
        self.mgmt.up()
        self.mgmt.bridge_if.add_address(netif.InterfaceAddress(
            netif.AddressFamily.INET,
            ipaddress.ip_interface('169.254.169.254/32')
        ))

    def init_nat(self):
        default_if = self.client.call_sync('networkd.configuration.get_default_interface')
        if not default_if:
            self.logger.warning('No default route interface; not configuring NAT')
            return

        p = pf.PF()

        # Try to find and remove existing NAT rules for the same subnet
        oldrule = first_or_default(
            lambda r: r.src.address.address == MGMT_ADDR.network.network_address,
            p.get_rules('nat')
        )

        if oldrule:
            p.delete_rule('nat', oldrule.index)

        rule = pf.Rule()
        rule.src.address.address = MGMT_ADDR.network.network_address
        rule.src.address.netmask = MGMT_ADDR.netmask
        rule.action = pf.RuleAction.NAT
        rule.af = socket.AF_INET
        rule.ifname = default_if
        rule.redirect_pool.append(pf.Address(ifname=default_if))
        rule.proxy_ports = [50001, 65535]
        p.append_rule('nat', rule)

        try:
            p.enable()
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise err

    def init_ec2(self):
        self.ec2 = EC2MetadataServer(self)
        self.ec2.start()

    def vm_by_mgmt_mac(self, mac):
        for i in self.containers.values():
            for tapmac in i.tap_interfaces.values():
                if tapmac == mac:
                    return i

        return None

    def vm_by_mgmt_ip(self, ip):
        for i in self.mgmt.allocations.values():
            if i.lease.client_ip == ip:
                return i.vm()

    def die(self):
        self.logger.warning('Exiting')
        for i in self.containers.values():
            i.stop(True)

        self.client.disconnect()
        sys.exit(0)

    def generate_id(self):
        return ''.join([random.choice(string.ascii_letters + string.digits) for n in range(32)])

    def dispatcher_error(self, error):
        self.die()

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        parser.add_argument('-p', type=int, metavar='PORT', default=5500, help="WebSockets server port")
        args = parser.parse_args()
        configure_logging('/var/log/containerd.log', 'DEBUG')
        setproctitle.setproctitle('containerd')

        gevent.signal(signal.SIGTERM, self.die)
        gevent.signal(signal.SIGQUIT, self.die)

        self.config = args.c
        self.init_datastore()
        self.init_dispatcher()
        self.init_mgmt()
        self.init_nat()
        self.init_ec2()
        self.logger.info('Started')

        # WebSockets server
        kwargs = {}
        s4 = WebSocketServer(('', args.p), ServerResource({
            '/console': ConsoleConnection,
        }, context=self), **kwargs)

        s6 = WebSocketServer(('::', args.p), ServerResource({
            '/console': ConsoleConnection,
        }, context=self), **kwargs)

        serv_threads = [gevent.spawn(s4.serve_forever), gevent.spawn(s6.serve_forever)]
        gevent.joinall(serv_threads)
Ejemplo n.º 45
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('neighbord')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.config = None
        self.logger = logging.getLogger()
        self.plugin_dirs = []
        self.plugins = {}

    def parse_config(self, filename):
        try:
            with open(filename, 'r') as f:
                self.config = json.load(f)
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error(
                'Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['neighbord']['plugin-dirs']

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore()
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for f in os.listdir(dir):
            name, ext = os.path.splitext(os.path.basename(f))
            if ext != '.py':
                continue

            try:
                plugin = load_module_from_file(name, os.path.join(dir, f))
                plugin._init(self)
            except:
                self.logger.error('Cannot initialize plugin {0}'.format(f),
                                  exc_info=True)

    def register_plugin(self, name, cls):
        self.plugins[name] = cls(self)
        self.logger.info('Registered plugin {0} (class {1})'.format(name, cls))

    def register_service(self, name, regtype, port, properties=None):
        for plugin in self.plugins.values():
            plugin.register(regtype, name, port, properties)

    def register(self):
        try:
            hostname = socket.gethostname()
            general = self.client.call_sync('system.general.get_config')
            properties = {
                'version': self.client.call_sync('system.info.version'),
                'description': general['description'],
                'tags': ','.join(general['tags'])
            }

            self.register_service(hostname, 'freenas', 80, properties)
            self.register_service(hostname, 'http', 80)
            self.register_service(hostname, 'ssh', 22)
            self.register_service(hostname, 'sftp-ssh', 22)
        except BaseException as err:
            self.logger.error('Failed to register services: {0}'.format(
                str(err)))

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('neighbord')
                self.client.enable_server()
                self.client.register_service('neighbord.management',
                                             ManagementService(self))
                self.client.register_service('neighbord.discovery',
                                             DiscoveryService(self))
                self.client.register_service('neighbord.debug', DebugService())
                self.client.resume_service('neighbord.management')
                self.client.resume_service('neighbord.discovery')
                self.client.resume_service('neighbord.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c',
                            metavar='CONFIG',
                            default=DEFAULT_CONFIGFILE,
                            help='Middleware config file')
        args = parser.parse_args()
        self.config = args.c
        configure_logging('/var/log/neighbord.log', 'DEBUG')

        setproctitle.setproctitle('neighbord')
        self.parse_config(self.config)
        self.init_datastore()
        self.init_dispatcher()
        self.scan_plugins()
        self.register()
        self.client.wait_forever()
Ejemplo n.º 46
0
class Context(object):
    def __init__(self):
        self.logger = logging.getLogger('schedulerd')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.scheduler = None
        self.active_tasks = {}

    def init_datastore(self):
        try:
            self.datastore = get_datastore(self.config)
        except DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_scheduler(self):
        store = MongoDBJobStore(database='freenas', collection='calendar_tasks', client=self.datastore.client)
        self.scheduler = BackgroundScheduler(jobstores={'default': store}, timezone=pytz.utc)
        self.scheduler.start()

    def register_schemas(self):
        self.client.register_schema('calendar-task', {
            'type': 'object',
            'additionalProperties': False,
            'properties': {
                'id': {'type': 'string'},
                'name': {'type': 'string'},
                'args': {'type': 'array'},
                'description': {'type': 'string'},
                'enabled': {'type': 'boolean'},
                'hidden': {'type': 'boolean'},
                'protected': {'type': 'boolean'},
                'status': {'$ref': 'calendar-task-status'},
                'schedule': {
                    'type': 'object',
                    'additionalProperties': False,
                    'properties': {
                        'coalesce': {'type': ['boolean', 'integer', 'null']},
                        'year': {'type': ['string', 'integer', 'null']},
                        'month': {'type': ['string', 'integer', 'null']},
                        'day': {'type': ['string', 'integer', 'null']},
                        'week': {'type': ['string', 'integer', 'null']},
                        'day_of_week': {'type': ['string', 'integer', 'null']},
                        'hour': {'type': ['string', 'integer', 'null']},
                        'minute': {'type': ['string', 'integer', 'null']},
                        'second': {'type': ['string', 'integer', 'null']},
                        'timezone': {'type': ['string', 'null']}
                    }
                }
            }
        })

        self.client.register_schema('calendar-task-status', {
            'type': 'object',
            'properties': {
                'next_run_time': {'type': 'string'},
                'last_run_status': {'type': 'string'},
                'current_run_status': {'type': ['string', 'null']},
                'current_run_progress': {'type': ['object', 'null']}
            }
        })

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('schedulerd')
                self.client.enable_server()
                self.client.register_service('scheduler.management', ManagementService(self))
                self.client.register_service('scheduler.debug', DebugService())
                self.client.resume_service('scheduler.management')
                self.client.resume_service('scheduler.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def run_job(self, *args, **kwargs):
        tid = self.client.submit_task(*args)
        self.active_tasks[kwargs['id']] = tid
        self.client.call_sync('task.wait', tid, timeout=None)
        result = self.client.call_sync('task.status', tid)
        if result['state'] != 'FINISHED':
            try:
                self.client.call_sync('alerts.emit', {
                    'name': 'scheduler.task.failed',
                    'severity': 'CRITICAL',
                    'description': 'Task {0} has failed: {1}'.format(kwargs['name'], result['error']['message']),
                })
            except RpcException as e:
                self.logger.error('Failed to emit alert', exc_info=True)

        del self.active_tasks[kwargs['id']]
        self.datastore.insert('schedulerd.runs', {
            'job_id': kwargs['id'],
            'task_id': result['id']
        })

    def emit_event(self, name, params):
        self.client.emit_event(name, params)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        parser.add_argument('-f', action='store_true', default=False, help='Run in foreground')
        args = parser.parse_args()
        configure_logging('/var/log/schedulerd.log', 'DEBUG')
        setproctitle.setproctitle('schedulerd')
        self.config = args.c
        self.init_datastore()
        self.init_scheduler()
        self.init_dispatcher()
        self.register_schemas()
        self.client.wait_forever()
Ejemplo n.º 47
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('alertd')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.plugin_dirs = []
        self.emitters = {}

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore()
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_reminder(self):
        t = threading.Thread(target=self.reminder_thread)
        t.daemon = True
        t.start()

    def parse_config(self, filename):
        try:
            f = open(filename, 'r')
            self.config = json.load(f)
            f.close()
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error('Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['alertd']['plugin-dirs']

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('alertd')
                self.client.enable_server()
                self.client.register_service('alertd.management', ManagementService(self))
                self.client.register_service('alertd.alert', AlertService(self))
                self.client.register_service('alertd.debug', DebugService())
                self.client.resume_service('alertd.management')
                self.client.resume_service('alertd.alert')
                self.client.resume_service('alertd.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for f in os.listdir(dir):
            name, ext = os.path.splitext(os.path.basename(f))
            if ext != '.py':
                continue

            try:
                plugin = imp.load_source(name, os.path.join(dir, f))
                plugin._init(self)
            except:
                self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True)

    def emit_alert(self, alert):
        self.logger.debug('Emitting alert <id:{0}> (class {1})'.format(alert['id'], alert['class']))
        for i in self.datastore.query('alert.filters'):
            for predicate in i.get('predicates', []):
                if predicate['operator'] not in operators_table:
                    continue

                if not operators_table[predicate['operator']](alert[predicate['property']], predicate['value']):
                    break
            else:
                try:
                    emitter = self.emitters.get(i['emitter'])
                    if not emitter:
                        self.logger.warning('Invalid emitter {0} for alert filter {1}'.format(i['emitter'], i['id']))
                        continue

                    self.logger.debug('Alert <id:{0}> matched filter {1}'.format(alert['id'], i['id']))
                    if alert['send_count'] > 0:
                        emitter.emit_again(alert, i['parameters'])
                    else:
                        emitter.emit_first(alert, i['parameters'])
                except BaseException as err:
                    # Failed to emit alert using alert emitter
                    # XXX: generate another alert about that
                    self.logger.error('Cannot emit alert <id:{0}> using {1}: {2}'.format(
                        alert['id'],
                        i['emitter'],
                        str(err))
                    )

        alert['send_count'] += 1
        alert['last_emitted_at'] = datetime.utcnow()

        if alert['one_shot']:
            alert['active'] = False

        self.datastore.update('alerts', alert['id'], alert)

    def cancel_alert(self, alert):
        self.logger.debug('Cancelling alert <id:{0}> (class {1})'.format(alert['id'], alert['class']))

        alert.update({
            'active': False,
            'cancelled': datetime.utcnow()
        })

        self.datastore.update('alerts', alert['id'], alert)

    def register_emitter(self, name, cls):
        self.emitters[name] = cls(self)
        self.logger.info('Registered emitter {0} (class {1})'.format(name, cls))

    def reminder_thread(self):
        while True:
            time.sleep(REMINDER_SECONDS)
            for i in self.datastore.query('alerts'):
                if not i['active'] or i['dismissed']:
                    continue

                last_emission = i.get('last_emitted_at') or i['created_at']
                interval = REMINDER_SCHEDULE[i['severity']]

                if not interval:
                    continue

                if last_emission + timedelta(seconds=interval) <= datetime.utcnow():
                    self.emit_alert(i)

    def checkin(self):
        checkin()

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        args = parser.parse_args()
        configure_logging('/var/log/alertd.log', 'DEBUG')

        setproctitle('alertd')
        self.config = args.c
        self.parse_config(self.config)
        self.init_datastore()
        self.init_dispatcher()
        self.scan_plugins()
        self.init_reminder()
        self.checkin()
        self.client.wait_forever()
Ejemplo n.º 48
0
class RESTApi(object):

    def __init__(self):
        self.logger = logging.getLogger('restd')
        self._cruds = []
        self._threads = []
        self._rpcs = {}
        self._schemas = {}
        self._used_schemas = set()
        self._services = {}
        self._tasks = {}
        self.api = falcon.API(middleware=[
            AuthMiddleware(),
            JSONTranslator(),
        ])
        self.api.add_route('/', SwaggerResource(self))

        gevent.signal(signal.SIGINT, self.die)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.dispatcher = Client()
        self.dispatcher.on_error(on_error)
        self.connect()

    def init_metadata(self):
        self._tasks = self.dispatcher.call_sync('discovery.get_tasks')
        self._schemas = self.dispatcher.call_sync('discovery.get_schema')
        for service in self.dispatcher.call_sync('discovery.get_services'):
            self._services[service] = self.dispatcher.call_sync('discovery.get_methods', service)
            for method in self._services[service]:
                self._rpcs['{0}.{1}'.format(service, method['name'])] = method

    def load_plugins(self):
        pluginsdir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'plugins'))
        for i in glob.glob1(pluginsdir, "*.py"):
            try:
                loader = importlib.machinery.SourceFileLoader(i.split('.')[0], os.path.join(pluginsdir, i))
                mod = loader.load_module()
            except:
                self.logger.error('Failed to load plugin %s', i, exc_info=True)
                raise
            mod._init(self)

    def connect(self):
        while True:
            try:
                self.dispatcher.connect('unix:')
                self.dispatcher.login_service('restd')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def __call__(self, environ, start_response):
        if 'HTTP_X_REAL_IP' in environ:
            environ['PATH_INFO'] = environ.get('PATH_INFO', '').replace('/api/v2.0', '', 1)
        return self.api.__call__(environ, start_response)

    def register_crud(self, klass):
        ins = klass(self, self.dispatcher)
        self._cruds.append(ins)

    def register_singleitem(self, klass):
        klass(self, self.dispatcher)

    def register_resource(self, klass):
        klass(self)

    def run(self):
        self.init_dispatcher()
        self.init_metadata()
        self.load_plugins()

        server4 = WSGIServer(('', 8889), self, handler_class=RESTWSGIHandler)
        self._threads = [gevent.spawn(server4.serve_forever)]
        checkin()
        gevent.joinall(self._threads)

    def die(self, *args):
        gevent.killall(self._threads)
        sys.exit(0)
Ejemplo n.º 49
0
 def initialize(self, context):
     self.client = Client()
     self.client.connect('unix:///var/run/logd.sock')
Ejemplo n.º 50
0
    def run(self, peer):
        if self.datastore.exists('peers', ('address', '=', peer['address']), ('type', '=', 'replication')):
            raise TaskException(errno.EEXIST, 'Replication peer entry for {0} already exists'.format(peer['address']))

        if peer['credentials']['type'] != 'ssh':
            raise TaskException(errno.EINVAL, 'SSH credentials type is needed to perform replication peer pairing')

        remote = peer.get('address')
        credentials = peer['credentials']
        username = credentials.get('username')
        port = credentials.get('port', 22)
        password = credentials.get('password')

        if not username:
            raise TaskException(errno.EINVAL, 'Username has to be specified')

        if not remote:
            raise TaskException(errno.EINVAL, 'Address of remote host has to be specified')

        if not password:
            raise TaskException(errno.EINVAL, 'Password has to be specified')

        remote_client = Client()
        try:
            try:
                remote_client.connect('ws+ssh://{0}@{1}'.format(username, remote), port=port, password=password)
                remote_client.login_service('replicator')
            except (AuthenticationException, OSError, ConnectionRefusedError):
                raise TaskException(errno.ECONNABORTED, 'Cannot connect to {0}:{1}'.format(remote, port))

            local_keys = self.dispatcher.call_sync('peer.get_ssh_keys')
            remote_keys = remote_client.call_sync('peer.get_ssh_keys')
            ip_at_remote_side = remote_client.call_sync('management.get_sender_address').split(',', 1)[0]

            remote_host_key = remote + ' ' + remote_keys[0].rsplit(' ', 1)[0]
            local_host_key = ip_at_remote_side + ' ' + local_keys[0].rsplit(' ', 1)[0]

            local_ssh_config = self.dispatcher.call_sync('service.sshd.get_config')

            if remote_client.call_sync('peer.query', [('name', '=', peer['name'])]):
                raise TaskException(errno.EEXIST, 'Peer entry {0} already exists at {1}'.format(peer['name'], remote))

            peer['credentials'] = {
                'pubkey': remote_keys[1],
                'hostkey': remote_host_key,
                'port': port,
                'type': 'replication'
            }

            self.join_subtasks(self.run_subtask(
                'peer.replication.create_local',
                peer
            ))

            peer['address'] = ip_at_remote_side
            peer['credentials'] = {
                'pubkey': local_keys[1],
                'hostkey': local_host_key,
                'port': local_ssh_config['port'],
                'type': 'replication'
            }

            id = self.datastore.query('peers', ('name', '=', peer['name']), select='id')
            try:
                call_task_and_check_state(
                    remote_client,
                    'peer.replication.create_local',
                    peer
                )
            except TaskException:
                self.datastore.delete('peers', id)
                self.dispatcher.dispatch_event('peer.changed', {
                    'operation': 'delete',
                    'ids': [id]
                })
                raise
        finally:
            remote_client.disconnect()
Ejemplo n.º 51
0
 def __init__(self, level=logging.NOTSET, address=None, ident=None):
     super(LogdLogHandler, self).__init__(level)
     self.address = address or 'unix:///var/run/logd.sock'
     self.ident = ident or os.path.basename(sys.executable)
     self.client = Client()
     self.client.connect(self.address)
Ejemplo n.º 52
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('clid')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.client = None
        self.config = None
        self.logger = logging.getLogger()
        self.plugin_dirs = []
        self.ml = None
        self.context = None

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_cli(self):
        self.logger.info('Initializing CLI instance')
        self.context = Context()
        self.context.connection = self.client
        self.context.plugin_dirs = PLUGIN_DIRS
        self.context.discover_plugins()
        self.context.start_entity_subscribers()
        self.context.login_plugins()
        self.ml = MainLoop(self.context)
        self.logger.info('CLI instance ready')

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('clid')
                self.client.enable_server()
                self.client.call_sync('management.enable_features', ['streaming_responses'])
                self.client.register_service('clid.management', ManagementService(self))
                self.client.register_service('clid.eval', EvalService(self))
                self.client.register_service('clid.debug', DebugService())
                self.client.resume_service('clid.management')
                self.client.resume_service('clid.eval')
                self.client.resume_service('clid.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        args = parser.parse_args()
        self.config = args.c
        configure_logging('/var/log/clid.log', 'DEBUG')

        setproctitle('clid')
        self.init_dispatcher()
        self.init_cli()
        self.client.wait_forever()
Ejemplo n.º 53
0
class Context(object):
    def __init__(self):
        self.server = None
        self.client = None
        self.jobs = {}
        self.provides = set()
        self.lock = RLock()
        self.kq = select.kqueue()
        self.devnull = os.open('/dev/null', os.O_RDWR)
        self.logger = logging.getLogger('Context')
        self.rpc = RpcContext()
        self.rpc.register_service_instance('serviced.management',
                                           ManagementService(self))
        self.rpc.register_service_instance('serviced.job', JobService(self))

    def init_dispatcher(self):
        if self.client and self.client.connected:
            return

        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_server(self, address):
        self.server = Server(self)
        self.server.rpc = self.rpc
        self.server.streaming = True
        self.server.start(address, transport_options={'permissions': 0o777})
        thread = Thread(target=self.server.serve_forever)
        thread.name = 'ServerThread'
        thread.daemon = True
        thread.start()

    def provide(self, targets):
        def doit():
            self.logger.debug('Adding dependency targets: {0}'.format(
                ', '.join(targets)))
            with self.lock:
                self.provides |= targets
                for job in list(self.jobs.values()):
                    if job.state == JobState.STOPPED and job.requires <= self.provides:
                        job.start()

        if targets:
            Timer(2, doit).start()

    def job_by_pid(self, pid):
        job = first_or_default(lambda j: j.pid == pid, self.jobs.values())
        return job

    def event_loop(self):
        while True:
            with contextlib.suppress(InterruptedError):
                for ev in self.kq.control(None, MAX_EVENTS):
                    self.logger.log(TRACE, 'New event: {0}'.format(ev))
                    if ev.filter == select.KQ_FILTER_PROC:
                        job = self.job_by_pid(ev.ident)
                        if job:
                            job.pid_event(ev)
                            continue

                        if ev.fflags & select.KQ_NOTE_CHILD:
                            if ev.fflags & select.KQ_NOTE_EXIT:
                                continue

                            pjob = self.job_by_pid(ev.data)
                            if not pjob:
                                self.untrack_pid(ev.ident)
                                continue

                            # Stop tracking at session ID boundary
                            try:
                                if pjob.pgid != os.getpgid(ev.ident):
                                    self.untrack_pid(ev.ident)
                                    continue
                            except ProcessLookupError:
                                continue

                            with self.lock:
                                job = Job(self)
                                job.load_anonymous(pjob, ev.ident)
                                self.jobs[job.id] = job
                                self.logger.info('Added job {0}'.format(
                                    job.label))

    def track_pid(self, pid):
        ev = select.kevent(
            pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE,
            select.KQ_NOTE_EXIT | select.KQ_NOTE_EXEC | select.KQ_NOTE_FORK
            | select.KQ_NOTE_TRACK, 0, 0)

        self.kq.control([ev], 0)

    def untrack_pid(self, pid):
        ev = select.kevent(pid, select.KQ_FILTER_PROC, select.KQ_EV_DELETE, 0,
                           0, 0)

        with contextlib.suppress(FileNotFoundError):
            self.kq.control([ev], 0)

    def emit_event(self, name, args):
        self.server.broadcast_event(name, args)
        if self.client and self.client.connected:
            self.client.emit_event(name, args)

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('serviced')
                self.client.enable_server(self.rpc)
                self.client.resume_service('serviced.job')
                self.client.resume_service('serviced.management')
                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def bootstrap(self):
        def doit():
            with self.lock:
                job = Job(self)
                job.load({
                    'Label': 'org.freenas.serviced.bootstrap',
                    'ProgramArguments': BOOTSTRAP_JOB,
                    'OneShot': True,
                    'RunAtLoad': True,
                })

                self.jobs[job.id] = job

        Thread(target=doit).start()

    def shutdown(self):
        self.client.disconnect()
        self.server.close()
        sys.exit(0)

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-s',
                            metavar='SOCKET',
                            default=DEFAULT_SOCKET_ADDRESS,
                            help='Socket address to listen on')
        args = parser.parse_args()

        configure_logging('/var/log/serviced.log', 'DEBUG', file=True)
        bsd.setproctitle('serviced')
        self.logger.info('Started')
        self.init_server(args.s)
        self.bootstrap()
        self.event_loop()
Ejemplo n.º 54
0
class Main(object):
    def __init__(self):
        self.logger = logging.getLogger('dscached')
        self.config = None
        self.datastore = None
        self.configstore = None
        self.rpc = RpcContext()
        self.rpc.streaming_enabled = True
        self.rpc.streaming_burst = 16
        self.client = None
        self.server = None
        self.plugin_dirs = []
        self.plugins = {}
        self.directories = []
        self.users_cache = TTLCacheStore()
        self.groups_cache = TTLCacheStore()
        self.hosts_cache = TTLCacheStore()
        self.cache_ttl = 7200
        self.search_order = []
        self.cache_enumerations = True
        self.cache_lookups = True
        self.rpc.register_service_instance('dscached.account', AccountService(self))
        self.rpc.register_service_instance('dscached.group', GroupService(self))
        self.rpc.register_service_instance('dscached.host', HostService(self))
        self.rpc.register_service_instance('dscached.management', ManagementService(self))
        self.rpc.register_service_instance('dscached.debug', DebugService())

    def get_enabled_directories(self):
        return list(filter(None, (self.get_directory_by_name(n) for n in self.get_search_order())))

    def get_search_order(self):
        return ['local', 'system'] + self.search_order

    def get_directory_by_domain(self, domain_name):
        return first_or_default(lambda d: d.domain_name == domain_name, self.directories)

    def get_directory_by_name(self, name):
        return first_or_default(lambda d: d.name == name, self.directories)

    def get_directory_for_id(self, uid=None, gid=None):
        if uid is not None:
            if uid == 0:
                # Special case for root user
                return first_or_default(lambda d: d.plugin_type == 'local', self.directories)

            return first_or_default(
                lambda d: d.max_uid and d.max_uid >= uid >= d.min_uid,
                self.directories
            )

        if gid is not None:
            if gid == 0:
                # Special case for wheel group
                return first_or_default(lambda d: d.plugin_type == 'local', self.directories)

            return first_or_default(
                lambda d: d.max_gid and d.max_gid >= gid >= d.min_gid,
                self.directories
            )

    def init_datastore(self):
        try:
            self.datastore = datastore.get_datastore()
        except datastore.DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

        self.configstore = ConfigStore(self.datastore)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def init_server(self, address):
        self.server = Server(self)
        self.server.rpc = self.rpc
        self.server.start(address)
        thread = Thread(target=self.server.serve_forever)
        thread.name = 'ServerThread'
        thread.daemon = True
        thread.start()

    def parse_config(self, filename):
        try:
            with open(filename, 'r') as f:
                self.config = json.load(f)
        except IOError as err:
            self.logger.error('Cannot read config file: %s', err.message)
            sys.exit(1)
        except ValueError:
            self.logger.error('Config file has unreadable format (not valid JSON)')
            sys.exit(1)

        self.plugin_dirs = self.config['dscached']['plugin-dirs']

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('dscached')
                self.client.enable_server(self.rpc)
                self.client.resume_service('dscached.account')
                self.client.resume_service('dscached.group')
                self.client.resume_service('dscached.host')
                self.client.resume_service('dscached.management')
                self.client.resume_service('dscached.debug')
                return
            except (OSError, RpcException) as err:
                self.logger.warning('Cannot connect to dispatcher: {0}, retrying in 1 second'.format(str(err)))
                time.sleep(1)

    def scan_plugins(self):
        for i in self.plugin_dirs:
            self.scan_plugin_dir(i)

    def scan_plugin_dir(self, dir):
        self.logger.debug('Scanning plugin directory %s', dir)
        for f in os.listdir(dir):
            name, ext = os.path.splitext(os.path.basename(f))
            if ext != '.py':
                continue

            try:
                plugin = imp.load_source(name, os.path.join(dir, f))
                plugin._init(self)
            except:
                self.logger.error('Cannot initialize plugin {0}'.format(f), exc_info=True)

    def register_plugin(self, name, cls):
        self.plugins[name] = cls
        self.logger.info('Registered plugin {0} (class {1})'.format(name, cls))

    def register_schema(self, name, schema):
        self.client.register_schema(name, schema)

    def init_directories(self):
        for i in self.datastore.query('directories'):
            try:
                directory = Directory(self, i)
                directory.configure()
                self.directories.append(directory)
            except BaseException as err:
                continue

    def load_config(self):
        self.search_order = self.configstore.get('directory.search_order')
        self.cache_ttl = self.configstore.get('directory.cache_ttl')
        self.cache_enumerations = self.configstore.get('directory.cache_enumerations')
        self.cache_lookups = self.configstore.get('directory.cache_lookups')

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c', metavar='CONFIG', default=DEFAULT_CONFIGFILE, help='Middleware config file')
        parser.add_argument('-s', metavar='SOCKET', default=DEFAULT_SOCKET_ADDRESS, help='Socket address to listen on')
        args = parser.parse_args()
        configure_logging('/var/log/dscached.log', 'DEBUG')

        setproctitle.setproctitle('dscached')
        self.config = args.c
        self.parse_config(self.config)
        self.init_datastore()
        self.init_dispatcher()
        self.load_config()
        self.init_server(args.s)
        self.scan_plugins()
        self.init_directories()
        self.client.wait_forever()
Ejemplo n.º 55
0
    def run(self, peer, initial_credentials):
        hostid = self.dispatcher.call_sync('system.info.host_uuid')
        hostname = self.dispatcher.call_sync('system.general.get_config')['hostname']
        remote_peer_name = hostname
        credentials = peer['credentials']
        remote = credentials.get('address')
        port = credentials.get('port', 22)
        username = initial_credentials.get('username')
        password = initial_credentials.get('password')
        auth_code = initial_credentials.get('auth_code')
        key_auth = initial_credentials.get('key_auth')

        local_ssh_config = self.dispatcher.call_sync('service.sshd.get_config')

        if self.datastore.exists('peers', ('credentials.address', '=', remote), ('type', '=', 'freenas')):
            raise TaskException(
                errno.EEXIST,
                'FreeNAS peer entry for {0} already exists'.format(remote)
            )

        remote_client = Client()

        try:
            if auth_code:
                try:
                    remote_client.connect('ws://{0}'.format(wrap_address(remote)))
                except (AuthenticationException, OSError, ConnectionRefusedError):
                    raise TaskException(errno.ECONNABORTED, 'Cannot connect to {0}:{1}'.format(remote, port))

                try:
                    remote_host_uuid, pubkey = remote_client.call_sync(
                        'peer.freenas.auth_with_code',
                        auth_code,
                        hostname,
                        local_ssh_config['port']
                    )
                except RpcException as err:
                    raise TaskException(err.code, err.message)

                try:
                    self.dispatcher.call_sync('peer.freenas.put_temp_pubkey', pubkey)
                    if not self.dispatcher.test_or_wait_for_event(
                        'peer.changed',
                        lambda ar: ar['operation'] == 'create' and remote_host_uuid in ar['ids'],
                        lambda: self.datastore.exists('peers', ('id', '=', remote_host_uuid)),
                        timeout=30
                    ):
                        raise TaskException(
                            errno.EAUTH,
                            'FreeNAS peer creation failed. Check connection to host {0}.'.format(remote)
                        )
                finally:
                    self.dispatcher.call_sync('peer.freenas.remove_temp_pubkey', pubkey)

            else:
                try:
                    if key_auth:
                        with io.StringIO() as f:
                            f.write(self.configstore.get('peer.freenas.key.private'))
                            f.seek(0)
                            pkey = RSAKey.from_private_key(f)

                        max_tries = 50
                        while True:
                            try:
                                remote_client.connect('ws+ssh://freenas@{0}'.format(
                                    wrap_address(remote)), pkey=pkey, port=port
                                )
                                break
                            except AuthenticationException:
                                if max_tries:
                                    max_tries -= 1
                                    time.sleep(1)
                                else:
                                    raise
                    else:
                        remote_client.connect(
                            'ws+ssh://{0}@{1}'.format(username, wrap_address(remote)),
                            port=port,
                            password=password
                        )

                    remote_client.login_service('replicator')
                except (AuthenticationException, OSError, ConnectionRefusedError):
                    raise TaskException(errno.ECONNABORTED, 'Cannot connect to {0}:{1}'.format(remote, port))

                local_host_key, local_pub_key = self.dispatcher.call_sync('peer.freenas.get_ssh_keys')
                remote_host_key, remote_pub_key = remote_client.call_sync('peer.freenas.get_ssh_keys')
                ip_at_remote_side = remote_client.local_address[0]

                remote_hostname = remote_client.call_sync('system.general.get_config')['hostname']

                remote_host_key = remote_host_key.rsplit(' ', 1)[0]
                local_host_key = local_host_key.rsplit(' ', 1)[0]

                if remote_client.call_sync('peer.query', [('id', '=', hostid)]):
                    raise TaskException(errno.EEXIST, 'Peer entry of {0} already exists at {1}'.format(hostname, remote))

                peer['credentials'] = {
                    '%type': 'freenas-credentials',
                    'pubkey': remote_pub_key,
                    'hostkey': remote_host_key,
                    'port': port,
                    'address': remote_hostname
                }

                local_id = remote_client.call_sync('system.info.host_uuid')
                peer['id'] = local_id
                peer['name'] = remote_hostname
                ip = socket.gethostbyname(remote)

                created_id = self.run_subtask_sync(
                    'peer.freenas.create_local',
                    peer,
                    ip,
                    True
                )

                peer['id'] = hostid
                peer['name'] = remote_peer_name
                peer['credentials'] = {
                    '%type': 'freenas-credentials',
                    'pubkey': local_pub_key,
                    'hostkey': local_host_key,
                    'port': local_ssh_config['port'],
                    'address': hostname
                }

                try:
                    call_task_and_check_state(
                        remote_client,
                        'peer.freenas.create_local',
                        peer,
                        ip_at_remote_side
                    )
                except TaskException:
                    self.datastore.delete('peers', local_id)
                    self.dispatcher.dispatch_event('peer.changed', {
                        'operation': 'delete',
                        'ids': [local_id]
                    })
                    raise

                return created_id
        finally:
            remote_client.disconnect()
Ejemplo n.º 56
0
    def main(self):
        if len(sys.argv) != 2:
            print("Invalid number of arguments", file=sys.stderr)
            sys.exit(errno.EINVAL)

        key = sys.argv[1]
        configure_logging(None, logging.DEBUG)

        self.datastore = get_datastore()
        self.configstore = ConfigStore(self.datastore)
        self.conn = Client()
        self.conn.connect('unix:')
        self.conn.login_service('task.{0}'.format(os.getpid()))
        self.conn.enable_server()
        self.conn.call_sync('management.enable_features',
                            ['streaming_responses'])
        self.conn.rpc.register_service_instance('taskproxy', self.service)
        self.conn.register_event_handler('task.progress',
                                         self.task_progress_handler)
        self.conn.call_sync('task.checkin', key)
        setproctitle.setproctitle('task executor (idle)')

        while True:
            try:
                task = self.task.get()
                logging.root.setLevel(
                    self.conn.call_sync('management.get_logging_level'))
                setproctitle.setproctitle('task executor (tid {0})'.format(
                    task['id']))

                if task['debugger']:
                    sys.path.append('/usr/local/lib/dispatcher/pydev')

                    import pydevd
                    host, port = task['debugger']
                    pydevd.settrace(host,
                                    port=port,
                                    stdoutToServer=True,
                                    stderrToServer=True)

                name, _ = os.path.splitext(os.path.basename(task['filename']))
                module = self.module_cache.get(task['filename'])
                if not module:
                    module = load_module_from_file(name, task['filename'])
                    self.module_cache[task['filename']] = module

                setproctitle.setproctitle('task executor (tid {0})'.format(
                    task['id']))
                fds = list(self.collect_fds(task['args']))

                try:
                    dispatcher = DispatcherWrapper(self.conn)
                    self.instance = getattr(module,
                                            task['class'])(dispatcher,
                                                           self.datastore)
                    self.instance.configstore = self.configstore
                    self.instance.user = task['user']
                    self.instance.environment = task['environment']
                    self.running.set()
                    self.run_task_hooks(self.instance, task, 'before')
                    result = self.instance.run(*task['args'])
                    self.run_task_hooks(self.instance,
                                        task,
                                        'after',
                                        result=result)
                except BaseException as err:
                    print("Task exception: {0}".format(str(err)),
                          file=sys.stderr)
                    traceback.print_exc(file=sys.stderr)

                    if hasattr(self.instance, 'rollback'):
                        self.put_status('ROLLBACK')
                        try:
                            self.instance.rollback(*task['args'])
                        except BaseException as rerr:
                            print("Task exception during rollback: {0}".format(
                                str(rerr)),
                                  file=sys.stderr)
                            traceback.print_exc(file=sys.stderr)

                    # Main task is already failed at this point, so ignore hook errors
                    with contextlib.suppress(RpcException):
                        self.run_task_hooks(self.instance,
                                            task,
                                            'error',
                                            error=serialize_error(err))

                    self.put_status('FAILED', exception=err)
                else:
                    self.put_status('FINISHED', result=result)
                finally:
                    self.close_fds(fds)
                    self.running.clear()

            except RpcException as err:
                print("RPC failed: {0}".format(str(err)), file=sys.stderr)
                print(traceback.format_exc(), flush=True)
                sys.exit(errno.EBADMSG)
            except socket.error as err:
                print("Cannot connect to dispatcher: {0}".format(str(err)),
                      file=sys.stderr)
                sys.exit(errno.ETIMEDOUT)

            if task['debugger']:
                import pydevd
                pydevd.stoptrace()

            setproctitle.setproctitle('task executor (idle)')
Ejemplo n.º 57
0
class Main(object):
    def __init__(self):
        self.client = None
        self.server = None
        self.datastore = None
        self.hdf = None
        self.hdf_group = None
        self.config = None
        self.logger = logging.getLogger('statd')
        self.data_sources = {}

    def init_datastore(self):
        try:
            self.datastore = get_datastore(self.config)
        except DatastoreException as err:
            self.logger.error('Cannot initialize datastore: %s', str(err))
            sys.exit(1)

    def init_database(self):
        # adding this try/except till system-dataset plugin is added back in in full fidelity
        # just a hack (since that directory's data will not persist)
        # Please remove this when system-dataset plugin is added back in
        try:
            directory = self.client.call_sync(
                'system_dataset.request_directory', 'statd')
        except RpcException:
            directory = '/var/tmp/statd'
            if not os.path.exists(directory):
                os.makedirs(directory)
        self.hdf = tables.open_file(os.path.join(directory, DEFAULT_DBFILE),
                                    mode='a')
        if not hasattr(self.hdf.root, 'stats'):
            self.hdf.create_group('/', 'stats')

        self.hdf_group = self.hdf.root.stats

    def request_table(self, name):
        try:
            if hasattr(self.hdf_group, name):
                return getattr(self.hdf_group, name)

            return self.hdf.create_table(self.hdf_group, name, DataPoint, name)
        except Exception as e:
            self.logger.error(str(e))

    def init_alert_config(self, name):
        config_name = name if self.datastore.exists('statd.alerts',
                                                    ('id', '=',
                                                     name)) else 'default'
        alert_config = self.datastore.get_by_id('statd.alerts', config_name)
        return alert_config

    def get_data_source(self, name):
        if name not in list(self.data_sources.keys()):
            config = DataSourceConfig(self.datastore, name)
            alert_config = self.init_alert_config(name)
            ds = DataSource(self, name, config, alert_config)
            self.data_sources[name] = ds
            self.client.call_sync('plugin.register_event_type', 'statd.output',
                                  'statd.{0}.pulse'.format(name))

        return self.data_sources[name]

    def register_schemas(self):
        self.client.register_schema(
            'GetStatsParams', {
                'type': 'object',
                'additionalProperties': False,
                'properties': {
                    'start': {
                        'type': 'datetime'
                    },
                    'end': {
                        'type': 'datetime'
                    },
                    'timespan': {
                        'type': 'integer'
                    },
                    'frequency': {
                        'type': 'string'
                    }
                }
            })

        self.client.register_schema(
            'GetStatsResult', {
                'type': 'object',
                'additionalProperties': False,
                'properties': {
                    'data': {
                        'type': 'array',
                    }
                }
            })

    def connect(self):
        while True:
            try:
                self.client.connect('unix:')
                self.client.login_service('statd')
                self.client.enable_server()
                self.register_schemas()
                self.client.register_service('statd.output',
                                             OutputService(self))
                self.client.register_service('statd.alert', AlertService(self))
                self.client.register_service('statd.debug',
                                             DebugService(gevent=True))
                self.client.resume_service('statd.output')
                self.client.resume_service('statd.alert')
                self.client.resume_service('statd.debug')
                for i in list(self.data_sources.keys()):
                    self.client.call_sync('plugin.register_event_type',
                                          'statd.output',
                                          'statd.{0}.pulse'.format(i))

                return
            except (OSError, RpcException) as err:
                self.logger.warning(
                    'Cannot connect to dispatcher: {0}, retrying in 1 second'.
                    format(str(err)))
                time.sleep(1)

    def init_dispatcher(self):
        def on_error(reason, **kwargs):
            if reason in (ClientError.CONNECTION_CLOSED, ClientError.LOGOUT):
                self.logger.warning('Connection to dispatcher lost')
                self.connect()

        self.client = Client()
        self.client.on_error(on_error)
        self.connect()

    def die(self):
        self.logger.warning('Exiting')
        self.server.stop()
        self.client.disconnect()
        sys.exit(0)

    def dispatcher_error(self, error):
        self.die()

    def checkin(self):
        checkin()

    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('-c',
                            metavar='CONFIG',
                            default=DEFAULT_CONFIGFILE,
                            help='Middleware config file')
        args = parser.parse_args()
        configure_logging('/var/log/fnstatd.log', 'DEBUG')
        setproctitle('fnstatd')

        # Signal handlers
        gevent.signal(signal.SIGQUIT, self.die)
        gevent.signal(signal.SIGTERM, self.die)
        gevent.signal(signal.SIGINT, self.die)

        self.server = InputServer(self)
        self.config = args.c
        self.init_datastore()
        self.init_dispatcher()
        self.init_database()
        self.server.start()
        self.logger.info('Started')
        self.checkin()
        self.client.wait_forever()