Beispiel #1
0
    def make_app(global_conf, **app_conf):
        """Returns a Sync Server Application."""
        global_conf.update(app_conf)
        params = Config(global_conf)
        app = klass(urls, controllers, params, auth_class)

        if params.get('debug', False):
            app = TransLogger(app, logger_name='syncserver',
                              setup_console_handler=True)

        if params.get('profile', False):
            from repoze.profile.profiler import AccumulatingProfileMiddleware
            app = AccumulatingProfileMiddleware(app,
                                          log_filename='profile.log',
                                          cachegrind_filename='cachegrind.out',
                                          discard_first_request=True,
                                          flush_at_shutdown=True,
                                          path='/__profile__')

        if params.get('client_debug', False):
            # errors are displayed in the user client
            app = ErrorMiddleware(app, debug=True,
                                  show_exceptions_in_wsgi_errors=True)
        else:
            # errors are logged and a 500 is returned with an empty body
            # to avoid any security whole
            app = CatchErrorMiddleware(app, logger_name='syncserver')

        if wrapper is not None:
            app = wrapper(app, config=params)
        return app
def make_app(global_conf, **app_conf):
    """Returns a Key Exchange Application."""
    global_conf.update(app_conf)
    config = Config(global_conf)
    app = KeyExchangeApp(config)
    blacklisted = app.blacklisted

    # hooking a profiler
    if global_conf.get('profile', 'false').lower() == 'true':
        from repoze.profile.profiler import AccumulatingProfileMiddleware
        app = AccumulatingProfileMiddleware(app, log_filename='profile.log',
                                          cachegrind_filename='cachegrind.out',
                                          discard_first_request=True,
                                          flush_at_shutdown=True,
                                           path='/__profile__')

    # hooking a client debugger
    if global_conf.get('client_debug', 'false').lower() == 'true':
        from paste.exceptions.errormiddleware import ErrorMiddleware
        app = ErrorMiddleware(app, debug=True,
                              show_exceptions_in_wsgi_errors=True)

    # hooking a stdout logger
    if global_conf.get('debug', 'false').lower() == 'true':
        from paste.translogger import TransLogger
        app = TransLogger(app, logger_name='jpakeapp',
                          setup_console_handler=True)

    # IP Filtering middleware
    if config.get('filtering.use', False):
        del config['filtering.use']
        params = config.get_section('filtering')
        app = IPFiltering(app, callback=blacklisted, **params)

    return app
    def make_app(global_conf, **app_conf):
        """Returns a Sync Server Application."""
        global_conf.update(app_conf)
        params = Config(global_conf)
        app = klass(urls, controllers, params, auth_class)

        if params.get('debug', False):
            app = TransLogger(app,
                              logger_name='syncserver',
                              setup_console_handler=True)

        if params.get('profile', False):
            from repoze.profile.profiler import AccumulatingProfileMiddleware
            app = AccumulatingProfileMiddleware(
                app,
                log_filename='profile.log',
                cachegrind_filename='cachegrind.out',
                discard_first_request=True,
                flush_at_shutdown=True,
                path='/__profile__')

        if params.get('client_debug', False):
            # errors are displayed in the user client
            app = ErrorMiddleware(app,
                                  debug=True,
                                  show_exceptions_in_wsgi_errors=True)
        else:
            # errors are logged and a 500 is returned with an empty body
            # to avoid any security whole
            app = CatchErrorMiddleware(app, logger_name='syncserver')

        if wrapper is not None:
            app = wrapper(app, config=params)
        return app
Beispiel #4
0
def make_sse_server(config_filename):
    configItems = ['configuration', 
            'broker_host', 
            'broker_amqp_port', 
            'broker_username', 
            'broker_password', 
            'broker_virtual_host', 
            'incoming_exchange_name', 
            'notifs_queue_name']
    SSEConfig = Config(config_filename)
    configMap = dict([(item, SSEConfig.get("app:post_office", item)) 
        for item in configItems])

    return ServerEventController(configMap)
Beispiel #5
0
    def setUp(self):
        test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                     'tests.conf')
        self.config = Config(cfgfile = test_cfg)

        self.app = TestApp(make_app(self.config))
        self.app.reset()
    def _init(cls):
        cls.log = logging.getLogger(__name__)
        cls.log.info("Initializing databases")
        config = Config().get_configuration()

        cls.db = PostgresqlDatabase(config.get("RESOURCES_DB", "name"),
                                    user=config.get("RESOURCES_DB", "user"),
                                    password=config.get(
                                        "RESOURCES_DB", "password"),
                                    host=config.get("RESOURCES_DB", "host"))

        cls._influx_token = config.get("INFLUX_DB", "token")
        cls.influx_org = config.get("INFLUX_DB", "org")
        cls.influx_bucket = config.get("INFLUX_DB", "bucket")
        cls.influx_client = InfluxDBClient(url=config.get(
            "INFLUX_DB", "server"),
                                           token=cls._influx_token)
        cls.influx_write_api = cls.influx_client.write_api(
            write_options=SYNCHRONOUS)
Beispiel #7
0
    def test_config(self):
        cfg_in = {'one': '1', 'two': 'bla', 'three': 'false'}
        config = Config(cfg_in)

        self.assertTrue(config['one'])
        self.assertEqual(config['two'], 'bla')
        self.assertFalse(config['three'])

        # config also reads extra config files.
        __, filename = tempfile.mkstemp()
        try:
            with open(filename, 'w') as f:
                f.write(_EXTRA)

            cfg_in = {'one': '1', 'two': 'file:%s' % filename}
            config.load_config(cfg_in)
            self.assertTrue(config['some.stuff'])
            self.assertEquals(config['other.thing'], 'ok')
        finally:
            os.remove(filename)
    def test_config(self):
        cfg_in = {'one': '1', 'two': 'bla', 'three': 'false'}
        config = Config(cfg_in)

        self.assertTrue(config['one'])
        self.assertEqual(config['two'], 'bla')
        self.assertFalse(config['three'])

        # config also reads extra config files.
        __, filename = tempfile.mkstemp()
        try:
            with open(filename, 'w') as f:
                f.write(_EXTRA)

            cfg_in = {'one': '1', 'two': 'file:%s' % filename}
            config.load_config(cfg_in)
            self.assertTrue(config['some.stuff'])
            self.assertEquals(config['other.thing'], 'ok')
        finally:
            os.remove(filename)
Beispiel #9
0
def convert_config(config):
    """Loads the configuration.

    If a "configuration" option is found, reads it using config.Config.
    Each section/option is then converted to "section.option" in the resulting
    mapping.
    """
    res = {}
    for key, value in config.items():
        if not isinstance(value, basestring) or not value.startswith('file:'):
            res[key] = convert(value)
            continue
        # we load the configuration and inject it in the mapping
        filename = value[len('file:'):]
        if not os.path.exists(filename):
            raise ValueError('The configuration file was not found. "%s"' % \
                            filename)

        conf = Config(filename)
        res.update(conf.get_map())

    return res
def convert_config(config):
    """Loads the configuration.

    If a "configuration" option is found, reads it using config.Config.
    Each section/option is then converted to "section.option" in the resulting
    mapping.
    """
    res = {}
    for key, value in config.items():
        if not isinstance(value, basestring) or not value.startswith('file:'):
            res[key] = convert(value)
            continue
        # we load the configuration and inject it in the mapping
        filename = value[len('file:'):]
        if not os.path.exists(filename):
            raise ValueError('The configuration file was not found. "%s"' % \
                            filename)

        conf = Config(filename)
        res.update(conf.get_map())

    return res
Beispiel #11
0
    def test_reader(self):
        config = Config(self.file_one)

        # values conversion
        self.assertEquals(config.get('one', 'foo'), 'bar')
        self.assertEquals(config.get('one', 'num'), -12)
        self.assertEquals(config.get('one', 'st'), 'o=k')
        self.assertEquals(config.get('one', 'lines'), [1, 'two', 3])
        self.assertEquals(config.get('one', 'env'), 'some stuff')

        # getting a map
        map = config.get_map()
        self.assertEquals(map['one.foo'], 'bar')

        map = config.get_map('one')
        self.assertEquals(map['foo'], 'bar')

        del os.environ['__STUFF__']
        self.assertRaises(EnvironmentNotFoundError, config.get, 'one', 'env')

        # extends
        self.assertEquals(config.get('three', 'more'), 'stuff')
        self.assertEquals(config.get('one', 'two'), 'a')
    def test_load_direct(self):
        from services.tests.test_pluginreg import Dummy
        bad_config = Config({'backend': 'xxx'})
        good_config = Config({'test.backend':
                              'services.tests.test_pluginreg.Dummy',
                              'test.foo': 'bar'})
        self.assertRaises(KeyError, load_and_configure, bad_config)
        obj = load_and_configure(good_config, 'test')
        self.assertTrue(isinstance(obj, Dummy))
        self.assertTrue(obj.foo == 'bar')

        missing_interface = {'interface': 'services.tests.test_pluginreg.Nope'}
        bad_interface = {'backend':
                         'services.tests.test_pluginreg.ImplementsBadly',
                         'interface':
                         'services.tests.test_pluginreg.ClassInterface'}
        good_interface = {'backend':
                         'services.tests.test_pluginreg.ImplementsCorrectly',
                         'interface':
                         'services.tests.test_pluginreg.ClassInterface'}
        obj = load_and_configure(good_interface)
        self.assertTrue(isinstance(obj, ImplementsCorrectly))
        self.assertRaises(TypeError, load_and_configure, bad_interface)
        self.assertRaises(ImportError, load_and_configure, missing_interface)
    def setUp(self):
        if not MEMCACHED:
            raise SkipTest

        # Ensure we have metlog loaded so the timers will work.
        config_file = os.path.join(os.path.dirname(__file__), "sync.conf")
        config = Config(cfgfile=config_file)
        load_and_configure(config, "metlog_loader")

        fd, self.dbfile = mkstemp()
        os.close(fd)

        self.fn = 'syncstorage.storage.memcachedsql.MemcachedSQLStorage'

        kwds = self.STORAGE_CONFIG.copy()
        kwds['sqluri'] = 'sqlite:///%s' % self.dbfile
        self.storage = SyncStorage.get(self.fn, **kwds)

        # make sure we have the standard collections in place

        for name in ('client', 'crypto', 'forms', 'history'):
            self.storage.set_collection(_UID, name)
Beispiel #14
0
class ClientAgentTest(unittest.TestCase):
    """Test class for all Client Agent tests.

    This handles the creation of an in-process Client Agent server to run
    tests against.

    """

    def setUp(self):
        test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests.conf")
        self.config = Config(cfgfile=test_cfg)
        # self.appdir = env.topdir
        self.app = TestApp(make_app(self.config))
        self.app.reset()

    def tearDown(self):
        pass

    def set_credentials(self, username=None, password=None):
        self.username = username
        if password is None:
            jws = JWS()
            password = jws.sign(
                header={"alg": "NONE", "typ": "IDAssertion"},
                payload={"exp": int(time.time() + 300), "moz-vep-id": username},
            )
        self.password = password

    def create_queue(self):
        res = self.app.post("/%s/new_queue" % VERSION, extra_environ={"test_session.uid": self.username})
        assert res.status_int == 200
        return json.loads(res.body)

    def create_subscription(self, token):
        res = self.app.post(
            "/%s/new_subscription" % VERSION,
            json.dumps({"token": token}),
            content_type="application/json",
            extra_environ={"test_session.uid": self.username},
        )
        assert res.status_int == 200

    def remove_subscription(self, token, success_response=200):
        res = self.app.post(
            "/%s/remove_subscription" % VERSION,
            json.dumps({"token": token}),
            status=success_response,
            content_type="application/json",
            extra_environ={"test_session.uid": self.username},
        )
        assert res.status_int == success_response

    def send_broadcast(self, message=None):
        if message is None:
            message = "'this is a broadcast message'"
        self.app.post(
            "/%s/broadcast" % (VERSION),
            message,
            content_type="application/json",
            extra_environ={"test_session.uid": self.username},
        )
        # TODO: Figure out a way to extract messages from queues in tests
        # For now we assume all is well if "200 OK" returned

    def test_create_queue(self):
        self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
        queue_info = self.create_queue()
        assert "queue_id" in queue_info
        assert "host" in queue_info
        assert "port" in queue_info
        assert len(queue_info["queue_id"]) > 0
        assert len(queue_info["host"]) > 0
        assert queue_info["port"]

    def test_subscriptions(self):
        self.set_credentials(self.config.get("tests.user", "*****@*****.**"), self.config.get("tests.password", None))
        token = "TEST123"

        # Can't delete subscription if it doesn't exist
        try:
            self.remove_subscription(token, success_response=400)
        except Exception, e:
            print str(e)
        # Normal create and delete
        self.create_subscription(token)
        self.remove_subscription(token)

        # Can't delete subscription if already deleted
        self.remove_subscription(token, success_response=400)
Beispiel #15
0
 def setUp(self):
     test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests.conf")
     self.config = Config(cfgfile=test_cfg)
     # self.appdir = env.topdir
     self.app = TestApp(make_app(self.config))
     self.app.reset()
Beispiel #16
0
def start_processes(config_ini):
    procs = []
    stoplist = []

    logger.info("Starting config service")
    conf = Config()
    proc = Process(target=conf.run, args=(config_ini,))
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_CONFIG) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_CONFIG)

    logger.info("Starting service-detection service")
    service_detection = ServiceDetection()
    proc = Process(target=service_detection.run, args=())
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_SD) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_SD)

    logger.info("Starting DNS service")
    dns = DNSAPI()
    proc = Process(target=dns.run, args=())
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_DNS) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_DNS)

    logger.info("Starting web-server service")
    mgmt_server = ManageWebservers()
    proc = Process(target=mgmt_server.run, args=())
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_WEBSERVER) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_WEBSERVER)

    logger.info("Starting module-loader service")
    module_ldr = ModuleLoader()
    proc = Process(target=module_ldr.run, args=())
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_MODULES) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_MODULES)

    logger.info("Starting database service")
    database = Database()
    proc = Process(target=database.run, args=())
    procs.append(proc)
    proc.start()

    if wait_service_up(SOCK_DATABASE) is False:
        stop_services(stoplist)
        sys.exit(1)
    stoplist.append(SOCK_DATABASE)

    logger.info("Starting shell-receiver service")
    shell = ShellReceiver()
    proc = Process(target=shell.run, args=())
    procs.append(proc)
    proc.start()

    # No need to wait for shell-service to be up
    logger.info("All processes have started")
    return procs
Beispiel #17
0
def ajax_config_setconfig(rd, user, data):
	filterOperation('setConfig', user, data.attr)
	Config.SetValue(data.attr, data.data)
    def test_config_merge(self):
        config = Config(cfgfile=self.file_four)
        global_ = {'foo': 'bar', 'baz': 'bawlp'}
        self.assertEqual(config.get_section(''), dict())
        self.assertEqual(config.get_section('global'), global_)

        self.assertEqual(config['auth.a'], 'b')
        self.assertEqual(config['auth.c'], 'd')
        self.assertEqual(config.get_section('auth'), {'a': 'b', 'c': 'd'})

        storage = {'e': 'f', 'g': 'h'}
        self.assertEqual(config['storage.e'], 'f')
        self.assertEqual(config['storage.g'], 'h')
        self.assertEqual(config.get_section('storage'), storage)

        storage_once = {'i': 'j', 'k': 'l'}
        storage_once.update(storage)
        once_merged = config.merge('multi:once')
        self.assertEqual(once_merged.get_section('storage'), storage_once)

        storage_thrice = {'i': 'jjj', 'k': 'lll'}
        storage_thrice.update(storage)
        thrice_merged = config.merge('multi:thrice')
        self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)

        # merge cache should guarantee same result
        config['storage.m'] = 'n'
        config['multi:thrice.storage.o'] = 'ppp'
        thrice_merged = config.merge('multi:thrice')
        self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)

        # but not after merge cache is cleared
        config.clear_merge_cache()
        thrice_merged = config.merge('multi:thrice')
        self.assertNotEqual(thrice_merged.get_section('storage'),
                            storage_thrice)
        storage_thrice.update(dict(m='n', o='ppp'))
        self.assertEqual(thrice_merged.get_section('storage'), storage_thrice)
Beispiel #19
0
    def test_config_merge(self):
        config = Config(cfgfile=self.file_four)
        global_ = {'foo': 'bar', 'baz': 'bawlp'}
        self.assertEqual(config.get_section(''), dict())
        self.assertEqual(config.get_section('global'), global_)

        self.assertEqual(config['auth.a'], 'b')
        self.assertEqual(config['auth.c'], 'd')
        self.assertEqual(config.get_section('auth'),
                         {'a': 'b', 'c': 'd'})

        storage = {'e': 'f', 'g': 'h'}
        self.assertEqual(config['storage.e'], 'f')
        self.assertEqual(config['storage.g'], 'h')
        self.assertEqual(config.get_section('storage'),
                         storage)

        storage_once = {'i': 'j', 'k': 'l'}
        storage_once.update(storage)
        once_merged = config.merge('multi:once')
        self.assertEqual(once_merged.get_section('storage'),
                         storage_once)

        storage_thrice = {'i': 'jjj', 'k': 'lll'}
        storage_thrice.update(storage)
        thrice_merged = config.merge('multi:thrice')
        self.assertEqual(thrice_merged.get_section('storage'),
                         storage_thrice)

        # merge cache should guarantee same result
        config['storage.m'] = 'n'
        config['multi:thrice.storage.o'] = 'ppp'
        thrice_merged = config.merge('multi:thrice')
        self.assertEqual(thrice_merged.get_section('storage'),
                         storage_thrice)

        # but not after merge cache is cleared
        config.clear_merge_cache()
        thrice_merged = config.merge('multi:thrice')
        self.assertNotEqual(thrice_merged.get_section('storage'),
                            storage_thrice)
        storage_thrice.update(dict(m='n', o='ppp'))
        self.assertEqual(thrice_merged.get_section('storage'),
                         storage_thrice)
Beispiel #20
0
    def __init__(self, urls, controllers, config=None,
                 auth_class=None):
        self.mapper = Mapper()
        if config is None:
            self.config = Config()
        elif isinstance(config, Config):
            self.config = config
        else:
            # try to convert to config object
            self.config = Config(config)

        # global config
        self.retry_after = self.config.get('global.retry_after', 1800)

        # heartbeat page
        self.heartbeat_page = self.config.get('global.heartbeat_page',
                                              '__heartbeat__')

        # debug page, if any
        self.debug_page = self.config.get('global.debug_page')

        # check if we want to clean when the app ends
        self.sigclean = self.config.get('global.clean_shutdown', True)

        # load the specified plugin modules
        self.modules = dict()
        app_modules = self.config.get('app.modules', [])
        if isinstance(app_modules, basestring):
            app_modules = [app_modules]
        for module in app_modules:
            self.modules[module] = load_and_configure(self.config, module)

        if self.modules.get('metlog_loader') is not None:
            # stash the metlog client in a more convenient spot
            self.logger = self.modules.get('metlog_loader').default_client
        else:
            # there was no metlog config, default to using StdLibLoggingSender
            sender = StdLibLoggingSender('syncserver', json_types=[])
            metlog = MetlogClient(sender, 'syncserver')
            CLIENT_HOLDER.set_client(metlog.logger, metlog)
            self.logger = metlog
        if not hasattr(self.logger, "cef"):
            log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
            self.logger.add_method(log_cef_fn)

        # XXX: this should be converted to auto-load in self.modules
        # loading the authentication tool
        self.auth = None if auth_class is None else auth_class(self.config)

        # loading and connecting controllers
        self.controllers = dict([(name, klass(self)) for name, klass in
                                 controllers.items()])

        for url in urls:
            if len(url) == 4:
                verbs, match, controller, action = url
                extras = {}
            elif len(url) == 5:
                verbs, match, controller, action, extras = url
            else:
                msg = "Each URL description needs 4 or 5 elements. Got %s" \
                    % str(url)
                raise ValueError(msg)

            if isinstance(verbs, str):
                verbs = [verbs]

            # wrap action methods w/ metlog decorators
            controller_instance = self.controllers.get(controller)
            if controller_instance is not None:
                wrapped_name = '_%s_wrapped' % action
                method = getattr(controller_instance, action, None)
                if ((method is not None) and
                    (not hasattr(controller_instance, wrapped_name))):
                    # add wrapped method
                    wrapped = svc_timeit(method)
                    wrapped = incr_count(wrapped)
                    wrapped = send_services_data(wrapped)
                    setattr(controller_instance, wrapped_name, wrapped)
            self.mapper.connect(None, match, controller=controller,
                                action=action, conditions=dict(method=verbs),
                                **extras)

        # loads host-specific configuration
        self._host_configs = {}

        # heartbeat & debug pages
        self.standard_controller = StandardController(self)

        # rehooked overridable points so they can be overridden in the base app
        self.standard_controller._debug_server = self._debug_server
        self.standard_controller._check_server = self._check_server

        # hooking callbacks when the app shuts down
        self.killing = self.shutting = False
        self.graceful_shutdown_interval = self.config.get(
                                      'global.graceful_shutdown_interval', 1.)
        self.hard_shutdown_interval = self.config.get(
                                          'global.hard_shutdown_interval', 1.)
        if self.sigclean:
            signal.signal(signal.SIGTERM, self._sigterm)
            signal.signal(signal.SIGINT, self._sigterm)
Beispiel #21
0
    def __init__(self, urls, controllers, config=None,
                 auth_class=None):
        self.mapper = Mapper()
        if config is None:
            self.config = Config()
        elif isinstance(config, Config):
            self.config = config
        else:
            # try to convert to config object
            self.config = Config(config)

        # global config
        self.retry_after = self.config.get('global.retry_after', 1800)

        # heartbeat page
        self.heartbeat_page = self.config.get('global.heartbeat_page',
                                              '__heartbeat__')

        # debug page, if any
        self.debug_page = self.config.get('global.debug_page')

        # check if we want to clean when the app ends
        self.sigclean = self.config.get('global.clean_shutdown', True)

        self.modules = dict()
        for module in self.config.get('app.modules', []):
            self.modules[module] = load_and_configure(self.config, module)

        # XXX: this should be converted to auto-load in self.modules
        # loading the authentication tool
        self.auth = None if auth_class is None else auth_class(self.config)

        # loading and connecting controllers
        self.controllers = dict([(name, klass(self)) for name, klass in
                                 controllers.items()])

        for url in urls:
            if len(url) == 4:
                verbs, match, controller, action = url
                extras = {}
            elif len(url) == 5:
                verbs, match, controller, action, extras = url
            else:
                msg = "Each URL description needs 4 or 5 elements. Got %s" \
                    % str(url)
                raise ValueError(msg)

            if isinstance(verbs, str):
                verbs = [verbs]

            self.mapper.connect(None, match, controller=controller,
                                action=action, conditions=dict(method=verbs),
                                **extras)

        # loads host-specific configuration
        self._host_configs = {}

        # heartbeat & debug pages
        self.standard_controller = StandardController(self)

        # rehooked overridable points so they can be overridden in the base app
        self.standard_controller._debug_server = self._debug_server
        self.standard_controller._check_server = self._check_server

        # hooking callbacks when the app shuts down
        self.killing = self.shutting = False
        self.graceful_shutdown_interval = self.config.get(
                                      'global.graceful_shutdown_interval', 1.)
        self.hard_shutdown_interval = self.config.get(
                                          'global.hard_shutdown_interval', 1.)
        if self.sigclean:
            signal.signal(signal.SIGTERM, self._sigterm)
            signal.signal(signal.SIGINT, self._sigterm)
Beispiel #22
0
 def convert_config(self, ini_cfg, ini_path):
     here = {'here': os.path.dirname(os.path.realpath(ini_path))}
     cfg_in = dict([(key, value % here)
                    for key, value in ini_cfg.items('DEFAULT') +
                    ini_cfg.items('app:main')])
     return Config(cfg_in)
class SyncServerApp(object):
    """ Dispatches the request to the right controller by using Routes.
    """
    def __init__(self, urls, controllers, config=None, auth_class=None):
        self.mapper = Mapper()
        if config is None:
            self.config = Config()
        elif isinstance(config, Config):
            self.config = config
        else:
            # try to convert to config object
            self.config = Config(config)

        # global config
        self.retry_after = self.config.get('global.retry_after', 1800)

        # heartbeat page
        self.heartbeat_page = self.config.get('global.heartbeat_page',
                                              '__heartbeat__')

        # debug page, if any
        self.debug_page = self.config.get('global.debug_page')

        # check if we want to clean when the app ends
        self.sigclean = self.config.get('global.clean_shutdown', True)

        # load the specified plugin modules
        self.modules = dict()
        app_modules = self.config.get('app.modules', [])
        if isinstance(app_modules, basestring):
            app_modules = [app_modules]
        for module in app_modules:
            self.modules[module] = load_and_configure(self.config, module)

        if self.modules.get('metlog_loader') is not None:
            # stash the metlog client in a more convenient spot
            self.logger = self.modules.get('metlog_loader').default_client
        else:
            # there was no metlog config, default to using StdLibLoggingSender
            sender = StdLibLoggingSender('syncserver', json_types=[])
            metlog = MetlogClient(sender, 'syncserver')
            CLIENT_HOLDER.set_client(metlog.logger, metlog)
            self.logger = metlog
        if not hasattr(self.logger, "cef"):
            log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
            self.logger.add_method(log_cef_fn)

        # XXX: this should be converted to auto-load in self.modules
        # loading the authentication tool
        self.auth = None if auth_class is None else auth_class(self.config)

        # loading and connecting controllers
        self.controllers = dict([(name, klass(self))
                                 for name, klass in controllers.items()])

        for url in urls:
            if len(url) == 4:
                verbs, match, controller, action = url
                extras = {}
            elif len(url) == 5:
                verbs, match, controller, action, extras = url
            else:
                msg = "Each URL description needs 4 or 5 elements. Got %s" \
                    % str(url)
                raise ValueError(msg)

            if isinstance(verbs, str):
                verbs = [verbs]

            # wrap action methods w/ metlog decorators
            controller_instance = self.controllers.get(controller)
            if controller_instance is not None:
                wrapped_name = '_%s_wrapped' % action
                method = getattr(controller_instance, action, None)
                if ((method is not None)
                        and (not hasattr(controller_instance, wrapped_name))):
                    # add wrapped method
                    wrapped = svc_timeit(method)
                    wrapped = incr_count(wrapped)
                    wrapped = send_services_data(wrapped)
                    setattr(controller_instance, wrapped_name, wrapped)
            self.mapper.connect(None,
                                match,
                                controller=controller,
                                action=action,
                                conditions=dict(method=verbs),
                                **extras)

        # loads host-specific configuration
        self._host_configs = {}

        # heartbeat & debug pages
        self.standard_controller = StandardController(self)

        # rehooked overridable points so they can be overridden in the base app
        self.standard_controller._debug_server = self._debug_server
        self.standard_controller._check_server = self._check_server

        # hooking callbacks when the app shuts down
        self.killing = self.shutting = False
        self.graceful_shutdown_interval = self.config.get(
            'global.graceful_shutdown_interval', 1.)
        self.hard_shutdown_interval = self.config.get(
            'global.hard_shutdown_interval', 1.)
        if self.sigclean:
            signal.signal(signal.SIGTERM, self._sigterm)
            signal.signal(signal.SIGINT, self._sigterm)

    def _sigterm(self, signal, frame):
        self.shutting = True

        # wait for a bit
        sleep(self.graceful_shutdown_interval)

        # no more queries
        self.killing = True

        # wait for a bit
        sleep(self.hard_shutdown_interval)

        # now we can notify the end -- so pending stuff can be cleaned up
        notify(APP_ENDS)

        # bye-bye
        sys.exit(0)

    def _before_call(self, request):
        return {}

    def _host_specific(self, host, config):
        """Will compute host-specific requests"""
        return config.merge('host:%s' % host)

    #
    # Debug & heartbeat pages
    #
    def _debug_server(self, request):
        return []

    def _check_server(self, request):
        pass

    def _debug(self, request):
        return self.standard_controller._debug(request)

    def _heartbeat(self, request):
        return self.standard_controller._heartbeat(request)

    # events fired when a request is handled
    def _notified(func):
        def __notified(self, request):
            notify(REQUEST_STARTS, request)
            response = None
            try:
                response = func(self, request)
                return response
            finally:
                notify(REQUEST_ENDS, response)

        return __notified

    # information dumped in error logs
    def get_infos(self, request):
        """Returns a mapping containing useful info.

        It can be related to the request, or global to the app.
        """
        return {'user': str(request.user)}

    #
    # entry point
    #
    @wsgify
    @_notified
    def __call__(self, request):
        """Entry point for the WSGI app."""
        # the app is being killed, no more requests please
        if self.killing:
            raise HTTPServiceUnavailable()

        request.server_time = round_time()

        # gets request-specific config
        request.config = self._host_specific(request.host, self.config)

        # pre-hook
        before_headers = self._before_call(request)

        try:
            response = self._dispatch_request(request)
        except HTTPException, response:
            # set before-call headers on all responses
            response.headers.update(before_headers)
            raise
        else:
Beispiel #24
0
from services.bilibili_client import BilibiliClient
from services.config import Config
from os import linesep

content = ""

config = Config()
config.print()
bilibili = BilibiliClient(config)
reward = bilibili.get_reward()
print(reward)
avs = bilibili.get_regions(rid=20, num=5)
print(avs)
share = bilibili.share(avs[0]['aid'])
print(share)


def add_content(line):
    global content
    content = content.__add__(line).__add__(linesep)


reward_after = bilibili.get_reward()
add_content(
    f"投币获得经验:{reward_after['coins_av']},分享视频{'成功' if reward_after['share_av'] else '失败'}!!!"
)
add_content(
    f"当前等级:{reward_after['level_info']['current_level']},当前经验:{reward_after['level_info']['current_exp']}"
)

print(content)
Beispiel #25
0
class TestStorage(unittest.TestCase):

    def setUp(self):
        test_cfg = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                     'tests.conf')
        self.config = Config(cfgfile = test_cfg)

        self.app = TestApp(make_app(self.config))
        self.app.reset()

    def tearDown(self):
        pass

    def test_all(self):
        storage = get_message_backend(self.config)
        username = self.config.get('tests.user', '*****@*****.**')
        test_token = 'TEST123'
        origin = 'example.com'
        test_message = json.dumps({"body": json.dumps({
                                                "token": test_token,
                                                "timestamp": int(time.time()),
                                                "ciphertext": "test",
                                                "ttl": 100}),
                                   "HMAC": "123abc"})
        queue_info = storage.create_client_queue(username)
        subs = storage.create_subscription(username, test_token)

        #Send a message to the user.
        storage.send_broadcast(test_message, 
                    username, 
                    origin = origin, 
                    queue = test_token)
        msgs = storage.get_pending_messages(username)
        self.assertEqual(json.loads(msgs[0].get('body')).get('token'),
                         test_token)
        storage._purge(username = username)
        #Send a message to a user based on their queue id (most common path)
        storage.publish_message(test_message,
                                queue_info.get('queue_id'),
                                origin = origin)
        msgs = storage.get_pending_messages(username)
        self.failUnless(test_token in msgs[0].get('body'))
        storage._purge(username = username)

        # Add a message to a subscription queue
        storage.queue_message(test_message,
                subs.get('queue_id'),
                origin = origin)
        msgs = storage.get_pending_messages(username)
        # clean all the messages out of the user queue
        storage._purge(username = username)

        # drop the subscription.
        storage.delete_subscription(username, test_token)
        # Should not be able to add message to a deleted queue
        self.failIf(storage.queue_message(test_message,
                              subs.get('queue_id'),
                              origin = origin))
        # Should not be able to read messages from an empty queue
        msgs = storage.get_pending_messages(username)
        self.failIf(len(msgs))
Beispiel #26
0
def init_config(logger):
    from services.config import Config
    config = Config().get_configuration()
    logger.info("Cluster ID: " + config.get("CLUSTER", "id"))
Beispiel #27
0
class SyncServerApp(object):
    """ Dispatches the request to the right controller by using Routes.
    """
    def __init__(self, urls, controllers, config=None,
                 auth_class=None):
        self.mapper = Mapper()
        if config is None:
            self.config = Config()
        elif isinstance(config, Config):
            self.config = config
        else:
            # try to convert to config object
            self.config = Config(config)

        # global config
        self.retry_after = self.config.get('global.retry_after', 1800)

        # heartbeat page
        self.heartbeat_page = self.config.get('global.heartbeat_page',
                                              '__heartbeat__')

        # debug page, if any
        self.debug_page = self.config.get('global.debug_page')

        # check if we want to clean when the app ends
        self.sigclean = self.config.get('global.clean_shutdown', True)

        # load the specified plugin modules
        self.modules = dict()
        app_modules = self.config.get('app.modules', [])
        if isinstance(app_modules, basestring):
            app_modules = [app_modules]
        for module in app_modules:
            self.modules[module] = load_and_configure(self.config, module)

        if self.modules.get('metlog_loader') is not None:
            # stash the metlog client in a more convenient spot
            self.logger = self.modules.get('metlog_loader').default_client
        else:
            # there was no metlog config, default to using StdLibLoggingSender
            sender = StdLibLoggingSender('syncserver', json_types=[])
            metlog = MetlogClient(sender, 'syncserver')
            CLIENT_HOLDER.set_client(metlog.logger, metlog)
            self.logger = metlog
        if not hasattr(self.logger, "cef"):
            log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
            self.logger.add_method(log_cef_fn)

        # XXX: this should be converted to auto-load in self.modules
        # loading the authentication tool
        self.auth = None if auth_class is None else auth_class(self.config)

        # loading and connecting controllers
        self.controllers = dict([(name, klass(self)) for name, klass in
                                 controllers.items()])

        for url in urls:
            if len(url) == 4:
                verbs, match, controller, action = url
                extras = {}
            elif len(url) == 5:
                verbs, match, controller, action, extras = url
            else:
                msg = "Each URL description needs 4 or 5 elements. Got %s" \
                    % str(url)
                raise ValueError(msg)

            if isinstance(verbs, str):
                verbs = [verbs]

            # wrap action methods w/ metlog decorators
            controller_instance = self.controllers.get(controller)
            if controller_instance is not None:
                wrapped_name = '_%s_wrapped' % action
                method = getattr(controller_instance, action, None)
                if ((method is not None) and
                    (not hasattr(controller_instance, wrapped_name))):
                    # add wrapped method
                    wrapped = svc_timeit(method)
                    wrapped = incr_count(wrapped)
                    wrapped = send_services_data(wrapped)
                    setattr(controller_instance, wrapped_name, wrapped)
            self.mapper.connect(None, match, controller=controller,
                                action=action, conditions=dict(method=verbs),
                                **extras)

        # loads host-specific configuration
        self._host_configs = {}

        # heartbeat & debug pages
        self.standard_controller = StandardController(self)

        # rehooked overridable points so they can be overridden in the base app
        self.standard_controller._debug_server = self._debug_server
        self.standard_controller._check_server = self._check_server

        # hooking callbacks when the app shuts down
        self.killing = self.shutting = False
        self.graceful_shutdown_interval = self.config.get(
                                      'global.graceful_shutdown_interval', 1.)
        self.hard_shutdown_interval = self.config.get(
                                          'global.hard_shutdown_interval', 1.)
        if self.sigclean:
            signal.signal(signal.SIGTERM, self._sigterm)
            signal.signal(signal.SIGINT, self._sigterm)

    def _sigterm(self, signal, frame):
        self.shutting = True

        # wait for a bit
        sleep(self.graceful_shutdown_interval)

        # no more queries
        self.killing = True

        # wait for a bit
        sleep(self.hard_shutdown_interval)

        # now we can notify the end -- so pending stuff can be cleaned up
        notify(APP_ENDS)

        # bye-bye
        sys.exit(0)

    def _before_call(self, request):
        return {}

    def _host_specific(self, host, config):
        """Will compute host-specific requests"""
        return config.merge('host:%s' % host)

    #
    # Debug & heartbeat pages
    #
    def _debug_server(self, request):
        return []

    def _check_server(self, request):
        pass

    def _debug(self, request):
        return self.standard_controller._debug(request)

    def _heartbeat(self, request):
        return self.standard_controller._heartbeat(request)

    # events fired when a request is handled
    def _notified(func):
        def __notified(self, request):
            notify(REQUEST_STARTS, request)
            response = None
            try:
                response = func(self, request)
                return response
            finally:
                notify(REQUEST_ENDS, response)
        return __notified

    # information dumped in error logs
    def get_infos(self, request):
        """Returns a mapping containing useful info.

        It can be related to the request, or global to the app.
        """
        return {'user': str(request.user)}

    #
    # entry point
    #
    @wsgify
    @_notified
    def __call__(self, request):
        """Entry point for the WSGI app."""
        # the app is being killed, no more requests please
        if self.killing:
            raise HTTPServiceUnavailable()

        request.server_time = round_time()

        # gets request-specific config
        request.config = self._host_specific(request.host, self.config)

        # pre-hook
        before_headers = self._before_call(request)

        try:
            response = self._dispatch_request(request)
        except HTTPException, response:
            # set before-call headers on all responses
            response.headers.update(before_headers)
            raise
        else:
Beispiel #28
0
def ajax_config_listconfig(rd, user, data):
	filterOperation('listConfig', user)
	return makeResponseSuccess(Config.ListAttrs())
    def __init__(self, urls, controllers, config=None, auth_class=None):
        self.mapper = Mapper()
        if config is None:
            self.config = Config()
        elif isinstance(config, Config):
            self.config = config
        else:
            # try to convert to config object
            self.config = Config(config)

        # global config
        self.retry_after = self.config.get('global.retry_after', 1800)

        # heartbeat page
        self.heartbeat_page = self.config.get('global.heartbeat_page',
                                              '__heartbeat__')

        # debug page, if any
        self.debug_page = self.config.get('global.debug_page')

        # check if we want to clean when the app ends
        self.sigclean = self.config.get('global.clean_shutdown', True)

        # load the specified plugin modules
        self.modules = dict()
        app_modules = self.config.get('app.modules', [])
        if isinstance(app_modules, basestring):
            app_modules = [app_modules]
        for module in app_modules:
            self.modules[module] = load_and_configure(self.config, module)

        if self.modules.get('metlog_loader') is not None:
            # stash the metlog client in a more convenient spot
            self.logger = self.modules.get('metlog_loader').default_client
        else:
            # there was no metlog config, default to using StdLibLoggingSender
            sender = StdLibLoggingSender('syncserver', json_types=[])
            metlog = MetlogClient(sender, 'syncserver')
            CLIENT_HOLDER.set_client(metlog.logger, metlog)
            self.logger = metlog
        if not hasattr(self.logger, "cef"):
            log_cef_fn = metlog_cef.cef_plugin.config_plugin(dict())
            self.logger.add_method(log_cef_fn)

        # XXX: this should be converted to auto-load in self.modules
        # loading the authentication tool
        self.auth = None if auth_class is None else auth_class(self.config)

        # loading and connecting controllers
        self.controllers = dict([(name, klass(self))
                                 for name, klass in controllers.items()])

        for url in urls:
            if len(url) == 4:
                verbs, match, controller, action = url
                extras = {}
            elif len(url) == 5:
                verbs, match, controller, action, extras = url
            else:
                msg = "Each URL description needs 4 or 5 elements. Got %s" \
                    % str(url)
                raise ValueError(msg)

            if isinstance(verbs, str):
                verbs = [verbs]

            # wrap action methods w/ metlog decorators
            controller_instance = self.controllers.get(controller)
            if controller_instance is not None:
                wrapped_name = '_%s_wrapped' % action
                method = getattr(controller_instance, action, None)
                if ((method is not None)
                        and (not hasattr(controller_instance, wrapped_name))):
                    # add wrapped method
                    wrapped = svc_timeit(method)
                    wrapped = incr_count(wrapped)
                    wrapped = send_services_data(wrapped)
                    setattr(controller_instance, wrapped_name, wrapped)
            self.mapper.connect(None,
                                match,
                                controller=controller,
                                action=action,
                                conditions=dict(method=verbs),
                                **extras)

        # loads host-specific configuration
        self._host_configs = {}

        # heartbeat & debug pages
        self.standard_controller = StandardController(self)

        # rehooked overridable points so they can be overridden in the base app
        self.standard_controller._debug_server = self._debug_server
        self.standard_controller._check_server = self._check_server

        # hooking callbacks when the app shuts down
        self.killing = self.shutting = False
        self.graceful_shutdown_interval = self.config.get(
            'global.graceful_shutdown_interval', 1.)
        self.hard_shutdown_interval = self.config.get(
            'global.hard_shutdown_interval', 1.)
        if self.sigclean:
            signal.signal(signal.SIGTERM, self._sigterm)
            signal.signal(signal.SIGINT, self._sigterm)
Beispiel #30
0
            self.delivery_channel.basic_publish(
                exchange=self.incoming_exchange_name,
                routing_key=token,
                body=message
            )

            print "Notification routed to user exchange"

        except Exception as ex:
            # TODO: Either put the message back in the incoming queue, or send
            # a NACK to the broker if we're going to do ACK/NACK crap
            raise ex


if __name__ == '__main__':
    # Load configuration settings
    config = Config(os.getcwd() + '/etc/notifs.conf')
    config_map = config.get_map()

    print "Starting routing server..."
    router = RouterServer(config_map)
    print "Routing server started."

    try:
        router.start()
    except KeyboardInterrupt:
        router.shutdown()
        print "Routing server shut down."

Beispiel #31
0
import json, argparse, os, sys, importlib

import gym
from services.environments.minigrid_test import *
from services.util import load_class, load_object, load_model, load_policy, load_algorithm
from services.arguments import Arguments
from services.config import Config
from services.constants import *

pwd = os.path.dirname(os.path.realpath(__file__))
PARSER = Arguments(pwd=pwd).parser
PARSER.add_argument("--test_name")

if __name__ == "__main__":
    args = PARSER.parse_args()
    config = Config(pwd=pwd, args=args).config

    env = gym.make(config.get(ENVIRONMENT_NAME, args.environment_name),
                   **config)
    environment_wrapper_config = config.get(ENVIRONTMENT_WRAPPER)
    if environment_wrapper_config:
        for idx, module in enumerate(environment_wrapper_config[MODULES]
                                     ):  # for ability to wrap mulitple
            environment_wrapper = load_class(
                module, environment_wrapper_config[CLASSES][idx])
            env = environment_wrapper(env)
    env.reset()

    model = load_model(config[MODEL_MODULE])(environment=env, **config).model
    policy = load_policy(config[POLICY_MODULE])(environment=env)
Beispiel #32
0
def ajax_config_getconfig(rd, user, data):
	filterOperation('getConfig', user, data.attr)
	return makeResponseSuccess(Config.__getattr__(data.attr))