def setup_backend(self, config): self.redis = RedisManager.from_config(config['redis_manager']) self.riak = RiakManager.from_config(config['riak_manager']) # this prefix is hard coded in VumiApi self.tagpool = TagpoolManager( self.redis.sub_manager('tagpool_store')) self.api = VumiApi(self.riak, self.redis)
def setUp(self): self.persistence_helper = self.add_helper(PersistenceHelper()) self.redis = yield self.persistence_helper.get_redis_manager() self.tagpool = TagpoolManager(self.redis) site = Site(TagpoolApiServer(self.tagpool)) self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1') self.add_cleanup(self.server.loseConnection) addr = self.server.getHost() self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port)) yield self.setup_tags()
def __init__(self, options): self.options = options self.config = yaml.safe_load(open(options['config'], "rb")) self.pools = self.config.get('pools', {}) redis = RedisManager.from_config(self.config.get('redis_manager', {})) self.tagpool = TagpoolManager( redis.sub_manager(self.config.get('tagpool_prefix', 'vumi')))
def setup_worker(self): config = self.get_static_config() self.redis_manager = yield TxRedisManager.from_config( config.redis_manager) tagpool = TagpoolManager(self.redis_manager) rpc = TagpoolApiServer(tagpool) addIntrospection(rpc) site = build_web_site({ config.web_path: rpc, config.health_path: httprpc.HttpRpcHealthResource(self), }) self.addService( StreamServerEndpointService(config.twisted_endpoint, site))
def __init__(self, manager, redis, sender=None, metric_publisher=None): # local import to avoid circular import since # go.api.go_api needs to access VumiApi from go.api.go_api.session_manager import SessionManager self.manager = manager self.redis = redis self.tpm = TagpoolManager(self.redis.sub_manager('tagpool_store')) self.mdb = MessageStore(self.manager, self.redis.sub_manager('message_store')) self.account_store = AccountStore(self.manager) self.token_manager = TokenManager( self.redis.sub_manager('token_manager')) self.session_manager = SessionManager( self.redis.sub_manager('session_manager')) self.mapi = sender self.metric_publisher = metric_publisher
class Command(BaseCommand): help = "Bootstrap a Vumi Go environment, primarily intended for testing." LOCAL_OPTIONS = [ make_option( '--config-file', dest='config_file', default=False, help='Config file telling us how to connect to Riak & Redis'), make_option( '--tagpool-file', dest='tagpool_files', action='append', default=[], help='YAML file with tagpools to create.'), make_option( '--workers-file', dest='workers_files', action='append', default=[], help='YAML file with transports, apps, and routers to create.'), make_option( '--account-setup-file', dest='account_setup_files', action='append', default=[], help='YAML file with details and contents for a single account.'), make_option( '--dest-dir', dest='dest_dir', default='setup_env/build/', help='Directory to write config files to.'), make_option( '--file-name-template', dest='file_name_template', default='go_%(file_name)s.%(suffix)s', help='Template to use when generating config files.'), make_option( '--supervisord-host', dest='supervisord_host', default='127.0.0.1', help='The host supervisord should bind to.'), make_option( '--supervisord-port', dest='supervisord_port', default='7101', help='The port supervisord should listen on.'), make_option( '--webapp-bind', dest='webapp_bind', default='127.0.0.1:8000', help='The host:addr that the Django webapp should bind to.'), make_option( '--go-api-endpoint', dest='go_api_endpoint', default='tcp:interface=127.0.0.1:port=8001', help='The Twisted endpoint that the Go API worker should use.'), make_option( '--message-store-api-port', dest='message_store_api_port', type=int, default=8002, help='The port that the message store API worker should use.'), ] option_list = BaseCommand.option_list + tuple(LOCAL_OPTIONS) auto_gen_warning = ("# This file has been automatically generated by: \n" + "# %s.\n\n") % (__file__,) def handle(self, *apps, **options): config_file = options['config_file'] if not config_file: raise CommandError('Please provide --config-file') self.config = self.read_yaml(config_file) self.setup_backend(self.config) self.file_name_template = options['file_name_template'] self.dest_dir = options['dest_dir'] self.supervisord_host = options['supervisord_host'] self.supervisord_port = options['supervisord_port'] self.webapp_bind = options['webapp_bind'] self.go_api_endpoint = options['go_api_endpoint'] self.message_store_api_port = options['message_store_api_port'] self.contact_group_info = [] self.conversation_info = [] self.router_info = [] self.transport_names = [] self.router_names = [] self.application_names = [] for tagpool_file in options['tagpool_files']: self.setup_tagpools(tagpool_file) for workers_file in options['workers_files']: self.create_worker_configs(workers_file) for account_setup_file in options['account_setup_files']: self.setup_account_objects(account_setup_file) self.create_command_dispatcher_config( self.application_names, self.router_names) self.write_supervisor_config_file( 'command_dispatcher', 'go.vumitools.api_worker.CommandDispatcher') self.create_routing_table_dispatcher_config( self.application_names, self.transport_names) self.write_supervisor_config_file( 'routing_table_dispatcher', 'go.vumitools.routing.AccountRoutingTableDispatcher') self.create_billing_dispatcher_config() self.write_supervisor_config_file( 'billing_dispatcher', 'go.vumitools.billing_worker.BillingDispatcher') self.create_go_api_worker_config() self.write_supervisor_config_file( 'go_api_worker', 'go.api.go_api.GoApiWorker') self.create_message_store_api_worker_config() self.write_supervisor_config_file( 'message_store_api_worker', 'vumi.components.message_store_api.MessageStoreAPIWorker') self.write_supervisord_conf() self.create_webui_supervisord_conf() self.create_billing_api_supervisord_conf() self.write_startup_script() def setup_backend(self, config): self.redis = RedisManager.from_config(config['redis_manager']) self.riak = RiakManager.from_config(config['riak_manager']) # this prefix is hard coded in VumiApi self.tagpool = TagpoolManager( self.redis.sub_manager('tagpool_store')) self.api = VumiApi(self.riak, self.redis) def read_yaml(self, file_path): # We remove a top-level '__ignore__' key which can contain blocks # referenced elsewhere. yaml_data = yaml.safe_load(open(file_path, 'rb')) if isinstance(yaml_data, dict) and '__ignore__' in yaml_data: yaml_data.pop('__ignore__') return yaml_data def write_yaml(self, fp, data): yaml.safe_dump(data, stream=fp, default_flow_style=False) def dump_yaml_block(self, data, indent=0): dumped = yaml.safe_dump(data, default_flow_style=False) return '\n'.join('%s%s' % (' ' * indent, line) for line in dumped.splitlines()) def open_file(self, file_name, mode): "NOTE: this is only here to make testing easier" return open(file_name, mode) def render_template(self, template_name, context): template_dir = 'setup_env/templates/' with open(os.path.join(template_dir, template_name), 'r') as fp: template = Template(fp.read()) return template.render(Context(context)) def setup_tagpools(self, file_path): """ Create tag pools defined in a tagpool file. :param str file_path: The tagpools YAML file to load. """ tp_config = self.read_yaml(file_path) pools = tp_config['pools'] for pool_name, pool_data in pools.items(): listed_tags = pool_data['tags'] tags = (eval(listed_tags, {}, {}) if isinstance(listed_tags, basestring) else listed_tags) # release and remove old tags for tag in self.tagpool.inuse_tags(pool_name): self.tagpool.release_tag(tag) self.tagpool.purge_pool(pool_name) self.tagpool.declare_tags([(pool_name, tag) for tag in tags]) self.tagpool.set_metadata(pool_name, pool_data['metadata']) self.stdout.write('Tag pools created: %s\n' % ( ', '.join(sorted(pools.keys())),)) def setup_account_objects(self, file_path): account_objects = self.read_yaml(file_path) user = self.setup_account(account_objects['account']) if user: self.setup_channels(user, account_objects.get('channels', {})) self.setup_routers(user, account_objects.get('routers', {})) self.setup_conversations( user, account_objects.get('conversations', {})) self.setup_contact_groups( user, account_objects.get('contact_groups', {})) self.setup_routing(user, account_objects) def setup_account(self, user_info): user_model = get_user_model() email = user_info['email'] if user_model.objects.filter(email=email).exists(): self.stderr.write( 'User %s already exists. Skipping.\n' % (email,)) return None user = user_model.objects.create_user(email, user_info['password']) user.first_name = user_info.get('first_name', '') user.last_name = user_info.get('last_name', '') user.save() profile = user.get_profile() account = profile.get_user_account() for pool_name, max_keys in user_info['tagpools']: self.assign_tagpool(account, pool_name, max_keys) for application in user_info['applications']: self.assign_application(account, application) self.stdout.write('Account %s created\n' % (email,)) return user def assign_tagpool(self, account, pool_name, max_keys): if pool_name not in self.tagpool.list_pools(): raise CommandError( 'Tagpool %s does not exist' % (pool_name,)) permission = self.api.account_store.tag_permissions( uuid4().hex, tagpool=unicode(pool_name), max_keys=max_keys) permission.save() account.tagpools.add(permission) account.save() return permission def assign_application(self, account, application_module): app_permission = self.api.account_store.application_permissions( uuid4().hex, application=unicode(application_module)) app_permission.save() account.applications.add(app_permission) account.save() return app_permission def setup_channels(self, user, channels): user_api = vumi_api_for_user(user) for channel in channels: tag = tuple(channel.split(':')) user_api.acquire_specific_tag(tag) self.stdout.write('Tag %s acquired\n' % (tag,)) def setup_routers(self, user, routers): user_api = vumi_api_for_user(user) for router_info in routers: router_info = router_info.copy() # So we can modify it. self.router_info.append({ 'account': user.email, 'key': router_info['key'], 'start': router_info.pop('start', True), }) router_key = router_info.pop('key') if user_api.get_router(router_key): self.stderr.write( 'Router %s already exists. Skipping.\n' % ( router_key,)) continue router_type = router_info.pop('router_type') view_def = get_router_view_definition(router_type) config = router_info.pop('config', {}) extra_inbound_endpoints = view_def.get_inbound_endpoints(config) extra_outbound_endpoints = view_def.get_outbound_endpoints(config) batch_id = user_api.api.mdb.batch_start() # We bypass the usual mechanisms so we can set the key ourselves. router = user_api.router_store.routers( router_key, user_account=user_api.user_account_key, router_type=router_type, name=router_info.pop('name'), config=config, extra_inbound_endpoints=extra_inbound_endpoints, extra_outbound_endpoints=extra_outbound_endpoints, batch=batch_id, **router_info) router.save() self.stdout.write('Router %s created\n' % (router.key,)) def setup_conversations(self, user, conversations): user_api = vumi_api_for_user(user) for conv_info in conversations: conv_info = conv_info.copy() # So we can modify it. self.conversation_info.append({ 'account': user.email, 'key': conv_info['key'], 'start': conv_info.pop('start', True), # Don't pass to conv. }) conversation_key = conv_info.pop('key') if user_api.get_wrapped_conversation(conversation_key): self.stderr.write( 'Conversation %s already exists. Skipping.\n' % ( conversation_key,)) continue conversation_type = conv_info.pop('conversation_type') view_def = get_conversation_view_definition(conversation_type) config = conv_info.pop('config', {}) batch_id = user_api.api.mdb.batch_start() # We bypass the usual mechanisms so we can set the key ourselves. conv = user_api.conversation_store.conversations( conversation_key, user_account=user_api.user_account_key, conversation_type=conversation_type, name=conv_info.pop('name'), config=config, batch=batch_id, extra_endpoints=view_def.get_endpoints(config), **conv_info) conv.save() self.stdout.write('Conversation %s created\n' % (conv.key,)) def setup_routing(self, user, account_objects): connectors = {} for conv in account_objects['conversations']: connectors[conv['key']] = GoConnector.for_conversation( conv['conversation_type'], conv['key']) for tag in account_objects['channels']: connectors[tag] = GoConnector.for_transport_tag(*(tag.split(':'))) for router in account_objects['routers']: connectors[router['key'] + ':INBOUND'] = GoConnector.for_router( router['router_type'], router['key'], GoConnector.INBOUND) connectors[router['key'] + ':OUTBOUND'] = GoConnector.for_router( router['router_type'], router['key'], GoConnector.OUTBOUND) rt = RoutingTable() for src, src_ep, dst, dst_ep in account_objects['routing_entries']: rt.add_entry( str(connectors[src]), src_ep, str(connectors[dst]), dst_ep) user_account = vumi_api_for_user(user).get_user_account() user_account.routing_table = rt user_account.save() self.stdout.write('Routing table for %s built\n' % (user.email,)) def get_transport_name(self, data): return data['config']['transport_name'] def mk_filename(self, file_name, suffix): fn = self.file_name_template % { 'file_name': file_name, 'suffix': suffix, } return os.path.join(self.dest_dir, fn) def create_worker_configs(self, file_path): workers = self.read_yaml(file_path) self.create_transport_configs(workers.get('transports', {})) self.create_router_configs(workers.get('routers', {})) self.create_application_configs(workers.get('applications', {})) def create_transport_configs(self, transports): for transport_name, transport_info in transports.iteritems(): self.transport_names.append(transport_name) config = transport_info['config'] config.update({'transport_name': transport_name}) self.write_worker_config_file(transport_name, config) self.write_supervisor_config_file( transport_name, transport_info['class']) def create_router_configs(self, routers): for router_name, router_info in routers.iteritems(): self.router_names.append(router_name) worker_name = '%s_router' % (router_name,) ri_connector_name = '%s_router_ri' % (router_name,) ro_connector_name = '%s_router_ro' % (router_name,) config = router_info['config'] config.update({ 'ri_connector_name': ri_connector_name, 'ro_connector_name': ro_connector_name, 'worker_name': worker_name, }) self.write_worker_config_file(worker_name, config) self.write_supervisor_config_file( worker_name, router_info['class']) def create_application_configs(self, applications): for application_name, application_info in applications.iteritems(): self.application_names.append(application_name) transport_name = '%s_transport' % (application_name,) worker_name = '%s_application' % (application_name,) config = application_info['config'] config.update({ 'transport_name': transport_name, 'worker_name': worker_name, }) self.write_worker_config_file(worker_name, config) self.write_supervisor_config_file( worker_name, application_info['class']) def write_worker_config_file(self, transport_name, config): fn = self.mk_filename(transport_name, 'yaml') with self.open_file(fn, 'w') as fp: fp.write(self.auto_gen_warning) self.write_yaml(fp, config) self.stdout.write('Wrote %s.\n' % (fn,)) def write_supervisor_config_file(self, program_name, worker_class, config=None, enabled=True): fn = self.mk_filename(program_name, 'conf') if not enabled: fn += '.disabled' config = config or self.mk_filename(program_name, 'yaml') with self.open_file(fn, 'w') as fp: section = "program:%s" % (program_name,) fp.write(self.auto_gen_warning) cp = ConfigParser() cp.add_section(section) cp.set( section, "environment", "DJANGO_SETTINGS_MODULE=go.settings") cp.set(section, "command", " ".join([ "twistd -n --pidfile=./tmp/pids/%s.pid" % (program_name,), "start_worker", "--worker-class=%s" % (worker_class,), "--config=%s" % (config,), "--vhost=%s" % self.config.get('vhost', '/develop'), ])) cp.set(section, "stdout_logfile", "./logs/%(program_name)s_%(process_num)s.log") cp.set(section, "stderr_logfile", "./logs/%(program_name)s_%(process_num)s.log") cp.write(fp) self.stdout.write('Wrote %s.\n' % (fn,)) def create_command_dispatcher_config(self, applications, routers): worker_names = [] worker_names.extend( '%s_application' % (app_name,) for app_name in applications) worker_names.extend( '%s_router' % (router_name,) for router_name in routers) fn = self.mk_filename('command_dispatcher', 'yaml') with self.open_file(fn, 'w') as fp: self.write_yaml(fp, { 'transport_name': 'command_dispatcher', 'worker_names': worker_names, }) self.stdout.write('Wrote %s.\n' % (fn,)) def create_go_api_worker_config(self): fn = self.mk_filename('go_api_worker', 'yaml') with self.open_file(fn, 'w') as fp: self.write_yaml(fp, { 'worker_name': "go_api_worker", 'redis_manager': self.config['redis_manager'], 'riak_manager': self.config['riak_manager'], 'twisted_endpoint': self.go_api_endpoint, 'web_path': "/api/v1/go/api", }) self.stdout.write('Wrote %s.\n' % (fn,)) def create_message_store_api_worker_config(self): fn = self.mk_filename('message_store_api_worker', 'yaml') with self.open_file(fn, 'w') as fp: self.write_yaml(fp, { 'worker_name': "message_store_api_worker", 'redis_manager': self.config['redis_manager'], 'riak_manager': self.config['riak_manager'], 'web_path': "/api/v1", 'web_port': self.message_store_api_port, 'health_path': "/health/", }) self.stdout.write('Wrote %s.\n' % (fn,)) def create_routing_table_dispatcher_config(self, applications, transports): fn = self.mk_filename('routing_table_dispatcher', 'yaml') with self.open_file(fn, 'w') as fp: templ = 'routing_table_dispatcher.yaml.template' data = self.render_template(templ, { 'transport_names': transports, 'application_names': [ '%s_transport' % (app,) for app in applications], 'conversation_mappings': dict([ (app, '%s_transport' % (app,)) for app in applications]), 'redis_manager': self.dump_yaml_block( self.config['redis_manager'], 1), 'riak_manager': self.dump_yaml_block( self.config['riak_manager'], 1), }) fp.write(self.auto_gen_warning) fp.write(data) self.stdout.write('Wrote %s.\n' % (fn,)) def create_billing_dispatcher_config(self): fn = self.mk_filename('billing_dispatcher', 'yaml') with self.open_file(fn, 'w') as fp: templ = 'billing_dispatcher.yaml.template' data = self.render_template(templ, { 'redis_manager': self.dump_yaml_block( self.config['redis_manager'], 1), 'riak_manager': self.dump_yaml_block( self.config['riak_manager'], 1), }) fp.write(self.auto_gen_warning) fp.write(data) self.stdout.write('Wrote %s.\n' % (fn,)) def write_supervisord_conf(self): fn = self.mk_filename('supervisord', 'conf') with self.open_file(fn, 'w') as fp: templ = 'supervisord.conf.template' data = self.render_template(templ, { 'host': self.supervisord_host, 'port': self.supervisord_port, 'include_files': '*.conf', }) fp.write(self.auto_gen_warning) fp.write(data) self.stdout.write('Wrote %s.\n' % (fn,)) def create_webui_supervisord_conf(self): program_name = 'webui' fn = self.mk_filename(program_name, 'conf') with self.open_file(fn, 'w') as fp: section = "program:%s" % (program_name,) fp.write(self.auto_gen_warning) cp = ConfigParser() cp.add_section(section) cp.set( section, "command", "./go-admin.sh runserver %s --noreload" % ( self.webapp_bind,)) cp.set(section, "stdout_logfile", "./logs/%(program_name)s_%(process_num)s.log") cp.set(section, "redirect_stderr", "true") cp.write(fp) self.stdout.write('Wrote %s.\n' % (fn,)) def create_billing_api_supervisord_conf(self): program_name = 'billing_api' fn = self.mk_filename(program_name, 'conf') with self.open_file(fn, 'w') as fp: section = "program:%s" % (program_name,) fp.write(self.auto_gen_warning) cp = ConfigParser() cp.add_section(section) cp.set(section, "command", "./go-admin.sh runbillingserver") cp.set(section, "stdout_logfile", "./logs/%(program_name)s_%(process_num)s.log") cp.set(section, "redirect_stderr", "true") cp.write(fp) self.stdout.write('Wrote %s.\n' % (fn,)) def setup_contact_groups(self, user, contact_groups): user_api = vumi_api_for_user(user) for group_info in contact_groups: self.contact_group_info.append({ 'account': user.email, 'key': group_info['key'], 'contacts_csv': group_info['contacts_csv'], }) name = group_info['name'].decode('utf-8') account_key = user_api.user_account_key group = user_api.contact_store.groups( group_info['key'], name=name, user_account=account_key) group.save() self.stdout.write('Group %s created\n' % (group.key,)) def write_startup_script(self): """ Generate a script to import contacts and start conversations. """ fn = self.mk_filename('startup_env', 'sh') with self.open_file(fn, 'w') as fp: templ = 'startup_env.sh.template' data = self.render_template(templ, { 'contact_groups': self.contact_group_info, 'conversations': self.conversation_info, 'routers': self.router_info, }) fp.write(self.auto_gen_warning) fp.write(data) os.chmod(fn, os.stat(fn).st_mode | stat.S_IEXEC) self.stdout.write('Wrote %s.\n' % (fn,))
class TestTagpoolApiServer(VumiTestCase): @inlineCallbacks def setUp(self): self.persistence_helper = self.add_helper(PersistenceHelper()) self.redis = yield self.persistence_helper.get_redis_manager() self.tagpool = TagpoolManager(self.redis) site = Site(TagpoolApiServer(self.tagpool)) self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1') self.add_cleanup(self.server.loseConnection) addr = self.server.getHost() self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port)) yield self.setup_tags() @inlineCallbacks def setup_tags(self): # pool1 has two tags which are free yield self.tagpool.declare_tags([("pool1", "tag1"), ("pool1", "tag2")]) # pool2 has two tags which are used yield self.tagpool.declare_tags([("pool2", "tag1"), ("pool2", "tag2")]) yield self.tagpool.acquire_specific_tag(["pool2", "tag1"]) yield self.tagpool.acquire_specific_tag(["pool2", "tag2"]) # pool3 is empty but has metadata yield self.tagpool.set_metadata("pool3", {"meta": "data"}) def _check_reason(self, result, expected_owner, expected_reason): owner, reason = result self.assertEqual(owner, expected_owner) self.assertEqual(reason.pop('owner'), expected_owner) self.assertTrue(isinstance(reason.pop('timestamp'), float)) self.assertEqual(reason, expected_reason) @inlineCallbacks def test_acquire_tag(self): result = yield self.proxy.callRemote("acquire_tag", "pool1") self.assertEqual(result, ["pool1", "tag1"]) self.assertEqual((yield self.tagpool.inuse_tags("pool1")), [("pool1", "tag1")]) result = yield self.proxy.callRemote("acquire_tag", "pool2") self.assertEqual(result, None) @inlineCallbacks def test_acquire_tag_with_owner_and_reason(self): result = yield self.proxy.callRemote("acquire_tag", "pool1", "me", {"foo": "bar"}) self.assertEqual(result, ["pool1", "tag1"]) result = yield self.tagpool.acquired_by(["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_acquire_specific_tag(self): result = yield self.proxy.callRemote("acquire_specific_tag", ["pool1", "tag1"]) self.assertEqual(result, ["pool1", "tag1"]) self.assertEqual((yield self.tagpool.inuse_tags("pool1")), [("pool1", "tag1")]) result = yield self.proxy.callRemote("acquire_specific_tag", ["pool2", "tag1"]) self.assertEqual(result, None) @inlineCallbacks def test_acquire_specific_tag_with_owner_and_reason(self): result = yield self.proxy.callRemote("acquire_specific_tag", ["pool1", "tag1"], "me", {"foo": "bar"}) self.assertEqual(result, ["pool1", "tag1"]) result = yield self.tagpool.acquired_by(["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_release_tag(self): result = yield self.proxy.callRemote("release_tag", ["pool1", "tag1"]) self.assertEqual(result, None) result = yield self.proxy.callRemote("release_tag", ["pool2", "tag1"]) self.assertEqual(result, None) self.assertEqual((yield self.tagpool.inuse_tags("pool2")), [("pool2", "tag2")]) @inlineCallbacks def test_declare_tags(self): tags = [("newpool", "tag1"), ("newpool", "tag2")] result = yield self.proxy.callRemote("declare_tags", tags) self.assertEqual(result, None) free_tags = yield self.tagpool.free_tags("newpool") self.assertEqual(sorted(free_tags), sorted(tags)) @inlineCallbacks def test_get_metadata(self): result = yield self.proxy.callRemote("get_metadata", "pool3") self.assertEqual(result, {"meta": "data"}) result = yield self.proxy.callRemote("get_metadata", "pool1") self.assertEqual(result, {}) @inlineCallbacks def test_set_metadata(self): result = yield self.proxy.callRemote("set_metadata", "newpool", {"my": "data"}) self.assertEqual(result, None) self.assertEqual((yield self.tagpool.get_metadata("newpool")), {"my": "data"}) @inlineCallbacks def test_purge_pool(self): result = yield self.proxy.callRemote("purge_pool", "pool1") self.assertEqual(result, None) self.assertEqual((yield self.tagpool.free_tags("pool1")), []) @inlineCallbacks def test_purge_pool_with_keys_in_use(self): d = self.proxy.callRemote("purge_pool", "pool2") yield d.addErrback(lambda f: log.err(f)) # txJSON-RPC 0.5 adds support for py3 which means # different error classes are being logged, accept both errors = self.flushLoggedErrors('xmlrpclib.Fault', 'xmlrpc.client.Fault') self.assertEqual(len(errors), 1) server_errors = self.flushLoggedErrors( 'vumi.components.tagpool.TagpoolError') self.assertEqual(len(server_errors), 1) @inlineCallbacks def test_list_pools(self): result = yield self.proxy.callRemote("list_pools") self.assertEqual(sorted(result), ["pool1", "pool2", "pool3"]) @inlineCallbacks def test_free_tags(self): result = yield self.proxy.callRemote("free_tags", "pool1") self.assertEqual(sorted(result), [["pool1", "tag1"], ["pool1", "tag2"]]) result = yield self.proxy.callRemote("free_tags", "pool2") self.assertEqual(result, []) result = yield self.proxy.callRemote("free_tags", "pool3") self.assertEqual(result, []) @inlineCallbacks def test_inuse_tags(self): result = yield self.proxy.callRemote("inuse_tags", "pool1") self.assertEqual(result, []) result = yield self.proxy.callRemote("inuse_tags", "pool2") self.assertEqual(sorted(result), [["pool2", "tag1"], ["pool2", "tag2"]]) result = yield self.proxy.callRemote("inuse_tags", "pool3") self.assertEqual(result, []) @inlineCallbacks def test_acquired_by(self): result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"]) self.assertEqual(result, [None, None]) result = yield self.proxy.callRemote("acquired_by", ["pool2", "tag1"]) self._check_reason(result, None, {}) yield self.tagpool.acquire_tag("pool1", owner="me", reason={"foo": "bar"}) result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_owned_tags(self): result = yield self.proxy.callRemote("owned_tags", None) self.assertEqual(sorted(result), [[u'pool2', u'tag1'], [u'pool2', u'tag2']]) yield self.tagpool.acquire_tag("pool1", owner="me", reason={"foo": "bar"}) result = yield self.proxy.callRemote("owned_tags", "me") self.assertEqual(result, [["pool1", "tag1"]])
class TestTagpoolApiServer(VumiTestCase): @inlineCallbacks def setUp(self): self.persistence_helper = self.add_helper(PersistenceHelper()) self.redis = yield self.persistence_helper.get_redis_manager() self.tagpool = TagpoolManager(self.redis) site = Site(TagpoolApiServer(self.tagpool)) self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1') self.add_cleanup(self.server.loseConnection) addr = self.server.getHost() self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port)) yield self.setup_tags() @inlineCallbacks def setup_tags(self): # pool1 has two tags which are free yield self.tagpool.declare_tags([ ("pool1", "tag1"), ("pool1", "tag2")]) # pool2 has two tags which are used yield self.tagpool.declare_tags([ ("pool2", "tag1"), ("pool2", "tag2")]) yield self.tagpool.acquire_specific_tag(["pool2", "tag1"]) yield self.tagpool.acquire_specific_tag(["pool2", "tag2"]) # pool3 is empty but has metadata yield self.tagpool.set_metadata("pool3", {"meta": "data"}) def _check_reason(self, result, expected_owner, expected_reason): owner, reason = result self.assertEqual(owner, expected_owner) self.assertEqual(reason.pop('owner'), expected_owner) self.assertTrue(isinstance(reason.pop('timestamp'), float)) self.assertEqual(reason, expected_reason) @inlineCallbacks def test_acquire_tag(self): result = yield self.proxy.callRemote("acquire_tag", "pool1") self.assertEqual(result, ["pool1", "tag1"]) self.assertEqual((yield self.tagpool.inuse_tags("pool1")), [("pool1", "tag1")]) result = yield self.proxy.callRemote("acquire_tag", "pool2") self.assertEqual(result, None) @inlineCallbacks def test_acquire_tag_with_owner_and_reason(self): result = yield self.proxy.callRemote( "acquire_tag", "pool1", "me", {"foo": "bar"}) self.assertEqual(result, ["pool1", "tag1"]) result = yield self.tagpool.acquired_by(["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_acquire_specific_tag(self): result = yield self.proxy.callRemote("acquire_specific_tag", ["pool1", "tag1"]) self.assertEqual(result, ["pool1", "tag1"]) self.assertEqual((yield self.tagpool.inuse_tags("pool1")), [("pool1", "tag1")]) result = yield self.proxy.callRemote("acquire_specific_tag", ["pool2", "tag1"]) self.assertEqual(result, None) @inlineCallbacks def test_acquire_specific_tag_with_owner_and_reason(self): result = yield self.proxy.callRemote( "acquire_specific_tag", ["pool1", "tag1"], "me", {"foo": "bar"}) self.assertEqual(result, ["pool1", "tag1"]) result = yield self.tagpool.acquired_by(["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_release_tag(self): result = yield self.proxy.callRemote("release_tag", ["pool1", "tag1"]) self.assertEqual(result, None) result = yield self.proxy.callRemote("release_tag", ["pool2", "tag1"]) self.assertEqual(result, None) self.assertEqual((yield self.tagpool.inuse_tags("pool2")), [("pool2", "tag2")]) @inlineCallbacks def test_declare_tags(self): tags = [("newpool", "tag1"), ("newpool", "tag2")] result = yield self.proxy.callRemote("declare_tags", tags) self.assertEqual(result, None) free_tags = yield self.tagpool.free_tags("newpool") self.assertEqual(sorted(free_tags), sorted(tags)) @inlineCallbacks def test_get_metadata(self): result = yield self.proxy.callRemote("get_metadata", "pool3") self.assertEqual(result, {"meta": "data"}) result = yield self.proxy.callRemote("get_metadata", "pool1") self.assertEqual(result, {}) @inlineCallbacks def test_set_metadata(self): result = yield self.proxy.callRemote("set_metadata", "newpool", {"my": "data"}) self.assertEqual(result, None) self.assertEqual((yield self.tagpool.get_metadata("newpool")), {"my": "data"}) @inlineCallbacks def test_purge_pool(self): result = yield self.proxy.callRemote("purge_pool", "pool1") self.assertEqual(result, None) self.assertEqual((yield self.tagpool.free_tags("pool1")), []) @inlineCallbacks def test_purge_pool_with_keys_in_use(self): d = self.proxy.callRemote("purge_pool", "pool2") yield d.addErrback(lambda f: log.err(f)) # txJSON-RPC 0.5 adds support for py3 which means # different error classes are being logged, accept both errors = self.flushLoggedErrors('xmlrpclib.Fault', 'xmlrpc.client.Fault') self.assertEqual(len(errors), 1) server_errors = self.flushLoggedErrors( 'vumi.components.tagpool.TagpoolError') self.assertEqual(len(server_errors), 1) @inlineCallbacks def test_list_pools(self): result = yield self.proxy.callRemote("list_pools") self.assertEqual(sorted(result), ["pool1", "pool2", "pool3"]) @inlineCallbacks def test_free_tags(self): result = yield self.proxy.callRemote("free_tags", "pool1") self.assertEqual( sorted(result), [["pool1", "tag1"], ["pool1", "tag2"]]) result = yield self.proxy.callRemote("free_tags", "pool2") self.assertEqual(result, []) result = yield self.proxy.callRemote("free_tags", "pool3") self.assertEqual(result, []) @inlineCallbacks def test_inuse_tags(self): result = yield self.proxy.callRemote("inuse_tags", "pool1") self.assertEqual(result, []) result = yield self.proxy.callRemote("inuse_tags", "pool2") self.assertEqual( sorted(result), [["pool2", "tag1"], ["pool2", "tag2"]]) result = yield self.proxy.callRemote("inuse_tags", "pool3") self.assertEqual(result, []) @inlineCallbacks def test_acquired_by(self): result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"]) self.assertEqual(result, [None, None]) result = yield self.proxy.callRemote("acquired_by", ["pool2", "tag1"]) self._check_reason(result, None, {}) yield self.tagpool.acquire_tag("pool1", owner="me", reason={"foo": "bar"}) result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"]) self._check_reason(result, "me", {"foo": "bar"}) @inlineCallbacks def test_owned_tags(self): result = yield self.proxy.callRemote("owned_tags", None) self.assertEqual(sorted(result), [[u'pool2', u'tag1'], [u'pool2', u'tag2']]) yield self.tagpool.acquire_tag("pool1", owner="me", reason={"foo": "bar"}) result = yield self.proxy.callRemote("owned_tags", "me") self.assertEqual(result, [["pool1", "tag1"]])
def setUp(self): self.persistence_helper = self.add_helper(PersistenceHelper()) self.redis = yield self.persistence_helper.get_redis_manager() yield self.redis._purge_all() # Just in case self.tpm = TagpoolManager(self.redis)
class TestTxTagpoolManager(VumiTestCase): @inlineCallbacks def setUp(self): self.persistence_helper = self.add_helper(PersistenceHelper()) self.redis = yield self.persistence_helper.get_redis_manager() yield self.redis._purge_all() # Just in case self.tpm = TagpoolManager(self.redis) def pool_key_generator(self, pool): def tkey(x): return "tagpools:%s:%s" % (pool, x) return tkey @inlineCallbacks def test_declare_tags(self): tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2") yield self.tpm.declare_tags([tag1, tag2]) self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag1) self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag2) self.assertEqual((yield self.tpm.acquire_tag("poolA")), None) tag3 = ("poolA", "tag3") yield self.tpm.declare_tags([tag2, tag3]) self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag3) @inlineCallbacks def test_declare_unicode_tag(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag) @inlineCallbacks def test_purge_pool(self): tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2") yield self.tpm.declare_tags([tag1, tag2]) yield self.tpm.purge_pool('poolA') self.assertEqual((yield self.tpm.acquire_tag('poolA')), None) @inlineCallbacks def test_purge_unicode_pool(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) yield self.tpm.purge_pool(tag[0]) self.assertEqual((yield self.tpm.acquire_tag(tag[0])), None) @inlineCallbacks def test_purge_inuse_pool(self): tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2") yield self.tpm.declare_tags([tag1, tag2]) self.assertEqual((yield self.tpm.acquire_tag('poolA')), tag1) try: yield self.tpm.purge_pool('poolA') except TagpoolError: pass else: self.fail("Expected TagpoolError to be raised.") @inlineCallbacks def test_list_pools(self): tag1, tag2 = ("poolA", "tag1"), ("poolB", "tag2") yield self.tpm.declare_tags([tag1, tag2]) self.assertEqual((yield self.tpm.list_pools()), set(['poolA', 'poolB'])) @inlineCallbacks def test_list_unicode_pool(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) self.assertEqual((yield self.tpm.list_pools()), set([tag[0]])) @inlineCallbacks def test_acquire_tag(self): tkey = self.pool_key_generator("poolA") tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2") yield self.tpm.declare_tags([tag1, tag2]) self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag1) self.assertEqual((yield self.tpm.acquire_tag("poolB")), None) redis = self.redis self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)), ["tag2"]) self.assertEqual((yield redis.smembers(tkey("free:set"))), set(["tag2"])) self.assertEqual((yield redis.smembers(tkey("inuse:set"))), set(["tag1"])) @inlineCallbacks def test_acquire_unicode_tag(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag) self.assertEqual((yield self.tpm.acquire_tag(tag[0])), None) @inlineCallbacks def test_acquire_specific_tag(self): tkey = self.pool_key_generator("poolA") tags = [("poolA", "tag%d" % i) for i in range(10)] tag5 = tags[5] yield self.tpm.declare_tags(tags) self.assertEqual((yield self.tpm.acquire_specific_tag(tag5)), tag5) self.assertEqual((yield self.tpm.acquire_specific_tag(tag5)), None) free_local_tags = [t[1] for t in tags] free_local_tags.remove("tag5") redis = self.redis self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)), free_local_tags) self.assertEqual((yield redis.smembers(tkey("free:set"))), set(free_local_tags)) self.assertEqual((yield redis.smembers(tkey("inuse:set"))), set(["tag5"])) @inlineCallbacks def test_acquire_specific_unicode_tag(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) self.assertEqual((yield self.tpm.acquire_specific_tag(tag)), tag) self.assertEqual((yield self.tpm.acquire_specific_tag(tag)), None) @inlineCallbacks def test_release_tag(self): tkey = self.pool_key_generator("poolA") tag1, tag2, tag3 = [("poolA", "tag%d" % i) for i in (1, 2, 3)] yield self.tpm.declare_tags([tag1, tag2, tag3]) yield self.tpm.acquire_tag("poolA") yield self.tpm.acquire_tag("poolA") yield self.tpm.release_tag(tag1) redis = self.redis self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)), ["tag3", "tag1"]) self.assertEqual((yield redis.smembers(tkey("free:set"))), set(["tag1", "tag3"])) self.assertEqual((yield redis.smembers(tkey("inuse:set"))), set(["tag2"])) @inlineCallbacks def test_release_unicode_tag(self): tag = (u"poöl", u"tág") yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_tag(tag[0]) yield self.tpm.release_tag(tag) self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag) @inlineCallbacks def test_metadata(self): mkey = self.pool_key_generator("poolA")("metadata") metadata = { "transport_type": "sms", "default_msg_fields": { "transport_name": "sphex", "helper_metadata": { "even_more_nested": "foo", }, }, } yield self.tpm.set_metadata("poolA", metadata) self.assertEqual((yield self.tpm.get_metadata("poolA")), metadata) tt_json = yield self.redis.hget(mkey, "transport_type") transport_type = json.loads(tt_json) self.assertEqual(transport_type, "sms") short_md = {"foo": "bar"} yield self.tpm.set_metadata("poolA", short_md) self.assertEqual((yield self.tpm.get_metadata("poolA")), short_md) @inlineCallbacks def test_metadata_for_unicode_pool_name(self): pool = u"poöl" metadata = {"foo": "bar"} yield self.tpm.set_metadata(pool, metadata) self.assertEqual((yield self.tpm.get_metadata(pool)), metadata) @inlineCallbacks def test_unicode_metadata(self): metadata = {u"föo": u"báz"} yield self.tpm.set_metadata("pool", metadata) self.assertEqual((yield self.tpm.get_metadata("pool")), metadata) def _check_reason(self, expected_owner, owner, reason, expected_data): self.assertEqual(expected_owner, owner) self.assertEqual(expected_owner, reason.pop('owner')) timestamp = reason.pop('timestamp') self.assertTrue(isinstance(timestamp, float)) self.assertEqual(reason, expected_data) @inlineCallbacks def test_acquired_by(self): tag = ["pool", "tag"] yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_tag(tag[0], "me", {"foo": "bar"}) owner, reason = yield self.tpm.acquired_by(tag) self._check_reason("me", owner, reason, {"foo": "bar"}) @inlineCallbacks def test_acquired_by_undeclared_tags(self): tag = ["pool", "tag"] owner, reason = yield self.tpm.acquired_by(tag) self.assertEqual(owner, None) self.assertEqual(reason, None) @inlineCallbacks def test_acquired_by_no_owner(self): tag = ["pool", "tag"] yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_tag(tag[0]) owner, reason = yield self.tpm.acquired_by(tag) self._check_reason(None, owner, reason, {}) @inlineCallbacks def test_acquired_by_unicode_owner(self): tag = ["pool", "tag"] yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_tag(tag[0], u"mé") owner, reason = yield self.tpm.acquired_by(tag) self._check_reason(u"mé", owner, reason, {}) @inlineCallbacks def test_acquired_by_from_unicode_tag(self): tag = [u"poöl", u"tág"] yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_tag(tag[0], "me") owner, reason = yield self.tpm.acquired_by(tag) self._check_reason(u"me", owner, reason, {}) @inlineCallbacks def test_acquired_by_after_using_specific_tag(self): tag = ["pool", "tag"] yield self.tpm.declare_tags([tag]) yield self.tpm.acquire_specific_tag(tag, "me", {"foo": "bar"}) owner, reason = yield self.tpm.acquired_by(tag) self._check_reason("me", owner, reason, {"foo": "bar"}) @inlineCallbacks def test_owned_tags(self): tags = [["pool1", "tag1"], ["pool2", "tag2"]] yield self.tpm.declare_tags(tags) yield self.tpm.acquire_tag(tags[0][0], owner="me") my_tags = yield self.tpm.owned_tags("me") self.assertEqual(my_tags, [tags[0]]) @inlineCallbacks def test_owned_tags_no_owner(self): tags = [["pool1", "tag1"], ["pool2", "tag2"]] yield self.tpm.declare_tags(tags) yield self.tpm.acquire_tag(tags[0][0]) my_tags = yield self.tpm.owned_tags(None) self.assertEqual(my_tags, [tags[0]]) @inlineCallbacks def test_owned_tags_unicode_owner(self): tags = [["pool1", "tag1"], ["pool2", "tag2"]] yield self.tpm.declare_tags(tags) yield self.tpm.acquire_tag(tags[0][0], owner=u"mé") my_tags = yield self.tpm.owned_tags(u"mé") self.assertEqual(my_tags, [tags[0]]) @inlineCallbacks def test_owned_tags_unicode_tags(self): tags = [[u"poöl1", u"tág1"], [u"poöl2", u"tág2"]] yield self.tpm.declare_tags(tags) yield self.tpm.acquire_tag(tags[0][0], owner="me") my_tags = yield self.tpm.owned_tags(u"me") self.assertEqual(my_tags, [tags[0]])