def setup_backend(self, config):
self.redis = RedisManager.from_config(config['redis_manager'])
self.riak = RiakManager.from_config(config['riak_manager'])
# this prefix is hard coded in VumiApi
self.tagpool = TagpoolManager(
self.redis.sub_manager('tagpool_store'))
self.api = VumiApi(self.riak, self.redis)
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
self.tagpool = TagpoolManager(self.redis)
site = Site(TagpoolApiServer(self.tagpool))
self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1')
self.add_cleanup(self.server.loseConnection)
addr = self.server.getHost()
self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port))
yield self.setup_tags()
def __init__(self, options):
self.options = options
self.config = yaml.safe_load(open(options['config'], "rb"))
self.pools = self.config.get('pools', {})
redis = RedisManager.from_config(self.config.get('redis_manager', {}))
self.tagpool = TagpoolManager(
redis.sub_manager(self.config.get('tagpool_prefix', 'vumi')))
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
self.tagpool = TagpoolManager(self.redis)
site = Site(TagpoolApiServer(self.tagpool))
self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1')
self.add_cleanup(self.server.loseConnection)
addr = self.server.getHost()
self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port))
yield self.setup_tags()
def setup_worker(self):
config = self.get_static_config()
self.redis_manager = yield TxRedisManager.from_config(
config.redis_manager)
tagpool = TagpoolManager(self.redis_manager)
rpc = TagpoolApiServer(tagpool)
addIntrospection(rpc)
site = build_web_site({
config.web_path:
rpc,
config.health_path:
httprpc.HttpRpcHealthResource(self),
})
self.addService(
StreamServerEndpointService(config.twisted_endpoint, site))
def __init__(self, manager, redis, sender=None, metric_publisher=None):
# local import to avoid circular import since
# go.api.go_api needs to access VumiApi
from go.api.go_api.session_manager import SessionManager
self.manager = manager
self.redis = redis
self.tpm = TagpoolManager(self.redis.sub_manager('tagpool_store'))
self.mdb = MessageStore(self.manager,
self.redis.sub_manager('message_store'))
self.account_store = AccountStore(self.manager)
self.token_manager = TokenManager(
self.redis.sub_manager('token_manager'))
self.session_manager = SessionManager(
self.redis.sub_manager('session_manager'))
self.mapi = sender
self.metric_publisher = metric_publisher
class Command(BaseCommand):
help = "Bootstrap a Vumi Go environment, primarily intended for testing."
LOCAL_OPTIONS = [
make_option(
'--config-file',
dest='config_file',
default=False,
help='Config file telling us how to connect to Riak & Redis'),
make_option(
'--tagpool-file',
dest='tagpool_files',
action='append',
default=[],
help='YAML file with tagpools to create.'),
make_option(
'--workers-file',
dest='workers_files',
action='append',
default=[],
help='YAML file with transports, apps, and routers to create.'),
make_option(
'--account-setup-file',
dest='account_setup_files',
action='append',
default=[],
help='YAML file with details and contents for a single account.'),
make_option(
'--dest-dir',
dest='dest_dir',
default='setup_env/build/',
help='Directory to write config files to.'),
make_option(
'--file-name-template',
dest='file_name_template',
default='go_%(file_name)s.%(suffix)s',
help='Template to use when generating config files.'),
make_option(
'--supervisord-host',
dest='supervisord_host',
default='127.0.0.1',
help='The host supervisord should bind to.'),
make_option(
'--supervisord-port',
dest='supervisord_port',
default='7101',
help='The port supervisord should listen on.'),
make_option(
'--webapp-bind',
dest='webapp_bind',
default='127.0.0.1:8000',
help='The host:addr that the Django webapp should bind to.'),
make_option(
'--go-api-endpoint',
dest='go_api_endpoint',
default='tcp:interface=127.0.0.1:port=8001',
help='The Twisted endpoint that the Go API worker should use.'),
make_option(
'--message-store-api-port',
dest='message_store_api_port',
type=int,
default=8002,
help='The port that the message store API worker should use.'),
]
option_list = BaseCommand.option_list + tuple(LOCAL_OPTIONS)
auto_gen_warning = ("# This file has been automatically generated by: \n" +
"# %s.\n\n") % (__file__,)
def handle(self, *apps, **options):
config_file = options['config_file']
if not config_file:
raise CommandError('Please provide --config-file')
self.config = self.read_yaml(config_file)
self.setup_backend(self.config)
self.file_name_template = options['file_name_template']
self.dest_dir = options['dest_dir']
self.supervisord_host = options['supervisord_host']
self.supervisord_port = options['supervisord_port']
self.webapp_bind = options['webapp_bind']
self.go_api_endpoint = options['go_api_endpoint']
self.message_store_api_port = options['message_store_api_port']
self.contact_group_info = []
self.conversation_info = []
self.router_info = []
self.transport_names = []
self.router_names = []
self.application_names = []
for tagpool_file in options['tagpool_files']:
self.setup_tagpools(tagpool_file)
for workers_file in options['workers_files']:
self.create_worker_configs(workers_file)
for account_setup_file in options['account_setup_files']:
self.setup_account_objects(account_setup_file)
self.create_command_dispatcher_config(
self.application_names, self.router_names)
self.write_supervisor_config_file(
'command_dispatcher',
'go.vumitools.api_worker.CommandDispatcher')
self.create_routing_table_dispatcher_config(
self.application_names, self.transport_names)
self.write_supervisor_config_file(
'routing_table_dispatcher',
'go.vumitools.routing.AccountRoutingTableDispatcher')
self.create_billing_dispatcher_config()
self.write_supervisor_config_file(
'billing_dispatcher',
'go.vumitools.billing_worker.BillingDispatcher')
self.create_go_api_worker_config()
self.write_supervisor_config_file(
'go_api_worker',
'go.api.go_api.GoApiWorker')
self.create_message_store_api_worker_config()
self.write_supervisor_config_file(
'message_store_api_worker',
'vumi.components.message_store_api.MessageStoreAPIWorker')
self.write_supervisord_conf()
self.create_webui_supervisord_conf()
self.create_billing_api_supervisord_conf()
self.write_startup_script()
def setup_backend(self, config):
self.redis = RedisManager.from_config(config['redis_manager'])
self.riak = RiakManager.from_config(config['riak_manager'])
# this prefix is hard coded in VumiApi
self.tagpool = TagpoolManager(
self.redis.sub_manager('tagpool_store'))
self.api = VumiApi(self.riak, self.redis)
def read_yaml(self, file_path):
# We remove a top-level '__ignore__' key which can contain blocks
# referenced elsewhere.
yaml_data = yaml.safe_load(open(file_path, 'rb'))
if isinstance(yaml_data, dict) and '__ignore__' in yaml_data:
yaml_data.pop('__ignore__')
return yaml_data
def write_yaml(self, fp, data):
yaml.safe_dump(data, stream=fp, default_flow_style=False)
def dump_yaml_block(self, data, indent=0):
dumped = yaml.safe_dump(data, default_flow_style=False)
return '\n'.join('%s%s' % (' ' * indent, line)
for line in dumped.splitlines())
def open_file(self, file_name, mode):
"NOTE: this is only here to make testing easier"
return open(file_name, mode)
def render_template(self, template_name, context):
template_dir = 'setup_env/templates/'
with open(os.path.join(template_dir, template_name), 'r') as fp:
template = Template(fp.read())
return template.render(Context(context))
def setup_tagpools(self, file_path):
"""
Create tag pools defined in a tagpool file.
:param str file_path:
The tagpools YAML file to load.
"""
tp_config = self.read_yaml(file_path)
pools = tp_config['pools']
for pool_name, pool_data in pools.items():
listed_tags = pool_data['tags']
tags = (eval(listed_tags, {}, {})
if isinstance(listed_tags, basestring)
else listed_tags)
# release and remove old tags
for tag in self.tagpool.inuse_tags(pool_name):
self.tagpool.release_tag(tag)
self.tagpool.purge_pool(pool_name)
self.tagpool.declare_tags([(pool_name, tag) for tag in tags])
self.tagpool.set_metadata(pool_name, pool_data['metadata'])
self.stdout.write('Tag pools created: %s\n' % (
', '.join(sorted(pools.keys())),))
def setup_account_objects(self, file_path):
account_objects = self.read_yaml(file_path)
user = self.setup_account(account_objects['account'])
if user:
self.setup_channels(user, account_objects.get('channels', {}))
self.setup_routers(user, account_objects.get('routers', {}))
self.setup_conversations(
user, account_objects.get('conversations', {}))
self.setup_contact_groups(
user, account_objects.get('contact_groups', {}))
self.setup_routing(user, account_objects)
def setup_account(self, user_info):
user_model = get_user_model()
email = user_info['email']
if user_model.objects.filter(email=email).exists():
self.stderr.write(
'User %s already exists. Skipping.\n' %
(email,))
return None
user = user_model.objects.create_user(email, user_info['password'])
user.first_name = user_info.get('first_name', '')
user.last_name = user_info.get('last_name', '')
user.save()
profile = user.get_profile()
account = profile.get_user_account()
for pool_name, max_keys in user_info['tagpools']:
self.assign_tagpool(account, pool_name, max_keys)
for application in user_info['applications']:
self.assign_application(account, application)
self.stdout.write('Account %s created\n' % (email,))
return user
def assign_tagpool(self, account, pool_name, max_keys):
if pool_name not in self.tagpool.list_pools():
raise CommandError(
'Tagpool %s does not exist' % (pool_name,))
permission = self.api.account_store.tag_permissions(
uuid4().hex, tagpool=unicode(pool_name), max_keys=max_keys)
permission.save()
account.tagpools.add(permission)
account.save()
return permission
def assign_application(self, account, application_module):
app_permission = self.api.account_store.application_permissions(
uuid4().hex, application=unicode(application_module))
app_permission.save()
account.applications.add(app_permission)
account.save()
return app_permission
def setup_channels(self, user, channels):
user_api = vumi_api_for_user(user)
for channel in channels:
tag = tuple(channel.split(':'))
user_api.acquire_specific_tag(tag)
self.stdout.write('Tag %s acquired\n' % (tag,))
def setup_routers(self, user, routers):
user_api = vumi_api_for_user(user)
for router_info in routers:
router_info = router_info.copy() # So we can modify it.
self.router_info.append({
'account': user.email,
'key': router_info['key'],
'start': router_info.pop('start', True),
})
router_key = router_info.pop('key')
if user_api.get_router(router_key):
self.stderr.write(
'Router %s already exists. Skipping.\n' % (
router_key,))
continue
router_type = router_info.pop('router_type')
view_def = get_router_view_definition(router_type)
config = router_info.pop('config', {})
extra_inbound_endpoints = view_def.get_inbound_endpoints(config)
extra_outbound_endpoints = view_def.get_outbound_endpoints(config)
batch_id = user_api.api.mdb.batch_start()
# We bypass the usual mechanisms so we can set the key ourselves.
router = user_api.router_store.routers(
router_key, user_account=user_api.user_account_key,
router_type=router_type, name=router_info.pop('name'),
config=config, extra_inbound_endpoints=extra_inbound_endpoints,
extra_outbound_endpoints=extra_outbound_endpoints,
batch=batch_id, **router_info)
router.save()
self.stdout.write('Router %s created\n' % (router.key,))
def setup_conversations(self, user, conversations):
user_api = vumi_api_for_user(user)
for conv_info in conversations:
conv_info = conv_info.copy() # So we can modify it.
self.conversation_info.append({
'account': user.email,
'key': conv_info['key'],
'start': conv_info.pop('start', True), # Don't pass to conv.
})
conversation_key = conv_info.pop('key')
if user_api.get_wrapped_conversation(conversation_key):
self.stderr.write(
'Conversation %s already exists. Skipping.\n' % (
conversation_key,))
continue
conversation_type = conv_info.pop('conversation_type')
view_def = get_conversation_view_definition(conversation_type)
config = conv_info.pop('config', {})
batch_id = user_api.api.mdb.batch_start()
# We bypass the usual mechanisms so we can set the key ourselves.
conv = user_api.conversation_store.conversations(
conversation_key, user_account=user_api.user_account_key,
conversation_type=conversation_type,
name=conv_info.pop('name'), config=config, batch=batch_id,
extra_endpoints=view_def.get_endpoints(config), **conv_info)
conv.save()
self.stdout.write('Conversation %s created\n' % (conv.key,))
def setup_routing(self, user, account_objects):
connectors = {}
for conv in account_objects['conversations']:
connectors[conv['key']] = GoConnector.for_conversation(
conv['conversation_type'], conv['key'])
for tag in account_objects['channels']:
connectors[tag] = GoConnector.for_transport_tag(*(tag.split(':')))
for router in account_objects['routers']:
connectors[router['key'] + ':INBOUND'] = GoConnector.for_router(
router['router_type'], router['key'], GoConnector.INBOUND)
connectors[router['key'] + ':OUTBOUND'] = GoConnector.for_router(
router['router_type'], router['key'], GoConnector.OUTBOUND)
rt = RoutingTable()
for src, src_ep, dst, dst_ep in account_objects['routing_entries']:
rt.add_entry(
str(connectors[src]), src_ep, str(connectors[dst]), dst_ep)
user_account = vumi_api_for_user(user).get_user_account()
user_account.routing_table = rt
user_account.save()
self.stdout.write('Routing table for %s built\n' % (user.email,))
def get_transport_name(self, data):
return data['config']['transport_name']
def mk_filename(self, file_name, suffix):
fn = self.file_name_template % {
'file_name': file_name,
'suffix': suffix,
}
return os.path.join(self.dest_dir, fn)
def create_worker_configs(self, file_path):
workers = self.read_yaml(file_path)
self.create_transport_configs(workers.get('transports', {}))
self.create_router_configs(workers.get('routers', {}))
self.create_application_configs(workers.get('applications', {}))
def create_transport_configs(self, transports):
for transport_name, transport_info in transports.iteritems():
self.transport_names.append(transport_name)
config = transport_info['config']
config.update({'transport_name': transport_name})
self.write_worker_config_file(transport_name, config)
self.write_supervisor_config_file(
transport_name,
transport_info['class'])
def create_router_configs(self, routers):
for router_name, router_info in routers.iteritems():
self.router_names.append(router_name)
worker_name = '%s_router' % (router_name,)
ri_connector_name = '%s_router_ri' % (router_name,)
ro_connector_name = '%s_router_ro' % (router_name,)
config = router_info['config']
config.update({
'ri_connector_name': ri_connector_name,
'ro_connector_name': ro_connector_name,
'worker_name': worker_name,
})
self.write_worker_config_file(worker_name, config)
self.write_supervisor_config_file(
worker_name, router_info['class'])
def create_application_configs(self, applications):
for application_name, application_info in applications.iteritems():
self.application_names.append(application_name)
transport_name = '%s_transport' % (application_name,)
worker_name = '%s_application' % (application_name,)
config = application_info['config']
config.update({
'transport_name': transport_name,
'worker_name': worker_name,
})
self.write_worker_config_file(worker_name, config)
self.write_supervisor_config_file(
worker_name, application_info['class'])
def write_worker_config_file(self, transport_name, config):
fn = self.mk_filename(transport_name, 'yaml')
with self.open_file(fn, 'w') as fp:
fp.write(self.auto_gen_warning)
self.write_yaml(fp, config)
self.stdout.write('Wrote %s.\n' % (fn,))
def write_supervisor_config_file(self, program_name, worker_class,
config=None, enabled=True):
fn = self.mk_filename(program_name, 'conf')
if not enabled:
fn += '.disabled'
config = config or self.mk_filename(program_name, 'yaml')
with self.open_file(fn, 'w') as fp:
section = "program:%s" % (program_name,)
fp.write(self.auto_gen_warning)
cp = ConfigParser()
cp.add_section(section)
cp.set(
section,
"environment",
"DJANGO_SETTINGS_MODULE=go.settings")
cp.set(section, "command", " ".join([
"twistd -n --pidfile=./tmp/pids/%s.pid" % (program_name,),
"start_worker",
"--worker-class=%s" % (worker_class,),
"--config=%s" % (config,),
"--vhost=%s" % self.config.get('vhost', '/develop'),
]))
cp.set(section, "stdout_logfile",
"./logs/%(program_name)s_%(process_num)s.log")
cp.set(section, "stderr_logfile",
"./logs/%(program_name)s_%(process_num)s.log")
cp.write(fp)
self.stdout.write('Wrote %s.\n' % (fn,))
def create_command_dispatcher_config(self, applications, routers):
worker_names = []
worker_names.extend(
'%s_application' % (app_name,) for app_name in applications)
worker_names.extend(
'%s_router' % (router_name,) for router_name in routers)
fn = self.mk_filename('command_dispatcher', 'yaml')
with self.open_file(fn, 'w') as fp:
self.write_yaml(fp, {
'transport_name': 'command_dispatcher',
'worker_names': worker_names,
})
self.stdout.write('Wrote %s.\n' % (fn,))
def create_go_api_worker_config(self):
fn = self.mk_filename('go_api_worker', 'yaml')
with self.open_file(fn, 'w') as fp:
self.write_yaml(fp, {
'worker_name': "go_api_worker",
'redis_manager': self.config['redis_manager'],
'riak_manager': self.config['riak_manager'],
'twisted_endpoint': self.go_api_endpoint,
'web_path': "/api/v1/go/api",
})
self.stdout.write('Wrote %s.\n' % (fn,))
def create_message_store_api_worker_config(self):
fn = self.mk_filename('message_store_api_worker', 'yaml')
with self.open_file(fn, 'w') as fp:
self.write_yaml(fp, {
'worker_name': "message_store_api_worker",
'redis_manager': self.config['redis_manager'],
'riak_manager': self.config['riak_manager'],
'web_path': "/api/v1",
'web_port': self.message_store_api_port,
'health_path': "/health/",
})
self.stdout.write('Wrote %s.\n' % (fn,))
def create_routing_table_dispatcher_config(self, applications, transports):
fn = self.mk_filename('routing_table_dispatcher', 'yaml')
with self.open_file(fn, 'w') as fp:
templ = 'routing_table_dispatcher.yaml.template'
data = self.render_template(templ, {
'transport_names': transports,
'application_names': [
'%s_transport' % (app,) for app in applications],
'conversation_mappings': dict([
(app, '%s_transport' % (app,)) for app in applications]),
'redis_manager': self.dump_yaml_block(
self.config['redis_manager'], 1),
'riak_manager': self.dump_yaml_block(
self.config['riak_manager'], 1),
})
fp.write(self.auto_gen_warning)
fp.write(data)
self.stdout.write('Wrote %s.\n' % (fn,))
def create_billing_dispatcher_config(self):
fn = self.mk_filename('billing_dispatcher', 'yaml')
with self.open_file(fn, 'w') as fp:
templ = 'billing_dispatcher.yaml.template'
data = self.render_template(templ, {
'redis_manager': self.dump_yaml_block(
self.config['redis_manager'], 1),
'riak_manager': self.dump_yaml_block(
self.config['riak_manager'], 1),
})
fp.write(self.auto_gen_warning)
fp.write(data)
self.stdout.write('Wrote %s.\n' % (fn,))
def write_supervisord_conf(self):
fn = self.mk_filename('supervisord', 'conf')
with self.open_file(fn, 'w') as fp:
templ = 'supervisord.conf.template'
data = self.render_template(templ, {
'host': self.supervisord_host,
'port': self.supervisord_port,
'include_files': '*.conf',
})
fp.write(self.auto_gen_warning)
fp.write(data)
self.stdout.write('Wrote %s.\n' % (fn,))
def create_webui_supervisord_conf(self):
program_name = 'webui'
fn = self.mk_filename(program_name, 'conf')
with self.open_file(fn, 'w') as fp:
section = "program:%s" % (program_name,)
fp.write(self.auto_gen_warning)
cp = ConfigParser()
cp.add_section(section)
cp.set(
section, "command", "./go-admin.sh runserver %s --noreload" % (
self.webapp_bind,))
cp.set(section, "stdout_logfile",
"./logs/%(program_name)s_%(process_num)s.log")
cp.set(section, "redirect_stderr", "true")
cp.write(fp)
self.stdout.write('Wrote %s.\n' % (fn,))
def create_billing_api_supervisord_conf(self):
program_name = 'billing_api'
fn = self.mk_filename(program_name, 'conf')
with self.open_file(fn, 'w') as fp:
section = "program:%s" % (program_name,)
fp.write(self.auto_gen_warning)
cp = ConfigParser()
cp.add_section(section)
cp.set(section, "command", "./go-admin.sh runbillingserver")
cp.set(section, "stdout_logfile",
"./logs/%(program_name)s_%(process_num)s.log")
cp.set(section, "redirect_stderr", "true")
cp.write(fp)
self.stdout.write('Wrote %s.\n' % (fn,))
def setup_contact_groups(self, user, contact_groups):
user_api = vumi_api_for_user(user)
for group_info in contact_groups:
self.contact_group_info.append({
'account': user.email,
'key': group_info['key'],
'contacts_csv': group_info['contacts_csv'],
})
name = group_info['name'].decode('utf-8')
account_key = user_api.user_account_key
group = user_api.contact_store.groups(
group_info['key'], name=name, user_account=account_key)
group.save()
self.stdout.write('Group %s created\n' % (group.key,))
def write_startup_script(self):
"""
Generate a script to import contacts and start conversations.
"""
fn = self.mk_filename('startup_env', 'sh')
with self.open_file(fn, 'w') as fp:
templ = 'startup_env.sh.template'
data = self.render_template(templ, {
'contact_groups': self.contact_group_info,
'conversations': self.conversation_info,
'routers': self.router_info,
})
fp.write(self.auto_gen_warning)
fp.write(data)
os.chmod(fn, os.stat(fn).st_mode | stat.S_IEXEC)
self.stdout.write('Wrote %s.\n' % (fn,))
class TestTagpoolApiServer(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
self.tagpool = TagpoolManager(self.redis)
site = Site(TagpoolApiServer(self.tagpool))
self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1')
self.add_cleanup(self.server.loseConnection)
addr = self.server.getHost()
self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port))
yield self.setup_tags()
@inlineCallbacks
def setup_tags(self):
# pool1 has two tags which are free
yield self.tagpool.declare_tags([("pool1", "tag1"), ("pool1", "tag2")])
# pool2 has two tags which are used
yield self.tagpool.declare_tags([("pool2", "tag1"), ("pool2", "tag2")])
yield self.tagpool.acquire_specific_tag(["pool2", "tag1"])
yield self.tagpool.acquire_specific_tag(["pool2", "tag2"])
# pool3 is empty but has metadata
yield self.tagpool.set_metadata("pool3", {"meta": "data"})
def _check_reason(self, result, expected_owner, expected_reason):
owner, reason = result
self.assertEqual(owner, expected_owner)
self.assertEqual(reason.pop('owner'), expected_owner)
self.assertTrue(isinstance(reason.pop('timestamp'), float))
self.assertEqual(reason, expected_reason)
@inlineCallbacks
def test_acquire_tag(self):
result = yield self.proxy.callRemote("acquire_tag", "pool1")
self.assertEqual(result, ["pool1", "tag1"])
self.assertEqual((yield self.tagpool.inuse_tags("pool1")),
[("pool1", "tag1")])
result = yield self.proxy.callRemote("acquire_tag", "pool2")
self.assertEqual(result, None)
@inlineCallbacks
def test_acquire_tag_with_owner_and_reason(self):
result = yield self.proxy.callRemote("acquire_tag", "pool1", "me",
{"foo": "bar"})
self.assertEqual(result, ["pool1", "tag1"])
result = yield self.tagpool.acquired_by(["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_acquire_specific_tag(self):
result = yield self.proxy.callRemote("acquire_specific_tag",
["pool1", "tag1"])
self.assertEqual(result, ["pool1", "tag1"])
self.assertEqual((yield self.tagpool.inuse_tags("pool1")),
[("pool1", "tag1")])
result = yield self.proxy.callRemote("acquire_specific_tag",
["pool2", "tag1"])
self.assertEqual(result, None)
@inlineCallbacks
def test_acquire_specific_tag_with_owner_and_reason(self):
result = yield self.proxy.callRemote("acquire_specific_tag",
["pool1", "tag1"], "me",
{"foo": "bar"})
self.assertEqual(result, ["pool1", "tag1"])
result = yield self.tagpool.acquired_by(["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_release_tag(self):
result = yield self.proxy.callRemote("release_tag", ["pool1", "tag1"])
self.assertEqual(result, None)
result = yield self.proxy.callRemote("release_tag", ["pool2", "tag1"])
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.inuse_tags("pool2")),
[("pool2", "tag2")])
@inlineCallbacks
def test_declare_tags(self):
tags = [("newpool", "tag1"), ("newpool", "tag2")]
result = yield self.proxy.callRemote("declare_tags", tags)
self.assertEqual(result, None)
free_tags = yield self.tagpool.free_tags("newpool")
self.assertEqual(sorted(free_tags), sorted(tags))
@inlineCallbacks
def test_get_metadata(self):
result = yield self.proxy.callRemote("get_metadata", "pool3")
self.assertEqual(result, {"meta": "data"})
result = yield self.proxy.callRemote("get_metadata", "pool1")
self.assertEqual(result, {})
@inlineCallbacks
def test_set_metadata(self):
result = yield self.proxy.callRemote("set_metadata", "newpool",
{"my": "data"})
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.get_metadata("newpool")),
{"my": "data"})
@inlineCallbacks
def test_purge_pool(self):
result = yield self.proxy.callRemote("purge_pool", "pool1")
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.free_tags("pool1")), [])
@inlineCallbacks
def test_purge_pool_with_keys_in_use(self):
d = self.proxy.callRemote("purge_pool", "pool2")
yield d.addErrback(lambda f: log.err(f))
# txJSON-RPC 0.5 adds support for py3 which means
# different error classes are being logged, accept both
errors = self.flushLoggedErrors('xmlrpclib.Fault',
'xmlrpc.client.Fault')
self.assertEqual(len(errors), 1)
server_errors = self.flushLoggedErrors(
'vumi.components.tagpool.TagpoolError')
self.assertEqual(len(server_errors), 1)
@inlineCallbacks
def test_list_pools(self):
result = yield self.proxy.callRemote("list_pools")
self.assertEqual(sorted(result), ["pool1", "pool2", "pool3"])
@inlineCallbacks
def test_free_tags(self):
result = yield self.proxy.callRemote("free_tags", "pool1")
self.assertEqual(sorted(result),
[["pool1", "tag1"], ["pool1", "tag2"]])
result = yield self.proxy.callRemote("free_tags", "pool2")
self.assertEqual(result, [])
result = yield self.proxy.callRemote("free_tags", "pool3")
self.assertEqual(result, [])
@inlineCallbacks
def test_inuse_tags(self):
result = yield self.proxy.callRemote("inuse_tags", "pool1")
self.assertEqual(result, [])
result = yield self.proxy.callRemote("inuse_tags", "pool2")
self.assertEqual(sorted(result),
[["pool2", "tag1"], ["pool2", "tag2"]])
result = yield self.proxy.callRemote("inuse_tags", "pool3")
self.assertEqual(result, [])
@inlineCallbacks
def test_acquired_by(self):
result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"])
self.assertEqual(result, [None, None])
result = yield self.proxy.callRemote("acquired_by", ["pool2", "tag1"])
self._check_reason(result, None, {})
yield self.tagpool.acquire_tag("pool1",
owner="me",
reason={"foo": "bar"})
result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_owned_tags(self):
result = yield self.proxy.callRemote("owned_tags", None)
self.assertEqual(sorted(result),
[[u'pool2', u'tag1'], [u'pool2', u'tag2']])
yield self.tagpool.acquire_tag("pool1",
owner="me",
reason={"foo": "bar"})
result = yield self.proxy.callRemote("owned_tags", "me")
self.assertEqual(result, [["pool1", "tag1"]])
class TestTagpoolApiServer(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
self.tagpool = TagpoolManager(self.redis)
site = Site(TagpoolApiServer(self.tagpool))
self.server = yield reactor.listenTCP(0, site, interface='127.0.0.1')
self.add_cleanup(self.server.loseConnection)
addr = self.server.getHost()
self.proxy = Proxy("http://%s:%d/" % (addr.host, addr.port))
yield self.setup_tags()
@inlineCallbacks
def setup_tags(self):
# pool1 has two tags which are free
yield self.tagpool.declare_tags([
("pool1", "tag1"), ("pool1", "tag2")])
# pool2 has two tags which are used
yield self.tagpool.declare_tags([
("pool2", "tag1"), ("pool2", "tag2")])
yield self.tagpool.acquire_specific_tag(["pool2", "tag1"])
yield self.tagpool.acquire_specific_tag(["pool2", "tag2"])
# pool3 is empty but has metadata
yield self.tagpool.set_metadata("pool3", {"meta": "data"})
def _check_reason(self, result, expected_owner, expected_reason):
owner, reason = result
self.assertEqual(owner, expected_owner)
self.assertEqual(reason.pop('owner'), expected_owner)
self.assertTrue(isinstance(reason.pop('timestamp'), float))
self.assertEqual(reason, expected_reason)
@inlineCallbacks
def test_acquire_tag(self):
result = yield self.proxy.callRemote("acquire_tag", "pool1")
self.assertEqual(result, ["pool1", "tag1"])
self.assertEqual((yield self.tagpool.inuse_tags("pool1")),
[("pool1", "tag1")])
result = yield self.proxy.callRemote("acquire_tag", "pool2")
self.assertEqual(result, None)
@inlineCallbacks
def test_acquire_tag_with_owner_and_reason(self):
result = yield self.proxy.callRemote(
"acquire_tag", "pool1", "me", {"foo": "bar"})
self.assertEqual(result, ["pool1", "tag1"])
result = yield self.tagpool.acquired_by(["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_acquire_specific_tag(self):
result = yield self.proxy.callRemote("acquire_specific_tag",
["pool1", "tag1"])
self.assertEqual(result, ["pool1", "tag1"])
self.assertEqual((yield self.tagpool.inuse_tags("pool1")),
[("pool1", "tag1")])
result = yield self.proxy.callRemote("acquire_specific_tag",
["pool2", "tag1"])
self.assertEqual(result, None)
@inlineCallbacks
def test_acquire_specific_tag_with_owner_and_reason(self):
result = yield self.proxy.callRemote(
"acquire_specific_tag", ["pool1", "tag1"], "me", {"foo": "bar"})
self.assertEqual(result, ["pool1", "tag1"])
result = yield self.tagpool.acquired_by(["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_release_tag(self):
result = yield self.proxy.callRemote("release_tag",
["pool1", "tag1"])
self.assertEqual(result, None)
result = yield self.proxy.callRemote("release_tag",
["pool2", "tag1"])
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.inuse_tags("pool2")),
[("pool2", "tag2")])
@inlineCallbacks
def test_declare_tags(self):
tags = [("newpool", "tag1"), ("newpool", "tag2")]
result = yield self.proxy.callRemote("declare_tags", tags)
self.assertEqual(result, None)
free_tags = yield self.tagpool.free_tags("newpool")
self.assertEqual(sorted(free_tags), sorted(tags))
@inlineCallbacks
def test_get_metadata(self):
result = yield self.proxy.callRemote("get_metadata", "pool3")
self.assertEqual(result, {"meta": "data"})
result = yield self.proxy.callRemote("get_metadata", "pool1")
self.assertEqual(result, {})
@inlineCallbacks
def test_set_metadata(self):
result = yield self.proxy.callRemote("set_metadata", "newpool",
{"my": "data"})
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.get_metadata("newpool")),
{"my": "data"})
@inlineCallbacks
def test_purge_pool(self):
result = yield self.proxy.callRemote("purge_pool", "pool1")
self.assertEqual(result, None)
self.assertEqual((yield self.tagpool.free_tags("pool1")), [])
@inlineCallbacks
def test_purge_pool_with_keys_in_use(self):
d = self.proxy.callRemote("purge_pool", "pool2")
yield d.addErrback(lambda f: log.err(f))
# txJSON-RPC 0.5 adds support for py3 which means
# different error classes are being logged, accept both
errors = self.flushLoggedErrors('xmlrpclib.Fault',
'xmlrpc.client.Fault')
self.assertEqual(len(errors), 1)
server_errors = self.flushLoggedErrors(
'vumi.components.tagpool.TagpoolError')
self.assertEqual(len(server_errors), 1)
@inlineCallbacks
def test_list_pools(self):
result = yield self.proxy.callRemote("list_pools")
self.assertEqual(sorted(result), ["pool1", "pool2", "pool3"])
@inlineCallbacks
def test_free_tags(self):
result = yield self.proxy.callRemote("free_tags", "pool1")
self.assertEqual(
sorted(result), [["pool1", "tag1"], ["pool1", "tag2"]])
result = yield self.proxy.callRemote("free_tags", "pool2")
self.assertEqual(result, [])
result = yield self.proxy.callRemote("free_tags", "pool3")
self.assertEqual(result, [])
@inlineCallbacks
def test_inuse_tags(self):
result = yield self.proxy.callRemote("inuse_tags", "pool1")
self.assertEqual(result, [])
result = yield self.proxy.callRemote("inuse_tags", "pool2")
self.assertEqual(
sorted(result), [["pool2", "tag1"], ["pool2", "tag2"]])
result = yield self.proxy.callRemote("inuse_tags", "pool3")
self.assertEqual(result, [])
@inlineCallbacks
def test_acquired_by(self):
result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"])
self.assertEqual(result, [None, None])
result = yield self.proxy.callRemote("acquired_by", ["pool2", "tag1"])
self._check_reason(result, None, {})
yield self.tagpool.acquire_tag("pool1", owner="me",
reason={"foo": "bar"})
result = yield self.proxy.callRemote("acquired_by", ["pool1", "tag1"])
self._check_reason(result, "me", {"foo": "bar"})
@inlineCallbacks
def test_owned_tags(self):
result = yield self.proxy.callRemote("owned_tags", None)
self.assertEqual(sorted(result),
[[u'pool2', u'tag1'], [u'pool2', u'tag2']])
yield self.tagpool.acquire_tag("pool1", owner="me",
reason={"foo": "bar"})
result = yield self.proxy.callRemote("owned_tags", "me")
self.assertEqual(result, [["pool1", "tag1"]])
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
yield self.redis._purge_all() # Just in case
self.tpm = TagpoolManager(self.redis)
class TestTxTagpoolManager(VumiTestCase):
@inlineCallbacks
def setUp(self):
self.persistence_helper = self.add_helper(PersistenceHelper())
self.redis = yield self.persistence_helper.get_redis_manager()
yield self.redis._purge_all() # Just in case
self.tpm = TagpoolManager(self.redis)
def pool_key_generator(self, pool):
def tkey(x):
return "tagpools:%s:%s" % (pool, x)
return tkey
@inlineCallbacks
def test_declare_tags(self):
tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2")
yield self.tpm.declare_tags([tag1, tag2])
self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag1)
self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag2)
self.assertEqual((yield self.tpm.acquire_tag("poolA")), None)
tag3 = ("poolA", "tag3")
yield self.tpm.declare_tags([tag2, tag3])
self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag3)
@inlineCallbacks
def test_declare_unicode_tag(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag)
@inlineCallbacks
def test_purge_pool(self):
tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2")
yield self.tpm.declare_tags([tag1, tag2])
yield self.tpm.purge_pool('poolA')
self.assertEqual((yield self.tpm.acquire_tag('poolA')), None)
@inlineCallbacks
def test_purge_unicode_pool(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
yield self.tpm.purge_pool(tag[0])
self.assertEqual((yield self.tpm.acquire_tag(tag[0])), None)
@inlineCallbacks
def test_purge_inuse_pool(self):
tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2")
yield self.tpm.declare_tags([tag1, tag2])
self.assertEqual((yield self.tpm.acquire_tag('poolA')), tag1)
try:
yield self.tpm.purge_pool('poolA')
except TagpoolError:
pass
else:
self.fail("Expected TagpoolError to be raised.")
@inlineCallbacks
def test_list_pools(self):
tag1, tag2 = ("poolA", "tag1"), ("poolB", "tag2")
yield self.tpm.declare_tags([tag1, tag2])
self.assertEqual((yield self.tpm.list_pools()),
set(['poolA', 'poolB']))
@inlineCallbacks
def test_list_unicode_pool(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
self.assertEqual((yield self.tpm.list_pools()),
set([tag[0]]))
@inlineCallbacks
def test_acquire_tag(self):
tkey = self.pool_key_generator("poolA")
tag1, tag2 = ("poolA", "tag1"), ("poolA", "tag2")
yield self.tpm.declare_tags([tag1, tag2])
self.assertEqual((yield self.tpm.acquire_tag("poolA")), tag1)
self.assertEqual((yield self.tpm.acquire_tag("poolB")), None)
redis = self.redis
self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)),
["tag2"])
self.assertEqual((yield redis.smembers(tkey("free:set"))),
set(["tag2"]))
self.assertEqual((yield redis.smembers(tkey("inuse:set"))),
set(["tag1"]))
@inlineCallbacks
def test_acquire_unicode_tag(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag)
self.assertEqual((yield self.tpm.acquire_tag(tag[0])), None)
@inlineCallbacks
def test_acquire_specific_tag(self):
tkey = self.pool_key_generator("poolA")
tags = [("poolA", "tag%d" % i) for i in range(10)]
tag5 = tags[5]
yield self.tpm.declare_tags(tags)
self.assertEqual((yield self.tpm.acquire_specific_tag(tag5)), tag5)
self.assertEqual((yield self.tpm.acquire_specific_tag(tag5)), None)
free_local_tags = [t[1] for t in tags]
free_local_tags.remove("tag5")
redis = self.redis
self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)),
free_local_tags)
self.assertEqual((yield redis.smembers(tkey("free:set"))),
set(free_local_tags))
self.assertEqual((yield redis.smembers(tkey("inuse:set"))),
set(["tag5"]))
@inlineCallbacks
def test_acquire_specific_unicode_tag(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
self.assertEqual((yield self.tpm.acquire_specific_tag(tag)), tag)
self.assertEqual((yield self.tpm.acquire_specific_tag(tag)), None)
@inlineCallbacks
def test_release_tag(self):
tkey = self.pool_key_generator("poolA")
tag1, tag2, tag3 = [("poolA", "tag%d" % i) for i in (1, 2, 3)]
yield self.tpm.declare_tags([tag1, tag2, tag3])
yield self.tpm.acquire_tag("poolA")
yield self.tpm.acquire_tag("poolA")
yield self.tpm.release_tag(tag1)
redis = self.redis
self.assertEqual((yield redis.lrange(tkey("free:list"), 0, -1)),
["tag3", "tag1"])
self.assertEqual((yield redis.smembers(tkey("free:set"))),
set(["tag1", "tag3"]))
self.assertEqual((yield redis.smembers(tkey("inuse:set"))),
set(["tag2"]))
@inlineCallbacks
def test_release_unicode_tag(self):
tag = (u"poöl", u"tág")
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_tag(tag[0])
yield self.tpm.release_tag(tag)
self.assertEqual((yield self.tpm.acquire_tag(tag[0])), tag)
@inlineCallbacks
def test_metadata(self):
mkey = self.pool_key_generator("poolA")("metadata")
metadata = {
"transport_type": "sms",
"default_msg_fields": {
"transport_name": "sphex",
"helper_metadata": {
"even_more_nested": "foo",
},
},
}
yield self.tpm.set_metadata("poolA", metadata)
self.assertEqual((yield self.tpm.get_metadata("poolA")), metadata)
tt_json = yield self.redis.hget(mkey, "transport_type")
transport_type = json.loads(tt_json)
self.assertEqual(transport_type, "sms")
short_md = {"foo": "bar"}
yield self.tpm.set_metadata("poolA", short_md)
self.assertEqual((yield self.tpm.get_metadata("poolA")), short_md)
@inlineCallbacks
def test_metadata_for_unicode_pool_name(self):
pool = u"poöl"
metadata = {"foo": "bar"}
yield self.tpm.set_metadata(pool, metadata)
self.assertEqual((yield self.tpm.get_metadata(pool)),
metadata)
@inlineCallbacks
def test_unicode_metadata(self):
metadata = {u"föo": u"báz"}
yield self.tpm.set_metadata("pool", metadata)
self.assertEqual((yield self.tpm.get_metadata("pool")),
metadata)
def _check_reason(self, expected_owner, owner, reason, expected_data):
self.assertEqual(expected_owner, owner)
self.assertEqual(expected_owner, reason.pop('owner'))
timestamp = reason.pop('timestamp')
self.assertTrue(isinstance(timestamp, float))
self.assertEqual(reason, expected_data)
@inlineCallbacks
def test_acquired_by(self):
tag = ["pool", "tag"]
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_tag(tag[0], "me", {"foo": "bar"})
owner, reason = yield self.tpm.acquired_by(tag)
self._check_reason("me", owner, reason, {"foo": "bar"})
@inlineCallbacks
def test_acquired_by_undeclared_tags(self):
tag = ["pool", "tag"]
owner, reason = yield self.tpm.acquired_by(tag)
self.assertEqual(owner, None)
self.assertEqual(reason, None)
@inlineCallbacks
def test_acquired_by_no_owner(self):
tag = ["pool", "tag"]
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_tag(tag[0])
owner, reason = yield self.tpm.acquired_by(tag)
self._check_reason(None, owner, reason, {})
@inlineCallbacks
def test_acquired_by_unicode_owner(self):
tag = ["pool", "tag"]
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_tag(tag[0], u"mé")
owner, reason = yield self.tpm.acquired_by(tag)
self._check_reason(u"mé", owner, reason, {})
@inlineCallbacks
def test_acquired_by_from_unicode_tag(self):
tag = [u"poöl", u"tág"]
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_tag(tag[0], "me")
owner, reason = yield self.tpm.acquired_by(tag)
self._check_reason(u"me", owner, reason, {})
@inlineCallbacks
def test_acquired_by_after_using_specific_tag(self):
tag = ["pool", "tag"]
yield self.tpm.declare_tags([tag])
yield self.tpm.acquire_specific_tag(tag, "me", {"foo": "bar"})
owner, reason = yield self.tpm.acquired_by(tag)
self._check_reason("me", owner, reason, {"foo": "bar"})
@inlineCallbacks
def test_owned_tags(self):
tags = [["pool1", "tag1"], ["pool2", "tag2"]]
yield self.tpm.declare_tags(tags)
yield self.tpm.acquire_tag(tags[0][0], owner="me")
my_tags = yield self.tpm.owned_tags("me")
self.assertEqual(my_tags, [tags[0]])
@inlineCallbacks
def test_owned_tags_no_owner(self):
tags = [["pool1", "tag1"], ["pool2", "tag2"]]
yield self.tpm.declare_tags(tags)
yield self.tpm.acquire_tag(tags[0][0])
my_tags = yield self.tpm.owned_tags(None)
self.assertEqual(my_tags, [tags[0]])
@inlineCallbacks
def test_owned_tags_unicode_owner(self):
tags = [["pool1", "tag1"], ["pool2", "tag2"]]
yield self.tpm.declare_tags(tags)
yield self.tpm.acquire_tag(tags[0][0], owner=u"mé")
my_tags = yield self.tpm.owned_tags(u"mé")
self.assertEqual(my_tags, [tags[0]])
@inlineCallbacks
def test_owned_tags_unicode_tags(self):
tags = [[u"poöl1", u"tág1"], [u"poöl2", u"tág2"]]
yield self.tpm.declare_tags(tags)
yield self.tpm.acquire_tag(tags[0][0], owner="me")
my_tags = yield self.tpm.owned_tags(u"me")
self.assertEqual(my_tags, [tags[0]])
