class MaxServer(SupervisorHelpers, NginxHelpers, CommonSteps, PyramidServer):
def __init__(self, config, *args, **kwargs):
self.config = config
self.process_prefix = 'max_'
self._client = None
self._instances = {}
self.instance = None
self.remote = RemoteConnection(self.config.ssh_user, self.config.server)
self.buildout = RemoteBuildoutHelper(self.remote, self.config.python_interpreter, self)
if not self.remote.file_exists(self.config.instances_root):
self.remote.mkdir(self.config.instances_root)
def get_client(self, instance_name, username='', password=''):
instance_info = self.get_instance(instance_name)
client = MaxClient(instance_info['server']['dns'])
if username and password:
client.login(username=username, password=password)
return client
def get_running_version(self, instance_name):
instance_info = self.get_instance(instance_name)
return requests.get('{}/info'.format(instance_info['server']['dns'])).json().get('version', '???')
def get_expected_version(self, instance_name):
versions = self.remote.get_file('{}/versions.cfg'.format(self.buildout.folder))
return re.search(r'\smax\s=\s(.*?)\s', versions, re.MULTILINE).groups()[0]
def get_instance(self, instance_name):
if instance_name not in self._instances:
max_ini = self.buildout.config_files.get(instance_name, {}).get('max.ini', '')
if not max_ini:
return {}
maxconfig = parse_ini_from(max_ini)
port_index = int(maxconfig['server:main']['port']) - MAX_BASE_PORT
instance = OrderedDict()
instance['name'] = instance_name
instance['port_index'] = port_index
instance['mongo_database'] = maxconfig['app:main']['mongodb.db_name']
instance['server'] = {
'direct': 'http://{}:{}'.format(self.config.server, maxconfig['server:main']['port']),
'dns': maxconfig['app:main']['max.server']
}
instance['oauth'] = maxconfig['app:main']['max.oauth_server']
self._instances[instance_name] = instance
return self._instances[instance_name]
def test_activity(self, instance_name, ldap_branch, restricted_user_password):
progress_log('Testing UTalk activity notifications')
# Get a maxclient for this instance
padded_log("Getting instance information")
instance_info = self.get_instance(instance_name)
restricted_password = restricted_user_password
client = self.get_client(instance_name, username='******', password=restricted_password)
padded_log("Setting up test clients")
test_users = [
('ulearn.testuser1', 'UTestuser1'),
('ulearn.testuser2', 'UTestuser2')
]
utalk_clients = []
max_clients = []
# Syncronization primitives
wait_for_others = AsyncResult()
counter = ReadyCounter(wait_for_others)
for user, password in test_users:
max_clients.append(self.get_client(
instance_name,
username=user,
password=password)
)
# Create websocket clients
utalk_clients.append(getUtalkClient(
instance_info['server']['dns'],
instance_name,
user,
password,
quiet=False
))
counter.add()
# Create users
padded_log("Creating users and conversations")
client.people['ulearn.testuser1'].post()
client.people['ulearn.testuser2'].post()
# Admin creates context with notifications enabled and subscribes users to it
context = client.contexts.post(url='http://testcontext', displayName='Test Context', notifications=True)
client.people['ulearn.testuser1'].subscriptions.post(object_url='http://testcontext')
client.people['ulearn.testuser2'].subscriptions.post(object_url='http://testcontext')
def post_activity():
max_clients[0].people['ulearn.testuser1'].activities.post(
object_content='Hola',
contexts=[{"url": "http://testcontext", "objectType": "context"}]
)
# Prepare test messages for clients
# First argument are messages to send (conversation, message)
# Second argument are messages to expect (conversation, sender, message)
# Third argument is a syncronization event to wait for all clients to be listening
# Fourth argument is a method to trigger when client ready
arguments1 = [
[
],
[
('test', 'test')
],
counter,
post_activity
]
arguments2 = [
[
],
[
('test', 'test')
],
counter,
None
]
padded_log("Starting websockets and waiting for messages . . .")
greenlets = [
gevent.spawn(utalk_clients[0].test, *arguments1),
gevent.spawn(utalk_clients[1].test, *arguments2)
]
gevent.joinall(greenlets, timeout=20, raise_error=True)
success = None not in [g.value for g in greenlets]
if success:
padded_success('Websocket test passed')
else:
padded_error('Websocket test failed, Timed out')
# Steps used in commands. Some of them defined in gummanager.libs.mixins
def configure_instance(self):
customizations = {
'mongodb-config': {
'replica_set': self.config.mongodb.replica_set,
'cluster_hosts': self.config.mongodb.cluster
},
'max-config': {
'name': self.instance.name,
},
'ports': {
'port_index': '{:0>2}'.format(self.instance.index),
},
'rabbitmq-config': {
'username': self.config.rabbitmq.username,
'password': self.config.rabbitmq.password
},
'hosts': {
'max': self.config.server_dns,
'oauth': self.config.oauth.server_dns,
'rabbitmq': self.config.rabbitmq.server
}
}
self.buildout.configure_file('customizeme.cfg', customizations),
return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))
def set_mongodb_indexes(self):
new_instance_folder = '{}/{}'.format(
self.config.instances_root,
self.instance.name
)
code, stdout = self.remote.execute('cd {0} && ./bin/max.mongoindexes'.format(new_instance_folder))
added = 'Creating' in stdout or 'already exists' in stdout
if added:
return success_log("Succesfully added indexes")
else:
return error_log("Error on adding indexes")
def configure_max_security_settings(self):
new_instance_folder = '{}/{}'.format(
self.config.instances_root,
self.instance.name
)
self.buildout.folder = new_instance_folder
self.remote.execute('cd {} && ./bin/max.security reset'.format(new_instance_folder))
code, stdout = self.remote.execute('cd {} && ./bin/max.security add {}'.format(new_instance_folder, self.config.authorized_user))
added = 'restart max process' in stdout
# Force read the new configuration files
if added:
return success_log("Succesfully configured security settings")
else:
return error_log("Error configuring security settings")
def create_max_nginx_entry(self):
nginx_params = {
'instance_name': self.instance.name,
'server': self.config.server,
'server_dns': self.config.server_dns,
'bigmax_port': BIGMAX_BASE_PORT,
'max_port': int(self.instance.index) + MAX_BASE_PORT,
'nginx_root_folder': self.config.nginx.root,
'max_root_folder': self.buildout.folder
}
nginxentry = MAX_NGINX_ENTRY.format(**nginx_params)
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/max-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
nginx_remote.put_file(nginx_file_location, nginxentry)
return success_log("Succesfully created {}".format(nginx_file_location))
def commit_local_changes(self):
self.buildout.commit_to_local_branch(
self.config.local_git_branch,
files=[
'customizeme.cfg',
'mongoauth.cfg'
])
return success_log("Succesfully commited local changes")
def add_instance_to_bigmax(self):
instances_file = ''
if self.remote.file_exists(self.config.bigmax_instances_list):
instances_file = self.remote.get_file(self.config.bigmax_instances_list, do_raise=True)
if '[{}]'.format(self.instance.name) not in instances_file:
linebreak = '\n' if instances_file else ''
instances_file += linebreak + BIGMAX_INSTANCE_ENTRY.format(**{
"server_dns": self.config.server_dns,
"instance_name": self.instance.name,
})
self.remote.put_file(self.config.bigmax_instances_list, instances_file, do_raise=True)
else:
return success_log("Instance {} already in bigmax instance list".format(self.instance.name))
return success_log("Succesfully added {} to bigmax instance list".format(self.instance.name))
def update_buildout(self):
result = self.buildout.upgrade(self.config.maxserver_buildout_branch, self.config.local_git_branch)
return success(result, "Succesfully commited local changes")
def reload_instance(self):
self.restart(self.instance.name)
sleep(1)
status = self.get_status(self.instance.name)
if status['status'] == 'running':
return success_log("Succesfully restarted max {}".format(self.instance.name))
else:
return error_log('Max instance {} is not running'.format(self.instance.name))
def check_version(self):
running_version = self.get_running_version(self.instance.name)
expected_version = self.get_expected_version(self.instance.name)
if running_version == expected_version:
return success_log("Max {} is running on {}".format(self.instance.name, running_version))
else:
return success_log("Max {} is running on {}, but {} was expected".format(self.instance.name, running_version, expected_version))
# Commands
@command
def new_instance(self, instance_name, port_index, oauth_instance=None, logecho=None, rabbitmq_url=None):
self.buildout.cfgfile = self.config.max.cfg_file
self.buildout.logecho = logecho
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
self.set_instance(
name=instance_name,
index=port_index,
oauth=oauth_instance if oauth_instance is not None else instance_name,
)
yield step_log('Cloning buildout')
yield self.clone_buildout()
yield step_log('Bootstraping buildout')
yield self.bootstrap_buildout()
yield step_log('Configuring customizeme.cfg')
yield self.configure_instance()
yield step_log('Configuring mongoauth.cfg')
yield self.configure_mongoauth()
yield step_log('Executing buildout')
yield self.execute_buildout()
yield step_log('Adding indexes to mongodb')
yield self.set_mongodb_indexes()
yield step_log('Configuring default permissions settings')
yield self.configure_max_security_settings()
yield step_log('Creating nginx entry for max')
yield self.create_max_nginx_entry()
yield step_log('Commiting to local branch')
yield self.commit_local_changes()
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
yield step_log('Adding instance to supervisor config')
yield self.configure_supervisor()
@command
def upgrade(self, instance_name, logecho=None):
self.buildout.cfgfile = self.config.max.cfg_file
self.buildout.logecho = logecho
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
self.set_instance(
name=instance_name,
)
yield step_log('Updating buildout')
yield self.update_buildout()
yield step_log('Executing buildout')
yield self.execute_buildout(update=True)
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
# yield step_log('Reloading max')
yield self.reload_instance()
yield step_log('Checking running version')
yield self.check_version()
class OauthServer(SupervisorHelpers, NginxHelpers, CommonSteps, TokenHelper, PyramidServer):
def __init__(self, config, *args, **kwargs):
self.config = config
self._instances = {}
self.process_prefix = 'osiris_'
self.remote = RemoteConnection(self.config.ssh_user, self.config.server)
self.buildout = RemoteBuildoutHelper(self.remote, self.config.python_interpreter, self)
if not self.remote.file_exists(self.config.instances_root):
self.remote.mkdir(self.config.instances_root)
def update_buildout(self):
result = self.buildout.upgrade(self.config.maxserver_buildout_branch, self.config.local_git_branch)
return success(result, "Succesfully commited local changes")
def reload_instance(self):
self.restart(self.instance.name)
sleep(1)
status = self.get_status(self.instance.name)
if status['status'] == 'running':
return success_log("Succesfully restarted oauth {}".format(self.instance.name))
else:
return error_log('Oauth instance {} is not running'.format(self.instance.name))
def get_instance(self, instance_name):
if instance_name not in self._instances:
osiris_ini = self.buildout.config_files[instance_name].get('osiris.ini', '')
if not osiris_ini:
return {}
osiris = parse_ini_from(osiris_ini)
ldap_ini = self.buildout.config_files[instance_name].get('ldap.ini', '')
if not ldap_ini:
return {}
ldap = parse_ini_from(ldap_ini)
port_index = int(osiris['server:main']['port']) - OSIRIS_BASE_PORT
instance = OrderedDict()
instance['name'] = instance_name
instance['port_index'] = port_index
instance['mongo_database'] = osiris['app:main']['osiris.store.db']
instance['server'] = {
'direct': 'http://{}:{}'.format(self.config.server, osiris['server:main']['port']),
'dns': 'https://{}/{}'.format(self.config.server_dns, instance_name)
}
instance['ldap'] = {
'server': ldap['ldap']['server'],
'basedn': ldap['ldap']['userbasedn'],
'branch': re.match(r"ou=(.*?),", ldap['ldap']['userbasedn']).groups()[0]
}
self._instances[instance_name] = instance
return self._instances[instance_name]
def test(self, instance_name, username, password):
instance = self.get_instance(instance_name)
try:
yield step_log('Testing oauth server @ {}'.format(instance['server']['dns']))
yield message_log('Checking server health')
try:
status = requests.get(instance['server']['dns'], verify=True).status_code
except requests.exceptions.SSLError:
yield error_log('SSL certificate verification failed')
yield message_log('Continuing test without certificate check')
try:
status = requests.get(instance['server']['dns'], verify=False).status_code
except requests.ConnectionError:
yield raising_error_log('Connection error, check nginx is running, and dns resolves as expected.')
except:
yield raising_error_log('Unknown error trying to access oauth server. Check params and try again')
else:
if status == 500:
yield raising_error_log('Error on oauth server, Possible causes:\n - ldap configuration error (bad server url?)\n - Mongodb configuration error (bad replicaset name or hosts list?)\nCheck osiris log for more information.')
elif status == 502:
yield raising_error_log('Server not respoding at {}. Check that:\n - osiris process is running\n - nginx upstream definition is pointing to the right host:port.'.format(instance['server']['dns']))
elif status == 504:
yield raising_error_log('Gateway timeout. Probably oauth server is giving timeout trying to contact ldap server')
elif status == 404:
yield raising_error_log('There\'s no oauth server at {}. Chech there\'s an nginx entry for this server.'.format(instance['server']['dns']))
elif status != 200:
yield raising_error_log('Server {} responded with {} code. Check osiris logs.'.format(instance['server']['dns'], status))
yield message_log('Retrieving token for "{}"'.format(username))
token = self.get_token(instance['server']['dns'], username, password)
succeeded_retrieve_token = token is not None
if not succeeded_retrieve_token:
yield raising_error_log('Error retreiving token. Check username/password and try again')
yield message_log('Checking retreived token')
succeeded_check_token = self.check_token(instance['server']['dns'], username, token)
if not succeeded_check_token:
yield raising_error_log('Error retreiving token')
if succeeded_check_token and succeeded_retrieve_token:
yield success_log('Oauth server check passed')
else:
yield raising_error_log('Oauth server check failed')
except StepError as error:
yield error_log(error.message)
# Steps used in commands. Some of them defined in gummanager.libs.mixins
def configure_instance(self):
customizations = {
'mongodb-config': {
'replica_set': self.config.mongodb.replica_set,
'cluster_hosts': self.config.mongodb.cluster
},
'osiris-config': {
'name': self.instance.name,
},
'ports': {
'port_index': '{:0>2}'.format(self.instance.index),
},
}
self.buildout.configure_file('customizeme.cfg', customizations),
return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))
def configure_ldap(self):
"""
Configure the right settings for ldap based on if :
branches option enabled or disabled
"""
if self.config.ldap.branches.enabled:
effective_admin_dn = 'cn={admin_cn},ou={branch},{base_dn}'.format(branch=self.instance.ldap, **self.config.ldap.branches)
effective_admin_password = self.config.ldap.branches.admin_password
effective_users_base_dn = 'ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
effective_groups_base_dn = 'ou=groups,ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
else:
effective_admin_dn = self.config.ldap.admin_dn
effective_admin_password = self.config.ldap.admin_password
effective_users_base_dn = self.config.ldap.users_base_dn
effective_groups_base_dn = self.config.ldap.group_base_dn
ldapini = configure_ini(
string=LDAP_INI,
params={
'ldap': {
'server': self.config.ldap.server,
'password': effective_admin_password,
'userbind': effective_admin_dn,
'userbasedn': effective_users_base_dn,
'groupbasedn': effective_groups_base_dn
}
}
)
ldap_ini_location = "{}/config/ldap.ini".format(self.buildout.folder)
self.remote.put_file(ldap_ini_location, ldapini)
return success_log('Succesfully configured {}'.format(ldap_ini_location))
def create_oauth_nginx_entry(self):
global_allowed_ips = self.config.oauth.allowed_ips
instance_allowed_ips = self.get_instance_allowed_ips()
allowed_ips = instance_allowed_ips + global_allowed_ips
nginx_params = {
'instance_name': self.instance.name,
'server': self.config.oauth.server,
'server_dns': self.config.oauth.server_dns,
'osiris_port': int(self.instance.index) + OSIRIS_BASE_PORT,
'buildout_folder': self.config.nginx.root,
'allowed_ips': '\n '.join(['allow {};'.format(ip) for ip in allowed_ips])
}
nginxentry = OSIRIS_NGINX_ENTRY.format(**nginx_params)
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
nginx_remote.put_file(nginx_file_location, nginxentry)
return success_log("Succesfully created {}".format(nginx_file_location))
def backup_nginx_configuration(self):
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
backup_content = nginx_remote.get_file(nginx_file_location)
nginx_remote.put_file(backup_file_location, backup_content)
return success_log("Succesfully backed up to {}".format(backup_file_location))
def recover_nginx_configuration(self):
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
backup_content = nginx_remote.get_file(backup_file_location)
nginx_remote.put_file(nginx_file_location, backup_content)
return success_log("Succesfully recovered backup from".format(backup_file_location))
def commit_local_changes(self, message=None):
self.buildout.commit_to_local_branch(
self.config.local_git_branch,
files=[
'customizeme.cfg',
'mongoauth.cfg'
],
message=message)
return success_log("Succesfully commited local changes")
def get_instance_allowed_ips(self):
"""
Get the ips that are currently configured on [osiris-config] allowed_ips.
If there's only one ip configured, the value will be a string, so we adapt it.
"""
configured_ips = self.buildout.read_configuration_file('customizeme.cfg')['osiris-config'].get('allowed_ips', [])
configured_ips = [a.strip() for a in configured_ips.split(' ') if a.strip()] if isinstance(configured_ips, str) else configured_ips
return configured_ips
def add_new_bypass_allowed_ip(self, ip):
configured_ips = self.get_instance_allowed_ips()
allowed_ips = list(set(configured_ips + [ip]))
if not set(allowed_ips).symmetric_difference(set(configured_ips)):
return message_log('No changes to allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
customizations = {
'osiris-config': {
'allowed_ips': allowed_ips,
},
}
self.buildout.configure_file('customizeme.cfg', customizations)
return success_log('Succesfully updated allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
def remove_bypass_allowed_ip(self, ip):
configured_ips = self.get_instance_allowed_ips()
allowed_ips = list(set(configured_ips) - set([ip]))
if not set(allowed_ips).symmetric_difference(set(configured_ips)):
return message_log('No changes to allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
customizations = {
'osiris-config': {
'allowed_ips': allowed_ips,
}
}
self.buildout.configure_file('customizeme.cfg', customizations)
return success_log('Succesfully updated allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
# COMMANDS
@command
def new_instance(self, instance_name, port_index, ldap_branch=None, logecho=None):
self.buildout.cfgfile = self.config.oauth.cfg_file
self.buildout.logecho = logecho
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
self.set_instance(
name=instance_name,
index=port_index,
ldap=ldap_branch if ldap_branch is not None else instance_name
)
yield step_log('Cloning buildout')
yield self.clone_buildout()
yield step_log('Bootstraping buildout')
yield self.bootstrap_buildout()
yield step_log('Configuring customizeme.cfg')
yield self.configure_instance()
yield step_log('Configuring mongoauth.cfg')
yield self.configure_mongoauth()
yield step_log('Configuring ldap.ini')
yield self.configure_ldap()
yield step_log('Creating nginx entry for oauth')
yield self.create_oauth_nginx_entry()
yield step_log('Executing buildout')
yield self.execute_buildout()
yield step_log('Commiting to local branch')
yield self.commit_local_changes()
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
yield step_log('Adding instance to supervisor config')
yield self.configure_supervisor()
@command
def upgrade(self, instance_name, logecho=None):
self.buildout.cfgfile = self.config.oauth.cfg_file
self.buildout.logecho = logecho
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
self.set_instance(
name=instance_name,
)
yield step_log('Updating buildout')
yield self.update_buildout()
yield step_log('Executing buildout')
yield self.execute_buildout(update=True)
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
# yield step_log('Reloading oauth')
yield self.reload_instance()
@command
def reconfigure_nginx(self, instance_name):
self.buildout.cfgfile = self.config.oauth.cfg_file
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
instance = self.get_instance(instance_name)
self.set_instance(
name=instance_name,
index=instance['port_index']
)
yield step_log('Backing up current configuration')
yield self.backup_nginx_configuration()
yield step_log('Creating nginx entry for oauth')
yield self.create_oauth_nginx_entry()
yield step_log('Testing new nginx configuration')
status = self.test_nginx()
if status[0] == 0:
self.recover_nginx_configuration()
yield status
yield step_log('Reloading nginx')
yield self.reload_nginx()
@command
def add_allowed_ip(self, instance_name, new_ip):
instance = self.get_instance(instance_name)
self.set_instance(
name=instance_name,
index=instance['port_index']
)
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
yield step_log('Backing up current configuration')
yield self.backup_nginx_configuration()
yield step_log('Adding new allowed ip')
yield self.add_new_bypass_allowed_ip(new_ip)
yield step_log('Creating nginx entry for oauth')
yield self.create_oauth_nginx_entry()
yield step_log('Testing new nginx configuration')
status = self.test_nginx()
if status[0] == 0:
yield self.recover_nginx_configuration()
yield status
yield step_log('Commiting to local branch')
yield self.commit_local_changes(message='Added allowed ip')
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
yield step_log('Reloading nginx')
yield self.reload_nginx()
@command
def remove_allowed_ip(self, instance_name, existing_ip):
instance = self.get_instance(instance_name)
self.set_instance(
name=instance_name,
index=instance['port_index']
)
self.buildout.folder = '{}/{}'.format(
self.config.instances_root,
instance_name
)
yield step_log('Backing up current configuration')
yield self.backup_nginx_configuration()
yield step_log('Adding new allowed ip')
yield self.remove_bypass_allowed_ip(existing_ip)
yield step_log('Creating nginx entry for oauth')
yield self.create_oauth_nginx_entry()
yield step_log('Testing new nginx configuration')
status = self.test_nginx()
if status[0] == 0:
yield self.recover_nginx_configuration()
yield status
yield step_log('Commiting to local branch')
yield self.commit_local_changes(message='Removed allowed ip')
yield step_log('Changing permissions')
yield self.set_filesystem_permissions()
yield step_log('Reloading nginx')
yield self.reload_nginx()