def check_version(self):
running_version = self.get_running_version(self.instance.name)
expected_version = self.get_expected_version(self.instance.name)
if running_version == expected_version:
return success_log("Max {} is running on {}".format(self.instance.name, running_version))
else:
return success_log("Max {} is running on {}, but {} was expected".format(self.instance.name, running_version, expected_version))
def add_entry(self, **configuration):
instances_file = ''
if self.remote.file_exists(self.config.maxbunny.instances_list):
instances_file = self.remote.get_file(self.config.maxbunny.instances_list, do_raise=True)
if '[{name}]'.format(**configuration) not in instances_file:
linebreak = '\n' if instances_file else ''
instances_file += linebreak + MAXBUNNY_INSTANCE_ENTRY.format(**configuration)
self.remote.put_file(self.config.maxbunny.instances_list, instances_file, do_raise=True)
else:
return success_log("Instance {name} already in maxbunny instance list".format(**configuration))
return success_log("Succesfully added {name} to maxbunny instance list".format(**configuration))
def add_instance_to_bigmax(self):
instances_file = ''
if self.remote.file_exists(self.config.bigmax_instances_list):
instances_file = self.remote.get_file(self.config.bigmax_instances_list, do_raise=True)
if '[{}]'.format(self.instance.name) not in instances_file:
linebreak = '\n' if instances_file else ''
instances_file += linebreak + BIGMAX_INSTANCE_ENTRY.format(**{
"server_dns": self.config.server_dns,
"instance_name": self.instance.name,
})
self.remote.put_file(self.config.bigmax_instances_list, instances_file, do_raise=True)
else:
return success_log("Instance {} already in bigmax instance list".format(self.instance.name))
return success_log("Succesfully added {} to bigmax instance list".format(self.instance.name))
def configure_ldap(self):
"""
Configure the right settings for ldap based on if :
branches option enabled or disabled
"""
if self.config.ldap.branches.enabled:
effective_admin_dn = 'cn={admin_cn},ou={branch},{base_dn}'.format(branch=self.instance.ldap, **self.config.ldap.branches)
effective_admin_password = self.config.ldap.branches.admin_password
effective_users_base_dn = 'ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
effective_groups_base_dn = 'ou=groups,ou={},{}'.format(self.instance.ldap, self.config.ldap.branches.base_dn)
else:
effective_admin_dn = self.config.ldap.admin_dn
effective_admin_password = self.config.ldap.admin_password
effective_users_base_dn = self.config.ldap.users_base_dn
effective_groups_base_dn = self.config.ldap.group_base_dn
ldapini = configure_ini(
string=LDAP_INI,
params={
'ldap': {
'server': self.config.ldap.server,
'password': effective_admin_password,
'userbind': effective_admin_dn,
'userbasedn': effective_users_base_dn,
'groupbasedn': effective_groups_base_dn
}
}
)
ldap_ini_location = "{}/config/ldap.ini".format(self.buildout.folder)
self.remote.put_file(ldap_ini_location, ldapini)
return success_log('Succesfully configured {}'.format(ldap_ini_location))
def add_branch(self, branch, branch_admin_password):
if not self.config.branches.enabled:
yield error_log('Branches are not enabled on this LDAP')
if self.config.readonly:
yield error_log('This LDAP is configured as read-only')
self.connect(auth=False)
self.authenticate(
username=self.config.admin_dn,
password=self.config.admin_password
)
self.add_ou_by_dn("ou={branch},{branches.base_dn}".format(branch=branch, **self.config))
self.set_branch(branch)
branch_restricted_user_dn = "cn={branches.restricted_cn},ou={branch},{branches.base_dn}".format(branch=branch, **self.config)
branch_admin_user_dn = "cn={branches.admin_cn},ou={branch},{branches.base_dn}".format(branch=branch, **self.config)
self.set_branch(branch)
self.add_ldap_user_by_dn(branch_admin_user_dn, 'LDAP Access User', self.config.branches.admin_password)
self.add_ldap_user_by_dn(branch_restricted_user_dn, 'Restricted User', branch_admin_password)
self.add_group_by_dn('cn=Managers,ou={branch},{branches.base_dn}'.format(branch=branch, **self.config), user_dns=[branch_admin_user_dn])
self.add_ou_by_dn('ou=groups,ou={branch},{branches.base_dn}'.format(branch=branch, **self.config))
self.add_ou_by_dn('ou=users,ou={branch},{branches.base_dn}'.format(branch=branch, **self.config))
# Add plain users
for user in self.config.branches.base_users:
self.add_ldap_user(user.username, user.username, user.password)
self.disconnect()
yield success_log("Branch {} successfully created".format(branch))
def setup_max(self, max_name, oauth_name, ldap_branch):
"""
"""
username = "******"
password = "******".format(ldap_branch)
oauth_server = "https://oauth.upcnet.es/{}".format(oauth_name)
user_token = self.get_token(oauth_server, username, password)
if user_token is None:
return error_log('Error on getting token for user "{}" on {}'.format(username, oauth_server))
params = {
"form.widgets.oauth_server": oauth_server,
"form.widgets.oauth_grant_type": "password",
"form.widgets.max_server": "https://max.upcnet.es/{}".format(max_name),
"form.widgets.max_server_alias": "https://ulearn.upcnet.es/{}".format(self.plonesite),
"form.widgets.max_app_username": "******",
"form.widgets.max_app_token": "",
"form.widgets.max_restricted_username": username,
"form.widgets.max_restricted_token": user_token,
"form.buttons.save": "Save",
}
req = requests.post("{}/@@maxui-settings".format(self.site_url), data=params, auth=self.auth)
if req.status_code not in [302, 200, 204, 201]:
return error_log("Error on configuring max settings".format(self.site_url))
else:
return success_log("Successfully configured max settings".format(self.site_url))
def configure_instance(self):
customizations = {
'mongodb-config': {
'replica_set': self.config.mongodb.replica_set,
'cluster_hosts': self.config.mongodb.cluster
},
'max-config': {
'name': self.instance.name,
},
'ports': {
'port_index': '{:0>2}'.format(self.instance.index),
},
'rabbitmq-config': {
'username': self.config.rabbitmq.username,
'password': self.config.rabbitmq.password
},
'hosts': {
'max': self.config.server_dns,
'oauth': self.config.oauth.server_dns,
'rabbitmq': self.config.rabbitmq.server
}
}
self.buildout.configure_file('customizeme.cfg', customizations),
return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))
def rebuild_catalog(self):
recatalog_url = "{}/portal_catalog?manage_catalogRebuild:method=+Clear+and+Rebuild+".format(self.site_url)
resp = requests.get(recatalog_url, auth=self.auth)
if resp.status_code not in [302, 200, 204, 201] or "Catalog Rebuilt" not in resp.content:
return error_log("Error on rebuilding catalog".format(self.site_url))
else:
return success_log("Successfully rebuild site catalog".format(self.site_url))
def setup_homepage(self):
setup_view_url = "{}/setuphomepage".format(self.site_url)
req = requests.get(setup_view_url, auth=self.auth)
if req.status_code not in [302, 200, 204, 201]:
return error_log("Error on hompepage setup".format(self.site_url))
else:
return success_log("Successfully configured homepage".format(self.site_url))
def add_users(self, branch, usersfile):
if self.config.readonly:
yield error_log('This LDAP is configured as read-only')
self.set_branch(branch)
self.connect(auth=False)
self.authenticate(
username=self.effective_admin_dn,
password=self.effective_admin_password)
try:
users = read_users_file(usersfile, required_fields=['username', 'fullname', 'password'])
except Exception as exc:
error_message = 'Error parsing users file {}: {{}}'.format(usersfile)
yield raising_error_log(error_message.format(exc.message))
try:
self.check_users(users)
except Exception as exc:
yield raising_error_log(exc.message)
yield step_log('Creating {} users '.format(len(users)))
for count, user in enumerate(users, start=1):
if not user:
yield error_log('Error parsing user at line #{}'.format(count))
continue
try:
self.add_ldap_user(**user)
yield success_log('User {} created'.format(user['username']))
except ldap.ALREADY_EXISTS:
yield error_log('User {} already exists'.format(user['username']))
except Exception as exc:
yield error_log('Error creating user {}: {}'.format(user['username']), exc.__repr__())
self.disconnect()
def delete_user(self, branch, username):
if self.config.readonly:
yield error_log('This LDAP is configured as read-only')
self.set_branch(branch)
self.connect()
self.del_user(username)
self.disconnect()
yield success_log("User {} deleted from branch".format(username))
def commit_local_changes(self):
self.buildout.commit_to_local_branch(
self.config.local_git_branch,
files=[
'customizeme.cfg',
'mongoauth.cfg'
])
return success_log("Succesfully commited local changes")
def recover_nginx_configuration(self):
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
backup_file_location = "{}/config/osiris-instances/{}.conf.backup".format(self.config.nginx.root, self.instance.name)
backup_content = nginx_remote.get_file(backup_file_location)
nginx_remote.put_file(nginx_file_location, backup_content)
return success_log("Succesfully recovered backup from".format(backup_file_location))
def reload_instance(self):
self.restart(self.instance.name)
sleep(1)
status = self.get_status(self.instance.name)
if status['status'] == 'running':
return success_log("Succesfully restarted max {}".format(self.instance.name))
else:
return error_log('Max instance {} is not running'.format(self.instance.name))
def configure_mongoauth(self):
customizations = {
'mongo-auth': {
'authdb': self.config.mongodb.authdb,
'username': self.config.mongodb.username,
'password': self.config.mongodb.password
},
}
self.buildout.configure_file('mongoauth.cfg', customizations),
return success_log('Succesfully configured {}/mongoauth.cfg'.format(self.buildout.folder))
def set_mongodb_indexes(self):
new_instance_folder = '{}/{}'.format(
self.config.instances_root,
self.instance.name
)
code, stdout = self.remote.execute('cd {0} && ./bin/max.mongoindexes'.format(new_instance_folder))
added = 'Creating' in stdout or 'already exists' in stdout
if added:
return success_log("Succesfully added indexes")
else:
return error_log("Error on adding indexes")
def add_user(self, branch, username, password):
if self.config.readonly:
yield error_log('This LDAP is configured as read-only')
self.set_branch(branch)
self.connect(auth=False)
self.authenticate(
username=self.effective_admin_dn,
password=self.effective_admin_password,
)
self.add_ldap_user(username, username, password)
self.disconnect()
yield success_log("User {} successfully added".format(username))
def test(self, instance_name, username, password):
instance = self.get_instance(instance_name)
try:
yield step_log('Testing oauth server @ {}'.format(instance['server']['dns']))
yield message_log('Checking server health')
try:
status = requests.get(instance['server']['dns'], verify=True).status_code
except requests.exceptions.SSLError:
yield error_log('SSL certificate verification failed')
yield message_log('Continuing test without certificate check')
try:
status = requests.get(instance['server']['dns'], verify=False).status_code
except requests.ConnectionError:
yield raising_error_log('Connection error, check nginx is running, and dns resolves as expected.')
except:
yield raising_error_log('Unknown error trying to access oauth server. Check params and try again')
else:
if status == 500:
yield raising_error_log('Error on oauth server, Possible causes:\n - ldap configuration error (bad server url?)\n - Mongodb configuration error (bad replicaset name or hosts list?)\nCheck osiris log for more information.')
elif status == 502:
yield raising_error_log('Server not respoding at {}. Check that:\n - osiris process is running\n - nginx upstream definition is pointing to the right host:port.'.format(instance['server']['dns']))
elif status == 504:
yield raising_error_log('Gateway timeout. Probably oauth server is giving timeout trying to contact ldap server')
elif status == 404:
yield raising_error_log('There\'s no oauth server at {}. Chech there\'s an nginx entry for this server.'.format(instance['server']['dns']))
elif status != 200:
yield raising_error_log('Server {} responded with {} code. Check osiris logs.'.format(instance['server']['dns'], status))
yield message_log('Retrieving token for "{}"'.format(username))
token = self.get_token(instance['server']['dns'], username, password)
succeeded_retrieve_token = token is not None
if not succeeded_retrieve_token:
yield raising_error_log('Error retreiving token. Check username/password and try again')
yield message_log('Checking retreived token')
succeeded_check_token = self.check_token(instance['server']['dns'], username, token)
if not succeeded_check_token:
yield raising_error_log('Error retreiving token')
if succeeded_check_token and succeeded_retrieve_token:
yield success_log('Oauth server check passed')
else:
yield raising_error_log('Oauth server check failed')
except StepError as error:
yield error_log(error.message)
def remove_bypass_allowed_ip(self, ip):
configured_ips = self.get_instance_allowed_ips()
allowed_ips = list(set(configured_ips) - set([ip]))
if not set(allowed_ips).symmetric_difference(set(configured_ips)):
return message_log('No changes to allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
customizations = {
'osiris-config': {
'allowed_ips': allowed_ips,
}
}
self.buildout.configure_file('customizeme.cfg', customizations)
return success_log('Succesfully updated allowed ips on {}/customizeme.cfg'.format(self.buildout.folder))
def configure_max_security_settings(self):
new_instance_folder = '{}/{}'.format(
self.config.instances_root,
self.instance.name
)
self.buildout.folder = new_instance_folder
self.remote.execute('cd {} && ./bin/max.security reset'.format(new_instance_folder))
code, stdout = self.remote.execute('cd {} && ./bin/max.security add {}'.format(new_instance_folder, self.config.authorized_user))
added = 'restart max process' in stdout
# Force read the new configuration files
if added:
return success_log("Succesfully configured security settings")
else:
return error_log("Error configuring security settings")
def batch_subscribe_users(self, instance, subscriptionsfile):
site = UlearnSite(self.get_environment(instance["environment"]), instance["mountpoint"], instance["plonesite"])
try:
communities = read_subscriptions_file(subscriptionsfile, required_fields=["owners", "readers", "editors"])
except Exception as exc:
error_message = "Error parsing subscriptionsfile file {}: {{}}".format(subscriptionsfile)
yield raising_error_log(error_message.format(exc.message))
for community in communities:
yield step_log("Subscribing users to {}".format(community["url"]))
succeeded = site.subscribe_users(**community)
if not succeeded.get("error", False):
yield success_log(succeeded["message"])
else:
yield error_log(succeeded["message"])
def configure_instance(self):
customizations = {
'mongodb-config': {
'replica_set': self.config.mongodb.replica_set,
'cluster_hosts': self.config.mongodb.cluster
},
'osiris-config': {
'name': self.instance.name,
},
'ports': {
'port_index': '{:0>2}'.format(self.instance.index),
},
}
self.buildout.configure_file('customizeme.cfg', customizations),
return success_log('Succesfully configured {}/customizeme.cfg'.format(self.buildout.folder))
def create_max_nginx_entry(self):
nginx_params = {
'instance_name': self.instance.name,
'server': self.config.server,
'server_dns': self.config.server_dns,
'bigmax_port': BIGMAX_BASE_PORT,
'max_port': int(self.instance.index) + MAX_BASE_PORT,
'nginx_root_folder': self.config.nginx.root,
'max_root_folder': self.buildout.folder
}
nginxentry = MAX_NGINX_ENTRY.format(**nginx_params)
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/max-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
nginx_remote.put_file(nginx_file_location, nginxentry)
return success_log("Succesfully created {}".format(nginx_file_location))
def setup_nginx(self, site, max_url):
nginx_params = {"instance_name": site.plonesite, "max_server": max_url, "mountpoint_id": site.mountpoint}
nginxentry = ULEARN_NGINX_ENTRY.format(**nginx_params)
success = self.prefes.put_file(
"{}/config/ulearn-instances/{}.conf".format(self.config.prefe_nginx_root, site.plonesite), nginxentry
)
if success:
return success_log(
"Succesfully created {}/config/ulearn-instances/{}.conf".format(
self.config.prefe_nginx_root, site.plonesite
)
)
else:
return error_log("Error when generating nginx config file for ulean")
def create_oauth_nginx_entry(self):
global_allowed_ips = self.config.oauth.allowed_ips
instance_allowed_ips = self.get_instance_allowed_ips()
allowed_ips = instance_allowed_ips + global_allowed_ips
nginx_params = {
'instance_name': self.instance.name,
'server': self.config.oauth.server,
'server_dns': self.config.oauth.server_dns,
'osiris_port': int(self.instance.index) + OSIRIS_BASE_PORT,
'buildout_folder': self.config.nginx.root,
'allowed_ips': '\n '.join(['allow {};'.format(ip) for ip in allowed_ips])
}
nginxentry = OSIRIS_NGINX_ENTRY.format(**nginx_params)
nginx_remote = RemoteConnection(self.config.nginx.ssh_user, self.config.nginx.server)
nginx_file_location = "{}/config/osiris-instances/{}.conf".format(self.config.nginx.root, self.instance.name)
nginx_remote.put_file(nginx_file_location, nginxentry)
return success_log("Succesfully created {}".format(nginx_file_location))
def setup_ldap(self, branch, ldap_config):
setup_view_url = "{}/setupldap".format(self.site_url)
params = {
"ldap_name": ldap_config.name,
"ldap_server": re.sub(r"ldaps?:\/\/", "", ldap_config.server),
"branch_name": branch,
"base_dn": ldap_config.branches.base_dn,
"branch_admin_cn": ldap_config.branches.admin_cn,
"branch_admin_password": ldap_config.branches.admin_password,
"allow_manage_users": True,
}
req = requests.post(setup_view_url, data=params, auth=self.auth)
if req.status_code not in [302, 200, 204, 201]:
return error_log('Error on ldap branch "{}" setup'.format(branch))
else:
return success_log('Successfully configured ldap branch "{}"'.format(branch))
def create(self, packages=[]):
if self.exists():
return error_log("There is already a ulearn on {}".format(self.site_url))
params = {
"site_id": self.plonesite,
"title": self.title,
"default_language": self.language,
"setup_content:boolean": True,
"extension_ids:list": ["plonetheme.classic:default", "plonetheme.sunburst:default"] + packages,
"form.submitted:boolean": True,
"submit": "Crear lloc Plone",
}
create_plone_url = "{}/@@plone-addsite".format(self.mountpoint_url)
self.echo.start()
req = requests.post(create_plone_url, params, auth=self.auth)
self.echo.stop()
if req.status_code not in [302, 200, 204, 201]:
return error_log("Error creating Plone site at {}".format(self.site_url))
else:
return success_log("Successfully created Plone site at {}".format(self.site_url))
def batch_add_users(self, instance, usersfile):
site = UlearnSite(self.get_environment(instance["environment"]), instance["mountpoint"], instance["plonesite"])
try:
users = read_users_file(usersfile, required_fields=["username", "fullname", "email", "password"])
except Exception as exc:
error_message = "Error parsing users file {}: {{}}".format(usersfile)
yield raising_error_log(error_message.format(exc.message))
try:
self.check_users(users)
except Exception as exc:
yield raising_error_log(exc.message)
yield step_log("Creating {} users ".format(len(users)))
for count, user in enumerate(users, start=1):
if not user:
yield error_log("Error parsing user at line #{}".format(count))
continue
succeeded = site.add_user(**user)
if not succeeded.get("error", False):
yield success_log(succeeded["message"])
else:
yield error_log(succeeded["message"])
def set_filesystem_permissions(self):
self.buildout.change_permissions(self.config.process_uid)
return success_log("Succesfully changed permissions")
def execute_buildout(self, update=False):
self.buildout.execute(update=update)
return success_log("Succesfully executed {} buildout".format(self.buildout.cfgfile))
