def validate_delete_user(self, name, username):
context = {
'name': name,
'username': username,
}
self.assertEqual('', sshrun(self.MASTER_SERVER,
"""mysql mysql -e 'SELECT * FROM db WHERE db="%(name)s";'""" % context, display=False).stdout)
self.assertEqual('', sshrun(self.MASTER_SERVER,
"""mysql mysql -e 'SELECT * FROM user WHERE user="******";'""" % context, display=False).stdout)
def validate_add(self, name, address=None):
sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False)
if not address:
address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN)
subscribe_address = "{}-subscribe@{}".format(*address.split('@'))
request_address = "{}-request@{}".format(name, address.split('@')[1])
self.subscribe(subscribe_address)
time.sleep(3)
sshrun(self.MASTER_SERVER,
'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"'
% request_address, display=False)
def validate_add(self, name, address=None):
sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False)
if not address:
address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN)
subscribe_address = "{}-subscribe@{}".format(*address.split('@'))
request_address = "{}-request@{}".format(name, address.split('@')[1])
self.subscribe(subscribe_address)
time.sleep(3)
sshrun(
self.MASTER_SERVER,
'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"'
% request_address,
display=False)
def retrieve_state(servers):
uptimes = []
pings = []
for server in servers:
address = server.get_address()
ping = run('ping -c 1 -w 1 %s' % address, async=True)
pings.append(ping)
uptime = sshrun(address,
'uptime',
persist=True,
async=True,
options={'ConnectTimeout': 1})
uptimes.append(uptime)
state = {}
for server, ping, uptime in zip(servers, pings, uptimes):
ping = join(ping, silent=True)
ping = ping.stdout.splitlines()[-1].decode()
if ping.startswith('rtt'):
ping = '%s ms' % ping.split('/')[4]
else:
ping = '<span style="color:red">Offline</span>'
uptime = join(uptime, silent=True)
uptime_stderr = uptime.stderr.decode()
uptime = uptime.stdout.decode().split()
if uptime:
uptime = 'Up %s %s load %s %s %s' % (
uptime[2], uptime[3], uptime[-3], uptime[-2], uptime[-1])
else:
uptime = '<span style="color:red">%s</span>' % uptime_stderr
state[server.pk] = (ping, uptime)
return state
def retrieve_state(servers):
uptimes = []
pings = []
for server in servers:
address = server.get_address()
ping = run('ping -c 1 -w 1 %s' % address, async=True)
pings.append(ping)
uptime = sshrun(address, 'uptime', persist=True, async=True, options={'ConnectTimeout': 1})
uptimes.append(uptime)
state = {}
for server, ping, uptime in zip(servers, pings, uptimes):
ping = join(ping, silent=True)
ping = ping.stdout.splitlines()[-1].decode()
if ping.startswith('rtt'):
ping = '%s ms' % ping.split('/')[4]
else:
ping = '<span style="color:red">Offline</span>'
uptime = join(uptime, silent=True)
uptime_stderr = uptime.stderr.decode()
uptime = uptime.stdout.decode().split()
if uptime:
uptime = 'Up %s %s load %s %s %s' % (uptime[2], uptime[3], uptime[-3], uptime[-2], uptime[-1])
else:
uptime = '<span style="color:red">%s</span>' % uptime_stderr
state[server.pk] = (ping, uptime)
return state
def _save(self, saas, server):
admin_link = 'https://%s/admin/' % saas.get_site_domain()
sys.stdout.write('admin_link: %s\n' % admin_link)
admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
admin_content = admin_content.content.decode('utf8')
if admin_content.startswith('Cannot connect to Database'):
self.error("Database is not yet configured.")
install = re.search(r'([^"]+firstinstall[^"]+)', admin_content)
if install:
if not hasattr(saas, 'password'):
self.error("Password is missing.")
install_path = install.groups()[0]
install_link = admin_link + install_path[1:]
post = {
'adminname': saas.name,
'orgname': saas.account.username,
'adminemail': saas.account.username,
'adminpassword': saas.password,
}
response = requests.post(install_link, data=post, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
sys.stdout.write(response.content.decode('utf8')+'\n')
if response.status_code != 200:
self.error("Bad status code %i." % response.status_code)
else:
md5_password = hashlib.md5()
md5_password.update(saas.password.encode('ascii'))
context = {
'name': saas.name,
'site_name': saas.name,
'db_user': settings.SAAS_PHPLIST_DB_USER,
'db_pass': settings.SAAS_PHPLIST_DB_PASS,
'db_name': settings.SAAS_PHPLIST_DB_NAME,
'db_host': settings.SAAS_PHPLIST_DB_HOST,
'digest': md5_password.hexdigest(),
}
context['db_name'] = context['db_name'] % context
cmd = textwrap.dedent("""\
mysql \\
--host=%(db_host)s \\
--user=%(db_user)s \\
--password=%(db_pass)s \\
--execute='UPDATE phplist_admin SET password="******" where ID=1; \\
UPDATE phplist_user_user SET password="******" where ID=1;' \\
%(db_name)s""") % context
sys.stdout.write('cmd: %s\n' % cmd)
sshrun(server.get_address(), cmd)
def _install_or_change_password(self, saas, server):
""" configures the database for the new site through HTTP to /admin/ """
admin_link = "https://%s/admin/" % saas.get_site_domain()
sys.stdout.write("admin_link: %s\n" % admin_link)
admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
admin_content = admin_content.content.decode("utf8")
if admin_content.startswith("Cannot connect to Database"):
self.error("Database is not yet configured.")
install = re.search(r'([^"]+firstinstall[^"]+)', admin_content)
if install:
if not hasattr(saas, "password"):
self.error("Password is missing.")
install_path = install.groups()[0]
install_link = admin_link + install_path[1:]
post = {
"adminname": saas.name,
"orgname": saas.account.username,
"adminemail": saas.account.username,
"adminpassword": saas.password,
}
response = requests.post(install_link, data=post, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
sys.stdout.write(response.content.decode("utf8") + "\n")
if response.status_code != 200:
self.error("Bad status code %i." % response.status_code)
else:
md5_password = hashlib.md5()
md5_password.update(saas.password.encode("ascii"))
context = self.get_context(saas)
context["digest"] = md5_password.hexdigest()
cmd = (
textwrap.dedent(
"""\
mysql \\
--host=%(db_host)s \\
--user=%(db_user)s \\
--password=%(db_pass)s \\
--execute='UPDATE phplist_admin SET password="******" where ID=1; \\
UPDATE phplist_user_user SET password="******" where ID=1;' \\
%(db_name)s"""
)
% context
)
sys.stdout.write("cmd: %s\n" % cmd)
sshrun(server.get_address(), cmd)
def _install_or_change_password(self, saas, server):
""" configures the database for the new site through HTTP to /admin/ """
admin_link = 'https://%s/admin/' % saas.get_site_domain()
sys.stdout.write('admin_link: %s\n' % admin_link)
admin_content = requests.get(admin_link,
verify=settings.SAAS_PHPLIST_VERIFY_SSL)
admin_content = admin_content.content.decode('utf8')
if admin_content.startswith('Cannot connect to Database'):
self.error("Database is not yet configured.")
install = re.search(r'([^"]+firstinstall[^"]+)', admin_content)
if install:
if not hasattr(saas, 'password'):
self.error("Password is missing.")
install_path = install.groups()[0]
install_link = admin_link + install_path[1:]
post = {
'adminname': saas.name,
'orgname': saas.account.username,
'adminemail': saas.account.username,
'adminpassword': saas.password,
}
response = requests.post(install_link,
data=post,
verify=settings.SAAS_PHPLIST_VERIFY_SSL)
sys.stdout.write(response.content.decode('utf8') + '\n')
if response.status_code != 200:
self.error("Bad status code %i." % response.status_code)
else:
md5_password = hashlib.md5()
md5_password.update(saas.password.encode('ascii'))
context = self.get_context(saas)
context['digest'] = md5_password.hexdigest()
cmd = textwrap.dedent("""\
mysql \\
--host=%(db_host)s \\
--user=%(db_user)s \\
--password=%(db_pass)s \\
--execute='UPDATE phplist_admin SET password="******" where ID=1; \\
UPDATE phplist_user_user SET password="******" where ID=1;' \\
%(db_name)s""") % context
sys.stdout.write('cmd: %s\n' % cmd)
sshrun(server.get_address(), cmd, persist=True)
def test_custom_filtering(self):
username = '******' % random_ascii(10)
password = '******' % random_ascii(5)
folder = random_ascii(5)
filtering = textwrap.dedent("""
require "fileinto";
if true {
fileinto "%s";
stop;
}""" % folder)
self.add(username, password, filtering=filtering)
self.addCleanup(self.delete, username)
imap = self.login_imap(username, password)
imap.create(folder)
self.validate_mailbox(username)
token = random_ascii(100)
self.send_email("%s@%s" % (username, settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN), token)
home = Mailbox.objects.get(name=username).get_home()
sshrun(self.MASTER_SERVER,
"grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder), display=False)
def test_quota(self):
username = '******' % random_ascii(10)
password = '******' % random_ascii(5)
self.add_quota_resource()
quota = 100
self.add(username, password, quota=quota)
self.addCleanup(self.delete, username)
get_quota = "doveadm quota get -u %s 2>&1|grep STORAGE|awk {'print $5'}" % username
stdout = sshrun(self.MASTER_SERVER, get_quota, display=False).stdout
self.assertEqual(quota*1024, int(stdout))
imap = self.login_imap(username, password)
imap_quota = int(imap.getquotaroot("INBOX")[1][1][0].split(' ')[-1].split(')')[0])
self.assertEqual(quota*1024, imap_quota)
def test_quota(self):
username = '******' % random_ascii(10)
password = '******' % random_ascii(5)
self.add_quota_resource()
quota = 100
self.add(username, password, quota=quota)
self.addCleanup(self.delete, username)
get_quota = "doveadm quota get -u %s 2>&1|grep STORAGE|awk {'print $5'}" % username
stdout = sshrun(self.MASTER_SERVER, get_quota, display=False).stdout
self.assertEqual(quota * 1024, int(stdout))
imap = self.login_imap(username, password)
imap_quota = int(
imap.getquotaroot("INBOX")[1][1][0].split(' ')[-1].split(')')[0])
self.assertEqual(quota * 1024, imap_quota)
def OpenSSH(backend, log, server, cmds, run_async=False):
"""
Executes cmds to remote server using SSH with connection resuse for maximum performance
"""
script = '\n'.join(cmds)
script = script.replace('\r', '')
log.state = log.STARTED
log.script = '\n'.join((log.script, script))
log.save(update_fields=('script', 'state', 'updated_at'))
if not cmds:
return
try:
ssh = sshrun(server.get_address(),
script,
executable=backend.script_executable,
persist=True,
run_async=run_async,
silent=True)
logger.debug('%s running on %s' % (backend, server))
if run_async:
for state in ssh:
log.stdout += state.stdout.decode('utf8')
log.stderr += state.stderr.decode('utf8')
log.save(update_fields=('stdout', 'stderr', 'updated_at'))
exit_code = state.exit_code
else:
log.stdout += ssh.stdout.decode('utf8')
log.stderr += ssh.stderr.decode('utf8')
exit_code = ssh.exit_code
if not log.exit_code:
log.exit_code = exit_code
if exit_code == 255 and log.stderr.startswith(
'ssh: connect to host'):
log.state = log.TIMEOUT
else:
log.state = log.SUCCESS if exit_code == 0 else log.FAILURE
logger.debug('%s execution state on %s is %s' %
(backend, server, log.state))
log.save()
except:
log.state = log.ERROR
log.traceback = ExceptionInfo(sys.exc_info()).traceback
logger.error('Exception while executing %s on %s' % (backend, server))
logger.debug(log.traceback)
log.save()
finally:
if log.state == log.STARTED:
log.state = log.ABORTED
log.save(update_fields=('state', 'updated_at'))
def test_custom_filtering(self):
username = '******' % random_ascii(10)
password = '******' % random_ascii(5)
folder = random_ascii(5)
filtering = textwrap.dedent("""
require "fileinto";
if true {
fileinto "%s";
stop;
}""" % folder)
self.add(username, password, filtering=filtering)
self.addCleanup(self.delete, username)
imap = self.login_imap(username, password)
imap.create(folder)
self.validate_mailbox(username)
token = random_ascii(100)
self.send_email(
"%s@%s" %
(username, settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN),
token)
home = Mailbox.objects.get(name=username).get_home()
sshrun(self.MASTER_SERVER,
"grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder),
display=False)
def OpenSSH(backend, log, server, cmds, async=False):
"""
Executes cmds to remote server using SSH with connection resuse for maximum performance
"""
script = '\n'.join(cmds)
script = script.replace('\r', '')
log.state = log.STARTED
log.script = script
log.save(update_fields=('script', 'state', 'updated_at'))
if not cmds:
return
channel = None
ssh = None
try:
ssh = sshrun(server.get_address(), script, executable=backend.script_executable,
persist=True, async=async, silent=True)
logger.debug('%s running on %s' % (backend, server))
if async:
second = False
for state in ssh:
log.stdout += state.stdout.decode('utf8')
log.stderr += state.stderr.decode('utf8')
log.save(update_fields=('stdout', 'stderr', 'updated_at'))
log.exit_code = state.exit_code
else:
log.stdout = ssh.stdout.decode('utf8')
log.stderr = ssh.stderr.decode('utf8')
log.exit_code = ssh.exit_code
log.state = log.SUCCESS if log.exit_code == 0 else log.FAILURE
logger.debug('%s execution state on %s is %s' % (backend, server, log.state))
log.save()
def OpenSSH(backend, log, server, cmds, async=False):
"""
Executes cmds to remote server using SSH with connection resuse for maximum performance
"""
script = '\n'.join(cmds)
script = script.replace('\r', '')
log.state = log.STARTED
log.script = '\n'.join((log.script, script))
log.save(update_fields=('script', 'state', 'updated_at'))
if not cmds:
return
try:
ssh = sshrun(server.get_address(),
script,
executable=backend.script_executable,
persist=True,
async=async,
silent=True)
logger.debug('%s running on %s' % (backend, server))
if async:
for state in ssh:
log.stdout += state.stdout.decode('utf8')
log.stderr += state.stderr.decode('utf8')
log.save(update_fields=('stdout', 'stderr', 'updated_at'))
exit_code = state.exit_code
else:
log.stdout += ssh.stdout.decode('utf8')
log.stderr += ssh.stderr.decode('utf8')
exit_code = ssh.exit_code
if not log.exit_code:
def validate_mailbox(self, username):
sshrun(self.MASTER_SERVER, "doveadm search -u %s ALL" % username, display=False)
def validate_email(self, username, token):
home = Mailbox.objects.get(name=username).get_home()
sshrun(self.MASTER_SERVER, "grep '%s' %s/Maildir/new/*" % (token, home), display=False)
def validate_mailbox(self, username):
sshrun(self.MASTER_SERVER,
"doveadm search -u %s ALL" % username,
display=False)
def validate_email(self, username, token):
home = Mailbox.objects.get(name=username).get_home()
sshrun(self.MASTER_SERVER,
"grep '%s' %s/Maildir/new/*" % (token, home),
display=False)