def start_mailman(quiet=False, config=None):
"""Start the Mailman master qrunner.
The client of start_mailman() is responsible for ensuring that
stop_mailman() is called at the appropriate time.
:param quiet: when this is true, no output will happen unless, an error
happens.
:param config: The LaunchpadConfig object to take configuration from.
Defaults to the global one.
:raises RuntimeException: when Mailman fails to start successfully.
"""
if config is None:
config = lp.services.config.config
# We need the Mailman bin directory so we can run some of Mailman's
# command line scripts.
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
# Monkey-patch the installed Mailman 2.1 tree.
monkey_patch(mailman_path, config)
# Start Mailman. Pass in the -s flag so that any stale master pid files
# will get deleted. "Stale" means the process that owned the pid no
# longer exists, so this can't hurt anything.
mailmanctl('start', quiet, config, '-s')
def withlist(callable):
callable_path = '%s.%s' % (__name__, callable)
site_list = 'unused_mailman_site_list'
prefix_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(prefix_path, 'bin')
command = './withlist -q -r %s %s' % (callable_path, site_list)
return subprocess.call(command.split(), cwd=mailman_bin)
def withlist(callable):
callable_path = '%s.%s' % (__name__, callable)
site_list = 'unused_mailman_site_list'
prefix_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(prefix_path, 'bin')
command = './withlist -q -r %s %s' % (callable_path, site_list)
return subprocess.call(command.split(), cwd=mailman_bin)
def start_mailman(quiet=False, config=None):
"""Start the Mailman master qrunner.
The client of start_mailman() is responsible for ensuring that
stop_mailman() is called at the appropriate time.
:param quiet: when this is true, no output will happen unless, an error
happens.
:param config: The LaunchpadConfig object to take configuration from.
Defaults to the global one.
:raises RuntimeException: when Mailman fails to start successfully.
"""
if config is None:
config = lp.services.config.config
# We need the Mailman bin directory so we can run some of Mailman's
# command line scripts.
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
# Monkey-patch the installed Mailman 2.1 tree.
monkey_patch(mailman_path, config)
# Start Mailman. Pass in the -s flag so that any stale master pid files
# will get deleted. "Stale" means the process that owned the pid no
# longer exists, so this can't hurt anything.
mailmanctl('start', quiet, config, '-s')
def deleteMailmanList(self, list_name):
"""Delete all Mailman data structures for `list_name`."""
mailman_bindir = os.path.normpath(os.path.join(
configure_prefix(config.mailman.build_prefix), 'bin'))
process = subprocess.Popen(('./rmlist', '-a', list_name),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=mailman_bindir)
stdout, stderr = process.communicate()
if process.returncode == 0:
self.logger.info('%s', stdout)
else:
self.logger.error('rmlist command failed with exit status: %s',
process.returncode)
self.logger.error('STDOUT:\n%s\nSTDERR:\n%s', stdout, stderr)
def mailmanctl(command, quiet=False, config=None, *additional_arguments):
"""Run mailmanctl command.
:param command: the command to use.
:param quiet: when this is true, no output will happen unless, an error
happens.
:param config: The LaunchpadConfig object to take configuration from.
Defaults to the global one.
:param additional_arguments: additional command arguments to pass to the
mailmanctl program.
:raises RuntimeError: when quiet is True and the command failed.
"""
if config is None:
config = lp.services.config.config
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
args = ['./mailmanctl']
args.extend(additional_arguments)
args.append(command)
if quiet:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
else:
stdout = None
stderr = None
env = dict(os.environ)
env['LPCONFIG'] = config.instance_name
process = subprocess.Popen(args,
cwd=mailman_bin,
stdout=stdout,
stderr=stderr,
env=env)
code = process.wait()
if code:
if quiet:
raise RuntimeError('mailmanctl %s failed: %d\n%s' %
(command, code, process.stdout.read()))
else:
print >> sys.stderr, 'mailmanctl %s failed: %d' % (command, code)
def stop_mailman(quiet=False, config=None):
"""Alias for mailmanctl('stop')."""
mailmanctl('stop', quiet, config)
# Further, if the Mailman master pid file was not removed, then the
# master watcher, and probably one of its queue runners, did not die.
# Kill it hard and clean up after it.
if config is None:
config = lp.services.config.config
mailman_path = configure_prefix(config.mailman.build_prefix)
master_pid_path = os.path.join(mailman_path, 'data', 'master-qrunner.pid')
try:
master_pid_file = open(master_pid_path)
except IOError as error:
if error.errno == errno.ENOENT:
# It doesn't exist, so we're all done.
return
raise
try:
master_pid = int(master_pid_file.read().strip())
finally:
master_pid_file.close()
try:
# Kill the entire process group.
os.kill(master_pid, -signal.SIGKILL)
except OSError as error:
if error.errno == errno.ESRCH:
# The process does not exist. It could be a zombie that has yet
# to be waited on, but let's not worry about that.
return
raise
try:
os.remove(master_pid_path)
except OSError as error:
if error.errno != errno.ENOENT:
raise
lock_dir = os.path.join(mailman_path, 'locks')
for entry in scandir.scandir(lock_dir):
os.remove(entry.path)
def stop_mailman(quiet=False, config=None):
"""Alias for mailmanctl('stop')."""
mailmanctl('stop', quiet, config)
# Further, if the Mailman master pid file was not removed, then the
# master watcher, and probably one of its queue runners, did not die.
# Kill it hard and clean up after it.
if config is None:
config = lp.services.config.config
mailman_path = configure_prefix(config.mailman.build_prefix)
master_pid_path = os.path.join(mailman_path, 'data', 'master-qrunner.pid')
try:
master_pid_file = open(master_pid_path)
except IOError as error:
if error.errno == errno.ENOENT:
# It doesn't exist, so we're all done.
return
raise
try:
master_pid = int(master_pid_file.read().strip())
finally:
master_pid_file.close()
try:
# Kill the entire process group.
os.kill(master_pid, -signal.SIGKILL)
except OSError as error:
if error.errno == errno.ESRCH:
# The process does not exist. It could be a zombie that has yet
# to be waited on, but let's not worry about that.
return
raise
try:
os.remove(master_pid_path)
except OSError as error:
if error.errno != errno.ENOENT:
raise
lock_dir = os.path.join(mailman_path, 'locks')
for filename in os.listdir(lock_dir):
os.remove(os.path.join(lock_dir, filename))
def mailmanctl(command, quiet=False, config=None, *additional_arguments):
"""Run mailmanctl command.
:param command: the command to use.
:param quiet: when this is true, no output will happen unless, an error
happens.
:param config: The LaunchpadConfig object to take configuration from.
Defaults to the global one.
:param additional_arguments: additional command arguments to pass to the
mailmanctl program.
:raises RuntimeError: when quiet is True and the command failed.
"""
if config is None:
config = lp.services.config.config
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
args = ['./mailmanctl']
args.extend(additional_arguments)
args.append(command)
if quiet:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
else:
stdout = None
stderr = None
env = dict(os.environ)
env['LPCONFIG'] = config.instance_name
process = subprocess.Popen(
args, cwd=mailman_bin, stdout=stdout, stderr=stderr, env=env)
code = process.wait()
if code:
if quiet:
raise RuntimeError(
'mailmanctl %s failed: %d\n%s' % (
command, code, process.stdout.read()))
else:
print >> sys.stderr, 'mailmanctl %s failed: %d' % (command, code)
def main(self):
"""See `LaunchpadScript`."""
source_url = None
if len(self.args) == 0:
self.parser.error('Missing source_url')
elif len(self.args) > 1:
self.parser.error('Too many arguments')
else:
source_url = self.args[0]
# We need to get to the Mailman API. Set up the paths so that Mailman
# can be imported. This can't be done at module global scope.
mailman_path = configure_prefix(config.mailman.build_prefix)
sys.path.append(mailman_path)
retcode = self.syncMailmanDirectories(source_url)
if retcode != 0:
return retcode
self.fixHostnames()
# All done; commit the database changes.
self.txn.commit()
return 0
def setUp(self):
super(TestMMCfgLaunchpadConfigTestCase, self).setUp()
# Generate a mailman config using this environment's config.
mailman_path = configure_prefix(config.mailman.build_prefix)
monkey_patch(mailman_path, config)
reload(mm_cfg)
def build_mailman():
# Build and install Mailman if it is enabled and not yet built.
if not config.mailman.build:
# There's nothing to do.
return 0
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
var_dir = os.path.abspath(config.mailman.build_var_dir)
executable = os.path.abspath('bin/py')
# If we can import the package, we assume Mailman is properly built at
# the least. This does not catch re-installs that might be necessary
# should our copy in sourcecode be updated. Do that manually.
try:
import Mailman
except ImportError:
need_build = need_install = True
else:
need_build = need_install = False
# Make sure that the configure prefix is correct, in case this tree
# was moved after building Mailman.
try:
from Mailman import Defaults
except ImportError:
need_build = need_install = True
else:
if Defaults.PYTHON != executable:
need_build = need_install = True
# We'll need to remove this; "make install" won't overwrite
# the existing file, and then installation will fail due to
# it having the wrong sys.path.
try:
os.unlink(
os.path.join(mailman_path, 'Mailman', 'mm_cfg.py'))
except OSError as e:
if e.errno != errno.ENOENT:
raise
if not need_install:
# Also check for Launchpad-specific bits stuck into the source tree by
# monkey_patch(), in case this is half-installed. See
# <https://bugs.launchpad.net/launchpad-registry/+bug/683486>.
try:
from Mailman.Queue import XMLRPCRunner
from Mailman.Handlers import LPModerate
except ImportError:
# Monkey patches not present, redo install and patch steps.
need_install = True
# Make sure the target directories exist and have the correct
# permissions, otherwise configure will complain.
user, group = as_username_groupname(config.mailman.build_user_group)
# Now work backwards to get the uid and gid
try:
uid = pwd.getpwnam(user).pw_uid
except KeyError:
print('No user found:', user, file=sys.stderr)
sys.exit(1)
try:
gid = grp.getgrnam(group).gr_gid
except KeyError:
print('No group found:', group, file=sys.stderr)
sys.exit(1)
# Ensure that the var_dir exists, is owned by the user:group, and has
# the necessary permissions. Set the mode separately after the
# makedirs() call because some platforms ignore mkdir()'s mode (though
# I think Linux does not ignore it -- better safe than sorry).
try:
os.makedirs(var_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
else:
# Just created the var directory, will need to install mailmain bits.
need_install = True
os.chown(var_dir, uid, gid)
os.chmod(var_dir, 0o2775)
# Skip mailman setup if nothing so far has shown a reinstall needed.
if not need_install:
return 0
mailman_source = os.path.join('sourcecode', 'mailman')
if config.mailman.build_host_name:
build_host_name = config.mailman.build_host_name
else:
build_host_name = socket.getfqdn()
# Build and install the Mailman software. Note that we don't care about
# --with-cgi-gid because we're not going to use that Mailman subsystem.
configure_args = (
'./configure',
'--prefix',
mailman_path,
'--with-var-prefix=' + var_dir,
'--with-python=' + executable,
'--with-username='******'--with-groupname=' + group,
'--with-mail-gid=' + group,
'--with-mailhost=' + build_host_name,
'--with-urlhost=' + build_host_name,
)
if need_build:
# Configure.
retcode = subprocess.call(configure_args, cwd=mailman_source)
if retcode:
print('Could not configure Mailman:', file=sys.stderr)
sys.exit(retcode)
# Make.
retcode = subprocess.call(('make', ), cwd=mailman_source)
if retcode:
print('Could not make Mailman.', file=sys.stderr)
sys.exit(retcode)
retcode = subprocess.call(('make', 'install'), cwd=mailman_source)
if retcode:
print('Could not install Mailman.', file=sys.stderr)
sys.exit(retcode)
# Symlink Mailman's Python modules into the import path.
try:
os.unlink(os.path.join('lib', 'Mailman'))
except OSError as e:
if e.errno != errno.ENOENT:
raise
os.symlink(os.path.join('mailman', 'Mailman'),
os.path.join('lib', 'Mailman'))
# Try again to import the package.
try:
import Mailman
except ImportError:
print('Could not import the Mailman package', file=sys.stderr)
return 1
# Check to see if the site list exists. The output can go to /dev/null
# because we don't really care about it. The site list exists if
# config_list returns a zero exit status, otherwise it doesn't
# (probably). Before we can do this however, we must monkey patch
# Mailman, otherwise mm_cfg.py won't be set up correctly.
monkey_patch(mailman_path, config)
import Mailman.mm_cfg
retcode = subprocess.call(
('./config_list', '-o', '/dev/null', Mailman.mm_cfg.MAILMAN_SITE_LIST),
cwd=mailman_bin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if retcode:
addr, password = configure_siteowner(
config.mailman.build_site_list_owner)
# The site list does not yet exist, so create it now.
retcode = subprocess.call(
('./newlist', '--quiet', '--emailhost=' + build_host_name,
Mailman.mm_cfg.MAILMAN_SITE_LIST, addr, password),
cwd=mailman_bin)
if retcode:
print('Could not create site list', file=sys.stderr)
return retcode
retcode = configure_site_list(mailman_bin,
Mailman.mm_cfg.MAILMAN_SITE_LIST)
if retcode:
print('Could not configure site list', file=sys.stderr)
return retcode
# Create a directory to hold the gzip'd tarballs for the directories of
# deactivated lists.
try:
os.mkdir(os.path.join(Mailman.mm_cfg.VAR_PREFIX, 'backups'))
except OSError as e:
if e.errno != errno.EEXIST:
raise
return 0
def build_mailman():
# Build and install Mailman if it is enabled and not yet built.
if not config.mailman.build:
# There's nothing to do.
return 0
mailman_path = configure_prefix(config.mailman.build_prefix)
mailman_bin = os.path.join(mailman_path, 'bin')
var_dir = os.path.abspath(config.mailman.build_var_dir)
# If we can import the package, we assume Mailman is properly built at
# the least. This does not catch re-installs that might be necessary
# should our copy in sourcecode be updated. Do that manually.
sys.path.append(mailman_path)
try:
import Mailman
except ImportError:
# sys.path_importer_cache is a mapping of elements of sys.path to
# importer objects used to handle them. In Python2.5+ when an element
# of sys.path is found to not exist on disk, a NullImporter is created
# and cached - this causes Python to never bother re-inspecting the
# disk for that path element. We must clear that cache element so that
# our second attempt to import MailMan after building it will actually
# check the disk.
del sys.path_importer_cache[mailman_path]
need_build = need_install = True
else:
need_build = need_install = False
# Also check for Launchpad-specific bits stuck into the source tree by
# monkey_patch(), in case this is half-installed. See
# <https://bugs.launchpad.net/launchpad-registry/+bug/683486>.
try:
from Mailman.Queue import XMLRPCRunner
from Mailman.Handlers import LPModerate
except ImportError:
# Monkey patches not present, redo install and patch steps.
need_install = True
# Make sure the target directories exist and have the correct
# permissions, otherwise configure will complain.
user, group = as_username_groupname(config.mailman.build_user_group)
# Now work backwards to get the uid and gid
try:
uid = pwd.getpwnam(user).pw_uid
except KeyError:
print >> sys.stderr, 'No user found:', user
sys.exit(1)
try:
gid = grp.getgrnam(group).gr_gid
except KeyError:
print >> sys.stderr, 'No group found:', group
sys.exit(1)
# Ensure that the var_dir exists, is owned by the user:group, and has
# the necessary permissions. Set the mode separately after the
# makedirs() call because some platforms ignore mkdir()'s mode (though
# I think Linux does not ignore it -- better safe than sorry).
try:
os.makedirs(var_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
else:
# Just created the var directory, will need to install mailmain bits.
need_install = True
os.chown(var_dir, uid, gid)
os.chmod(var_dir, 02775)
# Skip mailman setup if nothing so far has shown a reinstall needed.
if not need_install:
return 0
mailman_source = os.path.join('sourcecode', 'mailman')
if config.mailman.build_host_name:
build_host_name = config.mailman.build_host_name
else:
build_host_name = socket.getfqdn()
# Build and install the Mailman software. Note that we don't care about
# --with-cgi-gid because we're not going to use that Mailman subsystem.
executable = os.path.abspath('bin/py')
configure_args = (
'./configure',
'--prefix', mailman_path,
'--with-var-prefix=' + var_dir,
'--with-python=' + executable,
'--with-username='******'--with-groupname=' + group,
'--with-mail-gid=' + group,
'--with-mailhost=' + build_host_name,
'--with-urlhost=' + build_host_name,
)
if need_build:
# Configure.
retcode = subprocess.call(configure_args, cwd=mailman_source)
if retcode:
print >> sys.stderr, 'Could not configure Mailman:'
sys.exit(retcode)
# Make.
retcode = subprocess.call(('make', ), cwd=mailman_source)
if retcode:
print >> sys.stderr, 'Could not make Mailman.'
sys.exit(retcode)
retcode = subprocess.call(('make', 'install'), cwd=mailman_source)
if retcode:
print >> sys.stderr, 'Could not install Mailman.'
sys.exit(retcode)
# Try again to import the package.
try:
import Mailman
except ImportError:
print >> sys.stderr, 'Could not import the Mailman package'
return 1
# Check to see if the site list exists. The output can go to /dev/null
# because we don't really care about it. The site list exists if
# config_list returns a zero exit status, otherwise it doesn't
# (probably). Before we can do this however, we must monkey patch
# Mailman, otherwise mm_cfg.py won't be set up correctly.
monkey_patch(mailman_path, config)
import Mailman.mm_cfg
retcode = subprocess.call(
('./config_list', '-o', '/dev/null',
Mailman.mm_cfg.MAILMAN_SITE_LIST),
cwd=mailman_bin,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if retcode:
addr, password = configure_siteowner(
config.mailman.build_site_list_owner)
# The site list does not yet exist, so create it now.
retcode = subprocess.call(
('./newlist', '--quiet',
'--emailhost=' + build_host_name,
Mailman.mm_cfg.MAILMAN_SITE_LIST,
addr, password),
cwd=mailman_bin)
if retcode:
print >> sys.stderr, 'Could not create site list'
return retcode
retcode = configure_site_list(
mailman_bin, Mailman.mm_cfg.MAILMAN_SITE_LIST)
if retcode:
print >> sys.stderr, 'Could not configure site list'
return retcode
# Create a directory to hold the gzip'd tarballs for the directories of
# deactivated lists.
try:
os.mkdir(os.path.join(Mailman.mm_cfg.VAR_PREFIX, 'backups'))
except OSError as e:
if e.errno != errno.EEXIST:
raise
return 0
