def do_deploy_async(self, values, sender=None):
log.debug9("TRACE do_deploy_async(databaseserver)")
# Do the magic
#
# In case of error raise an exception
first_instance = True
# Check whether this is the first instance of the database
for value in self._parent.get_instances().values():
if ('databaseserver' == value.get_type()
and self.get_name() != value.get_name()
and self.get_state() in deployed_states):
first_instance = False
break
# If the database name wasn't specified
if 'database' not in values:
# Use the instance name if it was manually specified
if self.get_name()[0].isalpha():
values['database'] = self.get_name()
else:
# Either it was autogenerated or begins with a
# non-alphabetic character; prefix it with db_
values['database'] = "db_%s" % self.get_name()
if 'owner' not in values:
# We'll default to db_owner
values['owner'] = "db_owner"
# We will assume the owner is new until adding them fails
new_owner = True
# Determine if a password was passed in, so we know whether to
# suppress it from the settings list later.
if 'password' in values:
password_provided = True
else:
password_provided = False
if 'postgresql_conf' not in values:
values['postgresql_conf'] = self._settings['postgresql_conf']
if 'pg_hba_conf' not in values:
values['pg_hba_conf'] = self._settings['pg_hba_conf']
# Get the UID and GID of the 'postgres' user
try:
self.pg_uid = pwd.getpwnam('postgres').pw_uid
except KeyError:
raise RolekitError(MISSING_ID,
"Could not retrieve UID for postgres user")
try:
self.pg_gid = grp.getgrnam('postgres').gr_gid
except KeyError:
raise RolekitError(MISSING_ID,
"Could not retrieve GID for postgres group")
if first_instance:
# Initialize the database on the filesystem
initdb_args = ["/usr/bin/postgresql-setup", "--initdb"]
log.debug2("TRACE: Initializing database")
result = yield async .subprocess_future(initdb_args)
if result.status:
# If this fails, it may be just that the filesystem
# has already been initialized. We'll log the message
# and continue.
log.debug1("INITDB: %s" % result.stdout)
# Now we have to start the service to set everything else up
# It's safe to start an already-running service, so we'll
# just always make this call, particularly in case other instances
# exist but aren't running.
log.debug2("TRACE: Starting postgresql.service unit")
try:
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit(
"postgresql.service", "replace")
job_handler.register_job(job_path)
log.debug2("TRACE: unit start job registered")
job_results = yield job_handler.all_jobs_done_future()
log.debug2("TRACE: unit start job concluded")
if any([
x for x in job_results.values()
if x not in ("skipped", "done")
]):
details = ", ".join(
["%s: %s" % item for item in job_results.items()])
log.error("Starting services failed: {}".format(details))
raise RolekitError(
COMMAND_FAILED,
"Starting services failed: %s" % details)
except Exception as e:
log.error("Error received starting unit: {}".format(e))
raise
# Next we create the owner
log.debug2("TRACE: Creating owner of new database")
createuser_args = ["/usr/bin/createuser", values['owner']]
result = yield async .subprocess_future(createuser_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, the user probably already exists
# (such as when we're using db_owner). If the caller was trying to set
# a password, they probably didn't realize this, so we need to throw
# an exception.
log.info1("User {} already exists in the database".format(
values['owner']))
if password_provided:
raise RolekitError(INVALID_SETTING,
"Cannot set password on pre-existing user")
# If no password was specified, we'll continue
new_owner = False
# If no password was requested, generate a random one here
if not password_provided:
values['password'] = generate_password()
log.debug2("TRACE: Creating new database")
createdb_args = [
"/usr/bin/createdb", values['database'], "-O", values['owner']
]
result = yield async .subprocess_future(createdb_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED,
"Creating database failed: %d" % result.status)
# Next, set the password on the owner
# We'll skip this phase if the the user already existed
if new_owner:
log.debug2("TRACE: Setting password for database owner")
pwd_args = [
ROLEKIT_ROLES + "/databaseserver/tools/rk_db_setpwd.py",
"--database", values['database'], "--user", values['owner']
]
result = yield async .subprocess_future(pwd_args,
stdin=values['password'],
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
log.error("Setting owner password failed: {}".format(
result.status))
raise RolekitError(
COMMAND_FAILED,
"Setting owner password failed: %d" % result.status)
# If this password was provided by the user, don't save it to
# the settings for later retrieval. That could be a security
# issue
if password_provided:
values.pop("password", None)
else: # Not a new owner
# Never save the password to settings for an existing owner
log.debug2("TRACE: Owner already exists, not setting password")
values.pop("password", None)
if first_instance:
# Then update the server configuration to accept network
# connections.
log.debug2("TRACE: Opening access to external addresses")
# edit postgresql.conf to add listen_addresses = '*'
conffile = values['postgresql_conf']
bakfile = conffile + ".rksave"
try:
linkfile(conffile, bakfile)
with open(conffile) as f:
conflines = f.readlines()
tweaking_rules = [{
'regex': r"^\s*#?\s*listen_addresses\s*=.*",
'replace': r"listen_addresses = '*'",
'append_if_missing': True
}]
overwrite_safely(
conffile, "".join(_tweak_lines(conflines, tweaking_rules)))
except Exception as e:
log.fatal("Couldn't write {!r}: {}".format(conffile, e))
# At this point, conffile is unmodified, otherwise
# overwrite_safely() would have succeeded
try:
os.unlink(bakfile)
except Exception as x:
if not (isinstance(x, OSError)
and x.errno == errno.ENOENT):
log.error("Couldn't remove {!r}: {}".format(
bakfile, x))
raise RolekitError(
COMMAND_FAILED,
"Opening access to external addresses in '{}'"
"failed: {}".format(conffile, e))
# Edit pg_hba.conf to allow 'md5' auth on IPv4 and
# IPv6 interfaces.
conffile = values['pg_hba_conf']
bakfile = conffile + ".rksave"
try:
linkfile(conffile, bakfile)
with open(conffile) as f:
conflines = f.readlines()
tweaking_rules = [{
'regex': r"^\s*host((?:\s.*)$)",
'replace': r"#host\1"
}, {
'regex':
r"^\s*local(?:\s.*|)$",
'append':
"# Use md5 method for all connections\nhost all all all md5"
}]
overwrite_safely(
conffile, "".join(_tweak_lines(conflines, tweaking_rules)))
except Exception as e:
log.fatal("Couldn't write {!r}: {}".format(conffile, e))
# At this point, conffile is unmodified, otherwise
# overwrite_safely() would have succeeded
try:
os.unlink(bakfile)
except Exception as x:
if not (isinstance(x, OSError)
and x.errno == errno.ENOENT):
log.error("Couldn't remove {!r}: {}".format(
bakfile, x))
# Restore previous postgresql.conf from the backup
conffile = values['postgresql_conf']
bakfile = conffile + ".rksave"
try:
os.rename(bakfile, conffile)
except Exception as x:
log.error(
"Couldn't restore {!r} from backup {!r}: {}".format(
conffile, bakfile, x))
raise RolekitError(
COMMAND_FAILED,
"Changing all connections to use md5 method in '{}'"
"failed: {}".format(values['pg_hba_conf'], e))
# Restart the postgresql server to accept the new configuration
log.debug2("TRACE: Restarting postgresql.service unit")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.RestartUnit(
"postgresql.service", "replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([
x for x in job_results.values()
if x not in ("skipped", "done")
]):
details = ", ".join(
["%s: %s" % item for item in job_results.items()])
raise RolekitError(
COMMAND_FAILED,
"Restarting service failed: %s" % details)
# Create the systemd target definition
target = RoleDeploymentValues(self.get_type(), self.get_name(),
"Database Server")
target.add_required_units(['postgresql.service'])
log.debug2("TRACE: Database server deployed")
yield target
def do_deploy_async(self, values, sender=None):
log.debug9("TRACE: do_deploy_async")
# Run whatever series of actions are needed to deploy
# this role in a meaningful way.
#
import docker
# Get the default cache size
# Find out how much RAM is available on the system
if 'cache_size' not in values:
# Do a late import of psutil. This will only get
# used during a deployment, so we don't need to
# have it as a dependency for rolekit itself
import psutil
# Get the total number of bytes in local system memory
total_ram = psutil.virtual_memory().total
# If 25% of the available memory is less than 1GB, use
# that for the cache.
if total_ram / 4 < GiB_SIZE:
# Set cache_size in MiB
values['cache_size'] = int(total_ram / 4 / MiB_SIZE)
else:
# Cap the default size at 1 GB in MiB
values['cache_size'] = int(GiB_SIZE / MiB_SIZE)
# Set defaults
if "connections" not in values:
values["connections"] = self._DEFAULTS["connections"]
if "threads" not in values:
values["threads"] = self._DEFAULTS["threads"]
# Create a container for memcached and launch that
log.debug2("Enabling the Docker container manager")
# Enable and start the docker service
enable_units(['docker.service'])
log.debug2("Starting the Docker container manager")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit("docker.service", "replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([x for x in job_results.values() if x not in ("skipped", "done")]):
details = ", ".join(["%s: %s" % item for item in job_results.items()])
raise RolekitError(COMMAND_FAILED, "Starting docker.service failed: %s" % details)
log.debug2("Pulling %s image from Docker Hub" % MEMCACHED_DOCKER_IMAGE)
dockerclient = docker.Client(base_url=docker.utils.utils.DEFAULT_UNIX_SOCKET,
version='auto')
# First, pull down the latest version of the memcached container
dockerclient.pull(MEMCACHED_DOCKER_IMAGE, tag="latest")
log.debug2("Creating systemd service unit")
# Generate a systemd service unit for this container
container_unit = SystemdContainerServiceUnit(
image_name = MEMCACHED_DOCKER_IMAGE,
container_name = "memcached_%s" % self.get_name(),
desc="memcached docker container - %s" % self.get_name(),
env = {
"MEMCACHED_CACHE_SIZE": str(values['cache_size']),
"MEMCACHED_CONNECTIONS": str(values['connections']),
"MEMCACHED_THREADS": str(values['threads'])
},
ports = ("{0}:{0}/tcp".format(MEMCACHED_DEFAULT_PORT),
"{0}:{0}/udp".format(MEMCACHED_DEFAULT_PORT))
)
container_unit.write()
# Make systemd load this new unit file
log.debug2("Running systemd daemon-reload")
with SystemdJobHandler() as job_handler:
job_handler.manager.Reload()
# Return the target information
target = RoleDeploymentValues(self.get_type(), self.get_name(),
"Memory Cache")
target.add_required_units(['memcached_%s.service' % self.get_name()])
log.debug9("TRACE: exiting do_deploy_async")
yield target
def do_deploy_async(self, values, sender=None):
log.debug9("TRACE do_deploy_async(databaseserver)")
# Do the magic
#
# In case of error raise an exception
first_instance = True
# Check whether this is the first instance of the database
for value in self._parent.get_instances().values():
if ('databaseserver' == value._type and self._name != value._name
and self.get_state() in deployed_states):
first_instance = False
break
# First, check for all mandatory arguments
if 'database' not in values:
raise RolekitError(INVALID_VALUE, "Database name unset")
if 'owner' not in values:
# We'll default to db_owner
values['owner'] = "db_owner"
# We will assume the owner is new until adding them fails
new_owner = True
# Determine if a password was passed in, so we know whether to
# suppress it from the settings list later.
if 'password' in values:
password_provided = True
else:
password_provided = False
if 'postgresql_conf' not in values:
values['postgresql_conf'] = self._settings['postgresql_conf']
if 'pg_hba_conf' not in values:
values['pg_hba_conf'] = self._settings['pg_hba_conf']
# Get the UID and GID of the 'postgres' user
try:
self.pg_uid = pwd.getpwnam('postgres').pw_uid
except KeyError:
raise RolekitError(MISSING_ID,
"Could not retrieve UID for postgress user")
try:
self.pg_gid = grp.getgrnam('postgres').gr_gid
except KeyError:
raise RolekitError(MISSING_ID,
"Could not retrieve GID for postgress group")
if first_instance:
# Initialize the database on the filesystem
initdb_args = ["/usr/bin/postgresql-setup", "--initdb"]
log.debug2("TRACE: Initializing database")
result = yield async .subprocess_future(initdb_args)
if result.status:
# If this fails, it may be just that the filesystem
# has already been initialized. We'll log the message
# and continue.
log.debug1("INITDB: %s" % result.stdout)
# Now we have to start the service to set everything else up
# It's safe to start an already-running service, so we'll
# just always make this call, particularly in case other instances
# exist but aren't running.
log.debug2("TRACE: Starting postgresql.service unit")
try:
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit(
"postgresql.service", "replace")
job_handler.register_job(job_path)
log.debug2("TRACE: unit start job registered")
job_results = yield job_handler.all_jobs_done_future()
log.debug2("TRACE: unit start job concluded")
if any([
x for x in job_results.values()
if x not in ("skipped", "done")
]):
details = ", ".join(
["%s: %s" % item for item in job_results.items()])
log.error("Starting services failed: {}".format(details))
raise RolekitError(
COMMAND_FAILED,
"Starting services failed: %s" % details)
except Exception as e:
log.error("Error received starting unit: {}".format(e))
raise
# Next we create the owner
log.debug2("TRACE: Creating owner of new database")
createuser_args = ["/usr/bin/createuser", values['owner']]
result = yield async .subprocess_future(createuser_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, the user probably already exists
# (such as when we're using db_owner). If the caller was trying to set
# a password, they probably didn't realize this, so we need to throw
# an exception.
log.info1("User {} already exists in the database".format(
values['owner']))
if password_provided:
raise RolekitError(INVALID_SETTING,
"Cannot set password on pre-existing user")
# If no password was specified, we'll continue
new_owner = False
# If no password was requested, generate a random one here
if not password_provided:
values['password'] = generate_password()
log.debug2("TRACE: Creating new database")
createdb_args = [
"/usr/bin/createdb", values['database'], "-O", values['owner']
]
result = yield async .subprocess_future(createdb_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED,
"Creating database failed: %d" % result.status)
# Next, set the password on the owner
# We'll skip this phase if the the user already existed
if new_owner:
log.debug2("TRACE: Setting password for database owner")
pwd_args = [
ROLEKIT_ROLES + "/databaseserver/tools/rk_db_setpwd.py",
"--database", values['database'], "--user", values['owner']
]
result = yield async .subprocess_future(pwd_args,
stdin=values['password'],
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
log.error("Setting owner password failed: {}".format(
result.status))
raise RolekitError(
COMMAND_FAILED,
"Setting owner password failed: %d" % result.status)
# If this password was provided by the user, don't save it to
# the settings for later retrieval. That could be a security
# issue
if password_provided:
values.pop("password", None)
else: # Not a new owner
# Never save the password to settings for an existing owner
log.debug2("TRACE: Owner already exists, not setting password")
values.pop("password", None)
if first_instance:
# Then update the server configuration to accept network
# connections.
# edit postgresql.conf to add listen_addresses = '*'
log.debug2("TRACE: Opening access to external addresses")
sed_args = [
"/bin/sed", "-e",
"s@^[#]listen_addresses\W*=\W*'.*'@listen_addresses = '\*'@",
"-i.rksave", values['postgresql_conf']
]
result = yield async .subprocess_future(sed_args)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(
COMMAND_FAILED,
"Changing listen_addresses in '%s' failed: %d" %
(values['postgresql_conf'], result.status))
# Edit pg_hba.conf to allow 'md5' auth on IPv4 and
# IPv6 interfaces.
sed_args = [
"/bin/sed", "-e", "s@^host@#host@", "-e",
'/^local/a # Use md5 method for all connections', "-e",
'/^local/a host all all all md5',
"-i.rksave", values['pg_hba_conf']
]
result = yield async .subprocess_future(sed_args)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(
COMMAND_FAILED,
"Changing all connections to use md5 method in '%s' failed: %d"
% (values['pg_hba_conf'], result.status))
# Restart the postgresql server to accept the new configuration
log.debug2("TRACE: Restarting postgresql.service unit")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.RestartUnit(
"postgresql.service", "replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([
x for x in job_results.values()
if x not in ("skipped", "done")
]):
details = ", ".join(
["%s: %s" % item for item in job_results.items()])
raise RolekitError(
COMMAND_FAILED,
"Restarting service failed: %s" % details)
# Create the systemd target definition
#
# We use all of BindsTo, Requires and RequiredBy so we can ensure that
# all database instances are started and stopped together, since
# they're really all a single daemon service.
#
# The intention here is that starting or stopping any role instance or
# the main postgresql server will result in the same action happening
# to all roles. This way, rolekit maintains an accurate view of what
# instances are running and can communicate that to anyone registered
# to listen for notifications.
target = {
'Role': 'databaseserver',
'Instance': self.get_name(),
'Description': "Database Server Role - %s" % self.get_name(),
'BindsTo': ['postgresql.service'],
'Requires': ['postgresql.service'],
'RequiredBy': ['postgresql.service'],
'After': ['syslog.target', 'network.target']
}
log.debug2("TRACE: Database server deployed")
yield target
def installFirewall(self):
"""install firewall"""
log.debug1("%s.installFirewall()", self._log_prefix)
# are there any firewall settings to apply?
if len(self._settings["firewall"]["services"]) + \
len(self._settings["firewall"]["ports"]) < 1:
return
# create firewall client
fw = FirewallClient()
log.debug2("TRACE: Firewall client created")
# Make sure firewalld is running by getting the
# default zone
try:
default_zone = fw.getDefaultZone()
except DBusException:
# firewalld is not running
log.error("Firewalld is not running or rolekit cannot access it")
raise
# save changes to the firewall
try:
fw_changes = self._settings["firewall-changes"]
except KeyError:
fw_changes = { }
log.debug2("TRACE: Checking for zones: {}".format(self._settings))
try:
zones = self._settings["firewall_zones"]
except KeyError:
zones = []
# if firewall_zones setting is empty, use default zone
if len(zones) < 1:
zones = [ default_zone ]
log.debug2("TRACE: default zone {}".format(zones[0]))
for zone in zones:
log.debug2("TRACE: Processing zone {0}".format(zone))
# get permanent zone settings, run-time settings do not need a
# special treatment
z_perm = fw.config().getZoneByName(zone).getSettings()
for service in self._settings["firewall"]["services"]:
try:
fw.addService(zone, service, 0)
except Exception as e:
if not "ALREADY_ENABLED" in str(e):
raise
else:
fw_changes.setdefault(zone, {}).setdefault("services", {}).setdefault(service, []).append("runtime")
if not z_perm.queryService(service):
z_perm.addService(service)
fw_changes.setdefault(zone, {}).setdefault("services", {}).setdefault(service, []).append("permanent")
for port_proto in self._settings["firewall"]["ports"]:
port, proto = port_proto.split("/")
try:
fw.addPort(zone, port, proto, 0)
except Exception as e:
if not "ALREADY_ENABLED" in str(e):
raise
else:
fw_changes.setdefault(zone, {}).setdefault("ports", {}).setdefault(port_proto, []).append("runtime")
if not z_perm.queryPort(port, proto):
z_perm.addPort(port, proto)
fw_changes.setdefault(zone, {}).setdefault("ports", {}).setdefault(port_proto, []).append("permanent")
fw.config().getZoneByName(zone).update(z_perm)
self._settings["firewall-changes"] = fw_changes
self._settings.write()
def do_deploy_async(self, values, sender=None):
log.debug9("TRACE do_deploy_async(databaseserver)")
# Do the magic
#
# In case of error raise an exception
first_instance = True
# Check whether this is the first instance of the database
for value in self._parent.get_instances().values():
if ('databaseserver' == value._type and
self._name != value._name and
self.get_state() in deployed_states):
first_instance = False
break
# First, check for all mandatory arguments
if 'database' not in values:
raise RolekitError(INVALID_VALUE, "Database name unset")
if 'owner' not in values:
# We'll default to db_owner
values['owner'] = "db_owner"
# We will assume the owner is new until adding them fails
new_owner = True
# Determine if a password was passed in, so we know whether to
# suppress it from the settings list later.
if 'password' in values:
password_provided = True
else:
password_provided = False
if 'postgresql_conf' not in values:
values['postgresql_conf'] = self._settings['postgresql_conf']
if 'pg_hba_conf' not in values:
values['pg_hba_conf'] = self._settings['pg_hba_conf']
# Get the UID and GID of the 'postgres' user
try:
self.pg_uid = pwd.getpwnam('postgres').pw_uid
except KeyError:
raise RolekitError(MISSING_ID, "Could not retrieve UID for postgress user")
try:
self.pg_gid = grp.getgrnam('postgres').gr_gid
except KeyError:
raise RolekitError(MISSING_ID, "Could not retrieve GID for postgress group")
if first_instance:
# Initialize the database on the filesystem
initdb_args = ["/usr/bin/postgresql-setup", "--initdb"]
log.debug2("TRACE: Initializing database")
result = yield async.subprocess_future(initdb_args)
if result.status:
# If this fails, it may be just that the filesystem
# has already been initialized. We'll log the message
# and continue.
log.debug1("INITDB: %s" % result.stdout)
# Now we have to start the service to set everything else up
# It's safe to start an already-running service, so we'll
# just always make this call, particularly in case other instances
# exist but aren't running.
log.debug2("TRACE: Starting postgresql.service unit")
try:
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit("postgresql.service", "replace")
job_handler.register_job(job_path)
log.debug2("TRACE: unit start job registered")
job_results = yield job_handler.all_jobs_done_future()
log.debug2("TRACE: unit start job concluded")
if any([x for x in job_results.values() if x not in ("skipped", "done")]):
details = ", ".join(["%s: %s" % item for item in job_results.items()])
log.error("Starting services failed: {}".format(details))
raise RolekitError(COMMAND_FAILED, "Starting services failed: %s" % details)
except Exception as e:
log.error("Error received starting unit: {}".format(e))
raise
# Next we create the owner
log.debug2("TRACE: Creating owner of new database")
createuser_args = ["/usr/bin/createuser", values['owner']]
result = yield async.subprocess_future(createuser_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, the user probably already exists
# (such as when we're using db_owner). If the caller was trying to set
# a password, they probably didn't realize this, so we need to throw
# an exception.
log.info1("User {} already exists in the database".format(
values['owner']))
if password_provided:
raise RolekitError(INVALID_SETTING,
"Cannot set password on pre-existing user")
# If no password was specified, we'll continue
new_owner = False
# If no password was requested, generate a random one here
if not password_provided:
values['password'] = generate_password()
log.debug2("TRACE: Creating new database")
createdb_args = ["/usr/bin/createdb", values['database'],
"-O", values['owner']]
result = yield async.subprocess_future(createdb_args,
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED,
"Creating database failed: %d" % result.status)
# Next, set the password on the owner
# We'll skip this phase if the the user already existed
if new_owner:
log.debug2("TRACE: Setting password for database owner")
pwd_args = [ROLEKIT_ROLES + "/databaseserver/tools/rk_db_setpwd.py",
"--database", values['database'],
"--user", values['owner']]
result = yield async.subprocess_future(pwd_args,
stdin=values['password'],
uid=self.pg_uid,
gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
log.error("Setting owner password failed: {}".format(result.status))
raise RolekitError(COMMAND_FAILED,
"Setting owner password failed: %d" %
result.status)
# If this password was provided by the user, don't save it to
# the settings for later retrieval. That could be a security
# issue
if password_provided:
values.pop("password", None)
else: # Not a new owner
# Never save the password to settings for an existing owner
log.debug2("TRACE: Owner already exists, not setting password")
values.pop("password", None)
if first_instance:
# Then update the server configuration to accept network
# connections.
# edit postgresql.conf to add listen_addresses = '*'
log.debug2("TRACE: Opening access to external addresses")
sed_args = [ "/bin/sed",
"-e", "s@^[#]listen_addresses\W*=\W*'.*'@listen_addresses = '\*'@",
"-i.rksave", values['postgresql_conf'] ]
result = yield async.subprocess_future(sed_args)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED,
"Changing listen_addresses in '%s' failed: %d" %
(values['postgresql_conf'], result.status))
# Edit pg_hba.conf to allow 'md5' auth on IPv4 and
# IPv6 interfaces.
sed_args = [ "/bin/sed",
"-e", "s@^host@#host@",
"-e", '/^local/a # Use md5 method for all connections',
"-e", '/^local/a host all all all md5',
"-i.rksave", values['pg_hba_conf'] ]
result = yield async.subprocess_future(sed_args)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED,
"Changing all connections to use md5 method in '%s' failed: %d" %
(values['pg_hba_conf'], result.status))
# Restart the postgresql server to accept the new configuration
log.debug2("TRACE: Restarting postgresql.service unit")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.RestartUnit("postgresql.service", "replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([x for x in job_results.values() if x not in ("skipped", "done")]):
details = ", ".join(["%s: %s" % item for item in job_results.items()])
raise RolekitError(COMMAND_FAILED, "Restarting service failed: %s" % details)
# Create the systemd target definition
#
# We use all of BindsTo, Requires and RequiredBy so we can ensure that
# all database instances are started and stopped together, since
# they're really all a single daemon service.
#
# The intention here is that starting or stopping any role instance or
# the main postgresql server will result in the same action happening
# to all roles. This way, rolekit maintains an accurate view of what
# instances are running and can communicate that to anyone registered
# to listen for notifications.
target = {'Role': 'databaseserver',
'Instance': self.get_name(),
'Description': "Database Server Role - %s" %
self.get_name(),
'BindsTo': ['postgresql.service'],
'Requires': ['postgresql.service'],
'RequiredBy': ['postgresql.service'],
'After': ['syslog.target', 'network.target']}
log.debug2("TRACE: Database server deployed")
yield target
def do_deploy_async(self, values, sender=None):
log.debug9("TRACE do_deploy_async(databaseserver)")
# Do the magic
#
# In case of error raise an exception
first_instance = True
# Check whether this is the first instance of the database
for value in self._parent.get_instances().values():
if (
"databaseserver" == value.get_type()
and self.get_name() != value.get_name()
and self.get_state() in deployed_states
):
first_instance = False
break
# If the database name wasn't specified
if "database" not in values:
# Use the instance name if it was manually specified
if self.get_name()[0].isalpha():
values["database"] = self.get_name()
else:
# Either it was autogenerated or begins with a
# non-alphabetic character; prefix it with db_
values["database"] = "db_%s" % self.get_name()
if "owner" not in values:
# We'll default to db_owner
values["owner"] = "db_owner"
# We will assume the owner is new until adding them fails
new_owner = True
# Determine if a password was passed in, so we know whether to
# suppress it from the settings list later.
if "password" in values:
password_provided = True
else:
password_provided = False
if "postgresql_conf" not in values:
values["postgresql_conf"] = self._settings["postgresql_conf"]
if "pg_hba_conf" not in values:
values["pg_hba_conf"] = self._settings["pg_hba_conf"]
# Get the UID and GID of the 'postgres' user
try:
self.pg_uid = pwd.getpwnam("postgres").pw_uid
except KeyError:
raise RolekitError(MISSING_ID, "Could not retrieve UID for postgres user")
try:
self.pg_gid = grp.getgrnam("postgres").gr_gid
except KeyError:
raise RolekitError(MISSING_ID, "Could not retrieve GID for postgres group")
if first_instance:
# Initialize the database on the filesystem
initdb_args = ["/usr/bin/postgresql-setup", "--initdb"]
log.debug2("TRACE: Initializing database")
result = yield async.subprocess_future(initdb_args)
if result.status:
# If this fails, it may be just that the filesystem
# has already been initialized. We'll log the message
# and continue.
log.debug1("INITDB: %s" % result.stdout)
# Now we have to start the service to set everything else up
# It's safe to start an already-running service, so we'll
# just always make this call, particularly in case other instances
# exist but aren't running.
log.debug2("TRACE: Starting postgresql.service unit")
try:
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit("postgresql.service", "replace")
job_handler.register_job(job_path)
log.debug2("TRACE: unit start job registered")
job_results = yield job_handler.all_jobs_done_future()
log.debug2("TRACE: unit start job concluded")
if any([x for x in job_results.values() if x not in ("skipped", "done")]):
details = ", ".join(["%s: %s" % item for item in job_results.items()])
log.error("Starting services failed: {}".format(details))
raise RolekitError(COMMAND_FAILED, "Starting services failed: %s" % details)
except Exception as e:
log.error("Error received starting unit: {}".format(e))
raise
# Next we create the owner
log.debug2("TRACE: Creating owner of new database")
createuser_args = ["/usr/bin/createuser", values["owner"]]
result = yield async.subprocess_future(createuser_args, uid=self.pg_uid, gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, the user probably already exists
# (such as when we're using db_owner). If the caller was trying to set
# a password, they probably didn't realize this, so we need to throw
# an exception.
log.info1("User {} already exists in the database".format(values["owner"]))
if password_provided:
raise RolekitError(INVALID_SETTING, "Cannot set password on pre-existing user")
# If no password was specified, we'll continue
new_owner = False
# If no password was requested, generate a random one here
if not password_provided:
values["password"] = generate_password()
log.debug2("TRACE: Creating new database")
createdb_args = ["/usr/bin/createdb", values["database"], "-O", values["owner"]]
result = yield async.subprocess_future(createdb_args, uid=self.pg_uid, gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
raise RolekitError(COMMAND_FAILED, "Creating database failed: %d" % result.status)
# Next, set the password on the owner
# We'll skip this phase if the the user already existed
if new_owner:
log.debug2("TRACE: Setting password for database owner")
pwd_args = [
ROLEKIT_ROLES + "/databaseserver/tools/rk_db_setpwd.py",
"--database",
values["database"],
"--user",
values["owner"],
]
result = yield async.subprocess_future(pwd_args, stdin=values["password"], uid=self.pg_uid, gid=self.pg_gid)
if result.status:
# If the subprocess returned non-zero, raise an exception
log.error("Setting owner password failed: {}".format(result.status))
raise RolekitError(COMMAND_FAILED, "Setting owner password failed: %d" % result.status)
# If this password was provided by the user, don't save it to
# the settings for later retrieval. That could be a security
# issue
if password_provided:
values.pop("password", None)
else: # Not a new owner
# Never save the password to settings for an existing owner
log.debug2("TRACE: Owner already exists, not setting password")
values.pop("password", None)
if first_instance:
# Then update the server configuration to accept network
# connections.
log.debug2("TRACE: Opening access to external addresses")
# edit postgresql.conf to add listen_addresses = '*'
conffile = values["postgresql_conf"]
bakfile = conffile + ".rksave"
try:
linkfile(conffile, bakfile)
with open(conffile) as f:
conflines = f.readlines()
tweaking_rules = [
{
"regex": r"^\s*#?\s*listen_addresses\s*=.*",
"replace": r"listen_addresses = '*'",
"append_if_missing": True,
}
]
overwrite_safely(conffile, "".join(_tweak_lines(conflines, tweaking_rules)))
except Exception as e:
log.fatal("Couldn't write {!r}: {}".format(conffile, e))
# At this point, conffile is unmodified, otherwise
# overwrite_safely() would have succeeded
try:
os.unlink(bakfile)
except Exception as x:
if not (isinstance(x, OSError) and x.errno == errno.ENOENT):
log.error("Couldn't remove {!r}: {}".format(bakfile, x))
raise RolekitError(
COMMAND_FAILED, "Opening access to external addresses in '{}'" "failed: {}".format(conffile, e)
)
# Edit pg_hba.conf to allow 'md5' auth on IPv4 and
# IPv6 interfaces.
conffile = values["pg_hba_conf"]
bakfile = conffile + ".rksave"
try:
linkfile(conffile, bakfile)
with open(conffile) as f:
conflines = f.readlines()
tweaking_rules = [
{"regex": r"^\s*host((?:\s.*)$)", "replace": r"#host\1"},
{
"regex": r"^\s*local(?:\s.*|)$",
"append": "# Use md5 method for all connections\nhost all all all md5",
},
]
overwrite_safely(conffile, "".join(_tweak_lines(conflines, tweaking_rules)))
except Exception as e:
log.fatal("Couldn't write {!r}: {}".format(conffile, e))
# At this point, conffile is unmodified, otherwise
# overwrite_safely() would have succeeded
try:
os.unlink(bakfile)
except Exception as x:
if not (isinstance(x, OSError) and x.errno == errno.ENOENT):
log.error("Couldn't remove {!r}: {}".format(bakfile, x))
# Restore previous postgresql.conf from the backup
conffile = values["postgresql_conf"]
bakfile = conffile + ".rksave"
try:
os.rename(bakfile, conffile)
except Exception as x:
log.error("Couldn't restore {!r} from backup {!r}: {}".format(conffile, bakfile, x))
raise RolekitError(
COMMAND_FAILED,
"Changing all connections to use md5 method in '{}'" "failed: {}".format(values["pg_hba_conf"], e),
)
# Restart the postgresql server to accept the new configuration
log.debug2("TRACE: Restarting postgresql.service unit")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.RestartUnit("postgresql.service", "replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([x for x in job_results.values() if x not in ("skipped", "done")]):
details = ", ".join(["%s: %s" % item for item in job_results.items()])
raise RolekitError(COMMAND_FAILED, "Restarting service failed: %s" % details)
# Create the systemd target definition
target = RoleDeploymentValues(self.get_type(), self.get_name(), "Database Server")
target.add_required_units(["postgresql.service"])
log.debug2("TRACE: Database server deployed")
yield target
def do_deploy_async(self, values, sender=None):
log.debug9("TRACE: do_deploy_async")
# Run whatever series of actions are needed to deploy
# this role in a meaningful way.
#
import docker
# Get the default cache size
# Find out how much RAM is available on the system
if 'cache_size' not in values:
# Do a late import of psutil. This will only get
# used during a deployment, so we don't need to
# have it as a dependency for rolekit itself
import psutil
# Get the total number of bytes in local system memory
total_ram = psutil.virtual_memory().total
# If 25% of the available memory is less than 1GB, use
# that for the cache.
if total_ram / 4 < GiB_SIZE:
# Set cache_size in MiB
values['cache_size'] = int(total_ram / 4 / MiB_SIZE)
else:
# Cap the default size at 1 GB in MiB
values['cache_size'] = int(GiB_SIZE / MiB_SIZE)
# Set defaults
if "connections" not in values:
values["connections"] = self._DEFAULTS["connections"]
if "threads" not in values:
values["threads"] = self._DEFAULTS["threads"]
# Create a container for memcached and launch that
log.debug2("Enabling the Docker container manager")
# Enable and start the docker service
enable_units(['docker.service'])
log.debug2("Starting the Docker container manager")
with SystemdJobHandler() as job_handler:
job_path = job_handler.manager.StartUnit("docker.service",
"replace")
job_handler.register_job(job_path)
job_results = yield job_handler.all_jobs_done_future()
if any([
x for x in job_results.values()
if x not in ("skipped", "done")
]):
details = ", ".join(
["%s: %s" % item for item in job_results.items()])
raise RolekitError(
COMMAND_FAILED,
"Starting docker.service failed: %s" % details)
log.debug2("Pulling %s image from Docker Hub" % MEMCACHED_DOCKER_IMAGE)
dockerclient = docker.Client(
base_url=docker.utils.utils.DEFAULT_UNIX_SOCKET, version='auto')
# First, pull down the latest version of the memcached container
dockerclient.pull(MEMCACHED_DOCKER_IMAGE, tag="latest")
log.debug2("Creating systemd service unit")
# Generate a systemd service unit for this container
container_unit = SystemdContainerServiceUnit(
image_name=MEMCACHED_DOCKER_IMAGE,
container_name="memcached_%s" % self.get_name(),
desc="memcached docker container - %s" % self.get_name(),
env={
"MEMCACHED_CACHE_SIZE": str(values['cache_size']),
"MEMCACHED_CONNECTIONS": str(values['connections']),
"MEMCACHED_THREADS": str(values['threads'])
},
ports=("{0}:{0}/tcp".format(MEMCACHED_DEFAULT_PORT),
"{0}:{0}/udp".format(MEMCACHED_DEFAULT_PORT)))
container_unit.write()
# Make systemd load this new unit file
log.debug2("Running systemd daemon-reload")
with SystemdJobHandler() as job_handler:
job_handler.manager.Reload()
# Return the target information
target = RoleDeploymentValues(self.get_type(), self.get_name(),
"Memory Cache")
target.add_required_units(['memcached_%s.service' % self.get_name()])
log.debug9("TRACE: exiting do_deploy_async")
yield target
