def load(self, envs=os.environ):
"""
Parses the ContainerPilot config file and interpolates the
environment into it the same way that ContainerPilot does.
The `state` attribute will be populated with the state
derived from the configuration file and not the state known
by Consul.
"""
self.path = env('CONTAINERPILOT', None, envs,
lambda x: x.replace('file://', ''))
with open(self.path, 'r') as f:
cfg = f.read()
# remove templating so that we can parse it as JSON; we'll
# override the attributes directly in the resulting dict
cfg = cfg.replace('[{{ if .CONSUL_AGENT }}', '[')
cfg = cfg.replace('}{{ end }}', '}')
# remove templating for SERVICE_NAME
service_name = env('SERVICE_NAME', self.DEFAULT_SERVICE_NAME)
cfg = cfg.replace('{{ if .SERVICE_NAME }}{{ .SERVICE_NAME }}{{ else }}'+self.DEFAULT_SERVICE_NAME+'{{ end }}',service_name)
config = json.loads(cfg)
# if env('CONSUL_AGENT', False, envs, to_flag):
# # config['consul'] = 'localhost:8500'
# # cmd = config['coprocesses'][0]['command']
# # host_cfg_idx = cmd.index('-retry-join') + 1
# # cmd[host_cfg_idx] = env('CONSUL', 'consul', envs)
# # config['coprocesses'][0]['command'] = cmd
# else:
# config['consul'] = env('CONSUL', 'consul', envs,
# fn='{}:8500'.format)
# config['coprocesses'] = []
self.config = config
def __init__(self, envs=os.environ):
self.host = env('SCP_HOST', None, envs)
self.path = env('SCP_PATH', '/srv/backups', envs)
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(self.host)
def load(self, envs=os.environ):
"""
Parses the ContainerPilot config file and interpolates the
environment into it the same way that ContainerPilot does.
The `state` attribute will be populated with the state
derived from the configuration file and not the state known
by Consul.
"""
self.path = env('CONTAINERPILOT', None, envs,
lambda x: x.replace('file://', ''))
with open(self.path, 'r') as f:
cfg = f.read()
# remove templating so that we can parse it as JSON; we'll
# override the attributes directly in the resulting dict
cfg = cfg.replace('[{{ if .CONSUL_AGENT }}', '[')
cfg = cfg.replace('}{{ end }}', '}')
# remove templating for SERVICE_NAME
service_name = env('SERVICE_NAME', 'mysql')
cfg = cfg.replace('{{ if .SERVICE_NAME }}{{ .SERVICE_NAME }}{{ else }}mysql{{ end }}',service_name)
config = json.loads(cfg)
if env('CONSUL_AGENT', False, envs, to_flag):
config['consul'] = 'localhost:8500'
cmd = config['coprocesses'][0]['command']
host_cfg_idx = cmd.index('-retry-join') + 1
cmd[host_cfg_idx] = env('CONSUL', 'consul', envs)
config['coprocesses'][0]['command'] = cmd
else:
config['consul'] = env('CONSUL', 'consul', envs,
fn='{}:8500'.format)
config['coprocesses'] = []
self.config = config
def __init__(self, envs=os.environ):
"""
Figures out the Consul client hostname based on whether or
not we're using a local Consul agent.
"""
if env('CONSUL_AGENT', False, envs, fn=to_flag):
self.host = 'localhost'
else:
self.host = env('CONSUL', 'consul', envs)
self.client = pyconsul.Consul(host=self.host)
def __init__(self, envs=os.environ):
"""
Figures out the Consul client hostname based on whether or
not we're using a local Consul agent.
"""
if env('CONSUL_AGENT', False, envs, fn=to_flag):
self.host = 'localhost'
else:
self.host = env('CONSUL', 'consul', envs)
self.client = pyconsul.Consul(host=self.host)
def __init__(self, envs=os.environ):
self.secret_key = env('AWS_SECRET_KEY', None, envs)
self.access_key = env('AWS_ACCESS_KEY', None, envs)
self.bucket_name = env('AWS_S3_BUCKET', None, envs)
self.region = env('AWS_S3_REGION', 'sa-east-1', envs)
self.client = s3.connect_to_region(
self.region,
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
)
self.bucket = self.client.get_bucket(self.bucket_name)
def __init__(self, envs=os.environ):
self.access = env('AWS_ACCESS_KEY', None, envs)
self.secret = env('AWS_SECRET_ACCESS_KEY', None, envs)
self.region = env('AWS_REGION', 'us-east-1', envs)
self.bucket = env('AWS_BUCKET', None, envs)
# we don't want to use `env` here because we have a different
# de-munging to do
self.client = aws.s3.connect_to_region(
self.region,
aws_access_key_id=self.access,
aws_secret_access_key=self.secret)
self.bucket = self.client.get_bucket(self.bucket)
def main():
"""
Parse argument as command and execute that command with
parameters containing the state of MySQL, ContainerPilot, etc.
Default behavior is to run `pre_start` DB initialization.
"""
if len(sys.argv) == 1:
consul = Consul(envs={'CONSUL': os.environ.get('CONSUL', 'consul')})
cmd = pre_start
else:
consul = Consul()
try:
cmd = globals()[sys.argv[1]]
except KeyError:
log.error('Invalid command: %s', sys.argv[1])
sys.exit(1)
storage_class = env('BACKUP_STORAGE_CLASS', 'manager.libmanta.Manta')
my = MySQL()
backup_store = get_class(storage_class)()
cp = ContainerPilot()
cp.load()
node = Node(mysql=my, consul=consul, backup_store=backup_store, cp=cp)
cmd(node)
def __init__(self, mysql=None, cp=None, consul=None):
self.mysql = mysql
self.consul = consul
self.cp = cp
self.hostname = socket.gethostname()
self.name = 'mysql-{}'.format(self.hostname)
self.ip = get_ip(env('INTERFACE', 'net1'))
def __init__(self, envs=os.environ):
self.bucket_name = env('AWS_S3_BUCKET', None, envs)
self.endpoint = env('AWS_S3_ENDPOINT',
'https://s3.cloud.syseleven.net', envs)
self.access = env('AWS_ACCESS_KEY_ID', None, envs)
self.secret = env('AWS_SECRET_ACCESS_KEY', None, envs)
assert self.bucket_name, "missing bucket"
assert self.endpoint, "missing endpoint"
assert self.access, "missing access key"
assert self.secret, "missing secret"
self.s3 = boto3.resource(
's3',
endpoint_url=self.endpoint,
config=botocore.client.Config(s3={'addressing_style': 'virtual'}))
self.bucket = self.s3.Bucket(self.bucket_name)
def __init__(self, envs=os.environ):
self.mysql_db = env('MYSQL_DATABASE', None, envs)
self.mysql_user = env('MYSQL_USER', None, envs)
self.mysql_password = env('MYSQL_PASSWORD', None, envs)
self.mysql_root_password = env('MYSQL_ROOT_PASSWORD', '', envs)
self.mysql_random_root_password = env('MYSQL_RANDOM_ROOT_PASSWORD',
True, envs, to_flag)
self.mysql_onetime_password = env('MYSQL_ONETIME_PASSWORD',
False, envs, to_flag)
self.repl_user = env('MYSQL_REPL_USER', None, envs)
self.repl_password = env('MYSQL_REPL_PASSWORD', None, envs)
self.datadir = env('MYSQL_DATADIR', '/var/lib/mysql', envs)
self.pool_size = env('INNODB_BUFFER_POOL_SIZE', 0, envs, fn=int)
# state
self.ip = get_ip()
self._conn = None
self._query_buffer = OrderedDict()
def __init__(self, envs=os.environ):
self.mysql_db = env('MYSQL_DATABASE', None, envs)
self.mysql_user = env('MYSQL_USER', None, envs)
self.mysql_password = env('MYSQL_PASSWORD', None, envs)
self.mysql_root_password = env('MYSQL_ROOT_PASSWORD', '', envs)
self.mysql_random_root_password = env('MYSQL_RANDOM_ROOT_PASSWORD',
True, envs, to_flag)
self.mysql_onetime_password = env('MYSQL_ONETIME_PASSWORD', False,
envs, to_flag)
self.repl_user = env('MYSQL_REPL_USER', None, envs)
self.repl_password = env('MYSQL_REPL_PASSWORD', None, envs)
self.datadir = env('MYSQL_DATADIR', '/var/lib/mysql', envs)
self.pool_size = env('INNODB_BUFFER_POOL_SIZE', 0, envs, fn=int)
# state
self.ip = get_ip()
self._conn = None
self._query_buffer = OrderedDict()
def __init__(self, envs=os.environ):
self.account = env('MANTA_USER', None, envs)
self.user = env('MANTA_SUBUSER', None, envs)
self.role = env('MANTA_ROLE', None, envs)
self.key_id = env('MANTA_KEY_ID', None, envs)
self.url = env('MANTA_URL', 'https://us-east.manta.joyent.com', envs)
self.bucket = env('MANTA_BUCKET', '/{}/stor'.format(self.account), envs)
is_tls = env('MANTA_TLS_INSECURE', False, envs, fn=to_flag)
# we don't want to use `env` here because we have a different
# de-munging to do
self.private_key = envs.get('MANTA_PRIVATE_KEY', '').replace('#', '\n')
self.signer = pymanta.PrivateKeySigner(self.key_id, self.private_key)
self.client = pymanta.MantaClient(
self.url,
self.account,
subuser=self.user,
role=self.role,
disable_ssl_certificate_validation=is_tls,
signer=self.signer)
def __init__(self, envs=os.environ):
self.account = env('MANTA_USER', None, envs)
self.user = env('MANTA_SUBUSER', None, envs)
self.role = env('MANTA_ROLE', None, envs)
self.key_id = env('MANTA_KEY_ID', None, envs)
self.url = env('MANTA_URL', 'https://us-east.manta.joyent.com', envs)
self.bucket = env('MANTA_BUCKET', '/{}/stor'.format(self.account),
envs)
is_tls = env('MANTA_TLS_INSECURE', False, envs, fn=to_flag)
# we don't want to use `env` here because we have a different
# de-munging to do
self.private_key = envs.get('MANTA_PRIVATE_KEY', '').replace('#', '\n')
self.signer = pymanta.PrivateKeySigner(self.key_id, self.private_key)
self.client = pymanta.MantaClient(
self.url,
self.account,
subuser=self.user,
role=self.role,
disable_ssl_certificate_validation=is_tls,
signer=self.signer)
""" Module for Consul client wrapper and related tooling. """
from datetime import datetime, timedelta
import fcntl
import json
import os
import time
from manager.utils import debug, env, log, to_flag
# pylint: disable=import-error,invalid-name,dangerous-default-value
import consul as pyconsul
SESSION_CACHE_FILE = env('SESSION_CACHE_FILE', '/tmp/consul-session')
SESSION_NAME = env('SESSION_NAME', 'consul-session')
SESSION_TTL = env('SESSION_TTL', 25, fn=int)
MAX_SESSION = 3600
class Consul(object):
""" Consul represents the Consul instance this node talks to """
def __init__(self, envs=os.environ):
"""
Figures out the Consul client hostname based on whether or
not we're using a local Consul agent.
"""
if env('CONSUL_AGENT', False, envs, fn=to_flag):
self.host = 'localhost'
else:
self.host = env('CONSUL', 'consul', envs)
self.client = pyconsul.Consul(host=self.host)
""" Module for Consul client wrapper and related tooling. """
from datetime import datetime, timedelta
import fcntl
import json
import os
import time
from manager.utils import debug, env, log, to_flag, \
WaitTimeoutError, UnknownPrimary, PRIMARY_KEY, LAST_BACKUP_KEY, \
BACKUP_TTL, BACKUP_LOCK_KEY, LAST_BINLOG_KEY
# pylint: disable=import-error,invalid-name,dangerous-default-value
import consul as pyconsul
SESSION_CACHE_FILE = env('SESSION_CACHE_FILE', '/tmp/mysql-session')
SESSION_NAME = env('SESSION_NAME', 'mysql-primary-lock')
SESSION_TTL = env('SESSION_TTL', 25, fn=int)
FAILOVER_KEY = env('FAILOVER_IN_PROGRESS', 'FAILOVER_IN_PROGRESS')
FAILOVER_SESSION_FILE = env('FAILOVER_SESSION_FILE', '/tmp/failover-session')
MAX_SESSION=3600
class Consul(object):
""" Consul represents the Consul instance this node talks to """
def __init__(self, envs=os.environ):
"""
Figures out the Consul client hostname based on whether or
not we're using a local Consul agent.
"""
if env('CONSUL_AGENT', False, envs, fn=to_flag):
self.host = 'localhost'
""" Module for Consul client wrapper and related tooling. """
from datetime import datetime, timedelta
import fcntl
import json
import os
import time
from manager.utils import debug, env, log, to_flag, \
WaitTimeoutError, UnknownPrimary, PRIMARY_KEY, LAST_BACKUP_KEY, \
BACKUP_TTL, BACKUP_LOCK_KEY, LAST_BINLOG_KEY
# pylint: disable=import-error,invalid-name,dangerous-default-value
import consul as pyconsul
SESSION_CACHE_FILE = env('SESSION_CACHE_FILE', '/tmp/mysql-session')
SESSION_NAME = env('SESSION_NAME', 'mysql-primary-lock')
SESSION_TTL = env('SESSION_TTL', 25, fn=int)
FAILOVER_KEY = env('FAILOVER_IN_PROGRESS', 'FAILOVER_IN_PROGRESS')
FAILOVER_SESSION_FILE = env('FAILOVER_SESSION_FILE', '/tmp/failover-session')
MAX_SESSION = 3600
class Consul(object):
""" Consul represents the Consul instance this node talks to """
def __init__(self, envs=os.environ):
"""
Figures out the Consul client hostname based on whether or
not we're using a local Consul agent.
"""
if env('CONSUL_AGENT', False, envs, fn=to_flag):
self.host = 'localhost'
