def create_xtrabackup_command(instance, timestamp, tmp_log):
""" Create a xtrabackup command
Args:
instance - A hostAddr object
timestamp - A timestamp
tmp_log - A path to where xtrabackup should log
Returns:
a list that can be easily ingested by subprocess
"""
if host_utils.get_hiera_role() in host_utils.MASTERFUL_PUPPET_ROLES:
cnf = host_utils.OLD_CONF_ROOT.format(port=instance.port)
cnf_group = 'mysqld'
else:
cnf = host_utils.MYSQL_CNF_FILE
cnf_group = 'mysqld{port}'.format(port=instance.port)
datadir = host_utils.get_cnf_setting('datadir', instance.port)
xtra_user, xtra_pass = mysql_lib.get_mysql_user_for_role(
USER_ROLE_XTRABACKUP)
return XTRABACKUP_CMD.format(datadir=datadir,
xtra_user=xtra_user,
xtra_pass=xtra_pass,
cnf=cnf,
cnf_group=cnf_group,
port=instance.port,
tmp_log=tmp_log).split()
def create_xtrabackup_command(instance, timestamp, tmp_log):
""" Create a xtrabackup command
Args:
instance - A hostAddr object
timestamp - A timestamp
tmp_log - A path to where xtrabackup should log
Returns:
a list that can be easily ingested by subprocess
"""
if host_utils.get_hiera_role() in host_utils.MASTERFUL_PUPPET_ROLES:
cnf = host_utils.OLD_CONF_ROOT.format(port=instance.port)
cnf_group = 'mysqld'
else:
cnf = host_utils.MYSQL_CNF_FILE
cnf_group = 'mysqld{port}'.format(port=instance.port)
datadir = host_utils.get_cnf_setting('datadir', instance.port)
(xtra_user,
xtra_pass) = mysql_lib.get_mysql_user_for_role(USER_ROLE_XTRABACKUP)
return XTRABACKUP_CMD.format(datadir=datadir,
xtra_user=xtra_user,
xtra_pass=xtra_pass,
cnf=cnf,
cnf_group=cnf_group,
port=instance.port,
tmp_log=tmp_log).split()
def xtrabackup_instance(instance):
""" Take a compressed mysql backup
Args:
instance - A hostaddr instance
Returns:
A string of the path to the finished backup
"""
# Prevent issues with too many open files
resource.setrlimit(resource.RLIMIT_NOFILE, (131072, 131072))
(temp_path, target_path) = get_paths(port=str(instance.port))
backup_file = ("mysql-{host}-{port}-{timestamp}.xbstream").format(
host=instance.hostname,
port=str(instance.port),
timestamp=time.strftime('%Y-%m-%d-%H:%M:%S'))
tmp_xtra_path = os.path.join(temp_path, backup_file)
target_xtra_path = os.path.join(target_path, backup_file)
tmp_log = ''.join((tmp_xtra_path, '.log'))
target_log = ''.join((tmp_xtra_path, '.log'))
if host_utils.get_hiera_role() in host_utils.MASTERFUL_PUPPET_ROLES:
cnf = host_utils.OLD_CONF_ROOT.format(port=instance.port)
cnf_group = 'mysqld'
else:
cnf = host_utils.MYSQL_CNF_FILE
cnf_group = 'mysqld{port}'.format(port=instance.port)
datadir = host_utils.get_cnf_setting('datadir', instance.port)
xtra_user, xtra_pass = mysql_lib.get_mysql_user_for_role('xtrabackup')
cmd = ('/bin/bash -c "/usr/bin/innobackupex {datadir} {XTRA_DEFAULTS} '
'--user={xtra_user} --password={xtra_pass} '
'--defaults-file={cnf} --defaults-group={cnf_group} '
'--port={port} 2>{tmp_log} '
'>{dest}"').format(datadir=datadir,
XTRA_DEFAULTS=XTRA_DEFAULTS,
xtra_user=xtra_user,
xtra_pass=xtra_pass,
cnf=cnf,
cnf_group=cnf_group,
port=instance.port,
tmp_log=tmp_log,
dest=tmp_xtra_path)
log.info(cmd)
xtra = subprocess.Popen(cmd, shell=True)
xtra.wait()
with open(tmp_log, 'r') as log_file:
xtra_log = log_file.readlines()
if 'innobackupex: completed OK!' not in xtra_log[-1]:
raise Exception('innobackupex failed. '
'log_file: {tmp_log}'.format(tmp_log=tmp_log))
log.info('Moving backup and log to {target}'.format(target=target_path))
os.rename(tmp_xtra_path, target_xtra_path)
os.rename(tmp_log, target_log)
log.info('Xtrabackup was successful')
return target_xtra_path
def xtrabackup_instance(instance, timestamp):
""" Take a compressed mysql backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
# Prevent issues with too many open files
resource.setrlimit(resource.RLIMIT_NOFILE, (131072, 131072))
(temp_path, target_path) = get_paths(port=str(instance.port))
backup_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime(
'%Y-%m-%d-%H:%M:%S', timestamp),
backup_type=BACKUP_TYPE_XBSTREAM)
tmp_xtra_path = os.path.join(temp_path, backup_file)
target_xtra_path = os.path.join(target_path, backup_file)
tmp_log = ''.join((tmp_xtra_path, '.log'))
target_log = ''.join((tmp_xtra_path, '.log'))
if host_utils.get_hiera_role() in host_utils.MASTERFUL_PUPPET_ROLES:
cnf = host_utils.OLD_CONF_ROOT.format(port=instance.port)
cnf_group = 'mysqld'
else:
cnf = host_utils.MYSQL_CNF_FILE
cnf_group = 'mysqld{port}'.format(port=instance.port)
datadir = host_utils.get_cnf_setting('datadir', instance.port)
xtra_user, xtra_pass = mysql_lib.get_mysql_user_for_role(
USER_ROLE_XTRABACKUP)
cmd = XTRABACKUP_CMD.format(datadir=datadir,
xtra_user=xtra_user,
xtra_pass=xtra_pass,
cnf=cnf,
cnf_group=cnf_group,
port=instance.port,
tmp_log=tmp_log,
tmp_xtra_path=tmp_xtra_path)
log.info(cmd)
xtra = subprocess.Popen(cmd, shell=True)
xtra.wait()
with open(tmp_log, 'r') as log_file:
xtra_log = log_file.readlines()
if INNOBACKUP_OK not in xtra_log[-1]:
raise Exception('innobackupex failed. '
'log_file: {tmp_log}'.format(tmp_log=tmp_log))
log.info('Moving backup and log to {target}'.format(target=target_path))
os.rename(tmp_xtra_path, target_xtra_path)
os.rename(tmp_log, target_log)
log.info('Xtrabackup was successful')
return target_xtra_path
def xtrabackup_instance(instance, timestamp):
""" Take a compressed mysql backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
# Prevent issues with too many open files
resource.setrlimit(resource.RLIMIT_NOFILE, (131072, 131072))
(temp_path, target_path) = get_paths(port=str(instance.port))
backup_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime('%Y-%m-%d-%H:%M:%S',
timestamp),
backup_type=BACKUP_TYPE_XBSTREAM)
tmp_xtra_path = os.path.join(temp_path, backup_file)
target_xtra_path = os.path.join(target_path, backup_file)
tmp_log = ''.join((tmp_xtra_path, '.log'))
target_log = ''.join((tmp_xtra_path, '.log'))
if host_utils.get_hiera_role() in host_utils.MASTERFUL_PUPPET_ROLES:
cnf = host_utils.OLD_CONF_ROOT.format(port=instance.port)
cnf_group = 'mysqld'
else:
cnf = host_utils.MYSQL_CNF_FILE
cnf_group = 'mysqld{port}'.format(port=instance.port)
datadir = host_utils.get_cnf_setting('datadir', instance.port)
xtra_user, xtra_pass = mysql_lib.get_mysql_user_for_role(USER_ROLE_XTRABACKUP)
cmd = XTRABACKUP_CMD.format(datadir=datadir,
xtra_user=xtra_user,
xtra_pass=xtra_pass,
cnf=cnf,
cnf_group=cnf_group,
port=instance.port,
tmp_log=tmp_log,
tmp_xtra_path=tmp_xtra_path)
log.info(cmd)
xtra = subprocess.Popen(cmd, shell=True)
xtra.wait()
with open(tmp_log, 'r') as log_file:
xtra_log = log_file.readlines()
if INNOBACKUP_OK not in xtra_log[-1]:
raise Exception('innobackupex failed. '
'log_file: {tmp_log}'.format(tmp_log=tmp_log))
log.info('Moving backup and log to {target}'.format(target=target_path))
os.rename(tmp_xtra_path, target_xtra_path)
os.rename(tmp_log, target_log)
log.info('Xtrabackup was successful')
return target_xtra_path
def logical_backup_instance(instance, timestamp):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
(temp_path, target_path) = get_paths(port=instance.port)
dump_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime(
'%Y-%m-%d-%H:%M:%S', timestamp),
backup_type=BACKUP_TYPE_LOGICAL)
dump_tmp_path = os.path.join(temp_path, dump_file)
dump_path = os.path.join(target_path, dump_file)
dump_user, dump_pass = mysql_lib.get_mysql_user_for_role(
USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port)
with open(dump_tmp_path, "w") as out:
log.info('Running dump')
log.info('{dump_cmd} | {pigz} > {dump_tmp_path}'
''.format(dump_cmd=dump_cmd,
pigz=PIGZ,
dump_tmp_path=dump_tmp_path))
dump = subprocess.Popen(dump_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=sys.stderr)
pigz = subprocess.Popen(PIGZ,
shell=True,
stdin=dump.stdout,
stdout=out,
stderr=sys.stderr)
dump.wait()
pigz.wait()
if dump.returncode != 0:
raise Exception("Error: mysqldump did not succeed, aborting")
log.info('Moving backup and to {target}'.format(target=target_path))
os.rename(dump_tmp_path, dump_path)
log.info('mysqldump was successful')
return dump_path
def logical_backup_instance(instance, timestamp):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
(temp_path, target_path) = get_paths(port=instance.port)
dump_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime('%Y-%m-%d-%H:%M:%S',
timestamp),
backup_type=BACKUP_TYPE_LOGICAL)
dump_tmp_path = os.path.join(temp_path, dump_file)
dump_path = os.path.join(target_path, dump_file)
dump_user, dump_pass = mysql_lib.get_mysql_user_for_role(USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port)
with open(dump_tmp_path, "w") as out:
log.info('Running dump')
log.info('{dump_cmd} | {pigz} > {dump_tmp_path}'
''.format(dump_cmd=dump_cmd,
pigz=PIGZ,
dump_tmp_path=dump_tmp_path))
dump = subprocess.Popen(dump_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=sys.stderr)
pigz = subprocess.Popen(PIGZ,
shell=True,
stdin=dump.stdout,
stdout=out,
stderr=sys.stderr)
dump.wait()
pigz.wait()
if dump.returncode != 0:
raise Exception("Error: mysqldump did not succeed, aborting")
log.info('Moving backup and to {target}'.format(target=target_path))
os.rename(dump_tmp_path, dump_path)
log.info('mysqldump was successful')
return dump_path
def logical_backup_instance(instance, timestamp, initial_build):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
initial_build - Boolean, if this is being created right after the server
was built
Returns:
A string of the path to the finished backup
"""
backup_file = create_backup_file_name(instance, timestamp,
initial_build,
BACKUP_TYPE_LOGICAL)
(dump_user,
dump_pass) = mysql_lib.get_mysql_user_for_role(USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port).split()
procs = dict()
try:
log.info(' '.join(dump_cmd + ['|']))
procs['mysqldump'] = subprocess.Popen(dump_cmd,
stdout=subprocess.PIPE)
procs['pv'] = create_pv_proc(procs['mysqldump'].stdout)
log.info(' '.join(PIGZ + ['|']))
procs['pigz'] = subprocess.Popen(PIGZ,
stdin=procs['pv'].stdout,
stdout=subprocess.PIPE)
log.info('Uploading backup to s3://{buk}/{key}'
''.format(buk=environment_specific.BACKUP_BUCKET_UPLOAD_MAP[host_utils.get_iam_role()],
key=backup_file))
safe_uploader.safe_upload(precursor_procs=procs,
stdin=procs['pigz'].stdout,
bucket=environment_specific.BACKUP_BUCKET_UPLOAD_MAP[host_utils.get_iam_role()],
key=backup_file)
log.info('mysqldump was successful')
return backup_file
except:
safe_uploader.kill_precursor_procs(procs)
raise
def upgrade_auth_tables(port):
""" Run mysql_upgrade
Args:
port - the port of the instance on localhost to act on
"""
start_mysql(port,
DEFAULTS_FILE_ARG.format(defaults_file=MYSQL_UPGRADE_CNF_FILE))
socket = get_cnf_setting('socket', port)
username, password = mysql_lib.get_mysql_user_for_role('admin')
cmd = ''.join((MYSQL_UPGRADE, ' ', '--upgrade-system-tables ', '-S ',
socket, ' ', '-u ', username, ' ', '-p', password))
log.info(cmd)
(std_out, std_err, return_code) = shell_exec(cmd)
log.info(std_out)
if return_code != 0:
log.warning(std_err)
raise Exception('MySQL Upgrade failed with return code '
'of: {ret}'.format(ret=return_code))
stop_mysql(port)
def logical_backup_instance(instance, timestamp):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
dump_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime('%Y-%m-%d-%H:%M:%S',
timestamp),
backup_type=BACKUP_TYPE_LOGICAL)
(dump_user,
dump_pass) = mysql_lib.get_mysql_user_for_role(USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port)
procs = dict()
try:
procs['mysqldump'] = subprocess.Popen(dump_cmd.split(),
stdout=subprocess.PIPE)
procs['pigz'] = subprocess.Popen(PIGZ,
stdin=procs['mysqldump'].stdout,
stdout=subprocess.PIPE)
log.info('Uploading backup to {buk}/{key}'
''.format(buk=environment_specific.S3_BUCKET,
key=dump_file))
safe_uploader.safe_upload(precursor_procs=procs,
stdin=procs['pigz'].stdout,
bucket=environment_specific.S3_BUCKET,
key=dump_file)
log.info('mysqldump was successful')
except:
safe_uploader.kill_precursor_procs(procs)
raise
def logical_backup_instance(instance, timestamp):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
Returns:
A string of the path to the finished backup
"""
dump_file = BACKUP_FILE.format(hostname=instance.hostname,
port=instance.port,
timestamp=time.strftime(
'%Y-%m-%d-%H:%M:%S', timestamp),
backup_type=BACKUP_TYPE_LOGICAL)
dump_user, dump_pass = mysql_lib.get_mysql_user_for_role(
USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port)
procs = dict()
try:
procs['mysqldump'] = subprocess.Popen(dump_cmd.split(),
stdout=subprocess.PIPE)
procs['pigz'] = subprocess.Popen(PIGZ,
stdin=procs['mysqldump'].stdout,
stdout=subprocess.PIPE)
log.info('Uploading backup to {buk}/{key}'
''.format(buk=environment_specific.S3_BUCKET, key=dump_file))
safe_uploader.safe_upload(precursor_procs=procs,
stdin=procs['pigz'].stdout,
bucket=environment_specific.S3_BUCKET,
key=dump_file)
log.info('mysqldump was successful')
except:
safe_uploader.kill_precursor_procs(procs)
raise
def upgrade_auth_tables(port):
""" Run mysql_upgrade
Args:
port - the port of the instance on localhost to act on
"""
start_mysql(port,
DEFAULTS_FILE_ARG.format(defaults_file=MYSQL_UPGRADE_CNF_FILE))
socket = get_cnf_setting('socket', port)
username, password = mysql_lib.get_mysql_user_for_role('admin')
cmd = '' .join((MYSQL_UPGRADE, ' ',
'--upgrade-system-tables ',
'-S ', socket, ' ',
'-u ', username, ' ',
'-p', password))
log.info(cmd)
(std_out, std_err, return_code) = shell_exec(cmd)
log.info(std_out)
if return_code != 0:
log.warning(std_err)
raise Exception('MySQL Upgrade failed with return code '
'of: {ret}'.format(ret=return_code))
stop_mysql(port)
def logical_backup_instance(instance, timestamp, blackhole=False,
initial_build=False, databases=None):
""" Take a compressed mysqldump backup
Args:
instance - A hostaddr instance
timestamp - A timestamp which will be used to create the backup filename
blackhole - Boolean, if set will backup DBs as blackhole tables
with no indexes or data
initial_build - Boolean, if this is being created right after the server
was built
databases - List, if set backup only a subset of databases
Returns:
An S3 key of the backup.
"""
zk = host_utils.MysqlZookeeper()
try:
replica_type = zk.get_replica_type_from_instance(instance)
except:
# instance is not in production
replica_type = host_utils.REPLICA_ROLE_MASTER
arg_repl_coordinate = ARG_MASTER_DATA \
if replica_type == host_utils.REPLICA_ROLE_MASTER else ARG_SLAVE_DATA
arg_no_data = ARG_NO_DATA if blackhole else ''
if databases:
backup_type = BACKUP_TYPE_PARTIAL_LOGICAL
db_args = ARG_DATABASES.format(dbs=' '.join(databases))
else:
backup_type = BACKUP_TYPE_LOGICAL
db_args = ARG_ALL_DATABASES
arg_replace = ARG_REPLACE if databases == [mysql_lib.METADATA_DB] else ''
dump_user, dump_pass = mysql_lib.get_mysql_user_for_role(USER_ROLE_MYSQLDUMP)
dump_cmd = MYSQLDUMP_CMD.format(dump_user=dump_user,
dump_pass=dump_pass,
host=instance.hostname,
port=instance.port,
db_args=db_args,
net_buffer_length=MAX_INSERT_LENGTH,
arg_repl_coordinate=arg_repl_coordinate,
arg_replace=arg_replace,
arg_no_data=arg_no_data).split()
backup_file = create_backup_file_name(instance, timestamp,
initial_build,
backup_type)
procs = dict()
log.info(' '.join(dump_cmd + ['|']))
procs['mysqldump'] = subprocess.Popen(dump_cmd,
stdout=subprocess.PIPE)
if blackhole:
procs['innodb_to_blackhole'] = subprocess.Popen(
INNODB_TO_BLACKHOLE,
shell=True,
stdin=procs['mysqldump'].stdout,
stdout=subprocess.PIPE)
log.info(' '.join([INNODB_TO_BLACKHOLE, '|']))
# Blackhole only supports indexes up to 1k long, which is shorter
# than InnoDB. We are therefore removing indexes and
# auto_inc columns.
procs['remove_auto_inc_col_arg'] = subprocess.Popen(
REMOVE_AUTO_INC_COL_ARG,
shell=True,
stdin=procs['innodb_to_blackhole'].stdout,
stdout=subprocess.PIPE)
log.info(' '.join([REMOVE_AUTO_INC_COL_ARG, '|']))
procs['remove_auto_inc_start_value'] = subprocess.Popen(
REMOVE_AUTO_INC_START_VALUE,
shell=True,
stdin=procs['remove_auto_inc_col_arg'].stdout,
stdout=subprocess.PIPE)
log.info(' '.join([REMOVE_AUTO_INC_START_VALUE, '|']))
procs['remove_indexes'] = subprocess.Popen(
REMOVE_INDEXES,
shell=True,
stdin=procs['remove_auto_inc_start_value'].stdout,
stdout=subprocess.PIPE)
log.info(' '.join([REMOVE_INDEXES, '|']))
stdout = procs['remove_indexes'].stdout
elif databases == [mysql_lib.METADATA_DB]:
# If we are backing up the metadata db, we don't want to nuke
# existing data, but need to copy existing data over for rbr
# to work.
procs['create_if_not_exists_sed'] = subprocess.Popen(
CREATE_IF_NOT_EXISTS_SED,
shell=True,
stdin=procs['mysqldump'].stdout,
stdout=subprocess.PIPE)
log.info(' '.join([CREATE_IF_NOT_EXISTS_SED, '|']))
stdout = procs['create_if_not_exists_sed'].stdout
else:
stdout = procs['mysqldump'].stdout
log.info(' '.join(PIGZ + ['|']))
procs['pigz'] = subprocess.Popen(PIGZ,
stdin=stdout,
stdout=subprocess.PIPE)
key = safe_uploader.safe_upload(
precursor_procs=procs,
stdin=procs['pigz'].stdout,
bucket=environment_specific.BACKUP_BUCKET_UPLOAD_MAP[host_utils.get_iam_role()],
key=backup_file,
verbose=True)
log.info('mysqldump was successful')
return key
