def initialize_gpg():
gpghome = os.path.expanduser('~/.dbsake/gpg')
pycompat.makedirs(gpghome, mode=0o0700, exist_ok=True)
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
if not gpg:
raise common.SandboxError("Failed to find gpg")
gpg_cmd = cmd.shell_format('{0} -k 5072E1F5', gpg)
debug(" # Verifying .dbsake/gpg is initialized")
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
if not ret.returncode:
return # all is well
else:
for line in ret.stderr.splitlines():
debug(" # gpg: %s", line)
# else import the mysql key
info(" - Importing mysql public key to %s", gpghome)
key_server = 'pgp.mit.edu'
mysql_key_id = '5072E1F5'
gpg_cmd = cmd.shell_format('{0} --keyserver={1} --recv-keys {2}',
gpg, key_server, mysql_key_id)
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
for line in ret.stderr.splitlines():
debug(" # %s", line)
if ret.returncode != 0:
raise common.SandboxError("Failed to import mysql public key")
def initialize_gpg():
gpghome = os.path.expanduser('~/.dbsake/gpg')
pycompat.makedirs(gpghome, mode=0o0700, exist_ok=True)
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
if not gpg:
raise common.SandboxError("Failed to find gpg")
gpg_cmd = cmd.shell_format('{0} -k 5072E1F5', gpg)
debug(" # Verifying .dbsake/gpg is initialized")
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
if not ret.returncode:
return # all is well
else:
for line in ret.stderr.splitlines():
debug(" # gpg: %s", line)
# else import the mysql key
info(" - Importing mysql public key to %s", gpghome)
key_server = 'pgp.mit.edu'
mysql_key_id = '5072E1F5'
gpg_cmd = cmd.shell_format('{0} --keyserver={1} --recv-keys {2}', gpg,
key_server, mysql_key_id)
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
for line in ret.stderr.splitlines():
debug(" # %s", line)
if ret.returncode != 0:
raise common.SandboxError("Failed to import mysql public key")
def bootstrap(options, distribution, additional_options=()):
start = time.time()
defaults_file = os.path.join(options.basedir, 'my.sandbox.cnf')
logfile = os.path.join(options.basedir, 'bootstrap.log')
info(" - Logging bootstrap output to %s", logfile)
bootstrap_cmd = cmd.shell_format("{0} --defaults-file={1}",
distribution.mysqld, defaults_file)
additional_options = ('--bootstrap',
'--default-storage-engine=myisam') + \
additional_options
additional = ' '.join(map(cmd.shell_quote, additional_options))
if additional:
bootstrap_cmd += ' ' + additional
bootstrap_sql = os.path.join(options.basedir, 'bootstrap.sql')
with codecs.open(bootstrap_sql, 'wb', 'utf8') as fileobj:
os.fchmod(fileobj.fileno(), 0o0660)
for line in mysql_install_db(options, distribution):
print(line, file=fileobj)
info(" - Generated bootstrap SQL")
with open(logfile, 'wb') as stderr:
with open(bootstrap_sql, 'rb') as fileobj:
info(" - Running %s", bootstrap_cmd)
returncode = cmd.run(bootstrap_cmd,
stdin=fileobj,
stdout=stderr,
stderr=stderr)
if returncode != 0:
raise SandboxError("Bootstrapping failed. Details in %s" % stderr.name)
info(" * Bootstrapped sandbox in %.2f seconds", time.time() - start)
def bootstrap(options, distribution, additional_options=()):
start = time.time()
defaults_file = os.path.join(options.basedir, 'my.sandbox.cnf')
logfile = os.path.join(options.basedir, 'bootstrap.log')
info(" - Logging bootstrap output to %s", logfile)
bootstrap_cmd = cmd.shell_format("{0} --defaults-file={1}",
distribution.mysqld, defaults_file)
additional_options = ('--bootstrap',
'--default-storage-engine=myisam') + \
additional_options
additional = ' '.join(map(cmd.shell_quote, additional_options))
if additional:
bootstrap_cmd += ' ' + additional
bootstrap_sql = os.path.join(options.basedir, 'bootstrap.sql')
with codecs.open(bootstrap_sql, 'wb', 'utf8') as fileobj:
os.fchmod(fileobj.fileno(), 0o0660)
for line in mysql_install_db(options, distribution):
print(line, file=fileobj)
info(" - Generated bootstrap SQL")
with open(logfile, 'wb') as stderr:
with open(bootstrap_sql, 'rb') as fileobj:
info(" - Running %s", bootstrap_cmd)
returncode = cmd.run(bootstrap_cmd,
stdin=fileobj,
stdout=stderr,
stderr=stderr)
if returncode != 0:
raise SandboxError("Bootstrapping failed. Details in %s" % stderr.name)
info(" * Bootstrapped sandbox in %.2f seconds", time.time() - start)
def test_unpack_xb_uncompressed_pipe():
runner = CliRunner()
src = os.path.join(os.path.dirname(__file__), 'backup.xb.gz')
with runner.isolated_filesystem():
cat_cmd = cmd.shell_format("zcat {0}", src)
with cmd.piped_stdout(cat_cmd) as stdin:
result = runner.invoke(_unpack, ["-"], obj={}, input=stdin)
assert result.exit_code == 0
def test_unpack_xb_uncompressed_file():
runner = CliRunner()
src = os.path.join(os.path.dirname(__file__), 'backup.xb.gz')
with runner.isolated_filesystem():
cat_cmd = cmd.shell_format("zcat {0}", src)
with cmd.piped_stdout(cat_cmd) as stdin:
with open("backup.xb", "wb") as stdout:
shutil.copyfileobj(stdin, stdout)
result = runner.invoke(_unpack, [src], obj={})
assert result.exit_code == 0
def mysqld_version(mysqld):
"""Discover the MySQL version from a mysqld binary
This method runs mysqld --version and extract out the
version string to create a MySQLVersion tuple. This is used
to allow other portions of dbsake to check the mysql version
in use and take appropriate action. This is largely used by
the generate_defaults method in dbsake.core.mysql.sandbox.common
to conditionally enable my.cnf options based on the target
MySQL version.
:param mysqld: path to mysqld binary
:returns: MySQLVersion instance
"""
version_cmd = cmd.shell_format('{0} --no-defaults --version', mysqld)
try:
result = cmd.capture_stdout(version_cmd)
except OSError as exc: # pragma: no cover
if exc.errno == errno.ENOENT:
if os.path.exists(mysqld):
raise common.SandboxError(("%s is not executable. You may "
"have a tarball not compatible "
"with this system") % mysqld)
else:
raise common.SandboxError(("%s does not exist. Be sure you "
"use a precompiled binary tarball, "
"not a source tarball.") % mysqld)
else:
raise common.SandboxError("%s Failed (errno: %d[%s])" %
(version_cmd,
exc.errno,
errno.errorcode[exc.errno]))
if result.returncode != 0:
raise common.SandboxError("%s failed (exit status: %d)" %
(version_cmd, result.returncode))
m = re.search('(\d+[.]\d+[.]\d+)', result.stdout)
if not m:
raise common.SandboxError("Failed to discover version for %s" %
version_cmd)
return MySQLVersion.from_string(result.stdout)
def mysqld_version(mysqld):
"""Discover the MySQL version from a mysqld binary
This method runs mysqld --version and extract out the
version string to create a MySQLVersion tuple. This is used
to allow other portions of dbsake to check the mysql version
in use and take appropriate action. This is largely used by
the generate_defaults method in dbsake.core.mysql.sandbox.common
to conditionally enable my.cnf options based on the target
MySQL version.
:param mysqld: path to mysqld binary
:returns: MySQLVersion instance
"""
version_cmd = cmd.shell_format('{0} --no-defaults --version', mysqld)
try:
result = cmd.capture_stdout(version_cmd)
except OSError as exc: # pragma: no cover
if exc.errno == errno.ENOENT:
if os.path.exists(mysqld):
raise common.SandboxError(("%s is not executable. You may "
"have a tarball not compatible "
"with this system") % mysqld)
else:
raise common.SandboxError(("%s does not exist. Be sure you "
"use a precompiled binary tarball, "
"not a source tarball.") % mysqld)
else:
raise common.SandboxError(
"%s Failed (errno: %d[%s])" %
(version_cmd, exc.errno, errno.errorcode[exc.errno]))
if result.returncode != 0:
raise common.SandboxError("%s failed (exit status: %d)" %
(version_cmd, result.returncode))
m = re.search('(\d+[.]\d+[.]\d+)', result.stdout)
if not m:
raise common.SandboxError("Failed to discover version for %s" %
version_cmd)
return MySQLVersion.from_string(result.stdout)
def gpg_verify_stream(signature):
gpghome = os.path.expanduser('~/.dbsake/gpg')
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
verify_cmd = cmd.shell_format('{0} --verify {1} -', gpg, signature)
info(" - Verifying gpg signature via: %s", verify_cmd)
try:
with tempfile.TemporaryFile() as output:
try:
with cmd.stream_command(verify_cmd,
stdout=output,
stderr=output,
env={'GNUPGHOME': gpghome}) as stdin:
yield stdin
finally:
output.flush()
output.seek(0)
for line in output:
debug(" # %s", line.rstrip())
except cmd.CommandError:
raise common.SandboxError("gpg signature not valid")
else:
info(" - GPG signature validated")
def prepare_datadir(datadir, options):
innobackupex = pycompat.which('innobackupex')
if not innobackupex:
raise common.SandboxError("innobackupex not found in path. Aborting.")
xb_log = os.path.join(datadir, 'innobackupex.log')
xb_cmd = cmd.shell_format('{0} --apply-log {1!s} .',
innobackupex, options.innobackupex_options)
info(" - Running: %s", xb_cmd)
info(" - (cwd: %s)", datadir)
with open(xb_log, 'wb') as fileobj:
returncode = cmd.run(xb_cmd,
stdout=fileobj,
stderr=fileobj,
cwd=datadir)
if returncode != 0:
info(" ! innobackupex --apply-log failed. See details in %s",
xb_log)
raise common.SandboxError("Data preloading failed")
else:
info(" - innobackupex --apply-log succeeded. datadir is ready.")
def gpg_verify_stream(signature):
gpghome = os.path.expanduser('~/.dbsake/gpg')
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
verify_cmd = cmd.shell_format('{0} --verify {1} -', gpg, signature)
info(" - Verifying gpg signature via: %s", verify_cmd)
try:
with tempfile.TemporaryFile() as output:
try:
with cmd.stream_command(verify_cmd,
stdout=output,
stderr=output,
env={'GNUPGHOME': gpghome}) as stdin:
yield stdin
finally:
output.flush()
output.seek(0)
for line in output:
debug(" # %s", line.rstrip())
except cmd.CommandError:
raise common.SandboxError("gpg signature not valid")
else:
info(" - GPG signature validated")