def initialize_gpg():
gpghome = os.path.expanduser('~/.dbsake/gpg')
pycompat.makedirs(gpghome, mode=0o0700, exist_ok=True)
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
if not gpg:
raise common.SandboxError("Failed to find gpg")
gpg_cmd = cmd.shell_format('{0} -k 5072E1F5', gpg)
debug(" # Verifying .dbsake/gpg is initialized")
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
if not ret.returncode:
return # all is well
else:
for line in ret.stderr.splitlines():
debug(" # gpg: %s", line)
# else import the mysql key
info(" - Importing mysql public key to %s", gpghome)
key_server = 'pgp.mit.edu'
mysql_key_id = '5072E1F5'
gpg_cmd = cmd.shell_format('{0} --keyserver={1} --recv-keys {2}',
gpg, key_server, mysql_key_id)
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
for line in ret.stderr.splitlines():
debug(" # %s", line)
if ret.returncode != 0:
raise common.SandboxError("Failed to import mysql public key")
def initialize_gpg():
gpghome = os.path.expanduser('~/.dbsake/gpg')
pycompat.makedirs(gpghome, mode=0o0700, exist_ok=True)
gpg = pycompat.which('gpg') or pycompat.which('gpg2')
if not gpg:
raise common.SandboxError("Failed to find gpg")
gpg_cmd = cmd.shell_format('{0} -k 5072E1F5', gpg)
debug(" # Verifying .dbsake/gpg is initialized")
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
if not ret.returncode:
return # all is well
else:
for line in ret.stderr.splitlines():
debug(" # gpg: %s", line)
# else import the mysql key
info(" - Importing mysql public key to %s", gpghome)
key_server = 'pgp.mit.edu'
mysql_key_id = '5072E1F5'
gpg_cmd = cmd.shell_format('{0} --keyserver={1} --recv-keys {2}', gpg,
key_server, mysql_key_id)
ret = cmd.capture_both(gpg_cmd, env={'GNUPGHOME': gpghome})
for line in ret.stderr.splitlines():
debug(" # %s", line)
if ret.returncode != 0:
raise common.SandboxError("Failed to import mysql public key")
def download_tarball_asc(options):
"""Download the signature for a tarball"""
version = options.distribution
cdn = MySQLCDNInfo.from_version(version)
for url in cdn:
try:
stream = _urllib.urlopen(url + '.asc')
except _urllib.HTTPError as exc:
if exc.code != 404:
raise common.SandboxError("Failed to download: %s" % exc)
else:
continue
except _urllib.URLError as exc:
raise common.SandboxError("Failed to download: %s" % exc)
else:
break # stream was opened successfully
else:
raise common.SandboxError("GPG signature not found for %s" % version)
asc_path = discover_cache_path(cdn.name + '.asc')
pycompat.makedirs(os.path.dirname(asc_path), exist_ok=True)
with open(asc_path, 'wb') as fileobj:
fileobj.write(stream.read())
debug(" # Wrote %s", fileobj.name)
return fileobj.name
def download_tarball_asc(options):
"""Download the signature for a tarball"""
version = options.distribution
cdn = MySQLCDNInfo.from_version(version)
for url in cdn:
try:
stream = _urllib.urlopen(url + '.asc')
except _urllib.HTTPError as exc:
if exc.code != 404:
raise common.SandboxError("Failed to download: %s" % exc)
else:
continue
except _urllib.URLError as exc:
raise common.SandboxError("Failed to download: %s" % exc)
else:
break # stream was opened successfully
else:
raise common.SandboxError("GPG signature not found for %s" % version)
asc_path = discover_cache_path(cdn.name + '.asc')
pycompat.makedirs(os.path.dirname(asc_path), exist_ok=True)
with open(asc_path, 'wb') as fileobj:
fileobj.write(stream.read())
debug(" # Wrote %s", fileobj.name)
return fileobj.name
def test_exist_ok_existing_directory(base):
path = os.path.join(base, 'dir1')
mode = 0o777
old_mask = os.umask(0o022)
pycompat.makedirs(path, mode)
with pytest.raises(OSError):
pycompat.makedirs(path, mode)
pycompat.makedirs(path, mode, exist_ok=False)
pycompat.makedirs(path, 0o776, exist_ok=True)
pycompat.makedirs(path, mode=mode, exist_ok=True)
os.umask(old_mask)
def cache_download(name):
"""Cache a download in the specified path
This is a context manager that provides a file object to write a cached
download to. This is used internally by the distribution_from_download
method.
:param name: path to write a cached download ot
"""
pycompat.makedirs(os.path.dirname(name), exist_ok=True)
with open(name, 'wb') as fileobj:
yield fileobj
def cache_download(name):
"""Cache a download in the specified path
This is a context manager that provides a file object to write a cached
download to. This is used internally by the distribution_from_download
method.
:param name: path to write a cached download ot
"""
pycompat.makedirs(os.path.dirname(name), exist_ok=True)
with open(name, 'wb') as fileobj:
yield fileobj
def test_makedir(base):
path = os.path.join(base, 'dir1', 'dir2', 'dir3', '')
pycompat.makedirs(path)
path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4')
pycompat.makedirs(path)
# Try paths with a '.' in them
with pytest.raises(OSError):
pycompat.makedirs(os.curdir)
path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5',
os.curdir)
pycompat.makedirs(path)
path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4',
'dir5', 'dir6')
pycompat.makedirs(path)
def prepare_sandbox_paths(sbopts):
start = time.time()
for path in (sbopts.datadir, os.path.join(sbopts.basedir, 'tmp')):
try:
if pycompat.makedirs(path, exist_ok=True):
info(" - Created %s", path)
except OSError as exc:
raise SandboxError("%s" % exc)
info(" * Prepared sandbox in %.2f seconds", time.time() - start)
def prepare_sandbox_paths(sbopts):
start = time.time()
for path in (sbopts.datadir, os.path.join(sbopts.basedir, 'tmp')):
try:
if pycompat.makedirs(path, exist_ok=True):
info(" - Created %s", path)
except OSError as exc:
raise SandboxError("%s" % exc)
info(" * Prepared sandbox in %.2f seconds", time.time() - start)
def test_exist_ok_s_isgid_directory(base):
path = os.path.join(base, 'dir1')
S_ISGID = stat.S_ISGID
mode = 0o777
old_mask = os.umask(0o022)
try:
existing_testfn_mode = stat.S_IMODE(os.lstat(base).st_mode)
try:
os.chmod(base, existing_testfn_mode | S_ISGID)
except OSError as exc:
if exc.errno == errno.EPERM:
pytest.skip('Cannot set S_ISGID for dir.')
else:
raise
if (os.lstat(base).st_mode & S_ISGID != S_ISGID):
pytest.skip('No support for S_ISGID dir mode.')
# The os should apply S_ISGID from the parent dir for us, but
# this test need not depend on that behavior. Be explicit.
pycompat.makedirs(path, mode | S_ISGID)
# http://bugs.python.org/issue14992
# Should not fail when the bit is already set.
pycompat.makedirs(path, mode, exist_ok=True)
# remove the bit.
os.chmod(path, stat.S_IMODE(os.lstat(path).st_mode) & ~S_ISGID)
# May work even when the bit is not already set when demanded.
pycompat.makedirs(path, mode | S_ISGID, exist_ok=True)
finally:
os.umask(old_mask)
def sieve(options):
if options.output_format == 'directory':
pycompat.makedirs(options.directory, exist_ok=True)
if not options.table_schema:
options.exclude_section('tablestructure')
options.exclude_section('view_temporary')
options.exclude_section('view')
if not options.table_data:
options.exclude_section('tabledata')
if options.routines is False:
options.exclude_section('routines')
if options.events is False:
options.exclude_section('events')
if options.triggers is False:
options.exclude_section('triggers')
with compression.decompressed(options.input_stream) as input_stream:
dump_parser = parser.DumpParser(stream=input_stream)
filter_section = filters.SectionFilter(options)
transform_section = transform.SectionTransform(options)
write_section = writers.load(options, context=transform_section)
stats = collections.defaultdict(int)
for section in dump_parser:
if filter_section(section):
continue
stats[section.name] += 1
transform_section(section)
write_section(section)
return stats
def test_exist_ok_existing_regular_file(base):
base = base
path = os.path.join(base, 'dir1')
f = open(path, 'w')
f.write('abc')
f.close()
with pytest.raises(OSError):
pycompat.makedirs(path)
pycompat.makedirs(path, exist_ok=False)
pycompat.makedirs(path, exist_ok=True)
os.remove(path)
def distribution_from_system(options):
"""Deploy a MySQL distribution already installed on the system
"""
info(" - Deploying MySQL distributed from system binaries")
envpath = os.pathsep.join(['/usr/libexec',
'/usr/sbin',
os.environ['PATH']])
mysqld = pycompat.which('mysqld', path=envpath)
mysql = pycompat.which('mysql', path=envpath)
mysqld_safe = pycompat.which('mysqld_safe', path=envpath)
if None in (mysqld, mysql, mysqld_safe):
raise common.SandboxError("Unable to find MySQL binaries")
debug(" # Found mysqld: %s", mysqld)
debug(" # Found mysqld_safe: %s", mysqld_safe)
debug(" # Found mysql: %s", mysql)
version = mysqld_version(mysqld)
debug(" # MySQL server version: %s", version)
# XXX: we might be able to look this up from mysqld --help --verbose,
# but I really want to avoid that. This shold cover 99% of local
# cases and I think it's fine to abort if this doesn't exist
basedir = '/usr'
debug(" # MySQL --basedir %s", basedir)
# sharedir is absolutely required as we need it to bootstrap mysql
# and mysql will fail to start withtout it
join = os.path.join
exists = os.path.exists
share_search_path = ['share/mysql']
if 'Percona Server' in version.comment:
share_search_path = ['share/percona-server'] + share_search_path
sharedir = first_subdir(basedir, *share_search_path)
if not sharedir:
raise common.SandboxError("MySQL share directory not found (%s)" %
','.join(os.path.join(os.sep, 'usr', path)
for path in share_search_path))
for script in ('fill_help_tables.sql',
'mysql_system_tables_data.sql',
'mysql_system_tables.sql'):
if not exists(join(sharedir, script)):
raise common.SandboxError("MySQL bootstrap SQL '%s' not found." %
os.path.join(sharedir, script))
debug(" # MySQL share found in %s", sharedir)
# Note: plugindir may be None, if using mysql < 5.1
plugindir = first_subdir(basedir, 'lib64/mysql/plugin', 'lib/mysql/plugin')
if plugindir:
debug(" # Found MySQL plugin directory: %s", plugindir)
# now copy mysqld, mysql, mysqld_safe to sandbox_dir/bin
# then return an appropriate MySQLDistribution instance
bindir = os.path.join(options.basedir, 'bin')
pycompat.makedirs(bindir, 0o0770, exist_ok=True)
for name in [mysqld]:
shutil.copy2(name, bindir)
debug(" # Copied minimal MySQL commands to %s", bindir)
return MySQLDistribution(
version=version,
mysqld=os.path.join(bindir, os.path.basename(mysqld)),
mysqld_safe=mysqld_safe,
mysql=mysql,
basedir=basedir,
sharedir=sharedir,
libexecdir=bindir,
plugindir=plugindir
)
def distribution_from_system(options):
"""Deploy a MySQL distribution already installed on the system
"""
info(" - Deploying MySQL distributed from system binaries")
envpath = os.pathsep.join(
['/usr/libexec', '/usr/sbin', os.environ['PATH']])
mysqld = pycompat.which('mysqld', path=envpath)
mysql = pycompat.which('mysql', path=envpath)
mysqld_safe = pycompat.which('mysqld_safe', path=envpath)
if None in (mysqld, mysql, mysqld_safe):
raise common.SandboxError("Unable to find MySQL binaries")
debug(" # Found mysqld: %s", mysqld)
debug(" # Found mysqld_safe: %s", mysqld_safe)
debug(" # Found mysql: %s", mysql)
version = mysqld_version(mysqld)
debug(" # MySQL server version: %s", version)
# XXX: we might be able to look this up from mysqld --help --verbose,
# but I really want to avoid that. This shold cover 99% of local
# cases and I think it's fine to abort if this doesn't exist
basedir = '/usr'
debug(" # MySQL --basedir %s", basedir)
# sharedir is absolutely required as we need it to bootstrap mysql
# and mysql will fail to start withtout it
join = os.path.join
exists = os.path.exists
share_search_path = ['share/mysql']
if 'Percona Server' in version.comment:
share_search_path = ['share/percona-server'] + share_search_path
sharedir = first_subdir(basedir, *share_search_path)
if not sharedir:
raise common.SandboxError("MySQL share directory not found (%s)" %
','.join(
os.path.join(os.sep, 'usr', path)
for path in share_search_path))
for script in ('fill_help_tables.sql', 'mysql_system_tables_data.sql',
'mysql_system_tables.sql'):
if not exists(join(sharedir, script)):
raise common.SandboxError("MySQL bootstrap SQL '%s' not found." %
os.path.join(sharedir, script))
debug(" # MySQL share found in %s", sharedir)
# Note: plugindir may be None, if using mysql < 5.1
plugindir = first_subdir(basedir, 'lib64/mysql/plugin', 'lib/mysql/plugin')
if plugindir:
debug(" # Found MySQL plugin directory: %s", plugindir)
# now copy mysqld, mysql, mysqld_safe to sandbox_dir/bin
# then return an appropriate MySQLDistribution instance
bindir = os.path.join(options.basedir, 'bin')
pycompat.makedirs(bindir, 0o0770, exist_ok=True)
for name in [mysqld]:
shutil.copy2(name, bindir)
debug(" # Copied minimal MySQL commands to %s", bindir)
return MySQLDistribution(version=version,
mysqld=os.path.join(bindir,
os.path.basename(mysqld)),
mysqld_safe=mysqld_safe,
mysql=mysql,
basedir=basedir,
sharedir=sharedir,
libexecdir=bindir,
plugindir=plugindir)
