def test_src_gen_per_db(self, popen, transfer, mysql_svc, *args, **kwargs):
mysqldump = backup.Backup(type='mysqldump',
cloudfs_dir='s3://scalr-1a8f341e/backups/mysql/1265/',
chunk_size=512)
mysqldump._databases = ['db1', 'db2']
backups = list(mysqldump._gen_src())
assert len(backups) == popen.stdout.call_count
assert len(backups) == len(mysqldump._databases)
assert mock.call(linux.build_cmd_args(
executable='/usr/bin/mysqldump',
params=mysql2.__mysql__['mysqldump_options'].split() + ['db1'])) in popen.call_list()
assert mock.call(linux.build_cmd_args(
executable='/usr/bin/mysqldump',
params=mysql2.__mysql__['mysqldump_options'].split() + ['db2'])) in popen.call_list()
def dd(**kwds):
short = []
for k, v in kwds.items():
short.append('%s=%s' % (k, v))
return linux.system(linux.build_cmd_args(
executable='/bin/dd',
short=short))
def mysqldump(*databases, **long_kwds):
output = long_kwds.pop("output", None)
cmd = linux.build_cmd_args(executable="/usr/bin/mysqldump", long=long_kwds, params=databases)
kwds = {}
if output:
kwds["stdout"] = open(output, "w+")
return linux.system(cmd, **kwds)
def args(self, *params, **long_kwds):
self.cmd = linux.build_cmd_args(
executable=self.executable,
long=long_kwds,
params=params)
LOG.debug('cmd: %s', self.cmd)
return self
def dd(**kwds):
short = []
for k, v in kwds.items():
short.append('%s=%s' % (k, v))
return linux.system(linux.build_cmd_args(
executable='/bin/dd',
short=short))
def args(self, *params, **long_kwds):
self.cmd = linux.build_cmd_args(
executable=self.executable,
long=long_kwds,
params=params)
LOG.debug('cmd: %s', self.cmd)
return self
def innobackupex(*params, **long_kwds):
if not os.path.exists('/usr/bin/innobackupex'):
pkgmgr.installed('percona-xtrabackup')
return linux.system(linux.build_cmd_args(
executable='/usr/bin/innobackupex',
long=long_kwds,
params=params))
def iptables_restore(filename, *short_args, **long_kwds):
if isinstance(filename, basestring):
filename = open(filename)
linux.system(linux.build_cmd_args(executable=IPTABLES_RESTORE,
short=short_args,
long=long_kwds),
stdin=filename)
def mkfs(self, device, *short_args):
short_args = list(short_args)
short_args = ['-t', self.type] + short_args
args = linux.build_cmd_args(executable='/sbin/mkfs',
short=short_args,
params=[device])
system(args, error_text=self.error_messages['mkfs'] % device)
def _gen_src(self): if self.file_per_database: for db_name in self._databases: self._current_db = db_name cmd = linux.build_cmd_args( executable='/usr/bin/mysqldump', params=__mysql__['mysqldump_options'].split() + [db_name]) mysql_dump = subprocess.Popen(cmd, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) yield mysql_dump.stdout else: cmd = linux.build_cmd_args( executable='/usr/bin/mysqldump', params=__mysql__['mysqldump_options'].split() + ['--all-databases']) mysql_dump = subprocess.Popen(cmd, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) yield mysql_dump.stdout
def mkfs(self, device, *short_args):
short_args = list(short_args)
short_args.extend(('-t', self.type))
args = linux.build_cmd_args(
executable='/sbin/mkfs',
short=short_args,
params=[device])
system(args, error_text=self.error_messages['mkfs'] % device)
def mysqldump(*databases, **long_kwds):
output = long_kwds.pop('output', None)
cmd = linux.build_cmd_args(executable='/usr/bin/mysqldump',
long=long_kwds,
params=databases)
kwds = {}
if output:
kwds['stdout'] = open(output, 'w+')
return linux.system(cmd, **kwds)
def modprobe(module_name, **long_kwds):
if not os_info['mods_enabled']:
return (None, None, 0)
return linux.system(linux.build_cmd_args(executable='/sbin/modprobe',
long=long_kwds,
params=[module_name]),
error_text='Kernel module %s is not available' %
module_name)
def modprobe(module_name, **long_kwds):
if not os_info['mods_enabled']:
return (None, None, 0)
return linux.system(linux.build_cmd_args(
executable='/sbin/modprobe',
long=long_kwds,
params=[module_name]),
error_text='Kernel module %s is not available' % module_name)
def rsync(src, dst, **long_kwds):
linux.system(['sync'])
output = linux.system(linux.build_cmd_args(
executable=os.path.join(__node__['embedded_bin_dir'], 'rsync'),
long=long_kwds,
params=[src, dst],
duplicate_keys=True))
linux.system(['sync'])
return output
def iptables(**long_kwds):
ordered_long = OrderedDict()
for key in ("protocol", "match"):
if key in long_kwds:
ordered_long[key] = long_kwds.pop(key)
ordered_long.update(long_kwds)
return linux.system(linux.build_cmd_args(executable=IPTABLES_BIN,
long=ordered_long))
def mysqldump(*databases, **long_kwds):
output = long_kwds.pop('output', None)
cmd = linux.build_cmd_args(
executable='/usr/bin/mysqldump',
long=long_kwds,
params=databases)
kwds = {}
if output:
kwds['stdout'] = open(output, 'w+')
return linux.system(cmd, **kwds)
def vgchange(*volume_group_names, **long_kwds): try: return linux.system(linux.build_cmd_args( executable='/sbin/vgchange', long=long_kwds, params=volume_group_names)) except linux.LinuxError, e: if e.returncode == 5: raise NotFound() raise
def pvchange(*physical_volume_paths, **long_kwds): try: return linux.system(linux.build_cmd_args( executable='/sbin/pvchange', long=long_kwds, params=physical_volume_paths)) except linux.LinuxError, e: if e.returncode == 5: raise NotFound() raise
def my_print_defaults(*option_groups):
out = linux.system(linux.build_cmd_args(executable="/usr/bin/my_print_defaults", params=option_groups))[0]
ret = {}
for line in out.splitlines():
cols = line.split("=")
ret[cols[0][2:]] = cols[1] if len(cols) > 1 else True
for key in __mysql__["defaults"]:
if key not in ret:
ret[key] = __mysql__["defaults"][key]
return ret
def vgchange(*volume_group_names, **long_kwds):
try:
return linux.system(
linux.build_cmd_args(executable='/sbin/vgchange',
long=long_kwds,
params=volume_group_names))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def pvchange(*physical_volume_paths, **long_kwds):
try:
return linux.system(
linux.build_cmd_args(executable='/sbin/pvchange',
long=long_kwds,
params=physical_volume_paths))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def umount(device_or_mpoint, **long_kwds):
args = linux.build_cmd_args(executable='/bin/umount',
short=('-f', device_or_mpoint),
long=long_kwds)
try:
linux.system(args, error_text='Cannot umount %s' % device_or_mpoint)
except linux.LinuxError, e:
if 'not mounted' in e.err or 'not found' in e.err:
return
raise
def rsync(src, dst, **long_kwds):
if not os.path.exists('/usr/bin/rsync'):
pkgmgr.installed('rsync')
system(['sync'])
output = system(build_cmd_args(executable='/usr/bin/rsync',
long=long_kwds,
params=[src, dst],
duplicate_keys=True))
system(['sync'])
return output
def iptables_save(filename=None, *short_args, **long_kwds):
# file name is a path string or file-like object
# if filename is None return output
kwds = {}
if isinstance(filename, basestring):
filename = open(filename, 'w+')
if hasattr(filename, 'write'):
kwds['stdout'] = filename
out = linux.system(linux.build_cmd_args(executable=IPTABLES_SAVE,
short=short_args, long=long_kwds), **kwds)[0]
return out
def lvchange(*logical_volume_path, **long_kwds):
try:
long_kwds.update({'yes': True})
return linux.system(linux.build_cmd_args(
executable='/sbin/lvchange',
long=long_kwds,
params=logical_volume_path))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def lvchange(*logical_volume_path, **long_kwds):
try:
long_kwds.update({'yes': True})
return linux.system(
linux.build_cmd_args(executable='/sbin/lvchange',
long=long_kwds,
params=logical_volume_path))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def lvcreate(*params, **long_kwds):
try:
return linux.system(
linux.build_cmd_args(executable='/sbin/lvcreate',
long=long_kwds,
params=params))
finally:
if linux.os['name'] == 'GCEL':
# Logical volumes not available for mount immediately
# Problem posted to Google at 29 Apr 2013.
time.sleep(1)
def vgremove(*volume_group_names, **long_kwds):
try:
long_kwds.update({'force': True})
return linux.system(
linux.build_cmd_args(executable='/sbin/vgremove',
long=long_kwds,
params=volume_group_names))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def vgremove(*volume_group_names, **long_kwds):
try:
long_kwds.update({'force': True})
return linux.system(linux.build_cmd_args(
executable='/sbin/vgremove',
long=long_kwds,
params=volume_group_names))
except linux.LinuxError, e:
if e.returncode == 5:
raise NotFound()
raise
def get_children(self):
try:
pgrep = linux.system(
linux.build_cmd_args(executable="pgrep",
short=["-P"],
params=[str(self.pid)]))
except linux.LinuxError:
children = []
else:
children = map(int, pgrep[0].splitlines())
return children
def umount(device_or_mpoint, **long_kwds):
args = linux.build_cmd_args(
executable='/bin/umount',
short=('-f', device_or_mpoint),
long=long_kwds)
try:
linux.system(args, error_text='Cannot umount %s' % device_or_mpoint)
except linux.LinuxError, e:
if 'not mounted' in e.err or 'not found' in e.err:
return
raise
def rsync(src, dst, **long_kwds):
if not os.path.exists('/usr/bin/rsync'):
pkgmgr.installed('rsync')
system(['sync'])
output = system(
build_cmd_args(executable='/usr/bin/rsync',
long=long_kwds,
params=[src, dst],
duplicate_keys=True))
system(['sync'])
return output
def lvremove(*logical_volume_paths, **long_kwds):
try:
long_kwds.update({'force': True})
return linux.system(linux.build_cmd_args(
executable='/sbin/lvremove',
long=long_kwds,
params=logical_volume_paths))
except linux.LinuxError, e:
if e.returncode == 5:
raise NotFound()
raise
def iptables_save(filename=None, *short_args, **long_kwds):
# file name is a path string or file-like object
# if filename is None return output
kwds = {}
if isinstance(filename, basestring):
filename = open(filename, 'w+')
if hasattr(filename, 'write'):
kwds['stdout'] = filename
out = linux.system(linux.build_cmd_args(executable=IPTABLES_SAVE,
short=short_args, long=long_kwds), **kwds)[0]
return out
def get_children(self):
try:
pgrep = linux.system(linux.build_cmd_args(
executable="pgrep",
short=["-P"],
params=[str(self.pid)]))
except linux.LinuxError:
children = []
else:
children = map(int, pgrep[0].splitlines())
return children
def lvcreate(*params, **long_kwds):
try:
return linux.system(linux.build_cmd_args(
executable='/sbin/lvcreate',
long=long_kwds,
params=params))
finally:
if linux.os['name'] == 'GCEL':
# Logical volumes not available for mount immediately
# Problem posted to Google at 29 Apr 2013.
time.sleep(1)
def my_print_defaults(*option_groups):
out = linux.system(
linux.build_cmd_args(executable='/usr/bin/my_print_defaults',
params=option_groups))[0]
ret = {}
for line in out.splitlines():
cols = line.split('=')
ret[cols[0][2:]] = cols[1] if len(cols) > 1 else True
for key in __mysql__['defaults']:
if key not in ret:
ret[key] = __mysql__['defaults'][key]
return ret
def mdadm(mode, md_device=None, *devices, **long_kwds):
"""
Example:
mdadm.mdadm('create', '/dev/md0', '/dev/loop0', '/dev/loop1',
level=0, metadata='default',
assume_clean=True, raid_devices=2)
"""
raise_exc = long_kwds.pop('raise_exc', True)
return linux.system(linux.build_cmd_args(
mdadm_binary, ['--%s' % mode] + ([md_device] if md_device else []),
long_kwds, devices),
raise_exc=raise_exc)
def mdadm(mode, md_device=None, *devices, **long_kwds):
"""
Example:
mdadm.mdadm('create', '/dev/md0', '/dev/loop0', '/dev/loop1',
level=0, metadata='default',
assume_clean=True, raid_devices=2)
"""
raise_exc = long_kwds.pop('raise_exc', True)
return linux.system(linux.build_cmd_args(
mdadm_binary,
['--%s' % mode] + ([md_device] if md_device else []),
long_kwds, devices), raise_exc=raise_exc)
def vgextend(volume_group_name, *physical_volumes, **long_kwds):
try:
long_kwds.update({'force': True, 'yes': True})
return linux.system(
linux.build_cmd_args(executable='/sbin/vgextend',
long=long_kwds,
params=[volume_group_name] +
list(physical_volumes)))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def dmsetup(action, *params, **long_kwds):
if not os.path.exists('/sbin/dmsetup'):
if linux.os.debian_family:
package = 'dmsetup'
else:
package = 'device-mapper'
pkgmgr.installed(package)
return linux.system(linux.build_cmd_args(
executable='/sbin/dmsetup',
short=[action],
long=long_kwds,
params=params))
def mount(device, mpoint, *short_args, **long_kwds): args = linux.build_cmd_args( executable='/bin/mount', short=short_args, long=long_kwds, params=(device, mpoint) ) try: msg = 'Cannot mount %s -> %s' % (device, mpoint) linux.system(args, error_text=msg) except linux.LinuxError, e: if 'you must specify the filesystem type' in e.err: raise NoFileSystem(device) raise
def vgextend(volume_group_name, *physical_volumes, **long_kwds):
try:
long_kwds.update({
'force': True,
'yes': True
})
return linux.system(linux.build_cmd_args(
executable='/sbin/vgextend',
long=long_kwds,
params=[volume_group_name] + list(physical_volumes)))
except linux.LinuxError, e:
if e.returncode == 5:
raise NotFound()
raise
def mount(device, mpoint, *short_args, **long_kwds):
args = linux.build_cmd_args(executable='/bin/mount',
short=short_args,
long=long_kwds,
params=(device, mpoint))
if not os.path.exists(mpoint):
os.makedirs(mpoint)
try:
msg = 'Cannot mount %s -> %s' % (device, mpoint)
return linux.system(args, error_text=msg)
except linux.LinuxError, e:
if 'you must specify the filesystem type' in e.err:
raise NoFileSystem(device)
raise
def dmsetup(action, *params, **long_kwds):
if not os.path.exists('/sbin/dmsetup'):
from scalarizr.linux import pkgmgr
if linux.os.debian_family:
package = 'dmsetup'
else:
package = 'device-mapper'
pkgmgr.installed(package)
return linux.system(
linux.build_cmd_args(executable='/sbin/dmsetup',
short=[action],
long=long_kwds,
params=params))
def pvremove(*physical_volumes, **long_kwds):
try:
long_kwds.update({
'force': True,
'yes': True
})
return linux.system(linux.build_cmd_args(
executable='/sbin/pvremove',
long=long_kwds,
params=physical_volumes))
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def iptables(**long_kwds):
ordered_long = OrderedDict()
for key in ("protocol", "match"):
if key in long_kwds:
ordered_long[key] = long_kwds.pop(key)
ordered_long.update(long_kwds)
args0 = linux.build_cmd_args(executable=IPTABLES_BIN, long=ordered_long)
args = []
for arg in args0:
if arg.startswith('--not-'):
args.extend(('!', arg.replace('not-', '')))
else:
args.append(arg)
return linux.system(args)
def iptables(**long_kwds):
ordered_long = OrderedDict()
for key in ("protocol", "match"):
if key in long_kwds:
ordered_long[key] = long_kwds.pop(key)
ordered_long.update(long_kwds)
args0 = linux.build_cmd_args(
executable=IPTABLES_BIN,
long=ordered_long)
args = []
for arg in args0:
if arg.startswith('--not-'):
args.extend(('!', arg.replace('not-', '')))
else:
args.append(arg)
return linux.system(args)
def save():
'''
on RHEL call 'service iptables save'
on Ubuntu:
- touch or create /etc/network/if-pre-up.d/iptables.sh
$ cat /etc/network/if-pre-up.d/iptables.sh
#!/bin/bash
iptables-restore < /etc/iptables.rules
- iptables-save > /etc/iptables.rules
'''
if linux.os["family"] in ("RedHat", "Oracle"):
linux.system(linux.build_cmd_args(executable="service", short=['iptables',
'save']))
elif linux.os["family"] == "Debian":
with open('/etc/network/if-pre-up.d/iptables.sh', 'w') as fp:
fp.write('#!/bin/bash\n'
'iptables-restore < /etc/iptables.rules')
iptables_save('/etc/iptables.rules')
def save():
'''
on RHEL call 'service iptables save'
on Ubuntu:
- touch or create /etc/network/if-pre-up.d/iptables.sh
$ cat /etc/network/if-pre-up.d/iptables.sh
#!/bin/bash
iptables-restore < /etc/iptables.rules
- iptables-save > /etc/iptables.rules
'''
if linux.os["family"] in ("RedHat", "Oracle"):
linux.system(
linux.build_cmd_args(executable="service",
short=['iptables', 'save']))
elif linux.os["family"] == "Debian":
with open('/etc/network/if-pre-up.d/iptables.sh', 'w') as fp:
fp.write('#!/bin/bash\n' 'iptables-restore < /etc/iptables.rules')
iptables_save('/etc/iptables.rules')
def lvs(*volume_groups, **long_kwds):
try:
long_kwds.update({
'options': LVInfo.COLUMNS,
'separator': '|',
'noheadings': True
})
out = linux.system(linux.build_cmd_args(
executable='/sbin/lvs',
long=long_kwds,
params=volume_groups))[0]
ret = {}
for line in out.splitlines():
item = LVInfo(*line.strip().split('|'))
ret['%s/%s' % (item.vg_name, item.lv_name)] = item
return ret
except linux.LinuxError, e:
if e.returncode == 5:
raise NotFound()
raise
def lvs(*volume_groups, **long_kwds):
try:
long_kwds.update({
'options': LVInfo.COLUMNS,
'separator': '|',
'noheadings': True
})
out = linux.system(
linux.build_cmd_args(executable='/sbin/lvs',
long=long_kwds,
params=volume_groups))[0]
ret = {}
for line in out.splitlines():
item = LVInfo(*line.strip().split('|'))
ret['%s/%s' % (item.vg_name, item.lv_name)] = item
return ret
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def pvs(*physical_volumes, **long_kwds):
try:
long_kwds.update({
'options': PVInfo.COLUMNS,
'separator': '|',
'noheadings': True
})
out = linux.system(linux.build_cmd_args(
executable='/sbin/pvs',
long=long_kwds,
params=physical_volumes))[0]
ret = {}
for line in out.splitlines():
item = PVInfo(*line.strip().split('|'))
ret[item.pv_name] = item
return ret
except linux.LinuxError, e:
if e.returncode == 5:
raise NotFound()
raise
def pvs(*physical_volumes, **long_kwds):
try:
long_kwds.update({
'options': PVInfo.COLUMNS,
'separator': '|',
'noheadings': True
})
out = linux.system(
linux.build_cmd_args(executable='/sbin/pvs',
long=long_kwds,
params=physical_volumes))[0]
ret = {}
for line in out.splitlines():
item = PVInfo(*line.strip().split('|'))
ret[os.path.realpath(item.pv_name)] = item
return ret
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def vgs(*volume_groups, **long_kwds):
try:
long_kwds.update({
'options': VGInfo.COLUMNS,
'separator': '|',
'noheadings': True
})
out = linux.system(linux.build_cmd_args(
executable='/sbin/vgs',
long=long_kwds,
params=volume_groups))[0]
ret = {}
for line in out.splitlines():
item = VGInfo(*line.strip().split('|'))
ret[item.vg_name] = item
return ret
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def lvremove(*logical_volume_paths, **long_kwds):
try:
long_kwds.update({'force': True})
ret = linux.system(
linux.build_cmd_args(executable='/sbin/lvremove',
long=long_kwds,
params=logical_volume_paths))
'''
if linux.os['name'] == 'GCEL':
# Remove COW files
for path in logical_volume_paths:
path = '/dev/mapper/%s' % os.path.basename(path)
possible_cow = '%s-cow' % path
if os.path.exists(possible_cow):
linux.system('/sbin/dmsetup', 'remove', possible_cow)
# Wait for sync changes properly
time.sleep(1)
'''
return ret
except linux.LinuxError, e:
if 'not found' in str(e).lower():
raise NotFound()
raise
def chown_r(path, owner, group=None):
return linux.system(
linux.build_cmd_args(
executable='/bin/chown',
long={'recursive': True},
params=[owner if not group else owner + ':' + group, path]))
def mysqlbinlog(log_file, **log_kwds):
return linux.system(
linux.build_cmd_args(executable='/usr/bin/mysqlbinlog',
long=log_kwds,
params=[log_file]))
def losetup(*args, **long_kwds):
return linux.system(
linux.build_cmd_args(executable='/sbin/losetup',
long=long_kwds,
params=args))