def _testDetachAttachRaid(self):
mpoint = '/tmp/mpoint'
if not os.path.isdir(mpoint):
os.makedirs(mpoint)
self.array = Storage.create(type='raid',
disks=self.vols,
level=1,
vg='dbstorage',
snap_pv=self.snap_vol,
fstype='ext3')
self.array.mkfs()
self.array.mount(mpoint)
bigfile_path = os.path.join(mpoint, 'bigfile')
system('dd if=/dev/random of=%s bs=1M count=5' % bigfile_path)
md5sum = system(
('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
self.assertTrue(os.path.ismount(mpoint))
config = self.array.detach(force=True)
self.assertFalse(os.path.ismount(mpoint))
self.assertEqual(self.array.devname, None)
self.array = Storage.create(**config)
self.array.mount(mpoint)
md5sum2 = system(
('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
def testBackupRestoreRaid(self):
mpoint = '/tmp/mpoint'
if not os.path.isdir(mpoint):
os.makedirs(mpoint)
self.array = Storage.create(type='raid',
disks=self.vols,
level=1,
vg='dbstorage',
snap_pv=self.snap_vol,
fstype='ext3')
self.array.mkfs()
self.array.mount(mpoint)
# Create big file
bigfile_path = os.path.join(mpoint, 'bigfile')
system('dd if=/dev/random of=%s bs=1M count=5' % bigfile_path)
md5sum = system(
('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
array_snap = self.array.snapshot()
self.array.destroy(remove_disks=True)
self.array = Storage.create(snapshot=array_snap)
new_mpoint = '/tmp/mpoint2'
if not os.path.isdir(new_mpoint):
os.makedirs(new_mpoint)
self.array.mount(new_mpoint)
bigfile_path2 = os.path.join(new_mpoint, 'bigfile')
md5sum2 = system(
('/usr/bin/md5sum %s' % bigfile_path2))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
self.array.destroy(remove_disks=True)
def testBackupRestoreRaid(self):
mpoint = '/tmp/mpoint'
if not os.path.isdir(mpoint):
os.makedirs(mpoint)
self.array = Storage.create(type='raid', disks=self.vols, level=1, vg='dbstorage', snap_pv=self.snap_vol, fstype='ext3')
self.array.mkfs()
self.array.mount(mpoint)
# Create big file
bigfile_path = os.path.join(mpoint, 'bigfile')
system('dd if=/dev/random of=%s bs=1M count=5' % bigfile_path)
md5sum = system(('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
array_snap = self.array.snapshot()
self.array.destroy(remove_disks=True)
self.array = Storage.create(snapshot=array_snap)
new_mpoint = '/tmp/mpoint2'
if not os.path.isdir(new_mpoint):
os.makedirs(new_mpoint)
self.array.mount(new_mpoint)
bigfile_path2 = os.path.join(new_mpoint, 'bigfile')
md5sum2 = system(('/usr/bin/md5sum %s' % bigfile_path2))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
self.array.destroy(remove_disks=True)
def tearDown(self):
if self.vol:
self.vol.destroy()
if self.device:
system(('/sbin/losetup', '-d', self.device))
if os.path.exists(self.filename):
os.remove(self.filename)
os.removedirs(self.mpoint)
def tearDown(self):
if self.vol:
self.vol.destroy()
if self.device:
system(('/sbin/losetup', '-d', self.device))
if os.path.exists(self.filename):
os.remove(self.filename)
os.removedirs(self.mpoint)
def upload(self, files, remote_dst):
remote_path = os.path.normpath(remote_dst[len(self.SCHEMA):])
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, remote_path)
shutil.copy(file, remote_path)
ret.append('file://%s/%s' % (remote_path, os.path.basename(file)))
print system(('ls', '-la', remote_path))[0]
return tuple(ret)
def tearDown(self):
for vol in self.vols:
if vol:
vol.destroy()
for device in self.devices:
system(('/sbin/losetup', '-d', device))
for file in self.filenames:
os.remove(file)
for mpoint in self.mpoints:
os.rmdir(mpoint)
def tearDown(self):
for vol in self.vols:
if vol:
vol.destroy()
for device in self.devices:
system(('/sbin/losetup', '-d', device))
for file in self.filenames:
os.remove(file)
for mpoint in self.mpoints:
os.rmdir(mpoint)
def upload(self, files, remote_dst):
remote_path = os.path.normpath(remote_dst[len(self.SCHEMA):])
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, remote_path)
shutil.copy(file, remote_path)
ret.append('file://%s/%s' %
(remote_path, os.path.basename(file)))
print system(('ls', '-la', remote_path))[0]
return tuple(ret)
def test_on_mysql_newmaster_up(self):
bus.queryenv_service = _QueryEnv()
bus.platform = _Platform()
config = bus.config
sect_name = configtool.get_behaviour_section_name(mysql.BEHAVIOUR)
config.set(sect_name, mysql.OPT_REPLICATION_MASTER, '0')
handler = _MysqlHandler()
root_pass, repl_pass, stat_pass = handler._add_mysql_users(mysql.ROOT_USER, mysql.REPL_USER, mysql.STAT_USER)
handler._update_config( {mysql.OPT_ROOT_PASSWORD : root_pass,
mysql.OPT_REPL_PASSWORD : repl_pass,
mysql.OPT_STAT_PASSWORD : stat_pass})
message = _Message()
if disttool.is_redhat_based():
daemon = "/usr/libexec/mysqld"
else:
daemon = "/usr/sbin/mysqld"
initd.stop("mysql")
myd = Popen([daemon, '--defaults-file=/etc/mysql2/my.cnf', '--skip-grant-tables'], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
ping_service(LOCAL_IP, 3306, 5)
myclient = pexpect.spawn('/usr/bin/mysql -h'+LOCAL_IP)
myclient.expect('mysql>')
repl_password = re.sub('[^\w]','', cryptotool.keygen(20))
sql = "update mysql.user set password = PASSWORD('"+repl_password+"') where user = '******';"
myclient.sendline(sql)
myclient.expect('mysql>')
result = myclient.before
if re.search('ERROR', result):
os.kill(myd.pid, signal.SIGTERM)
raise BaseException("Cannot update user", result)
myclient.sendline('FLUSH TABLES WITH READ LOCK;')
myclient.expect('mysql>')
# system('cp -pr /var/lib/mysql /var/lib/backmysql')
# system('rm -rf /var/lib/mysql && cp -pr /var/lib/mysql2 /var/lib/mysql')
myclient.sendline('SHOW MASTER STATUS;')
myclient.expect('mysql>')
# retrieve log file and position
try:
master_status = myclient.before.split('\r\n')[4].split('|')
except:
raise BaseException("Cannot get master status")
finally:
myclient.sendline('UNLOCK TABLES;')
os.kill(myd.pid, signal.SIGTERM)
myd = Popen([daemon, '--defaults-file=/etc/mysql2/my.cnf'], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
ping_service(LOCAL_IP, 3306, 5)
message.log_file = master_status[1].strip()
message.log_pos = master_status[2].strip()
message.repl_user = mysql.REPL_USER
message.repl_password = repl_password
message.root_password = root_pass
handler.on_Mysql_NewMasterUp(message)
os.kill(myd.pid, signal.SIGTERM)
initd.stop("mysql")
system ('rm -rf /var/lib/mysql && cp -pr /var/lib/backmysql /var/lib/mysql && rm -rf /var/lib/backmysql')
config.set(sect_name, mysql.OPT_REPLICATION_MASTER, '1')
def test_on_mysql_newmaster_up(self):
bus.queryenv_service = _QueryEnv()
bus.platform = _Platform()
config = bus.config
sect_name = configtool.get_behaviour_section_name(mysql.BEHAVIOUR)
config.set(sect_name, mysql.OPT_REPLICATION_MASTER, '0')
handler = _MysqlHandler()
root_pass, repl_pass, stat_pass = handler._add_mysql_users(mysql.ROOT_USER, mysql.REPL_USER, mysql.STAT_USER)
handler._update_config( {mysql.OPT_ROOT_PASSWORD : root_pass,
mysql.OPT_REPL_PASSWORD : repl_pass,
mysql.OPT_STAT_PASSWORD : stat_pass})
message = _Message()
if linux.os.redhat_family:
daemon = "/usr/libexec/mysqld"
else:
daemon = "/usr/sbin/mysqld"
initd.stop("mysql")
myd = Popen([daemon, '--defaults-file=/etc/mysql2/my.cnf', '--skip-grant-tables'], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
ping_service(LOCAL_IP, 3306, 5)
myclient = pexpect.spawn('/usr/bin/mysql -h'+LOCAL_IP)
myclient.expect('mysql>')
repl_password = re.sub('[^\w]','', cryptotool.keygen(20))
sql = "update mysql.user set password = PASSWORD('"+repl_password+"') where user = '******';"
myclient.sendline(sql)
myclient.expect('mysql>')
result = myclient.before
if re.search('ERROR', result):
os.kill(myd.pid, signal.SIGTERM)
raise BaseException("Cannot update user", result)
myclient.sendline('FLUSH TABLES WITH READ LOCK;')
myclient.expect('mysql>')
# system('cp -pr /var/lib/mysql /var/lib/backmysql')
# system('rm -rf /var/lib/mysql && cp -pr /var/lib/mysql2 /var/lib/mysql')
myclient.sendline('SHOW MASTER STATUS;')
myclient.expect('mysql>')
# retrieve log file and position
try:
master_status = myclient.before.split('\r\n')[4].split('|')
except:
raise BaseException("Cannot get master status")
finally:
myclient.sendline('UNLOCK TABLES;')
os.kill(myd.pid, signal.SIGTERM)
myd = Popen([daemon, '--defaults-file=/etc/mysql2/my.cnf'], stdin=PIPE, stdout=PIPE, stderr=STDOUT)
ping_service(LOCAL_IP, 3306, 5)
message.log_file = master_status[1].strip()
message.log_pos = master_status[2].strip()
message.repl_user = mysql.REPL_USER
message.repl_password = repl_password
message.root_password = root_pass
handler.on_Mysql_NewMasterUp(message)
os.kill(myd.pid, signal.SIGTERM)
initd.stop("mysql")
system ('rm -rf /var/lib/mysql && cp -pr /var/lib/backmysql /var/lib/mysql && rm -rf /var/lib/backmysql')
config.set(sect_name, mysql.OPT_REPLICATION_MASTER, '1')
def tearDown(self):
for array in self.arrays:
system("mdadm -S -f %s" % array)
system("mdadm --remove -f %s" % array)
for device in self.devices:
system("/sbin/losetup -d %s" % device)
system("rm -f /tmp/device*")
def tearDown(self):
for array in self.arrays:
system('mdadm -S -f %s' % array)
system('mdadm --remove -f %s' % array)
for device in self.devices:
system('/sbin/losetup -d %s' % device)
system('rm -f /tmp/device*')
def _testDetachAttachRaid(self):
mpoint = '/tmp/mpoint'
if not os.path.isdir(mpoint):
os.makedirs(mpoint)
self.array = Storage.create(type='raid', disks=self.vols, level=1, vg='dbstorage', snap_pv=self.snap_vol, fstype='ext3')
self.array.mkfs()
self.array.mount(mpoint)
bigfile_path = os.path.join(mpoint, 'bigfile')
system('dd if=/dev/random of=%s bs=1M count=5' % bigfile_path)
md5sum = system(('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
self.assertTrue(os.path.ismount(mpoint))
config = self.array.detach(force=True)
self.assertFalse(os.path.ismount(mpoint))
self.assertEqual(self.array.devname, None)
self.array = Storage.create(**config)
self.array.mount(mpoint)
md5sum2 = system(('/usr/bin/md5sum %s' % bigfile_path))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
def test_create_on_exiting_file(self):
system('/bin/dd if=/dev/zero of=%s bs=1M seek=%d count=1' % (self.filename, self.SIZE-1), shell=True)
self.loop = mkloop(self.filename)
self.assert_size()
def tearDown(self):
initd.stop("mysql")
system('cp /etc/mysql/my.cnf /tmp/etc' +
time.strftime("%d%b%H%M%S", time.gmtime()))
system('rm -rf /etc/mysql/')
system('rm -rf /var/lib/mysql')
system('cp -pr /tmp/mysqletc/ /etc/mysql/ ')
system('cp -pr /tmp/mysqldata/ /var/lib/mysql ')
system('rm -rf /tmp/mysql*')
system('rm -rf /mnt/dbstorage/*')
system('rm -rf /mnt/tmpdir/*')
initd.start("mysql")
config = bus.config
section = configtool.get_behaviour_section_name(mysql.BEHAVIOUR)
try:
config.remove_option(section, mysql.OPT_ROOT_USER)
config.remove_option(section, mysql.OPT_ROOT_PASSWORD)
config.remove_option(section, mysql.OPT_REPL_USER)
config.remove_option(section, mysql.OPT_REPL_PASSWORD)
config.remove_option(section, mysql.OPT_STAT_USER)
config.remove_option(section, mysql.OPT_STAT_PASSWORD)
except:
pass
def setUp(self):
system('mkdir /mnt/tmpdir')
system('rsync -a /var/lib/mysql/ /mnt/tmpdir/mysql-data')
system('rsync -a /var/log/mysql/binarylog/ /mnt/tmpdir/mysql-misc')
system('cp -pr /etc/mysql/ /tmp/mysqletc/')
system('cp -pr /var/lib/mysql /tmp/mysqldata/')
def _get_loopback(self):
image = '/tmp/device%s' % firstmatched(lambda x: not os.path.exists('/tmp/device%s' % x), range(100))
system("dd if=/dev/zero of=%s bs=1M count=15" % image)
loop_dev = system('/sbin/losetup -f --show %s' % image)[0].strip()
return loop_dev
def _get_loopback(self):
image = "/tmp/device%s" % firstmatched(lambda x: not os.path.exists("/tmp/device%s" % x), range(100))
system("dd if=/dev/zero of=%s bs=1M count=15" % image)
loop_dev = system("/sbin/losetup -f --show %s" % image)[0].strip()
return loop_dev
def test_1(self):
class TransferMock(object):
SCHEMA = 'file://'
def __init__(self):
self._logger = logging.getLogger(__name__)
pass
def upload(self, files, remote_dst):
remote_path = os.path.normpath(remote_dst[len(self.SCHEMA):])
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, remote_path)
shutil.copy(file, remote_path)
ret.append('file://%s/%s' %
(remote_path, os.path.basename(file)))
print system(('ls', '-la', remote_path))[0]
return tuple(ret)
def download(self, remote_files, dst, recursive=False):
if isinstance(remote_files, basestring):
remote_files = (remote_files, )
files = list(
os.path.normpath(path[len(self.SCHEMA):])
for path in remote_files)
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, dst)
shutil.copy(file, dst)
ret.append(os.path.join(dst, os.path.basename(file)))
return ret
Storage.lookup_provider('eph')._snap_pvd._transfer = TransferMock()
# Create snapshot strage volume (Remote storage emulation)
self.vols[1] = Storage.create(device=self.devices[1],
mpoint=self.mpoints[1],
fstype='ext3')
self.vols[1].mkfs()
self.vols[1].mount()
# Create and mount EPH storage
self.vols[0] = Storage.create(type='eph',
disk=self.devices[0],
vg='casstorage',
snap_backend='%s%s' %
(TransferMock.SCHEMA, self.mpoints[1]),
fstype='ext3',
mpoint=self.mpoints[0])
self.vols[0].mkfs()
self.vols[0].mount()
# Create big file
bigfile = os.path.join(self.mpoints[0], 'bigfile')
system(
('dd', 'if=/dev/urandom', 'of=%s' % bigfile, 'bs=1M', 'count=15'))
bigsize = os.path.getsize(bigfile)
self.assertTrue(bigsize > 0)
md5sum = system(('/usr/bin/md5sum', bigfile))[0].strip().split(' ')[0]
# Snapshot storage
snap = self.vols[0].snapshot(description='Bigfile with us forever')
self.assertEqual(snap.type, 'eph')
self.assertEqual(snap.vg, 'casstorage')
self.assertEqual(snap.state, Snapshot.CREATING)
wait_until(lambda: snap.state in (Snapshot.COMPLETED, Snapshot.FAILED))
print snap.config()
if snap.state == Snapshot.FAILED:
raise Exception(
'Snapshot creation failed. See log for more details')
# Destroy original storage
self.vols[0].destroy()
self.vols[0] = None
# Restore snapshot
self.vols[2] = Storage.create(disk=self.devices[2], snapshot=snap)
self.vols[2].mount(self.mpoints[2])
bigfile2 = os.path.join(self.mpoints[2], 'bigfile')
self.assertTrue(os.path.exists(bigfile2))
md5sum2 = system(
('/usr/bin/md5sum', bigfile2))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
def setUp(self):
system('mkdir /mnt/tmpdir')
system('rsync -a /var/lib/mysql/ /mnt/tmpdir/mysql-data')
system('rsync -a /var/log/mysql/binarylog/ /mnt/tmpdir/mysql-misc')
system('cp -pr /etc/mysql/ /tmp/mysqletc/')
system('cp -pr /var/lib/mysql /tmp/mysqldata/')
def tearDown(self):
if self.loop:
system('/sbin/losetup -d %s' % self.loop, shell=True)
if os.path.exists(self.filename):
os.remove(self.filename)
def test_1(self):
class TransferMock(object):
SCHEMA = 'file://'
def __init__(self):
self._logger = logging.getLogger(__name__)
pass
def upload(self, files, remote_dst):
remote_path = os.path.normpath(remote_dst[len(self.SCHEMA):])
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, remote_path)
shutil.copy(file, remote_path)
ret.append('file://%s/%s' % (remote_path, os.path.basename(file)))
print system(('ls', '-la', remote_path))[0]
return tuple(ret)
def download(self, remote_files, dst, recursive=False):
if isinstance(remote_files, basestring):
remote_files = (remote_files,)
files = list(os.path.normpath(path[len(self.SCHEMA):]) for path in remote_files)
ret = []
for file in files:
self._logger.debug('Copy %s -> %s/', file, dst)
shutil.copy(file, dst)
ret.append(os.path.join(dst, os.path.basename(file)))
return ret
Storage.lookup_provider('eph')._snap_pvd._transfer = TransferMock()
# Create snapshot strage volume (Remote storage emulation)
self.vols[1] = Storage.create(
device=self.devices[1],
mpoint=self.mpoints[1],
fstype='ext3'
)
self.vols[1].mkfs()
self.vols[1].mount()
# Create and mount EPH storage
self.vols[0] = Storage.create(
type='eph',
disk=self.devices[0],
vg='casstorage',
snap_backend = '%s%s' % (TransferMock.SCHEMA, self.mpoints[1]),
fstype = 'ext3',
mpoint = self.mpoints[0]
)
self.vols[0].mkfs()
self.vols[0].mount()
# Create big file
bigfile = os.path.join(self.mpoints[0], 'bigfile')
system(('dd', 'if=/dev/urandom', 'of=%s' % bigfile, 'bs=1M', 'count=15'))
bigsize = os.path.getsize(bigfile)
self.assertTrue(bigsize > 0)
md5sum = system(('/usr/bin/md5sum', bigfile))[0].strip().split(' ')[0]
# Snapshot storage
snap = self.vols[0].snapshot(description='Bigfile with us forever')
self.assertEqual(snap.type, 'eph')
self.assertEqual(snap.vg, 'casstorage')
self.assertEqual(snap.state, Snapshot.CREATING)
wait_until(lambda: snap.state in (Snapshot.COMPLETED, Snapshot.FAILED))
print snap.config()
if snap.state == Snapshot.FAILED:
raise Exception('Snapshot creation failed. See log for more details')
# Destroy original storage
self.vols[0].destroy()
self.vols[0] = None
# Restore snapshot
self.vols[2] = Storage.create(disk=self.devices[2], snapshot=snap)
self.vols[2].mount(self.mpoints[2])
bigfile2 = os.path.join(self.mpoints[2], 'bigfile')
self.assertTrue(os.path.exists(bigfile2))
md5sum2 = system(('/usr/bin/md5sum', bigfile2))[0].strip().split(' ')[0]
self.assertEqual(md5sum, md5sum2)
def tearDown(self):
initd.stop("mysql")
system('cp /etc/mysql/my.cnf /tmp/etc'+time.strftime("%d%b%H%M%S", time.gmtime()))
system('rm -rf /etc/mysql/')
system('rm -rf /var/lib/mysql')
system('cp -pr /tmp/mysqletc/ /etc/mysql/ ')
system('cp -pr /tmp/mysqldata/ /var/lib/mysql ')
system('rm -rf /tmp/mysql*')
system('rm -rf /mnt/dbstorage/*')
system('rm -rf /mnt/tmpdir/*')
initd.start("mysql")
config = bus.config
section = configtool.get_behaviour_section_name(mysql.BEHAVIOUR)
try:
config.remove_option(section, mysql.OPT_ROOT_USER)
config.remove_option(section, mysql.OPT_ROOT_PASSWORD)
config.remove_option(section, mysql.OPT_REPL_USER)
config.remove_option(section, mysql.OPT_REPL_PASSWORD)
config.remove_option(section, mysql.OPT_STAT_USER)
config.remove_option(section, mysql.OPT_STAT_PASSWORD)
except:
pass
def tearDown(self):
if self.device:
system(('/sbin/losetup', '-d', self.device))
if os.path.exists(self.mpoint):
os.removedirs(self.mpoint)
