def test_incremental(self):
self._write_value_for_key_in_src(10)
main(['', self.cfgfile])
self._check_value_of_key_in_dest(10)
self._write_value_for_key_in_src("hi")
main(['', '--incremental', self.cfgfile])
self._check_value_of_key_in_dest("hi")
def test_incremental(self):
self._write_value_for_key_in_src(10)
main(['', self.cfgfile])
self._check_value_of_key_in_dest(10)
self._write_value_for_key_in_src("hi")
main(['', '--incremental', self.cfgfile])
self._check_value_of_key_in_dest("hi")
def test_incremental(self):
x = 10
self._write_value_for_key_in_src(x)
main(['', self.cfgfile])
self._check_value_of_key_in_dest(x)
x = "hi"
self._write_value_for_key_in_src(x)
main(['', '--incremental', self.cfgfile])
self._check_value_of_key_in_dest(x)
def test_incremental(self):
x = 10
self._write_value_for_x_in_src(x)
main(['', self.cfgfile])
self._check_value_of_x_in_dest(x)
x = "hi"
self._write_value_for_x_in_src(x)
main(['', '--incremental', self.cfgfile])
self._check_value_of_x_in_dest(x)
def test_clear_full_dest(self):
self._write_value_for_key_in_dest(999)
self._write_value_for_key_in_dest(666, key='y')
self._write_value_for_key_in_dest(8675309, key='z')
self._write_value_for_key_in_src(1, key='x')
self._write_value_for_key_in_src(2, key='y')
# omit z
main(['', '--clear', self.cfgfile])
self._check_value_of_key_in_dest(1, key='x')
self._check_value_of_key_in_dest(2, key='y')
self._check_value_of_key_in_dest(None, key='z')
def test_clear_full_dest(self):
self._write_value_for_key_in_dest(999)
self._write_value_for_key_in_dest(666, key="y")
self._write_value_for_key_in_dest(8675309, key="z")
self._write_value_for_key_in_src(1, key="x")
self._write_value_for_key_in_src(2, key="y")
# omit z
main(["", "--clear", self.cfgfile])
self._check_value_of_key_in_dest(1, key="x")
self._check_value_of_key_in_dest(2, key="y")
self._check_value_of_key_in_dest(None, key="z")
def test_clear_full_dest(self):
self._write_value_for_key_in_dest(999)
self._write_value_for_key_in_dest(666, key='y')
self._write_value_for_key_in_dest(8675309, key='z')
self._write_value_for_key_in_src(1, key='x')
self._write_value_for_key_in_src(2, key='y')
# omit z
main(['', '--clear', self.cfgfile])
self._check_value_of_key_in_dest(1, key='x')
self._check_value_of_key_in_dest(2, key='y')
self._check_value_of_key_in_dest(None, key='z')
def test_dry_run(self):
from ZODB.DB import DB
from ZODB.FileStorage import FileStorage
from relstorage.zodbconvert import main
from relstorage.zodbconvert import storage_has_data
import transaction
src = FileStorage(self.srcfile)
db = DB(src)
conn = db.open()
conn.root()['x'] = 10
transaction.commit()
conn.close()
db.close()
main(['', '--dry-run', self.cfgfile])
dest = FileStorage(self.destfile)
db2 = DB(dest)
conn2 = db2.open()
self.assertEqual(conn2.root().get('x'), None)
conn2.close()
db2.close()
def test_convert(self):
from ZODB.DB import DB
from ZODB.FileStorage import FileStorage
from relstorage.zodbconvert import main
from relstorage.zodbconvert import storage_has_data
import transaction
src = FileStorage(self.srcfile)
db = DB(src)
conn = db.open()
conn.root()["x"] = 10
transaction.commit()
conn.close()
db.close()
main(["", self.cfgfile])
dest = FileStorage(self.destfile)
db2 = DB(dest)
conn2 = db2.open()
self.assertEqual(conn2.root().get("x"), 10)
conn2.close()
db2.close()
def test_dry_run(self):
from ZODB.DB import DB
from ZODB.FileStorage import FileStorage
from relstorage.zodbconvert import main
from relstorage.zodbconvert import storage_has_data
import transaction
src = FileStorage(self.srcfile)
db = DB(src)
conn = db.open()
conn.root()['x'] = 10
transaction.commit()
conn.close()
db.close()
main(['', '--dry-run', self.cfgfile])
dest = FileStorage(self.destfile)
db2 = DB(dest)
conn2 = db2.open()
self.assertEqual(conn2.root().get('x'), None)
conn2.close()
db2.close()
def test_clear_empty_dest(self):
x = 10
self._write_value_for_key_in_src(x)
main(['', '--clear', self.cfgfile])
self._check_value_of_key_in_dest(x)
def test_incremental_empty_src_dest(self):
# Should work and not raise a POSKeyError
main(['', '--incremental', self.cfgfile])
self._check_value_of_key_in_dest(None)
def test_clear_empty_dest(self):
x = 10
self._write_value_for_key_in_src(x)
main(['', '--clear', self.cfgfile])
self._check_value_of_key_in_dest(x)
def test_dry_run(self):
self._write_value_for_key_in_src(10)
main(['', '--dry-run', self.cfgfile])
self._check_value_of_key_in_dest(None)
def test_convert(self):
self._write_value_for_key_in_src(10)
main(['', self.cfgfile])
self._check_value_of_key_in_dest(10)
def test_incremental_empty_src_dest(self):
# Should work and not raise a POSKeyError
main(['', '--incremental', self.cfgfile])
self._check_value_of_x_in_dest(None)
def convertFromZeoToMySql():
host, port, db, user, passwd, root, socket = getMySqlSettings()
print "Creating database..."
print "-" * 79
createMySqlDatabase(host, port, db, user, passwd, root, socket)
print "%s database created successfully." % db
print
fd, fn = mkstemp()
zodbconvert_conf = os.fdopen(fd, 'w')
zodbconvert_conf.write('<filestorage source>\n')
zodbconvert_conf.write(' path %s\n' % findDataFs())
zodbconvert_conf.write('</filestorage>\n\n')
zodbconvert_conf.write('<relstorage destination>\n')
zodbconvert_conf.write(' keep-history false\n')
zodbconvert_conf.write(' <mysql>\n')
zodbconvert_conf.write(' host %s\n' % host)
zodbconvert_conf.write(' port %s\n' % port)
zodbconvert_conf.write(' db %s\n' % db)
zodbconvert_conf.write(' user %s\n' % user)
zodbconvert_conf.write(' passwd %s\n' % passwd)
if host == 'localhost' and socket:
zodbconvert_conf.write(' unix_socket %s\n' % socket)
zodbconvert_conf.write(' </mysql>\n')
zodbconvert_conf.write('</relstorage>\n')
zodbconvert_conf.close()
print "Shutting down ZEO for the last time..."
print "-" * 79
try:
call([
zenPath("bin", "zeoctl"), "-C",
zenPath("etc", "zeo.conf"), "stop"
])
except OSError:
pass
print
print "Analyzing ZODB..."
print "-" * 79
p = Popen([
zenPath("bin", "python"),
zenPath("bin", "zodbconvert"), "--dry-run", fn
],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = p.communicate()
# Last line stdout should be of the form:
# Would copy %d transactions.
try:
import re
match = re.search(r"Would copy (\d+) transactions", (stdout + stderr))
if not match:
raise Exception("could not find transaction count")
except Exception as ex:
print "zodbconvert could not be run:"
print stdout
print '-' * 40
print stderr
print "exception: ", ex
sys.exit(1)
totaltxs = int(match.group(1))
print "%d transactions to be copied." % totaltxs
print
# Monkeypatch in order to get a running count of transactions copied
counter = itertools.count()
counter.next()
from relstorage.storage import RelStorage
old_tpc_finish = RelStorage.tpc_finish
def new_tpc_finish(*args, **kwargs):
txnum = counter.next()
sys.stdout.write('\r - Copying transaction %d of %d' %
(txnum, totaltxs))
sys.stdout.flush()
if txnum == totaltxs:
print
return old_tpc_finish(*args, **kwargs)
RelStorage.tpc_finish = new_tpc_finish
print "Loading current ZODB into MySQL. This may take some time..."
print "-" * 79
from relstorage import zodbconvert
zodbconvert.main([None, '--clear', fn])
os.unlink(fn)
print
# Undo the monkeypatch
RelStorage.tpc_finish = old_tpc_finish
print "Removing ZEO from startup script..."
print "-" * 79
removeZeoctlFromStartup()
print "Done."
print
print "Converting Zenoss configuration files.."
print "-" * 79
print "Zope (%s)" % convertZopeConfToMySql(host, port, db, user, passwd,
socket)
print
for proc in zopeClients:
print "%s (%s)" % (proc,
updateConf(proc.lower(),
toRemove=[
'host', 'port', 'mysqldb',
'mysqluser', 'mysqlpasswd'
]))
toAdd = [('zodb-db-type', 'mysql'), ('zodb-host', host),
('zodb-port', port), ('zodb-db', db), ('zodb-user', user),
('zodb-password', passwd)]
if host == 'localhost' and socket:
toAdd.append((
'zodb-socket',
socket,
))
print "Global (%s)" % updateConf('global', toAdd=toAdd)
# sync the zodb_main.conf
call([zenPath("bin", "zenglobalconf"), "-s"])
def test_dry_run(self):
self._write_value_for_x_in_src(10)
main(['', '--dry-run', self.cfgfile])
self._check_value_of_x_in_dest(None)
def test_convert(self):
self._write_value_for_x_in_src(10)
main(['', self.cfgfile])
self._check_value_of_x_in_dest(10)
def convertFromZeoToMySql():
host, port, db, user, passwd, root, socket = getMySqlSettings()
print "Creating database..."
print "-"*79
createMySqlDatabase(host, port, db, user, passwd, root, socket)
print "%s database created successfully." % db
print
fd, fn = mkstemp()
zodbconvert_conf = os.fdopen(fd, 'w')
zodbconvert_conf.write('<filestorage source>\n')
zodbconvert_conf.write(' path %s\n' % findDataFs())
zodbconvert_conf.write('</filestorage>\n\n')
zodbconvert_conf.write('<relstorage destination>\n')
zodbconvert_conf.write(' keep-history false\n')
zodbconvert_conf.write(' <mysql>\n')
zodbconvert_conf.write(' host %s\n' % host)
zodbconvert_conf.write(' port %s\n' % port)
zodbconvert_conf.write(' db %s\n' % db)
zodbconvert_conf.write(' user %s\n' % user)
zodbconvert_conf.write(' passwd %s\n' % passwd)
if host=='localhost' and socket:
zodbconvert_conf.write(' unix_socket %s\n' % socket)
zodbconvert_conf.write(' </mysql>\n')
zodbconvert_conf.write('</relstorage>\n')
zodbconvert_conf.close()
print "Shutting down ZEO for the last time..."
print "-"*79
try:
call([zenPath("bin", "zeoctl"), "-C", zenPath("etc", "zeo.conf"), "stop"])
except OSError:
pass
print
print "Analyzing ZODB..."
print "-"*79
p = Popen([zenPath("bin", "python"), zenPath("bin", "zodbconvert"), "--dry-run", fn], stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
# Last line stdout should be of the form:
# Would copy %d transactions.
try:
import re
match = re.search(r"Would copy (\d+) transactions", (stdout+stderr))
if not match:
raise Exception("could not find transaction count")
except Exception as ex:
print "zodbconvert could not be run:"
print stdout
print '-'*40
print stderr
print "exception: ", ex
sys.exit(1)
totaltxs = int(match.group(1))
print "%d transactions to be copied." % totaltxs
print
# Monkeypatch in order to get a running count of transactions copied
counter = itertools.count()
counter.next()
from relstorage.storage import RelStorage
old_tpc_finish = RelStorage.tpc_finish
def new_tpc_finish(*args, **kwargs):
txnum = counter.next()
sys.stdout.write('\r - Copying transaction %d of %d' % (txnum, totaltxs))
sys.stdout.flush()
if txnum==totaltxs:
print
return old_tpc_finish(*args, **kwargs)
RelStorage.tpc_finish = new_tpc_finish
print "Loading current ZODB into MySQL. This may take some time..."
print "-"*79
from relstorage import zodbconvert
zodbconvert.main([None, '--clear', fn])
os.unlink(fn)
print
# Undo the monkeypatch
RelStorage.tpc_finish = old_tpc_finish
print "Removing ZEO from startup script..."
print "-"*79
removeZeoctlFromStartup()
print "Done."
print
print "Converting Zenoss configuration files.."
print "-"*79
print "Zope (%s)" % convertZopeConfToMySql(host, port, db, user, passwd, socket)
print
for proc in zopeClients:
print "%s (%s)" % (proc, updateConf(proc.lower(), toRemove=[
'host', 'port', 'mysqldb', 'mysqluser', 'mysqlpasswd']))
toAdd = [
('zodb-db-type', 'mysql'),
('zodb-host', host),
('zodb-port', port),
('zodb-db', db),
('zodb-user', user),
('zodb-password', passwd)
]
if host=='localhost' and socket:
toAdd.append(('zodb-socket', socket,))
print "Global (%s)" % updateConf('global', toAdd=toAdd)
# sync the zodb_main.conf
call([zenPath("bin", "zenglobalconf"), "-s"])
