Beispiel #1
0
    def test_incremental(self):
        self._write_value_for_key_in_src(10)
        main(['', self.cfgfile])
        self._check_value_of_key_in_dest(10)

        self._write_value_for_key_in_src("hi")
        main(['', '--incremental', self.cfgfile])
        self._check_value_of_key_in_dest("hi")
Beispiel #2
0
    def test_incremental(self):
        self._write_value_for_key_in_src(10)
        main(['', self.cfgfile])
        self._check_value_of_key_in_dest(10)

        self._write_value_for_key_in_src("hi")
        main(['', '--incremental', self.cfgfile])
        self._check_value_of_key_in_dest("hi")
    def test_incremental(self):
        x = 10
        self._write_value_for_key_in_src(x)
        main(['', self.cfgfile])
        self._check_value_of_key_in_dest(x)

        x = "hi"
        self._write_value_for_key_in_src(x)
        main(['', '--incremental', self.cfgfile])
        self._check_value_of_key_in_dest(x)
    def test_incremental(self):
        x = 10
        self._write_value_for_x_in_src(x)
        main(['', self.cfgfile])
        self._check_value_of_x_in_dest(x)

        x = "hi"
        self._write_value_for_x_in_src(x)
        main(['', '--incremental', self.cfgfile])
        self._check_value_of_x_in_dest(x)
Beispiel #5
0
    def test_clear_full_dest(self):
        self._write_value_for_key_in_dest(999)
        self._write_value_for_key_in_dest(666, key='y')
        self._write_value_for_key_in_dest(8675309, key='z')

        self._write_value_for_key_in_src(1, key='x')
        self._write_value_for_key_in_src(2, key='y')
        # omit z

        main(['', '--clear', self.cfgfile])

        self._check_value_of_key_in_dest(1, key='x')
        self._check_value_of_key_in_dest(2, key='y')
        self._check_value_of_key_in_dest(None, key='z')
Beispiel #6
0
    def test_clear_full_dest(self):
        self._write_value_for_key_in_dest(999)
        self._write_value_for_key_in_dest(666, key="y")
        self._write_value_for_key_in_dest(8675309, key="z")

        self._write_value_for_key_in_src(1, key="x")
        self._write_value_for_key_in_src(2, key="y")
        # omit z

        main(["", "--clear", self.cfgfile])

        self._check_value_of_key_in_dest(1, key="x")
        self._check_value_of_key_in_dest(2, key="y")
        self._check_value_of_key_in_dest(None, key="z")
Beispiel #7
0
    def test_clear_full_dest(self):
        self._write_value_for_key_in_dest(999)
        self._write_value_for_key_in_dest(666, key='y')
        self._write_value_for_key_in_dest(8675309, key='z')

        self._write_value_for_key_in_src(1, key='x')
        self._write_value_for_key_in_src(2, key='y')
        # omit z

        main(['', '--clear', self.cfgfile])

        self._check_value_of_key_in_dest(1, key='x')
        self._check_value_of_key_in_dest(2, key='y')
        self._check_value_of_key_in_dest(None, key='z')
Beispiel #8
0
    def test_dry_run(self):
        from ZODB.DB import DB
        from ZODB.FileStorage import FileStorage
        from relstorage.zodbconvert import main
        from relstorage.zodbconvert import storage_has_data
        import transaction

        src = FileStorage(self.srcfile)
        db = DB(src)
        conn = db.open()
        conn.root()['x'] = 10
        transaction.commit()
        conn.close()
        db.close()

        main(['', '--dry-run', self.cfgfile])

        dest = FileStorage(self.destfile)
        db2 = DB(dest)
        conn2 = db2.open()
        self.assertEqual(conn2.root().get('x'), None)
        conn2.close()
        db2.close()
Beispiel #9
0
    def test_convert(self):
        from ZODB.DB import DB
        from ZODB.FileStorage import FileStorage
        from relstorage.zodbconvert import main
        from relstorage.zodbconvert import storage_has_data
        import transaction

        src = FileStorage(self.srcfile)
        db = DB(src)
        conn = db.open()
        conn.root()["x"] = 10
        transaction.commit()
        conn.close()
        db.close()

        main(["", self.cfgfile])

        dest = FileStorage(self.destfile)
        db2 = DB(dest)
        conn2 = db2.open()
        self.assertEqual(conn2.root().get("x"), 10)
        conn2.close()
        db2.close()
Beispiel #10
0
    def test_dry_run(self):
        from ZODB.DB import DB
        from ZODB.FileStorage import FileStorage
        from relstorage.zodbconvert import main
        from relstorage.zodbconvert import storage_has_data
        import transaction

        src = FileStorage(self.srcfile)
        db = DB(src)
        conn = db.open()
        conn.root()['x'] = 10
        transaction.commit()
        conn.close()
        db.close()

        main(['', '--dry-run', self.cfgfile])

        dest = FileStorage(self.destfile)
        db2 = DB(dest)
        conn2 = db2.open()
        self.assertEqual(conn2.root().get('x'), None)
        conn2.close()
        db2.close()
Beispiel #11
0
 def test_clear_empty_dest(self):
     x = 10
     self._write_value_for_key_in_src(x)
     main(['', '--clear', self.cfgfile])
     self._check_value_of_key_in_dest(x)
Beispiel #12
0
 def test_incremental_empty_src_dest(self):
     # Should work and not raise a POSKeyError
     main(['', '--incremental', self.cfgfile])
     self._check_value_of_key_in_dest(None)
Beispiel #13
0
 def test_clear_empty_dest(self):
     x = 10
     self._write_value_for_key_in_src(x)
     main(['', '--clear', self.cfgfile])
     self._check_value_of_key_in_dest(x)
Beispiel #14
0
 def test_dry_run(self):
     self._write_value_for_key_in_src(10)
     main(['', '--dry-run', self.cfgfile])
     self._check_value_of_key_in_dest(None)
Beispiel #15
0
 def test_convert(self):
     self._write_value_for_key_in_src(10)
     main(['', self.cfgfile])
     self._check_value_of_key_in_dest(10)
 def test_incremental_empty_src_dest(self):
     # Should work and not raise a POSKeyError
     main(['', '--incremental', self.cfgfile])
     self._check_value_of_x_in_dest(None)
Beispiel #17
0
def convertFromZeoToMySql():
    host, port, db, user, passwd, root, socket = getMySqlSettings()
    print "Creating database..."
    print "-" * 79
    createMySqlDatabase(host, port, db, user, passwd, root, socket)
    print "%s database created successfully." % db
    print

    fd, fn = mkstemp()
    zodbconvert_conf = os.fdopen(fd, 'w')
    zodbconvert_conf.write('<filestorage source>\n')
    zodbconvert_conf.write('  path %s\n' % findDataFs())
    zodbconvert_conf.write('</filestorage>\n\n')

    zodbconvert_conf.write('<relstorage destination>\n')
    zodbconvert_conf.write('  keep-history false\n')
    zodbconvert_conf.write('  <mysql>\n')
    zodbconvert_conf.write('    host %s\n' % host)
    zodbconvert_conf.write('    port %s\n' % port)
    zodbconvert_conf.write('    db %s\n' % db)
    zodbconvert_conf.write('    user %s\n' % user)
    zodbconvert_conf.write('    passwd %s\n' % passwd)
    if host == 'localhost' and socket:
        zodbconvert_conf.write('    unix_socket %s\n' % socket)
    zodbconvert_conf.write('  </mysql>\n')
    zodbconvert_conf.write('</relstorage>\n')
    zodbconvert_conf.close()

    print "Shutting down ZEO for the last time..."
    print "-" * 79
    try:
        call([
            zenPath("bin", "zeoctl"), "-C",
            zenPath("etc", "zeo.conf"), "stop"
        ])
    except OSError:
        pass
    print

    print "Analyzing ZODB..."
    print "-" * 79
    p = Popen([
        zenPath("bin", "python"),
        zenPath("bin", "zodbconvert"), "--dry-run", fn
    ],
              stdout=PIPE,
              stderr=PIPE)
    stdout, stderr = p.communicate()
    # Last line stdout should be of the form:
    #     Would copy %d transactions.
    try:
        import re
        match = re.search(r"Would copy (\d+) transactions", (stdout + stderr))
        if not match:
            raise Exception("could not find transaction count")
    except Exception as ex:
        print "zodbconvert could not be run:"
        print stdout
        print '-' * 40
        print stderr
        print "exception: ", ex
        sys.exit(1)

    totaltxs = int(match.group(1))
    print "%d transactions to be copied." % totaltxs
    print

    # Monkeypatch in order to get a running count of transactions copied
    counter = itertools.count()
    counter.next()
    from relstorage.storage import RelStorage
    old_tpc_finish = RelStorage.tpc_finish

    def new_tpc_finish(*args, **kwargs):
        txnum = counter.next()
        sys.stdout.write('\r  -  Copying transaction %d of %d' %
                         (txnum, totaltxs))
        sys.stdout.flush()
        if txnum == totaltxs:
            print
        return old_tpc_finish(*args, **kwargs)

    RelStorage.tpc_finish = new_tpc_finish

    print "Loading current ZODB into MySQL. This may take some time..."
    print "-" * 79
    from relstorage import zodbconvert
    zodbconvert.main([None, '--clear', fn])
    os.unlink(fn)
    print

    # Undo the monkeypatch
    RelStorage.tpc_finish = old_tpc_finish

    print "Removing ZEO from startup script..."
    print "-" * 79
    removeZeoctlFromStartup()
    print "Done."
    print

    print "Converting Zenoss configuration files.."
    print "-" * 79
    print "Zope (%s)" % convertZopeConfToMySql(host, port, db, user, passwd,
                                               socket)
    print

    for proc in zopeClients:
        print "%s (%s)" % (proc,
                           updateConf(proc.lower(),
                                      toRemove=[
                                          'host', 'port', 'mysqldb',
                                          'mysqluser', 'mysqlpasswd'
                                      ]))

    toAdd = [('zodb-db-type', 'mysql'), ('zodb-host', host),
             ('zodb-port', port), ('zodb-db', db), ('zodb-user', user),
             ('zodb-password', passwd)]
    if host == 'localhost' and socket:
        toAdd.append((
            'zodb-socket',
            socket,
        ))
    print "Global (%s)" % updateConf('global', toAdd=toAdd)
    # sync the zodb_main.conf
    call([zenPath("bin", "zenglobalconf"), "-s"])
 def test_dry_run(self):
     self._write_value_for_x_in_src(10)
     main(['', '--dry-run', self.cfgfile])
     self._check_value_of_x_in_dest(None)
 def test_convert(self):
     self._write_value_for_x_in_src(10)
     main(['', self.cfgfile])
     self._check_value_of_x_in_dest(10)
Beispiel #20
0
def convertFromZeoToMySql():
    host, port, db, user, passwd, root, socket = getMySqlSettings()
    print "Creating database..."
    print "-"*79
    createMySqlDatabase(host, port, db, user, passwd, root, socket)
    print "%s database created successfully." % db
    print

    fd, fn = mkstemp()
    zodbconvert_conf = os.fdopen(fd, 'w')
    zodbconvert_conf.write('<filestorage source>\n')
    zodbconvert_conf.write('  path %s\n' % findDataFs())
    zodbconvert_conf.write('</filestorage>\n\n')

    zodbconvert_conf.write('<relstorage destination>\n')
    zodbconvert_conf.write('  keep-history false\n')
    zodbconvert_conf.write('  <mysql>\n')
    zodbconvert_conf.write('    host %s\n' % host)
    zodbconvert_conf.write('    port %s\n' % port)
    zodbconvert_conf.write('    db %s\n' % db)
    zodbconvert_conf.write('    user %s\n' % user)
    zodbconvert_conf.write('    passwd %s\n' % passwd)
    if host=='localhost' and socket:
        zodbconvert_conf.write('    unix_socket %s\n' % socket)
    zodbconvert_conf.write('  </mysql>\n')
    zodbconvert_conf.write('</relstorage>\n')
    zodbconvert_conf.close()

    print "Shutting down ZEO for the last time..."
    print "-"*79
    try:
        call([zenPath("bin", "zeoctl"), "-C", zenPath("etc", "zeo.conf"), "stop"])
    except OSError:
        pass
    print

    print "Analyzing ZODB..."
    print "-"*79
    p = Popen([zenPath("bin", "python"), zenPath("bin", "zodbconvert"), "--dry-run", fn], stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()
    # Last line stdout should be of the form:
    #     Would copy %d transactions.
    try:
        import re
        match = re.search(r"Would copy (\d+) transactions", (stdout+stderr))
        if not match:
            raise Exception("could not find transaction count")
    except Exception as ex:
        print "zodbconvert could not be run:"
        print stdout
        print '-'*40
        print stderr
        print "exception: ", ex
        sys.exit(1)

    totaltxs = int(match.group(1))
    print "%d transactions to be copied." % totaltxs
    print

    # Monkeypatch in order to get a running count of transactions copied
    counter = itertools.count()
    counter.next()
    from relstorage.storage import RelStorage
    old_tpc_finish = RelStorage.tpc_finish
    def new_tpc_finish(*args, **kwargs):
        txnum = counter.next()
        sys.stdout.write('\r  -  Copying transaction %d of %d' % (txnum, totaltxs))
        sys.stdout.flush()
        if txnum==totaltxs:
            print
        return old_tpc_finish(*args, **kwargs)
    RelStorage.tpc_finish = new_tpc_finish

    print "Loading current ZODB into MySQL. This may take some time..."
    print "-"*79
    from relstorage import zodbconvert
    zodbconvert.main([None, '--clear', fn])
    os.unlink(fn)
    print

    # Undo the monkeypatch
    RelStorage.tpc_finish = old_tpc_finish

    print "Removing ZEO from startup script..."
    print "-"*79
    removeZeoctlFromStartup()
    print "Done."
    print

    print "Converting Zenoss configuration files.."
    print "-"*79
    print "Zope (%s)" % convertZopeConfToMySql(host, port, db, user, passwd, socket)
    print

    for proc in zopeClients:
        print "%s (%s)" % (proc, updateConf(proc.lower(), toRemove=[
            'host', 'port', 'mysqldb', 'mysqluser', 'mysqlpasswd']))

    toAdd = [
        ('zodb-db-type', 'mysql'),
        ('zodb-host', host),
        ('zodb-port', port),
        ('zodb-db', db),
        ('zodb-user', user),
        ('zodb-password', passwd)
    ]
    if host=='localhost' and socket:
       toAdd.append(('zodb-socket', socket,))
    print "Global (%s)" % updateConf('global', toAdd=toAdd)
    # sync the zodb_main.conf
    call([zenPath("bin", "zenglobalconf"), "-s"])