def test_reindex(self):
crate_v3 = CrateNode(crate_dir=get_crate('3.x.x'),
keep_data=True,
settings=self.crate_settings)
self._to_stop.append(crate_v3)
crate_v3.start()
with client(crate_v3.http_url) as c:
aio.run(c.execute, "create table t (x int)")
args = (
(1, ),
(2, ),
(3, ),
)
aio.run(c.execute_many, "insert into t (x) values (?)", args)
crate_v3.stop()
self._to_stop.remove(crate_v3)
crate_v4 = CrateNode(crate_dir=get_crate('4.0.3'),
keep_data=True,
settings=self.crate_settings)
self._to_stop.append(crate_v4)
crate_v4.start()
reindex(crate_v4.http_url)
with client(crate_v4.http_url) as c:
result = aio.run(
c.execute,
"SELECT version FROM information_schema.tables WHERE table_name = 't'"
)
version = result['rows'][0][0]
self.assertEqual(version, {'upgraded': None, 'created': '4.0.3'})
cnt = aio.run(c.execute, 'SELECT count(*) FROM t')['rows'][0][0]
self.assertEqual(cnt, 3)
def test_reindex(self):
crate_v3 = CrateNode(crate_dir=get_crate('3.x.x'),
keep_data=True,
settings=self.crate_settings)
self._to_stop.append(crate_v3)
crate_v3.start()
with connect(crate_v3.http_url) as conn:
cur = conn.cursor()
cur.execute("create table t (x int)")
args = (
(1, ),
(2, ),
(3, ),
)
cur.executemany("insert into t (x) values (?)", args)
crate_v3.stop()
self._to_stop.remove(crate_v3)
crate_v4 = CrateNode(crate_dir=get_crate('4.0.3'),
keep_data=True,
settings=self.crate_settings)
self._to_stop.append(crate_v4)
crate_v4.start()
reindex(crate_v4.http_url)
with connect(crate_v4.http_url) as conn:
cur = conn.cursor()
cur.execute(
"SELECT version FROM information_schema.tables WHERE table_name = 't'"
)
version, = cur.fetchone()
self.assertEqual(version, {'upgraded': None, 'created': '4.0.3'})
cur.execute('SELECT count(*) FROM t')
cnt, = cur.fetchone()
self.assertEqual(cnt, 3)
def _run_spec(version, spec, result_hosts, env, settings, tmpdir):
crate_dir = get_crate(version)
settings.setdefault('cluster.name', str(uuid4()))
results = []
with Logger() as log, CrateNode(crate_dir=crate_dir, settings=settings, env=env) as n:
n.start()
do_run_spec(
spec=spec,
benchmark_hosts=n.http_url,
log=log,
result_hosts=result_hosts,
sample_mode='reservoir',
action='setup'
)
jfr_file = jfr_start(n.process.pid, tmpdir)
log.result = results.append
do_run_spec(
spec=spec,
benchmark_hosts=n.http_url,
log=log,
result_hosts=result_hosts,
sample_mode='reservoir',
action='queries'
)
jfr_stop(n.process.pid)
do_run_spec(
spec=spec,
benchmark_hosts=n.http_url,
log=log,
result_hosts=result_hosts,
sample_mode='reservoir',
action='teardown'
)
return (results, jfr_extract_metrics(jfr_file))
def new_node(version, settings=None, env=None):
crate_dir = get_crate(version)
version_tuple = _extract_version(crate_dir)
v = version_tuple_to_strict_version(version_tuple)
s = {
'path.data': self._path_data,
'cluster.name': 'crate-qa',
}
s.update(settings or {})
s.update(test_settings(v))
s = remove_unsupported_settings(v, s)
e = {
'CRATE_HEAP_SIZE': self.CRATE_HEAP_SIZE,
'CRATE_DISABLE_GC_LOGGING': '1',
'CRATE_HOME': crate_dir,
}
e.update(env or {})
if self.DEBUG:
print(f'# Running CrateDB {version} ({v}) ...')
s_nice = pformat(s)
print(f'with settings: {s_nice}')
e_nice = pformat(e)
print(f'with environment: {e_nice}')
n = CrateNode(
crate_dir=crate_dir,
keep_data=True,
settings=s,
env=e,
)
n._settings = s # CrateNode does not hold its settings
self._add_log_consumer(n)
self._on_stop.append(n)
return (n, version_tuple)
def run(version, spec, env, settings):
crate_dir = get_crate(version)
settings.setdefault('cluster.name', str(uuid4()))
with Logger() as log, CrateNode(crate_dir=crate_dir, settings=settings, env=env) as n:
n.start()
do_run_spec(
spec=spec,
log=log,
sample_mode='reservoir',
benchmark_hosts=n.http_url,
action='setup'
)
with connect(n.http_url) as conn:
optimize_tables(conn.cursor())
return gather_sizes(n.data_path)
def test_build_from_branch(self):
self.assertIsNotNone(get_crate('4.1'))
import os
import doctest
import subprocess
import functools
from unittest import TestCase
from cr8.run_crate import CrateNode, get_crate
from cr8.clients import client
from cr8 import aio
crate_dir = get_crate('latest-testing')
node = CrateNode(
crate_dir=crate_dir,
settings={
'cluster.name': 'cr8-tests',
'http.port': '44200-44250'
})
def setup(*args):
with client(node.http_url) as c:
aio.run(
c.execute,
'create table x.demo (id int, name string, country string) \
with (number_of_replicas = 0)'
)
aio.run(c.execute, 'create table y.demo (name text) with (number_of_replicas = 0)')
aio.run(c.execute, 'create blob table blobtable with (number_of_replicas = 0)')
import os
import doctest
import subprocess
import functools
from cr8.run_crate import CrateNode, get_crate
from crate.client import connect
crate_dir = get_crate('latest-nightly')
node = CrateNode(crate_dir=crate_dir,
settings={
'cluster.name': 'cr8-tests',
'http.port': '44200-44250',
'bootstrap.system_call_filter': False
})
def setup(*args):
with connect(node.http_url) as conn:
c = conn.cursor()
c.execute('create table x.demo (id int, name string, country string) \
with (number_of_replicas = 0)')
c.execute('create blob table blobtable with (number_of_replicas = 0)')
def teardown(*args):
try:
with connect(node.http_url) as conn:
c = conn.cursor()
c.execute('drop table x.demo')
c.execute('drop blob table blobtable')
finally:
import os
import doctest
import subprocess
import functools
from unittest import main
from cr8.run_crate import CrateNode, get_crate
from crate.client import connect
crate_dir = get_crate('latest-testing')
node = CrateNode(
crate_dir=crate_dir,
settings={
'cluster.name': 'cr8-tests',
'http.port': '44200-44250'
})
def setup(*args):
node.start()
with connect(node.http_url) as conn:
c = conn.cursor()
c.execute('create table x.demo (id int, name string, country string) \
with (number_of_replicas = 0)')
c.execute('create blob table blobtable with (number_of_replicas = 0)')
def teardown(*args):
try:
with connect(node.http_url) as conn:
c = conn.cursor()
import os
import doctest
import subprocess
import functools
from cr8.run_crate import CrateNode, get_crate
from crate.client import connect
crate_dir = get_crate('latest-nightly')
node = CrateNode(
crate_dir=crate_dir,
settings={
'cluster.name': 'cr8-tests',
'http.port': '44200-44250',
'bootstrap.system_call_filter': False
})
def setup(*args):
with connect(node.http_url) as conn:
c = conn.cursor()
c.execute('create table x.demo (id int, name string, country string) \
with (number_of_replicas = 0)')
c.execute('create blob table blobtable with (number_of_replicas = 0)')
def teardown(*args):
try:
with connect(node.http_url) as conn:
c = conn.cursor()
c.execute('drop table x.demo')
def _fetch_version_tuple(self, version: str) -> tuple:
crate_dir = get_crate(version)
return _extract_version(crate_dir)