def test_suite():
suite = unittest.TestSuite()
s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s1.layer = CrateLayer('crate-enterprise',
crate_home=crate_path(),
port=MonitoringIntegrationTest.CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(
MonitoringIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': True
})
suite.addTest(s1)
s2 = unittest.TestSuite(
unittest.makeSuite(MonitoringSettingIntegrationTest))
s2.layer = CrateLayer('crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(
MonitoringSettingIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': False
})
suite.addTest(s2)
return suite
def setUpClass(cls):
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=cls.NUM_SERVERS)
for i in range(cls.NUM_SERVERS):
http_port = bind_port()
layer = CrateLayer(
cls.node_name(i),
crate_path(),
host='localhost',
port=http_port,
transport_port=transport_port_range,
settings=CRATE_SETTINGS,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
cluster_name=cls.__class__.__name__)
layer.start()
cls.HTTP_PORTS.append(http_port)
cls.CRATES.append(layer)
dsn = cls.random_dns()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(cls.CRATES):
with connect(dsn) as conn:
c = conn.cursor()
c.execute("select * from sys.nodes")
num_nodes = len(c.fetchall())
time.sleep(5)
def test_suite():
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
crateLayer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env=env,
settings={
'license.enterprise': True,
'psql.port': GLOBAL_PORT_POOL.get(),
}
)
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ConnectionsBeanTest)
s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ThreadPoolsBeanTest)
s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(CircuitBreakersBeanTest)
s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
env = os.environ.copy()
env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env=env,
settings={
'license.enterprise': False,
}
)
suite.addTest(s)
return suite
def test_suite():
crateLayer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(JMX_PORT)
},
settings={
'license.enterprise': True,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)
},
settings={
'license.enterprise': False,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.addTest(s)
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SigarIntegrationTest))
suite.layer = CrateLayer('crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT)
return suite
def create_index(index_name, crate_home, output_dir):
crate_layer = CrateLayer(
'data',
crate_home=crate_home,
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'es.api.enabled': True,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage": "1k",
})
crate_layer.start()
crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT)
try:
with connect(crate_http) as conn:
cur = conn.cursor()
cmds = INDICES[index_name].split(';')
for cmd in cmds[:-1]:
LOGGER.info(cmd)
cur.execute(cmd)
cur.execute("select version['number'] from sys.nodes")
version = cur.fetchone()[0]
r = http.request('POST', crate_http + '/_flush')
r.read()
compress_index(index_name, version, crate_layer.wdPath(),
output_dir)
finally:
crate_layer.stop()
def create_index(index_name, crate_home, output_dir):
crate_layer = CrateLayer('data',
crate_home=crate_home,
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'es.api.enabled': True,
})
crate_layer.start()
crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT)
try:
with connect(crate_http) as conn:
cur = conn.cursor()
cmds = INDICES[index_name].split(';')
for cmd in cmds[:-1]:
LOGGER.info(cmd)
cur.execute(cmd)
cur.execute("select version['number'] from sys.nodes")
version = cur.fetchone()[0]
r = http.request('POST', crate_http + '/_flush')
r.read()
compress_index(index_name, version, crate_layer.wdPath(),
output_dir)
finally:
crate_layer.stop()
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest))
crate_layer = CrateLayer('crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={'stats.enabled': True})
suite.layer = crate_layer
return suite
def test_suite():
crateLayer = CrateLayer('crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT)},
settings={'license.enterprise': True})
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(
unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(
unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ConnectionsBeanTest)
s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(
unittest.makeSuite(MonitoringSettingIntegrationTest))
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)},
settings={
'license.enterprise': False,
})
suite.addTest(s)
return suite
def create_index(index_name, crate_home, output_dir):
crate_layer = CrateLayer(
'data',
crate_home=crate_home,
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'es.api.enabled': True,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
crate_layer.start()
crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT)
try:
with connect(crate_http) as conn:
cur = conn.cursor()
cmds = INDICES[index_name].split(';')
for cmd in cmds[:-1]:
LOGGER.info(cmd)
cur.execute(cmd)
cur.execute("select version['number'] from sys.nodes")
version = cur.fetchone()[0]
r = http.request('POST', crate_http + '/_flush')
r.read()
compress_index(index_name, version, crate_layer.wdPath(), output_dir)
finally:
crate_layer.stop()
def create_index(index_name, crate_home, output_dir):
crate_layer = CrateLayer(
'data',
crate_home=crate_home,
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'es.api.enabled': True,
}
)
crate_layer.start()
crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT)
try:
with connect(crate_http) as conn:
cur = conn.cursor()
cmds = INDICES[index_name].split(';')
for cmd in cmds[:-1]:
LOGGER.info(cmd)
cur.execute(cmd)
cur.execute("select version['number'] from sys.nodes")
version = cur.fetchone()[0]
r = http.request('POST', crate_http + '/_flush')
r.read()
compress_index(index_name, version, crate_layer.wdPath(), output_dir)
finally:
crate_layer.stop()
def setup_layer():
crate_path = os.path.join(here, 'parts/crate')
crate_http_port = get_port_number()
crate_transport_port = get_port_number()
crate_layer = CrateLayer('crate',
crate_home=crate_path,
port=crate_http_port,
transport_port=crate_transport_port)
engine = sa.create_engine('crate://localhost:' + str(crate_http_port))
session = scoped_session(
sessionmaker(autoflush=False, autocommit=False, bind=engine))
tornado_layer = TornadoLayer(
ScratyApplication(db_session=session, debug=True))
return crate_layer, tornado_layer, engine
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest))
crate_layer = CrateLayer(
'crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'psql.port': CRATE_PSQL_PORT,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.layer = crate_layer
return suite
def main():
num_nodes = 3
node0_http_port = bind_port()
dns_port = bind_port()
transport_ports = []
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
for i in range(0, num_nodes):
port = bind_port()
transport_ports.append(port)
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(
port=port)
dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
dns_server.start_thread()
crate_layers = []
for i in range(0, num_nodes):
crate_layer = CrateLayer(
'node-' + str(i),
cluster_name='crate-dns-discovery',
crate_home=crate_path(),
port=node0_http_port if i == 0 else bind_port(),
transport_port=transport_ports[i],
settings={
'psql.port': bind_port(),
"discovery.zen.hosts_provider": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
})
crate_layers.append(crate_layer)
crate_layer.start()
try:
conn = connect('localhost:{}'.format(node0_http_port))
c = conn.cursor()
c.execute('''select count() from sys.nodes''')
result = c.fetchone()
if result[0] != num_nodes:
raise AssertionError(
"Nodes could not join, expected number of nodes: " +
str(num_nodes) + ", found: " + str(result[0]))
finally:
for crate_layer in crate_layers:
crate_layer.stop()
dns_server.stop()
def create_index(cfg):
crate_layer = CrateLayer(
'data',
crate_home=cfg.crate_home,
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT
)
crate_layer.start()
try:
with connect('localhost:' + CRATE_HTTP_PORT) as conn:
cur = conn.cursor()
cmds = CREATE_INDEX_SQL.split(';')
for cmd in cmds[:-1]:
cur.execute(cmd)
cur.execute("select version['number'] from sys.nodes")
version = cur.fetchone()[0]
compress_index(version, crate_layer.wdPath(), cfg.output_dir)
finally:
crate_layer.stop()
def ensure_cratedb_layer():
"""
In order to skip individual tests by manually disabling them within
`def test_suite()`, it is crucial make the test layer not run on each
and every occasion. So, things like this will be possible::
./bin/test -vvvv --ignore_dir=testing
TODO: Through a subsequent patch, the possibility to individually
unselect specific tests might be added to `def test_suite()`
on behalf of environment variables.
A blueprint for this kind of logic can be found at
https://github.com/crate/crate/commit/414cd833.
"""
global crate_layer
if crate_layer is None:
crate_layer = CrateLayer('crate',
crate_home=crate_path(),
port=crate_port,
host=local,
transport_port=crate_transport_port,
settings=settings)
return crate_layer
def main():
crate_layer = CrateLayer(
'crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'psql.port': CRATE_PSQL_PORT,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage": "1k",
})
crate_layer.start()
logfiles = []
try:
with ProcessPoolExecutor() as executor:
futures = []
for i, filename in enumerate(tests_path.glob('**/*.test')):
filepath = tests_path / filename
relpath = str(filepath.relative_to(tests_path))
if not any(p.match(str(relpath)) for p in FILE_WHITELIST):
continue
logfile = f'sqllogic-{os.path.basename(relpath)}-{i}.log'
logfiles.append(logfile)
future = executor.submit(run_file,
filename=str(filepath),
host='localhost',
port=str(CRATE_PSQL_PORT),
log_level=logging.WARNING,
log_file=logfile,
failfast=True,
schema=f'x{i}')
futures.append(future)
for future in as_completed(futures):
future.result()
finally:
crate_layer.stop()
# instead of having dozens file merge to one which is in gitignore
merge_logfiles(logfiles)
from crate.crash.outputs import _val_len as val_len
from crate.crash.command import (CrateShell, main, get_stdin, noargs_command,
host_and_port, _create_shell, get_parser)
crate_http_port = 44209
crate_transport_port = 44309
crate_settings = {
'cluster.name': 'Testing44209',
'node.name': 'crate',
'psql.port': 45441,
'lang.js.enabled': True,
'http.port': crate_http_port,
'transport.tcp.port': crate_transport_port
}
node = CrateLayer.from_uri(
'https://cdn.crate.io/downloads/releases/crate-4.0.2.tar.gz',
'crate',
settings=crate_settings)
def setUpModule():
node.start()
def tearDownModule():
node.stop()
def fake_stdin(data):
stdin = TextIOWrapper(tempfile.TemporaryFile())
stdin.write(data)
stdin.flush()
def printjson(jsonstr):
d = json.loads(jsonstr.decode('utf-8'))
print(json.dumps(d, indent=4, sort_keys=True))
def get_rnd_port():
sock = socket.socket()
sock.bind(('', 0))
return sock.getsockname()[1]
crate_port = get_rnd_port()
crate_transport = get_rnd_port()
crate_layer = CrateLayer('crate',
crate_home=crate_path,
crate_exec=crate_exec,
port=crate_port,
transport_port=crate_transport)
tornado_port = get_rnd_port()
tornado_host = 'localhost:{port}'.format(port=tornado_port)
tornado_uri = 'http://{host}'.format(host=tornado_host)
engine = create_engine('crate://localhost:{}'.format(crate_port))
Session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=engine))
app = StreamMania(dbsession=Session, debug_mode=True)
tornado_layer = TornadoLayer(app, tornado_port, patches=[tvdb_mock])
crate_host = '127.0.0.1:{port}'.format(port=crate_port)
crate_uri = 'http://{0}'.format(crate_host)
def set_next_response(self, response):
self.response = response
def set_next_server_infos(self, server, server_name, version):
self._server_infos = (server, server_name, version)
def setUpMocked(test):
test.globs['connection_client_mocked'] = ClientMocked()
crate_port = 44209
crate_transport_port = 44309
crate_layer = CrateLayer('crate',
crate_home=crate_path(),
crate_exec=crate_path('bin', 'crate'),
port=crate_port,
transport_port=crate_transport_port)
crate_host = "127.0.0.1:{port}".format(port=crate_port)
crate_uri = "http://%s" % crate_host
def setUpWithCrateLayer(test):
test.globs['HttpClient'] = http.Client
test.globs['crate_host'] = crate_host
test.globs['pprint'] = pprint
test.globs['print'] = cprint
conn = connect(crate_host)
cursor = conn.cursor()
def set_next_response(self, response):
self.response = response
def set_next_server_infos(self, server, server_name, version):
self._server_infos = (server, server_name, version)
def setUpMocked(test):
test.globs['connection_client_mocked'] = ClientMocked()
crate_port = 44209
crate_transport_port = 44309
crate_layer = CrateLayer('crate',
crate_home=crate_path(),
port=crate_port,
transport_port=crate_transport_port)
crate_host = "127.0.0.1:{port}".format(port=crate_port)
crate_uri = "http://%s" % crate_host
def setUpForDjango(test):
connect(crate_host)
import os
os.environ["DJANGO_SETTINGS_MODULE"] = "crate.client.django.tests.settings"
from django.test.runner import setup_databases
setup_databases(3, False)
from crate.client.cursor import Cursor
from crate.testing.layer import CrateLayer
from uuid import uuid4
def rnd_port():
sock = socket.socket()
sock.bind(('127.0.0.1', 0))
port = sock.getsockname()[1]
return port
here = os.path.dirname(__file__)
root = os.path.join(here, '..')
crate_layer = CrateLayer('crate',
port=rnd_port(),
transport_port=rnd_port(),
crate_home=os.path.join(root, 'parts', 'crate'))
db = CrateDatabase(servers=crate_layer.crate_servers)
def gen_key():
return str(uuid4())
class BaseModel(Model):
class Meta:
database = db
class User(BaseModel):
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
'''
log = logging.getLogger('crate.testing.layer')
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
log.addHandler(ch)
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
enterprise_crate = CrateLayer('crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=0,
env=env,
settings={
'license.enterprise': True,
})
env = os.environ.copy()
env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)
community_crate = CrateLayer('crate',
crate_home=crate_path(),
port=bind_port(),
transport_port=0,
env=env,
settings={
'license.enterprise': False,
})
crash_parser = zc.customdoctests.DocTestParser(ps1='cr>',
comment_prefix='#',
transform=crash_transform)
crate_http_port = 44209
crate_transport_port = 44309
crate_settings = {
'psql.port': 45441,
'lang.js.enabled': True,
}
crate_layer = CrateLayer('crate',
crate_home=crate_path(),
crate_exec=crate_path('bin', 'crate'),
port=crate_http_port,
settings=crate_settings,
transport_port=crate_transport_port)
crate_host = "127.0.0.1:" + str(crate_http_port)
crate_uri = "http://" + crate_host
def setUp(test):
test.globs['cmd'] = CrateTestCmd(error_trace=True, is_tty=False)
def tearDown(test):
test.globs['cmd'].exit()
'license.enterprise': 'true',
'lang.js.enabled': 'true',
'auth.host_based.enabled': 'true',
'auth.host_based.config.0.user': '******',
'auth.host_based.config.0.method': 'trust',
'auth.host_based.config.98.user': '******',
'auth.host_based.config.98.method': 'trust',
'auth.host_based.config.99.user': '******',
'auth.host_based.config.99.method': 'password',
}
crate_port = 44209
crate_transport_port = 44309
local = '127.0.0.1'
crate_layer = CrateLayer('crate',
crate_home=crate_path(),
port=crate_port,
host=local,
transport_port=crate_transport_port,
settings=settings)
crate_host = "{host}:{port}".format(host=local, port=crate_port)
crate_uri = "http://%s" % crate_host
def refresh(table):
with connect(crate_host) as conn:
cursor = conn.cursor()
cursor.execute("refresh table %s" % table)
def setUpWithCrateLayer(test):
test.globs['HttpClient'] = http.Client
