def test_suite():
suite = unittest.TestSuite()
s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s1.layer = CrateLayer('crate-enterprise',
crate_home=crate_path(),
port=MonitoringIntegrationTest.CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(
MonitoringIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': True
})
suite.addTest(s1)
s2 = unittest.TestSuite(
unittest.makeSuite(MonitoringSettingIntegrationTest))
s2.layer = CrateLayer('crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(
MonitoringSettingIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': False
})
suite.addTest(s2)
return suite
def test_suite():
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
crateLayer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env=env,
settings={
'license.enterprise': True,
'psql.port': GLOBAL_PORT_POOL.get(),
}
)
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ConnectionsBeanTest)
s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ThreadPoolsBeanTest)
s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(CircuitBreakersBeanTest)
s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
env = os.environ.copy()
env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env=env,
settings={
'license.enterprise': False,
}
)
suite.addTest(s)
return suite
def test_suite():
crateLayer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(JMX_PORT)
},
settings={
'license.enterprise': True,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)
},
settings={
'license.enterprise': False,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.addTest(s)
return suite
def setUp(self):
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
transport_ports = [bind_port() for _ in range(self.num_nodes)]
for port in transport_ports:
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(
port=port)
dns_port = bind_port()
self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
self.dns_server.start_thread()
self.nodes = nodes = []
for i in range(self.num_nodes):
settings = {
'node.name': f'node-{i}',
'cluster.name': 'crate-dns-discovery',
'psql.port': 0,
'transport.tcp.port': transport_ports[i],
"discovery.seed_providers": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
}
if i is 0:
settings['cluster.initial_master_nodes'] = f'node-{i}'
node = CrateNode(crate_dir=crate_path(),
version=(4, 0, 0),
settings=settings,
env={'CRATE_HEAP_SIZE': '256M'})
node.start()
nodes.append(node)
def setUp(self):
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=self.num_servers)
for i in range(self.num_servers):
layer = GracefulStopCrateLayer(
crate_dir=crate_path(),
settings={
'cluster.name': self.__class__.__name__,
'node.name': self.node_name(i),
'transport.tcp.port': transport_port_range,
},
env={
**os.environ.copy(),
'CRATE_HEAP_SIZE': '256M'
},
version=(4, 0, 0)
)
layer.start()
self.clients.append(Client(layer.http_url))
self.crates.append(layer)
self.node_names.append(self.node_name(i))
client = self.random_client()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(self.crates):
response = client.sql("select * from sys.nodes")
num_nodes = response.get("rowcount", 0)
time.sleep(.5)
def setUpClass(cls):
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=cls.NUM_SERVERS)
for i in range(cls.NUM_SERVERS):
http_port = bind_port()
layer = CrateLayer(
cls.node_name(i),
crate_path(),
host='localhost',
port=http_port,
transport_port=transport_port_range,
settings=CRATE_SETTINGS,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
cluster_name=cls.__class__.__name__)
layer.start()
cls.HTTP_PORTS.append(http_port)
cls.CRATES.append(layer)
dsn = cls.random_dns()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(cls.CRATES):
with connect(dsn) as conn:
c = conn.cursor()
c.execute("select * from sys.nodes")
num_nodes = len(c.fetchall())
time.sleep(5)
def setUp(self):
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=self.num_servers)
for i in range(self.num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
host='localhost',
port=bind_port(),
transport_port=transport_port_range,
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage":
"1k",
},
env=os.environ.copy(),
cluster_name=self.__class__.__name__)
layer.start()
self.clients.append(Client(layer.crate_servers))
self.crates.append(layer)
self.node_names.append(self.node_name(i))
client = self.random_client()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(self.crates):
response = client.sql("select * from sys.nodes")
num_nodes = response.get("rowcount", 0)
time.sleep(.5)
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SigarIntegrationTest))
suite.layer = CrateLayer('crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT)
return suite
def setUpClass(cls):
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=cls.NUM_SERVERS)
for i in range(cls.NUM_SERVERS):
layer = CrateNode(
crate_dir=crate_path(),
version=(4, 0, 0),
settings={
'cluster.name': cls.__class__.__name__,
'node.name': cls.node_name(i),
'psql.port': 0,
'transport.tcp.port': transport_port_range,
},
env={
'JAVA_HOME': os.environ.get('JAVA_HOME', ''),
'CRATE_HEAP_SIZE': '256M'
}
)
layer.start()
cls.HTTP_PORTS.append(layer.addresses.http.port)
cls.CRATES.append(layer)
dsn = cls.random_dns()
num_nodes = 0
# wait until all nodes joined the cluster
with connect(dsn) as conn:
c = conn.cursor()
while num_nodes < len(cls.CRATES):
c.execute("select * from sys.nodes")
num_nodes = len(c.fetchall())
time.sleep(5)
def __init__(self, *args, **kwargs):
num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS))
super(GracefulStopTest, self).__init__(*args, **kwargs)
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = GLOBAL_PORT_POOL.get_range(range_size=num_servers)
for i in range(num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
host='localhost',
port=GLOBAL_PORT_POOL.get(),
transport_port=transport_port_range,
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage": "1k",
},
cluster_name=self.__class__.__name__)
client = Client(layer.crate_servers)
self.crates.append(layer)
self.clients.append(client)
self.node_names.append(self.node_name(i))
self.layer = CascadedLayer(
"{0}_{1}_crates".format(self.__class__.__name__, num_servers),
*self.crates
)
def __init__(self, *args, **kwargs):
num_servers = kwargs.pop(
"num_servers",
getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS))
super(GracefulStopTest, self).__init__(*args, **kwargs)
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = GLOBAL_PORT_POOL.get_range(
range_size=num_servers)
for i in range(num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
host='localhost',
port=GLOBAL_PORT_POOL.get(),
transport_port=transport_port_range,
cluster_name=self.__class__.__name__)
client = Client(layer.crate_servers)
self.crates.append(layer)
self.clients.append(client)
self.node_names.append(self.node_name(i))
self.layer = CascadedLayer(
"{0}_{1}_crates".format(self.__class__.__name__, num_servers),
*self.crates)
def setUp(self):
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=self.num_servers)
for i in range(self.num_servers):
layer = GracefulStopCrateLayer(
crate_dir=crate_path(),
settings={
'cluster.name': self.__class__.__name__,
'node.name': self.node_name(i),
'transport.tcp.port': transport_port_range,
},
env={
**os.environ.copy(),
'CRATE_HEAP_SIZE': '256M'
},
version=(4, 0, 0)
)
layer.start()
self.clients.append(Client(layer.http_url))
self.crates.append(layer)
self.node_names.append(self.node_name(i))
client = self.random_client()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(self.crates):
response = client.sql("select * from sys.nodes")
num_nodes = response.get("rowcount", 0)
time.sleep(.5)
def test_suite():
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=MonitoringIntegrationTest.CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(MonitoringIntegrationTest.JMX_PORT)
},
settings={
'license.enterprise': True,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
suite.addTest(s)
s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(MonitoringSettingIntegrationTest.JMX_PORT)
},
settings={
'license.enterprise': False,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.addTest(s)
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(HdfsIntegrationTest))
crate_layer = HdfsCrateLayer('crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT)
hadoop_layer = HadoopLayer()
suite.layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SigarIntegrationTest))
suite.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT
)
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest))
crate_layer = CrateLayer('crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={'stats.enabled': True})
suite.layer = crate_layer
return suite
def test_suite():
crateLayer = CrateLayer('crate-enterprise',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT)},
settings={'license.enterprise': True})
# Graceful tests
suite = unittest.TestSuite()
s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(
unittest.makeSuite(MonitoringNodeStatusIntegrationTest))
s.layer = crateLayer
suite.addTest(s)
s = unittest.TestSuite(
unittest.makeSuite(MonitoringNodeInfoIntegrationTest))
s.layer = s.layer = crateLayer
suite.addTest(s)
s = unittest.makeSuite(ConnectionsBeanTest)
s.layer = crateLayer
suite.addTest(s)
# JMX Disabled test
s = unittest.TestSuite(
unittest.makeSuite(MonitoringSettingIntegrationTest))
s.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)},
settings={
'license.enterprise': False,
})
suite.addTest(s)
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(HdfsIntegrationTest))
crate_layer = HdfsCrateLayer(
'crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT
)
hadoop_layer = HadoopLayer()
suite.layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
return suite
def main():
num_nodes = 3
node0_http_port = bind_port()
dns_port = bind_port()
transport_ports = []
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
for i in range(0, num_nodes):
port = bind_port()
transport_ports.append(port)
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(
port=port)
dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
dns_server.start_thread()
crate_layers = []
for i in range(0, num_nodes):
crate_layer = CrateLayer(
'node-' + str(i),
cluster_name='crate-dns-discovery',
crate_home=crate_path(),
port=node0_http_port if i == 0 else bind_port(),
transport_port=transport_ports[i],
settings={
'psql.port': bind_port(),
"discovery.zen.hosts_provider": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
})
crate_layers.append(crate_layer)
crate_layer.start()
try:
conn = connect('localhost:{}'.format(node0_http_port))
c = conn.cursor()
c.execute('''select count() from sys.nodes''')
result = c.fetchone()
if result[0] != num_nodes:
raise AssertionError(
"Nodes could not join, expected number of nodes: " +
str(num_nodes) + ", found: " + str(result[0]))
finally:
for crate_layer in crate_layers:
crate_layer.stop()
dns_server.stop()
def test_suite():
crate_layer = HdfsCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
)
hadoop_layer = HadoopLayer()
layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
suite = unittest.makeSuite(HdfsIntegrationTest)
suite.layer = layer
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest))
crate_layer = CrateLayer(
'crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'stats.enabled': True,
'psql.port': CRATE_PSQL_PORT
}
)
suite.layer = crate_layer
return suite
def test_suite():
suite = unittest.TestSuite()
s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest))
s1.layer = CrateLayer(
'crate-enterprise',
crate_home=crate_path(),
port=MonitoringIntegrationTest.CRATE_HTTP_PORT,
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(MonitoringIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': True
}
)
suite.addTest(s1)
s2 = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest))
s2.layer = CrateLayer(
'crate',
crate_home=crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
env={
"CRATE_JAVA_OPTS":
JMX_OPTS.format(MonitoringSettingIntegrationTest.JMX_PORT)
},
settings={
'stats.enabled': True,
'license.enterprise': False
}
)
suite.addTest(s2)
return suite
def test_suite():
crate_layer = HdfsCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'psql.port': GLOBAL_PORT_POOL.get(),
},
env=os.environ.copy()
)
hadoop_layer = HadoopLayer()
layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
suite = unittest.makeSuite(HdfsIntegrationTest)
suite.layer = layer
return suite
def test_suite():
suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest))
crate_layer = CrateLayer(
'crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'psql.port': CRATE_PSQL_PORT,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
suite.layer = crate_layer
return suite
def test_suite():
crate_layer = HdfsCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage": "1k",
})
hadoop_layer = HadoopLayer()
layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
suite = unittest.makeSuite(HdfsIntegrationTest)
suite.layer = layer
return suite
def test_suite():
crate_layer = HdfsCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low" : "100k",
"cluster.routing.allocation.disk.watermark.high" : "10k",
"cluster.routing.allocation.disk.watermark.flood_stage" : "1k",
}
)
hadoop_layer = HadoopLayer()
layer = HadoopAndCrateLayer(crate_layer, hadoop_layer)
suite = unittest.makeSuite(HdfsIntegrationTest)
suite.layer = layer
return suite
def main():
crate_layer = CrateLayer(
'crate-sqllogic',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'psql.port': CRATE_PSQL_PORT,
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage": "1k",
})
crate_layer.start()
logfiles = []
try:
with ProcessPoolExecutor() as executor:
futures = []
for i, filename in enumerate(tests_path.glob('**/*.test')):
filepath = tests_path / filename
relpath = str(filepath.relative_to(tests_path))
if not any(p.match(str(relpath)) for p in FILE_WHITELIST):
continue
logfile = f'sqllogic-{os.path.basename(relpath)}-{i}.log'
logfiles.append(logfile)
future = executor.submit(run_file,
filename=str(filepath),
host='localhost',
port=str(CRATE_PSQL_PORT),
log_level=logging.WARNING,
log_file=logfile,
failfast=True,
schema=f'x{i}')
futures.append(future)
for future in as_completed(futures):
future.result()
finally:
crate_layer.stop()
# instead of having dozens file merge to one which is in gitignore
merge_logfiles(logfiles)
def __init__(self, *args, **kwargs):
num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS))
super(GracefulStopTest, self).__init__(*args, **kwargs)
self.crates = []
self.clients = []
self.node_names = []
for i in range(num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
cluster_name=self.__class__.__name__)
client = Client(layer.crate_servers)
self.crates.append(layer)
self.clients.append(client)
self.node_names.append(self.node_name(i))
self.layer = CascadedLayer(
"{0}_{1}_crates".format(self.__class__.__name__, num_servers),
*self.crates
)
def __init__(self, *args, **kwargs):
num_servers = kwargs.pop(
"num_servers",
getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS))
super(GracefulStopTest, self).__init__(*args, **kwargs)
self.crates = []
self.clients = []
self.node_names = []
for i in range(num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
port=GLOBAL_PORT_POOL.get(),
transport_port=GLOBAL_PORT_POOL.get(),
cluster_name=self.__class__.__name__)
client = Client(layer.crate_servers)
self.crates.append(layer)
self.clients.append(client)
self.node_names.append(self.node_name(i))
self.layer = CascadedLayer(
"{0}_{1}_crates".format(self.__class__.__name__, num_servers),
*self.crates)
def setUp(self):
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
transport_ports = [bind_port() for _ in range(self.num_nodes)]
for port in transport_ports:
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(port=port)
dns_port = bind_port()
self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
self.dns_server.start_thread()
self.nodes = nodes = []
for i in range(self.num_nodes):
settings = {
'node.name': f'node-{i}',
'cluster.name': 'crate-dns-discovery',
'psql.port': 0,
'transport.tcp.port': transport_ports[i],
"discovery.seed_providers": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
}
if i is 0:
settings['cluster.initial_master_nodes'] = f'node-{i}'
node = CrateNode(
crate_dir=crate_path(),
version=(4, 0, 0),
settings=settings,
env={
'CRATE_HEAP_SIZE': '256M'
}
)
node.start()
nodes.append(node)
def __init__(self, *args, **kwargs):
num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS))
super(GracefulStopTest, self).__init__(*args, **kwargs)
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = GLOBAL_PORT_POOL.get_range(range_size=num_servers)
for i in range(num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
host='localhost',
port=GLOBAL_PORT_POOL.get(),
transport_port=transport_port_range,
cluster_name=self.__class__.__name__)
client = Client(layer.crate_servers)
self.crates.append(layer)
self.clients.append(client)
self.node_names.append(self.node_name(i))
self.layer = CascadedLayer(
"{0}_{1}_crates".format(self.__class__.__name__, num_servers),
*self.crates
)
def setUp(self):
add_hadoop_libs(hdfs_repo_libs_path, crate_path())
super(HdfsCrateLayer, self).setUp()
kwargs.setdefault('settings', {})
self.repo_path = kwargs['settings']['path.repo'] = tempfile.mkdtemp()
super().__init__(*args, **kwargs)
def start(self):
super().start()
cmd._connect(self.http_url)
def stop(self):
print('')
print('ConnectingCrateLayer.stop()')
shutil.rmtree(self.repo_path, ignore_errors=True)
super().stop()
crate = ConnectingCrateLayer(crate_dir=crate_path(),
env={
'JAVA_HOME':
os.environ.get('JAVA_HOME', ''),
'CRATE_JAVA_OPTS':
'-Dio.netty.leakDetection.level=paranoid',
},
settings=CRATE_SETTINGS,
version=(4, 0, 0))
def crash_transform(s):
# The examples in the docs show the real port '4200' to a reader.
# Our test suite requires the port to be '44200' to avoid conflicts.
# Therefore, we need to replace the ports before a test is being run.
if s.startswith('_'):
import platform
import shutil
import tempfile
import threading
import unittest
import socket
import json
from pathlib import Path
from crate.client import connect
from cr8.run_crate import CrateNode, wait_until
from subprocess import Popen, PIPE, DEVNULL
from testutils.paths import crate_path
from urllib.request import urlretrieve
from minio import Minio
crate_node = CrateNode(crate_dir=crate_path(),
settings={
'transport.tcp.port': 0,
'psql.port': 0,
},
env={'CRATE_HEAP_SIZE': '256M'},
version=(4, 0, 0))
def _is_up(host: str, port: int) -> bool:
try:
conn = socket.create_connection((host, port))
conn.close()
return True
except (socket.gaierror, ConnectionRefusedError):
return False
from subprocess import PIPE, Popen
from urllib.request import urlretrieve
JMX_PORT = bind_port()
JMX_OPTS = '''
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dio.netty.leakDetection.level=paranoid
'''
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
env['CRATE_HEAP_SIZE'] = '256M'
enterprise_crate = CrateNode(crate_dir=crate_path(),
settings={
'transport.tcp.port': 0,
'psql.port': 0,
'node.name': 'crate-enterprise',
},
env=env,
version=(4, 0, 0))
class JmxClient:
SJK_JAR_URL = "https://repository.sonatype.org/service/local/artifact/maven/redirect?r=central-proxy&g=org.gridkit.jvmtool&a=sjk&v=LATEST"
CACHE_DIR = os.environ.get(
'XDG_CACHE_HOME',
def start(self):
add_hadoop_libs(hdfs_repo_libs_path, crate_path())
super().start()
if returncode is not None:
raise SystemError('Failed to start server rc={0} cmd={1}'.format(
returncode, self.start_cmd))
self._wait_for_start()
self._wait_for_master()
cmd._connect(self.crate_servers[0])
def tearDown(self):
shutil.rmtree(self.repo_path, ignore_errors=True)
super(ConnectingCrateLayer, self).tearDown()
crate_layer = ConnectingCrateLayer(
'crate',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
settings={
'cluster.routing.schedule': '30ms',
}
)
def setUpLocations(test):
test.globs['cmd'] = cmd
cmd.stmt("""
create table locations (
id string primary key,
name string,
"date" timestamp,
from cr8.run_crate import CrateNode
from test_jmx import JmxClient
JMX_PORT = bind_port()
JMX_OPTS = '''
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
'''
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
crate_layer = CrateNode(
crate_dir=crate_path(),
version=(4, 0, 0),
settings={
'transport.tcp.port': 0,
'node.name': 'crate-ce',
},
env=env
)
class CeHasNoEnterpriseModulesITest(unittest.TestCase):
@classmethod
def setUpClass(cls):
crate_layer.start()
def create_doctest_suite():
crate_layer = ConnectingCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
'cluster.routing.allocation.disk.watermark.low': '100k',
'cluster.routing.allocation.disk.watermark.high': '10k',
'cluster.routing.allocation.disk.watermark.flood_stage': '1k',
'license.enterprise': 'true',
'lang.js.enabled': 'true',
'es.api.enabled': 'true',
})
tests = []
for fn in doctest_files(
'general/blobs.rst',
'interfaces/http.rst',
):
s = docsuite(fn, parser=bash_parser, setUp=setUpLocations)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files(
'general/ddl/create-table.rst',
'general/ddl/generated-columns.rst', 'general/ddl/constraints.rst',
'general/ddl/sharding.rst', 'general/ddl/replication.rst',
'general/ddl/column-policy.rst', 'general/ddl/system-columns.rst',
'general/ddl/alter-table.rst', 'general/ddl/storage.rst',
'general/ddl/fulltext-indices.rst', 'admin/runtime-config.rst',
'general/ddl/show-create-table.rst',
'general/user-defined-functions.rst', 'admin/user-management.rst',
'admin/snapshots.rst', 'admin/privileges.rst',
'admin/ingestion/rules.rst', 'general/dql/index.rst',
'general/dql/refresh.rst', 'admin/optimization.rst',
'general/dql/fulltext.rst', 'general/ddl/data-types.rst',
'general/occ.rst', 'general/information-schema.rst',
'general/ddl/partitioned-tables.rst',
'general/builtins/aggregation.rst',
'general/builtins/arithmetic.rst', 'general/builtins/scalar.rst',
'general/builtins/table-functions.rst',
'admin/system-information.rst', 'general/dql/selects.rst',
'interfaces/postgres.rst', 'general/ddl/views.rst'):
s = docsuite(fn, setUp=setUpLocationsAndQuotes)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/geo.rst', ):
s = docsuite(fn, setUp=setUpCountries)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files(
'general/dql/joins.rst',
'general/builtins/subquery-expressions.rst',
):
s = docsuite(fn, setUp=setUpEmpDeptAndColourArticlesAndGeo)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dml.rst', ):
s = docsuite(fn, setUp=setUpLocationsQuotesAndUserVisits)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/union.rst', ):
s = docsuite(fn, setUp=setUpPhotosAndCountries)
s.layer = crate_layer
tests.append(s)
# randomize order of tests to make sure they don't depend on each other
random.shuffle(tests)
suite = unittest.TestSuite()
suite.addTests(tests)
return suite
def start(self):
add_hadoop_libs(hdfs_repo_libs_path, crate_path())
super().start()
self.repo_path = kwargs['settings']['path.repo'] = tempfile.mkdtemp()
super().__init__(*args, **kwargs)
def start(self):
super().start()
cmd._connect(self.http_url)
def stop(self):
print('')
print('ConnectingCrateLayer.stop()')
shutil.rmtree(self.repo_path, ignore_errors=True)
super().stop()
crate = ConnectingCrateLayer(
crate_dir=crate_path(),
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
settings=CRATE_SETTINGS,
version=(4, 0, 0))
def crash_transform(s):
# The examples in the docs show the real port '4200' to a reader.
# Our test suite requires the port to be '44200' to avoid conflicts.
# Therefore, we need to replace the ports before a test is being run.
if s.startswith('_'):
return s[1:]
if hasattr(crate, 'addresses'):
s = s.replace(':4200', ':{0}'.format(crate.addresses.http.port))
return u'cmd.process({0})'.format(repr(s.strip().rstrip(';')))
doctest_file = partial(os.path.join, 'docs')
def doctest_files(*items):
return (doctest_file(item) for item in items)
def get_abspath(name):
return os.path.abspath(
os.path.join(os.path.dirname(__file__), 'testdata', name))
crate_layer = ConnectingCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=0,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
settings={
'license.enterprise': 'true',
'lang.js.enabled': 'true',
'psql.port': 0,
})
class DocTests(unittest.TestSuite):
def run(self, result, debug=False):
crate_layer.start()
try:
super().run(result, debug)
def create_doctest_suite():
crate_layer = ConnectingCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
'cluster.routing.allocation.disk.watermark.low': '100k',
'cluster.routing.allocation.disk.watermark.high': '10k',
'cluster.routing.allocation.disk.watermark.flood_stage': '1k',
'license.enterprise': 'true',
'lang.js.enabled': 'true',
'es.api.enabled': 'true',
}
)
tests = []
for fn in doctest_files('general/blobs.rst',
'interfaces/http.rst',):
s = docsuite(fn, parser=bash_parser, setUp=setUpLocations)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/ddl/create-table.rst',
'general/ddl/generated-columns.rst',
'general/ddl/constraints.rst',
'general/ddl/sharding.rst',
'general/ddl/replication.rst',
'general/ddl/column-policy.rst',
'general/ddl/system-columns.rst',
'general/ddl/alter-table.rst',
'general/ddl/storage.rst',
'general/ddl/fulltext-indices.rst',
'admin/runtime-config.rst',
'general/ddl/show-create-table.rst',
'general/user-defined-functions.rst',
'admin/user-management.rst',
'admin/snapshots.rst',
'admin/privileges.rst',
'admin/ingestion/rules.rst',
'general/dql/index.rst',
'general/dql/refresh.rst',
'admin/optimization.rst',
'general/dql/fulltext.rst',
'general/ddl/data-types.rst',
'general/occ.rst',
'general/information-schema.rst',
'general/ddl/partitioned-tables.rst',
'general/builtins/aggregation.rst',
'general/builtins/arithmetic.rst',
'general/builtins/scalar.rst',
'general/builtins/table-functions.rst',
'admin/system-information.rst',
'general/dql/selects.rst',
'interfaces/postgres.rst',
'general/ddl/views.rst'):
s = docsuite(fn, setUp=setUpLocationsAndQuotes)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/geo.rst',):
s = docsuite(fn, setUp=setUpCountries)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/joins.rst',
'general/builtins/subquery-expressions.rst',):
s = docsuite(fn, setUp=setUpEmpDeptAndColourArticlesAndGeo)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dml.rst',):
s = docsuite(fn, setUp=setUpLocationsQuotesAndUserVisits)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/union.rst',):
s = docsuite(fn, setUp=setUpPhotosAndCountries)
s.layer = crate_layer
tests.append(s)
# randomize order of tests to make sure they don't depend on each other
random.shuffle(tests)
suite = unittest.TestSuite()
suite.addTests(tests)
return suite
def setUp(self):
add_hadoop_libs(hdfs_repo_libs_path, crate_path())
super().setUp()
def create_doctest_suite():
crate_layer = ConnectingCrateLayer(
'crate',
host='localhost',
crate_home=crate_path(),
port=CRATE_HTTP_PORT,
transport_port=CRATE_TRANSPORT_PORT,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
settings={
'license.enterprise': 'true',
'lang.js.enabled': 'true',
'es.api.enabled': 'true',
'psql.port': GLOBAL_PORT_POOL.get(),
}
)
tests = []
for fn in doctest_files('general/blobs.rst',
'interfaces/http.rst',):
s = docsuite(fn, parser=bash_parser, setUp=setUpLocations, globs={
'sh': partial(
subprocess.run,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=60,
shell=True
),
'pretty_print': pretty_print
})
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/ddl/create-table.rst',
'general/ddl/generated-columns.rst',
'general/ddl/constraints.rst',
'general/ddl/sharding.rst',
'general/ddl/replication.rst',
'general/ddl/column-policy.rst',
'general/ddl/system-columns.rst',
'general/ddl/alter-table.rst',
'general/ddl/storage.rst',
'general/ddl/fulltext-indices.rst',
'admin/runtime-config.rst',
'general/ddl/show-create-table.rst',
'general/user-defined-functions.rst',
'admin/user-management.rst',
'admin/snapshots.rst',
'admin/privileges.rst',
'admin/ingestion/rules.rst',
'general/dql/index.rst',
'general/dql/refresh.rst',
'admin/optimization.rst',
'general/dql/fulltext.rst',
'general/ddl/data-types.rst',
'general/occ.rst',
'general/information-schema.rst',
'general/ddl/partitioned-tables.rst',
'general/builtins/aggregation.rst',
'general/builtins/arithmetic.rst',
'general/builtins/scalar.rst',
'general/builtins/table-functions.rst',
'admin/system-information.rst',
'general/dql/selects.rst',
'interfaces/postgres.rst',
'general/ddl/views.rst',
'sql/general/value-expressions.rst'):
s = docsuite(fn, setUp=setUpLocationsAndQuotes)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/geo.rst',):
s = docsuite(fn, setUp=setUpCountries)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/joins.rst',
'general/builtins/subquery-expressions.rst',
'general/builtins/window-functions.rst',):
s = docsuite(fn, setUp=setUpEmpDeptAndColourArticlesAndGeo)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dml.rst',):
s = docsuite(fn, setUp=setUpLocationsQuotesAndUserVisits)
s.layer = crate_layer
tests.append(s)
for fn in doctest_files('general/dql/union.rst',):
s = docsuite(fn, setUp=setUpPhotosAndCountries)
s.layer = crate_layer
tests.append(s)
# randomize order of tests to make sure they don't depend on each other
random.shuffle(tests)
suite = unittest.TestSuite()
suite.addTests(tests)
return suite
def setUp(self):
add_hadoop_libs(hdfs_repo_libs_path, crate_path())
super(HdfsCrateLayer, self).setUp()
def __init__(self, **kwargs):
super(CrateTestCmd, self).__init__(**kwargs)
doctest_print = PrintWrapper()
self.logger = ColorPrinter(False, stream=doctest_print, line_end='\n')
def stmt(self, stmt):
stmt = stmt.replace('\n', ' ')
if stmt.startswith('\\'):
self.process(stmt)
else:
self.execute(stmt)
cmd = CrateTestCmd(is_tty=False)
jars = [project_path('blackbox', 'tmp', 'crateClient', 'crate-client.jar')]
jars += glob.glob(crate_path('lib', '*.jar'))
java_repl = JavaRepl(
jars=jars,
port=GLOBAL_PORT_POOL.get()
)
def wait_for_schema_update(schema, table, column):
conn = connect('localhost:' + str(CRATE_HTTP_PORT))
c = conn.cursor()
count = 0
while count == 0:
c.execute(('select count(*) from information_schema.columns '
'where schema_name = ? and table_name = ? '
'and column_name = ?'),
(schema, table, column))
