def setUp(self):
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
transport_ports = [bind_port() for _ in range(self.num_nodes)]
for port in transport_ports:
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(
port=port)
dns_port = bind_port()
self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
self.dns_server.start_thread()
self.nodes = nodes = []
for i in range(self.num_nodes):
settings = {
'node.name': f'node-{i}',
'cluster.name': 'crate-dns-discovery',
'psql.port': 0,
'transport.tcp.port': transport_ports[i],
"discovery.seed_providers": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
}
if i is 0:
settings['cluster.initial_master_nodes'] = f'node-{i}'
node = CrateNode(crate_dir=crate_path(),
version=(4, 0, 0),
settings=settings,
env={'CRATE_HEAP_SIZE': '256M'})
node.start()
nodes.append(node)
def main():
num_nodes = 3
node0_http_port = bind_port()
dns_port = bind_port()
transport_ports = []
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
for i in range(0, num_nodes):
port = bind_port()
transport_ports.append(port)
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(
port=port)
dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
dns_server.start_thread()
crate_layers = []
for i in range(0, num_nodes):
crate_layer = CrateLayer(
'node-' + str(i),
cluster_name='crate-dns-discovery',
crate_home=crate_path(),
port=node0_http_port if i == 0 else bind_port(),
transport_port=transport_ports[i],
settings={
'psql.port': bind_port(),
"discovery.zen.hosts_provider": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
})
crate_layers.append(crate_layer)
crate_layer.start()
try:
conn = connect('localhost:{}'.format(node0_http_port))
c = conn.cursor()
c.execute('''select count() from sys.nodes''')
result = c.fetchone()
if result[0] != num_nodes:
raise AssertionError(
"Nodes could not join, expected number of nodes: " +
str(num_nodes) + ", found: " + str(result[0]))
finally:
for crate_layer in crate_layers:
crate_layer.stop()
dns_server.stop()
def setUp(self):
self.crates = []
self.clients = []
self.node_names = []
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=self.num_servers)
for i in range(self.num_servers):
layer = GracefulStopCrateLayer(
self.node_name(i),
crate_path(),
host='localhost',
port=bind_port(),
transport_port=transport_port_range,
settings={
# The disk.watermark settings can be removed once crate-python > 0.21.1 has been released
"cluster.routing.allocation.disk.watermark.low": "100k",
"cluster.routing.allocation.disk.watermark.high": "10k",
"cluster.routing.allocation.disk.watermark.flood_stage":
"1k",
},
env=os.environ.copy(),
cluster_name=self.__class__.__name__)
layer.start()
self.clients.append(Client(layer.crate_servers))
self.crates.append(layer)
self.node_names.append(self.node_name(i))
client = self.random_client()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(self.crates):
response = client.sql("select * from sys.nodes")
num_nodes = response.get("rowcount", 0)
time.sleep(.5)
def setUpClass(cls):
# auto-discovery with unicast on the same host only works if all nodes are configured with the same port range
transport_port_range = bind_range(range_size=cls.NUM_SERVERS)
for i in range(cls.NUM_SERVERS):
http_port = bind_port()
layer = CrateLayer(
cls.node_name(i),
crate_path(),
host='localhost',
port=http_port,
transport_port=transport_port_range,
settings=CRATE_SETTINGS,
env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')},
cluster_name=cls.__class__.__name__)
layer.start()
cls.HTTP_PORTS.append(http_port)
cls.CRATES.append(layer)
dsn = cls.random_dns()
num_nodes = 0
# wait until all nodes joined the cluster
while num_nodes < len(cls.CRATES):
with connect(dsn) as conn:
c = conn.cursor()
c.execute("select * from sys.nodes")
num_nodes = len(c.fetchall())
time.sleep(5)
def setUp(self):
zone_file = '''
crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h )
crate.internal. 400 IN NS localhost
crate.internal. 600 IN A 127.0.0.1'''
transport_ports = [bind_port() for _ in range(self.num_nodes)]
for port in transport_ports:
zone_file += '''
_test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(port=port)
dns_port = bind_port()
self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port)
self.dns_server.start_thread()
self.nodes = nodes = []
for i in range(self.num_nodes):
settings = {
'node.name': f'node-{i}',
'cluster.name': 'crate-dns-discovery',
'psql.port': 0,
'transport.tcp.port': transport_ports[i],
"discovery.seed_providers": "srv",
"discovery.srv.query": "_test._srv.crate.internal.",
"discovery.srv.resolver": "127.0.0.1:" + str(dns_port)
}
if i is 0:
settings['cluster.initial_master_nodes'] = f'node-{i}'
node = CrateNode(
crate_dir=crate_path(),
version=(4, 0, 0),
settings=settings,
env={
'CRATE_HEAP_SIZE': '256M'
}
)
node.start()
nodes.append(node)
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.
import os
import re
import unittest
from crate.client import connect
from testutils.ports import bind_port
from testutils.paths import crate_path
from cr8.run_crate import CrateNode
from subprocess import PIPE, Popen
from urllib.request import urlretrieve
JMX_PORT = bind_port()
JMX_OPTS = '''
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
-Dio.netty.leakDetection.level=paranoid
'''
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
env['CRATE_HEAP_SIZE'] = '256M'
enterprise_crate = CrateNode(crate_dir=crate_path(),
settings={
'transport.tcp.port': 0,
'psql.port': 0,
import logging
from testutils.ports import bind_port
from testutils.paths import crate_path, project_root
from cr8.run_crate import CrateNode
from crate.client import connect
from urllib.error import HTTPError
from urllib.request import urlretrieve
HADOOP_VERSION = '2.8.5'
HADOOP_SOURCE = ('http://www-eu.apache.org/dist/hadoop/common/'
'hadoop-{version}/hadoop-{version}.tar.gz'.format(version=HADOOP_VERSION))
CACHE_DIR = os.environ.get(
'XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache', 'crate-tests'))
NN_PORT = bind_port()
hdfs_repo_libs_path = os.path.join(
project_root,
'es',
'es-repository-hdfs',
'build',
'extraLibs')
log = logging.getLogger('crate.testing.layer')
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
log.addHandler(ch)
# However, to use any modules in this file marked as "Enterprise Features",
# Crate must have given you permission to enable and use such Enterprise
# Features and you must have a valid Enterprise or Subscription Agreement
# with Crate. If you enable or use the Enterprise Features, you represent
# and warrant that you have a valid Enterprise or Subscription Agreement
# with Crate. Your use of the Enterprise Features if governed by the terms
# and conditions of your Enterprise or Subscription Agreement with Crate.
import os
import unittest
from testutils.ports import bind_port
from testutils.paths import crate_path
from cr8.run_crate import CrateNode
from test_jmx import JmxClient
JMX_PORT = bind_port()
JMX_OPTS = '''
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
'''
env = os.environ.copy()
env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT)
crate_layer = CrateNode(
crate_dir=crate_path(),
version=(4, 0, 0),
settings={
'transport.tcp.port': 0,
import shutil
import json
import re
import random
import tempfile
import logging
import subprocess
import unittest
from functools import partial
from testutils.paths import crate_path, project_path
from testutils.ports import bind_port
from crate.crash.command import CrateShell
from crate.crash.printer import PrintWrapper, ColorPrinter
from crate.client import connect
CRATE_HTTP_PORT = bind_port()
CRATE_DSN = 'localhost:' + str(CRATE_HTTP_PORT)
log = logging.getLogger('crate.testing.layer')
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
log.addHandler(ch)
class CrateTestShell(CrateShell):
def __init__(self):
super(CrateTestShell, self).__init__(is_tty=False)
self.logger = ColorPrinter(False, stream=PrintWrapper(), line_end='\n')
def stmt(self, stmt):
stmt = stmt.replace('\n', ' ')
# with Crate. Your use of the Enterprise Features if governed by the terms
# and conditions of your Enterprise or Subscription Agreement with Crate.
import os
import re
import unittest
import time
import logging
from crate.client import connect
from testutils.ports import bind_port
from testutils.paths import crate_path
from crate.testing.layer import CrateLayer
from subprocess import PIPE, Popen
from urllib.request import urlretrieve
JMX_PORT = bind_port()
CRATE_HTTP_PORT = bind_port()
JMX_PORT_ENTERPRISE_DISABLED = bind_port()
JMX_OPTS = '''
-Dcom.sun.management.jmxremote
-Dcom.sun.management.jmxremote.port={}
-Dcom.sun.management.jmxremote.ssl=false
-Dcom.sun.management.jmxremote.authenticate=false
'''
log = logging.getLogger('crate.testing.layer')
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
log.addHandler(ch)
import logging
from testutils.ports import bind_port
from testutils.paths import crate_path, project_root
from cr8.run_crate import CrateNode
from crate.client import connect
from urllib.error import HTTPError
from urllib.request import urlretrieve
HADOOP_VERSION = '2.8.4'
HADOOP_SOURCE = ('http://www-eu.apache.org/dist/hadoop/common/'
'hadoop-{version}/hadoop-{version}.tar.gz'.format(version=HADOOP_VERSION))
CACHE_DIR = os.environ.get(
'XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache', 'crate-tests'))
NN_PORT = bind_port()
hdfs_repo_libs_path = os.path.join(
project_root,
'es',
'es-repository-hdfs',
'build',
'extraLibs')
log = logging.getLogger('crate.testing.layer')
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
log.addHandler(ch)
