def setUp(self): zone_file = ''' crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h ) crate.internal. 400 IN NS localhost crate.internal. 600 IN A 127.0.0.1''' transport_ports = [bind_port() for _ in range(self.num_nodes)] for port in transport_ports: zone_file += ''' _test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format( port=port) dns_port = bind_port() self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port) self.dns_server.start_thread() self.nodes = nodes = [] for i in range(self.num_nodes): settings = { 'node.name': f'node-{i}', 'cluster.name': 'crate-dns-discovery', 'psql.port': 0, 'transport.tcp.port': transport_ports[i], "discovery.seed_providers": "srv", "discovery.srv.query": "_test._srv.crate.internal.", "discovery.srv.resolver": "127.0.0.1:" + str(dns_port) } if i is 0: settings['cluster.initial_master_nodes'] = f'node-{i}' node = CrateNode(crate_dir=crate_path(), version=(4, 0, 0), settings=settings, env={'CRATE_HEAP_SIZE': '256M'}) node.start() nodes.append(node)
def main(): num_nodes = 3 node0_http_port = bind_port() dns_port = bind_port() transport_ports = [] zone_file = ''' crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h ) crate.internal. 400 IN NS localhost crate.internal. 600 IN A 127.0.0.1''' for i in range(0, num_nodes): port = bind_port() transport_ports.append(port) zone_file += ''' _test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format( port=port) dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port) dns_server.start_thread() crate_layers = [] for i in range(0, num_nodes): crate_layer = CrateLayer( 'node-' + str(i), cluster_name='crate-dns-discovery', crate_home=crate_path(), port=node0_http_port if i == 0 else bind_port(), transport_port=transport_ports[i], settings={ 'psql.port': bind_port(), "discovery.zen.hosts_provider": "srv", "discovery.srv.query": "_test._srv.crate.internal.", "discovery.srv.resolver": "127.0.0.1:" + str(dns_port) }) crate_layers.append(crate_layer) crate_layer.start() try: conn = connect('localhost:{}'.format(node0_http_port)) c = conn.cursor() c.execute('''select count() from sys.nodes''') result = c.fetchone() if result[0] != num_nodes: raise AssertionError( "Nodes could not join, expected number of nodes: " + str(num_nodes) + ", found: " + str(result[0])) finally: for crate_layer in crate_layers: crate_layer.stop() dns_server.stop()
def setUp(self): self.crates = [] self.clients = [] self.node_names = [] # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = bind_range(range_size=self.num_servers) for i in range(self.num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), host='localhost', port=bind_port(), transport_port=transport_port_range, settings={ # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low": "100k", "cluster.routing.allocation.disk.watermark.high": "10k", "cluster.routing.allocation.disk.watermark.flood_stage": "1k", }, env=os.environ.copy(), cluster_name=self.__class__.__name__) layer.start() self.clients.append(Client(layer.crate_servers)) self.crates.append(layer) self.node_names.append(self.node_name(i)) client = self.random_client() num_nodes = 0 # wait until all nodes joined the cluster while num_nodes < len(self.crates): response = client.sql("select * from sys.nodes") num_nodes = response.get("rowcount", 0) time.sleep(.5)
def setUpClass(cls): # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = bind_range(range_size=cls.NUM_SERVERS) for i in range(cls.NUM_SERVERS): http_port = bind_port() layer = CrateLayer( cls.node_name(i), crate_path(), host='localhost', port=http_port, transport_port=transport_port_range, settings=CRATE_SETTINGS, env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')}, cluster_name=cls.__class__.__name__) layer.start() cls.HTTP_PORTS.append(http_port) cls.CRATES.append(layer) dsn = cls.random_dns() num_nodes = 0 # wait until all nodes joined the cluster while num_nodes < len(cls.CRATES): with connect(dsn) as conn: c = conn.cursor() c.execute("select * from sys.nodes") num_nodes = len(c.fetchall()) time.sleep(5)
def setUp(self): zone_file = ''' crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h ) crate.internal. 400 IN NS localhost crate.internal. 600 IN A 127.0.0.1''' transport_ports = [bind_port() for _ in range(self.num_nodes)] for port in transport_ports: zone_file += ''' _test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(port=port) dns_port = bind_port() self.dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port) self.dns_server.start_thread() self.nodes = nodes = [] for i in range(self.num_nodes): settings = { 'node.name': f'node-{i}', 'cluster.name': 'crate-dns-discovery', 'psql.port': 0, 'transport.tcp.port': transport_ports[i], "discovery.seed_providers": "srv", "discovery.srv.query": "_test._srv.crate.internal.", "discovery.srv.resolver": "127.0.0.1:" + str(dns_port) } if i is 0: settings['cluster.initial_master_nodes'] = f'node-{i}' node = CrateNode( crate_dir=crate_path(), version=(4, 0, 0), settings=settings, env={ 'CRATE_HEAP_SIZE': '256M' } ) node.start() nodes.append(node)
# # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. import os import re import unittest from crate.client import connect from testutils.ports import bind_port from testutils.paths import crate_path from cr8.run_crate import CrateNode from subprocess import PIPE, Popen from urllib.request import urlretrieve JMX_PORT = bind_port() JMX_OPTS = ''' -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port={} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dio.netty.leakDetection.level=paranoid ''' env = os.environ.copy() env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT) env['CRATE_HEAP_SIZE'] = '256M' enterprise_crate = CrateNode(crate_dir=crate_path(), settings={ 'transport.tcp.port': 0, 'psql.port': 0,
import logging from testutils.ports import bind_port from testutils.paths import crate_path, project_root from cr8.run_crate import CrateNode from crate.client import connect from urllib.error import HTTPError from urllib.request import urlretrieve HADOOP_VERSION = '2.8.5' HADOOP_SOURCE = ('http://www-eu.apache.org/dist/hadoop/common/' 'hadoop-{version}/hadoop-{version}.tar.gz'.format(version=HADOOP_VERSION)) CACHE_DIR = os.environ.get( 'XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache', 'crate-tests')) NN_PORT = bind_port() hdfs_repo_libs_path = os.path.join( project_root, 'es', 'es-repository-hdfs', 'build', 'extraLibs') log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch)
# However, to use any modules in this file marked as "Enterprise Features", # Crate must have given you permission to enable and use such Enterprise # Features and you must have a valid Enterprise or Subscription Agreement # with Crate. If you enable or use the Enterprise Features, you represent # and warrant that you have a valid Enterprise or Subscription Agreement # with Crate. Your use of the Enterprise Features if governed by the terms # and conditions of your Enterprise or Subscription Agreement with Crate. import os import unittest from testutils.ports import bind_port from testutils.paths import crate_path from cr8.run_crate import CrateNode from test_jmx import JmxClient JMX_PORT = bind_port() JMX_OPTS = ''' -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port={} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false ''' env = os.environ.copy() env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT) crate_layer = CrateNode( crate_dir=crate_path(), version=(4, 0, 0), settings={ 'transport.tcp.port': 0,
import shutil import json import re import random import tempfile import logging import subprocess import unittest from functools import partial from testutils.paths import crate_path, project_path from testutils.ports import bind_port from crate.crash.command import CrateShell from crate.crash.printer import PrintWrapper, ColorPrinter from crate.client import connect CRATE_HTTP_PORT = bind_port() CRATE_DSN = 'localhost:' + str(CRATE_HTTP_PORT) log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch) class CrateTestShell(CrateShell): def __init__(self): super(CrateTestShell, self).__init__(is_tty=False) self.logger = ColorPrinter(False, stream=PrintWrapper(), line_end='\n') def stmt(self, stmt): stmt = stmt.replace('\n', ' ')
# with Crate. Your use of the Enterprise Features if governed by the terms # and conditions of your Enterprise or Subscription Agreement with Crate. import os import re import unittest import time import logging from crate.client import connect from testutils.ports import bind_port from testutils.paths import crate_path from crate.testing.layer import CrateLayer from subprocess import PIPE, Popen from urllib.request import urlretrieve JMX_PORT = bind_port() CRATE_HTTP_PORT = bind_port() JMX_PORT_ENTERPRISE_DISABLED = bind_port() JMX_OPTS = ''' -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port={} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false ''' log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch)
import logging from testutils.ports import bind_port from testutils.paths import crate_path, project_root from cr8.run_crate import CrateNode from crate.client import connect from urllib.error import HTTPError from urllib.request import urlretrieve HADOOP_VERSION = '2.8.4' HADOOP_SOURCE = ('http://www-eu.apache.org/dist/hadoop/common/' 'hadoop-{version}/hadoop-{version}.tar.gz'.format(version=HADOOP_VERSION)) CACHE_DIR = os.environ.get( 'XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache', 'crate-tests')) NN_PORT = bind_port() hdfs_repo_libs_path = os.path.join( project_root, 'es', 'es-repository-hdfs', 'build', 'extraLibs') log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch)