def test_suite(): suite = unittest.TestSuite() s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s1.layer = CrateLayer('crate-enterprise', crate_home=crate_path(), port=MonitoringIntegrationTest.CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format( MonitoringIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': True }) suite.addTest(s1) s2 = unittest.TestSuite( unittest.makeSuite(MonitoringSettingIntegrationTest)) s2.layer = CrateLayer('crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format( MonitoringSettingIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': False }) suite.addTest(s2) return suite
def setUpClass(cls): # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = bind_range(range_size=cls.NUM_SERVERS) for i in range(cls.NUM_SERVERS): http_port = bind_port() layer = CrateLayer( cls.node_name(i), crate_path(), host='localhost', port=http_port, transport_port=transport_port_range, settings=CRATE_SETTINGS, env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')}, cluster_name=cls.__class__.__name__) layer.start() cls.HTTP_PORTS.append(http_port) cls.CRATES.append(layer) dsn = cls.random_dns() num_nodes = 0 # wait until all nodes joined the cluster while num_nodes < len(cls.CRATES): with connect(dsn) as conn: c = conn.cursor() c.execute("select * from sys.nodes") num_nodes = len(c.fetchall()) time.sleep(5)
def test_suite(): env = os.environ.copy() env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT) crateLayer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env=env, settings={ 'license.enterprise': True, 'psql.port': GLOBAL_PORT_POOL.get(), } ) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ConnectionsBeanTest) s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ThreadPoolsBeanTest) s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(CircuitBreakersBeanTest) s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) env = os.environ.copy() env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env=env, settings={ 'license.enterprise': False, } ) suite.addTest(s) return suite
def test_suite(): crateLayer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT) }, settings={ 'license.enterprise': True, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED) }, settings={ 'license.enterprise': False, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) suite.addTest(s) return suite
def test_suite(): suite = unittest.TestSuite(unittest.makeSuite(SigarIntegrationTest)) suite.layer = CrateLayer('crate', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT) return suite
def create_index(index_name, crate_home, output_dir): crate_layer = CrateLayer( 'data', crate_home=crate_home, port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'es.api.enabled': True, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low": "100k", "cluster.routing.allocation.disk.watermark.high": "10k", "cluster.routing.allocation.disk.watermark.flood_stage": "1k", }) crate_layer.start() crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT) try: with connect(crate_http) as conn: cur = conn.cursor() cmds = INDICES[index_name].split(';') for cmd in cmds[:-1]: LOGGER.info(cmd) cur.execute(cmd) cur.execute("select version['number'] from sys.nodes") version = cur.fetchone()[0] r = http.request('POST', crate_http + '/_flush') r.read() compress_index(index_name, version, crate_layer.wdPath(), output_dir) finally: crate_layer.stop()
def create_index(index_name, crate_home, output_dir): crate_layer = CrateLayer('data', crate_home=crate_home, port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'es.api.enabled': True, }) crate_layer.start() crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT) try: with connect(crate_http) as conn: cur = conn.cursor() cmds = INDICES[index_name].split(';') for cmd in cmds[:-1]: LOGGER.info(cmd) cur.execute(cmd) cur.execute("select version['number'] from sys.nodes") version = cur.fetchone()[0] r = http.request('POST', crate_http + '/_flush') r.read() compress_index(index_name, version, crate_layer.wdPath(), output_dir) finally: crate_layer.stop()
def test_suite(): suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest)) crate_layer = CrateLayer('crate-sqllogic', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={'stats.enabled': True}) suite.layer = crate_layer return suite
def test_suite(): crateLayer = CrateLayer('crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT)}, settings={'license.enterprise': True}) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite( unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite( unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ConnectionsBeanTest) s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite( unittest.makeSuite(MonitoringSettingIntegrationTest)) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)}, settings={ 'license.enterprise': False, }) suite.addTest(s) return suite
def create_index(index_name, crate_home, output_dir): crate_layer = CrateLayer( 'data', crate_home=crate_home, port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'es.api.enabled': True, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) crate_layer.start() crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT) try: with connect(crate_http) as conn: cur = conn.cursor() cmds = INDICES[index_name].split(';') for cmd in cmds[:-1]: LOGGER.info(cmd) cur.execute(cmd) cur.execute("select version['number'] from sys.nodes") version = cur.fetchone()[0] r = http.request('POST', crate_http + '/_flush') r.read() compress_index(index_name, version, crate_layer.wdPath(), output_dir) finally: crate_layer.stop()
def create_index(index_name, crate_home, output_dir): crate_layer = CrateLayer( 'data', crate_home=crate_home, port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'es.api.enabled': True, } ) crate_layer.start() crate_http = 'localhost:{}'.format(CRATE_HTTP_PORT) try: with connect(crate_http) as conn: cur = conn.cursor() cmds = INDICES[index_name].split(';') for cmd in cmds[:-1]: LOGGER.info(cmd) cur.execute(cmd) cur.execute("select version['number'] from sys.nodes") version = cur.fetchone()[0] r = http.request('POST', crate_http + '/_flush') r.read() compress_index(index_name, version, crate_layer.wdPath(), output_dir) finally: crate_layer.stop()
def setup_layer(): crate_path = os.path.join(here, 'parts/crate') crate_http_port = get_port_number() crate_transport_port = get_port_number() crate_layer = CrateLayer('crate', crate_home=crate_path, port=crate_http_port, transport_port=crate_transport_port) engine = sa.create_engine('crate://localhost:' + str(crate_http_port)) session = scoped_session( sessionmaker(autoflush=False, autocommit=False, bind=engine)) tornado_layer = TornadoLayer( ScratyApplication(db_session=session, debug=True)) return crate_layer, tornado_layer, engine
def test_suite(): suite = unittest.TestSuite(unittest.makeSuite(SqlLogicTest)) crate_layer = CrateLayer( 'crate-sqllogic', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'psql.port': CRATE_PSQL_PORT, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) suite.layer = crate_layer return suite
def main(): num_nodes = 3 node0_http_port = bind_port() dns_port = bind_port() transport_ports = [] zone_file = ''' crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h ) crate.internal. 400 IN NS localhost crate.internal. 600 IN A 127.0.0.1''' for i in range(0, num_nodes): port = bind_port() transport_ports.append(port) zone_file += ''' _test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format( port=port) dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port) dns_server.start_thread() crate_layers = [] for i in range(0, num_nodes): crate_layer = CrateLayer( 'node-' + str(i), cluster_name='crate-dns-discovery', crate_home=crate_path(), port=node0_http_port if i == 0 else bind_port(), transport_port=transport_ports[i], settings={ 'psql.port': bind_port(), "discovery.zen.hosts_provider": "srv", "discovery.srv.query": "_test._srv.crate.internal.", "discovery.srv.resolver": "127.0.0.1:" + str(dns_port) }) crate_layers.append(crate_layer) crate_layer.start() try: conn = connect('localhost:{}'.format(node0_http_port)) c = conn.cursor() c.execute('''select count() from sys.nodes''') result = c.fetchone() if result[0] != num_nodes: raise AssertionError( "Nodes could not join, expected number of nodes: " + str(num_nodes) + ", found: " + str(result[0])) finally: for crate_layer in crate_layers: crate_layer.stop() dns_server.stop()
def create_index(cfg): crate_layer = CrateLayer( 'data', crate_home=cfg.crate_home, port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT ) crate_layer.start() try: with connect('localhost:' + CRATE_HTTP_PORT) as conn: cur = conn.cursor() cmds = CREATE_INDEX_SQL.split(';') for cmd in cmds[:-1]: cur.execute(cmd) cur.execute("select version['number'] from sys.nodes") version = cur.fetchone()[0] compress_index(version, crate_layer.wdPath(), cfg.output_dir) finally: crate_layer.stop()
def ensure_cratedb_layer(): """ In order to skip individual tests by manually disabling them within `def test_suite()`, it is crucial make the test layer not run on each and every occasion. So, things like this will be possible:: ./bin/test -vvvv --ignore_dir=testing TODO: Through a subsequent patch, the possibility to individually unselect specific tests might be added to `def test_suite()` on behalf of environment variables. A blueprint for this kind of logic can be found at https://github.com/crate/crate/commit/414cd833. """ global crate_layer if crate_layer is None: crate_layer = CrateLayer('crate', crate_home=crate_path(), port=crate_port, host=local, transport_port=crate_transport_port, settings=settings) return crate_layer
def main(): crate_layer = CrateLayer( 'crate-sqllogic', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'psql.port': CRATE_PSQL_PORT, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low": "100k", "cluster.routing.allocation.disk.watermark.high": "10k", "cluster.routing.allocation.disk.watermark.flood_stage": "1k", }) crate_layer.start() logfiles = [] try: with ProcessPoolExecutor() as executor: futures = [] for i, filename in enumerate(tests_path.glob('**/*.test')): filepath = tests_path / filename relpath = str(filepath.relative_to(tests_path)) if not any(p.match(str(relpath)) for p in FILE_WHITELIST): continue logfile = f'sqllogic-{os.path.basename(relpath)}-{i}.log' logfiles.append(logfile) future = executor.submit(run_file, filename=str(filepath), host='localhost', port=str(CRATE_PSQL_PORT), log_level=logging.WARNING, log_file=logfile, failfast=True, schema=f'x{i}') futures.append(future) for future in as_completed(futures): future.result() finally: crate_layer.stop() # instead of having dozens file merge to one which is in gitignore merge_logfiles(logfiles)
from crate.crash.outputs import _val_len as val_len from crate.crash.command import (CrateShell, main, get_stdin, noargs_command, host_and_port, _create_shell, get_parser) crate_http_port = 44209 crate_transport_port = 44309 crate_settings = { 'cluster.name': 'Testing44209', 'node.name': 'crate', 'psql.port': 45441, 'lang.js.enabled': True, 'http.port': crate_http_port, 'transport.tcp.port': crate_transport_port } node = CrateLayer.from_uri( 'https://cdn.crate.io/downloads/releases/crate-4.0.2.tar.gz', 'crate', settings=crate_settings) def setUpModule(): node.start() def tearDownModule(): node.stop() def fake_stdin(data): stdin = TextIOWrapper(tempfile.TemporaryFile()) stdin.write(data) stdin.flush()
def printjson(jsonstr): d = json.loads(jsonstr.decode('utf-8')) print(json.dumps(d, indent=4, sort_keys=True)) def get_rnd_port(): sock = socket.socket() sock.bind(('', 0)) return sock.getsockname()[1] crate_port = get_rnd_port() crate_transport = get_rnd_port() crate_layer = CrateLayer('crate', crate_home=crate_path, crate_exec=crate_exec, port=crate_port, transport_port=crate_transport) tornado_port = get_rnd_port() tornado_host = 'localhost:{port}'.format(port=tornado_port) tornado_uri = 'http://{host}'.format(host=tornado_host) engine = create_engine('crate://localhost:{}'.format(crate_port)) Session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=engine)) app = StreamMania(dbsession=Session, debug_mode=True) tornado_layer = TornadoLayer(app, tornado_port, patches=[tvdb_mock]) crate_host = '127.0.0.1:{port}'.format(port=crate_port) crate_uri = 'http://{0}'.format(crate_host)
def set_next_response(self, response): self.response = response def set_next_server_infos(self, server, server_name, version): self._server_infos = (server, server_name, version) def setUpMocked(test): test.globs['connection_client_mocked'] = ClientMocked() crate_port = 44209 crate_transport_port = 44309 crate_layer = CrateLayer('crate', crate_home=crate_path(), crate_exec=crate_path('bin', 'crate'), port=crate_port, transport_port=crate_transport_port) crate_host = "127.0.0.1:{port}".format(port=crate_port) crate_uri = "http://%s" % crate_host def setUpWithCrateLayer(test): test.globs['HttpClient'] = http.Client test.globs['crate_host'] = crate_host test.globs['pprint'] = pprint test.globs['print'] = cprint conn = connect(crate_host) cursor = conn.cursor()
def set_next_response(self, response): self.response = response def set_next_server_infos(self, server, server_name, version): self._server_infos = (server, server_name, version) def setUpMocked(test): test.globs['connection_client_mocked'] = ClientMocked() crate_port = 44209 crate_transport_port = 44309 crate_layer = CrateLayer('crate', crate_home=crate_path(), port=crate_port, transport_port=crate_transport_port) crate_host = "127.0.0.1:{port}".format(port=crate_port) crate_uri = "http://%s" % crate_host def setUpForDjango(test): connect(crate_host) import os os.environ["DJANGO_SETTINGS_MODULE"] = "crate.client.django.tests.settings" from django.test.runner import setup_databases setup_databases(3, False)
from crate.client.cursor import Cursor from crate.testing.layer import CrateLayer from uuid import uuid4 def rnd_port(): sock = socket.socket() sock.bind(('127.0.0.1', 0)) port = sock.getsockname()[1] return port here = os.path.dirname(__file__) root = os.path.join(here, '..') crate_layer = CrateLayer('crate', port=rnd_port(), transport_port=rnd_port(), crate_home=os.path.join(root, 'parts', 'crate')) db = CrateDatabase(servers=crate_layer.crate_servers) def gen_key(): return str(uuid4()) class BaseModel(Model): class Meta: database = db class User(BaseModel):
-Dcom.sun.management.jmxremote.port={} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false ''' log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch) env = os.environ.copy() env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT) enterprise_crate = CrateLayer('crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=0, env=env, settings={ 'license.enterprise': True, }) env = os.environ.copy() env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED) community_crate = CrateLayer('crate', crate_home=crate_path(), port=bind_port(), transport_port=0, env=env, settings={ 'license.enterprise': False, })
crash_parser = zc.customdoctests.DocTestParser(ps1='cr>', comment_prefix='#', transform=crash_transform) crate_http_port = 44209 crate_transport_port = 44309 crate_settings = { 'psql.port': 45441, 'lang.js.enabled': True, } crate_layer = CrateLayer('crate', crate_home=crate_path(), crate_exec=crate_path('bin', 'crate'), port=crate_http_port, settings=crate_settings, transport_port=crate_transport_port) crate_host = "127.0.0.1:" + str(crate_http_port) crate_uri = "http://" + crate_host def setUp(test): test.globs['cmd'] = CrateTestCmd(error_trace=True, is_tty=False) def tearDown(test): test.globs['cmd'].exit()
'license.enterprise': 'true', 'lang.js.enabled': 'true', 'auth.host_based.enabled': 'true', 'auth.host_based.config.0.user': '******', 'auth.host_based.config.0.method': 'trust', 'auth.host_based.config.98.user': '******', 'auth.host_based.config.98.method': 'trust', 'auth.host_based.config.99.user': '******', 'auth.host_based.config.99.method': 'password', } crate_port = 44209 crate_transport_port = 44309 local = '127.0.0.1' crate_layer = CrateLayer('crate', crate_home=crate_path(), port=crate_port, host=local, transport_port=crate_transport_port, settings=settings) crate_host = "{host}:{port}".format(host=local, port=crate_port) crate_uri = "http://%s" % crate_host def refresh(table): with connect(crate_host) as conn: cursor = conn.cursor() cursor.execute("refresh table %s" % table) def setUpWithCrateLayer(test): test.globs['HttpClient'] = http.Client