class MonitoringIntegrationTest(unittest.TestCase): JMX_PORT = GLOBAL_PORT_POOL.get() CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() def test_mbean_select_frq_attribute(self): jmx_client = JmxTermClient(MonitoringIntegrationTest.JMX_PORT) conn = connect('localhost:{}' .format(MonitoringIntegrationTest.CRATE_HTTP_PORT)) c = conn.cursor() to_sleep = 0.2 while True: c.execute("select 1") value, _ = jmx_client.query_jmx( 'io.crate.monitoring:type=QueryStats', 'SelectQueryFrequency' ) if float(value) > 0.0: break if to_sleep > 30: raise AssertionError('''The mbean attribute has not produced the expected result.''') time.sleep(to_sleep) to_sleep *= 2
def __init__(self, *args, **kwargs): num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS)) super(GracefulStopTest, self).__init__(*args, **kwargs) self.crates = [] self.clients = [] self.node_names = [] # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = GLOBAL_PORT_POOL.get_range(range_size=num_servers) for i in range(num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), host='localhost', port=GLOBAL_PORT_POOL.get(), transport_port=transport_port_range, settings={ # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low": "100k", "cluster.routing.allocation.disk.watermark.high": "10k", "cluster.routing.allocation.disk.watermark.flood_stage": "1k", }, cluster_name=self.__class__.__name__) client = Client(layer.crate_servers) self.crates.append(layer) self.clients.append(client) self.node_names.append(self.node_name(i)) self.layer = CascadedLayer( "{0}_{1}_crates".format(self.__class__.__name__, num_servers), *self.crates )
def test_suite(): suite = unittest.TestSuite() s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s1.layer = CrateLayer('crate-enterprise', crate_home=crate_path(), port=MonitoringIntegrationTest.CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format( MonitoringIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': True }) suite.addTest(s1) s2 = unittest.TestSuite( unittest.makeSuite(MonitoringSettingIntegrationTest)) s2.layer = CrateLayer('crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format( MonitoringSettingIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': False }) suite.addTest(s2) return suite
def __init__(self, *args, **kwargs): num_servers = kwargs.pop( "num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS)) super(GracefulStopTest, self).__init__(*args, **kwargs) self.crates = [] self.clients = [] self.node_names = [] # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = GLOBAL_PORT_POOL.get_range( range_size=num_servers) for i in range(num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), host='localhost', port=GLOBAL_PORT_POOL.get(), transport_port=transport_port_range, cluster_name=self.__class__.__name__) client = Client(layer.crate_servers) self.crates.append(layer) self.clients.append(client) self.node_names.append(self.node_name(i)) self.layer = CascadedLayer( "{0}_{1}_crates".format(self.__class__.__name__, num_servers), *self.crates)
def test_suite(): env = os.environ.copy() env['CRATE_JAVA_OPTS'] = JMX_OPTS.format(JMX_PORT) crateLayer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env=env, settings={ 'license.enterprise': True, 'psql.port': GLOBAL_PORT_POOL.get(), } ) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ConnectionsBeanTest) s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ThreadPoolsBeanTest) s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(CircuitBreakersBeanTest) s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) env = os.environ.copy() env["CRATE_JAVA_OPTS"] = JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env=env, settings={ 'license.enterprise': False, } ) suite.addTest(s) return suite
def test_suite(): crateLayer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT) }, settings={ 'license.enterprise': True, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED) }, settings={ 'license.enterprise': False, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) suite.addTest(s) return suite
def main(): num_nodes = 3 node0_http_port = GLOBAL_PORT_POOL.get() dns_port = GLOBAL_PORT_POOL.get() transport_ports = [] zone_file = ''' crate.internal. 600 IN SOA localhost localhost ( 2007120710 1d 2h 4w 1h ) crate.internal. 400 IN NS localhost crate.internal. 600 IN A 127.0.0.1''' for i in range(0, num_nodes): port = GLOBAL_PORT_POOL.get() transport_ports.append(port) zone_file += ''' _test._srv.crate.internal. 600 IN SRV 1 10 {port} 127.0.0.1.'''.format(port=port) dns_server = DNSServer(ZoneResolver(zone_file), port=dns_port) dns_server.start_thread() crate_layers = [] for i in range(0, num_nodes): crate_layer = CrateLayer( 'node-' + str(i), cluster_name='crate-dns-discovery', crate_home=crate_path(), port=node0_http_port if i == 0 else GLOBAL_PORT_POOL.get(), transport_port=transport_ports[i], settings={ 'psql.port': GLOBAL_PORT_POOL.get(), "discovery.zen.hosts_provider": "srv", "discovery.srv.query": "_test._srv.crate.internal.", "discovery.srv.resolver": "127.0.0.1:" + str(dns_port) } ) crate_layers.append(crate_layer) crate_layer.start() try: conn = connect('localhost:{}'.format(node0_http_port)) c = conn.cursor() c.execute('''select count() from sys.nodes''') result = c.fetchone() if result[0] != num_nodes: raise AssertionError("Nodes could not join, expected number of nodes: " + str(num_nodes) + ", found: " + str(result[0])) finally: for crate_layer in crate_layers: crate_layer.stop() dns_server.stop()
class MonitoringNodeStatusIntegrationTest(unittest.TestCase): JMX_PORT = GLOBAL_PORT_POOL.get() CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() def test_mbean_select_ready(self): jmx_client = JmxTermClient(MonitoringIntegrationTest.JMX_PORT) value, _ = jmx_client.query_jmx('io.crate.monitoring:type=NodeStatus', 'Ready') value = bool(value) if value is not True: raise AssertionError( "The mbean attribute has not produced the expected result. " + "Expected: True, Got: {}".format(value))
def test_suite(): suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=MonitoringIntegrationTest.CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(MonitoringIntegrationTest.JMX_PORT) }, settings={ 'license.enterprise': True, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) suite.addTest(s) s = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(MonitoringSettingIntegrationTest.JMX_PORT) }, settings={ 'license.enterprise': False, # The disk.watermark settings can be removed once crate-python > 0.21.1 has been released "cluster.routing.allocation.disk.watermark.low" : "100k", "cluster.routing.allocation.disk.watermark.high" : "10k", "cluster.routing.allocation.disk.watermark.flood_stage" : "1k", } ) suite.addTest(s) return suite
def test_suite(): crateLayer = CrateLayer('crate-enterprise', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT)}, settings={'license.enterprise': True}) # Graceful tests suite = unittest.TestSuite() s = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite( unittest.makeSuite(MonitoringNodeStatusIntegrationTest)) s.layer = crateLayer suite.addTest(s) s = unittest.TestSuite( unittest.makeSuite(MonitoringNodeInfoIntegrationTest)) s.layer = s.layer = crateLayer suite.addTest(s) s = unittest.makeSuite(ConnectionsBeanTest) s.layer = crateLayer suite.addTest(s) # JMX Disabled test s = unittest.TestSuite( unittest.makeSuite(MonitoringSettingIntegrationTest)) s.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={"CRATE_JAVA_OPTS": JMX_OPTS.format(JMX_PORT_ENTERPRISE_DISABLED)}, settings={ 'license.enterprise': False, }) suite.addTest(s) return suite
def __init__(self, *args, **kwargs): num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS)) super(GracefulStopTest, self).__init__(*args, **kwargs) self.crates = [] self.clients = [] self.node_names = [] for i in range(num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), cluster_name=self.__class__.__name__) client = Client(layer.crate_servers) self.crates.append(layer) self.clients.append(client) self.node_names.append(self.node_name(i)) self.layer = CascadedLayer( "{0}_{1}_crates".format(self.__class__.__name__, num_servers), *self.crates )
def __init__(self, *args, **kwargs): num_servers = kwargs.pop( "num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS)) super(GracefulStopTest, self).__init__(*args, **kwargs) self.crates = [] self.clients = [] self.node_names = [] for i in range(num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), cluster_name=self.__class__.__name__) client = Client(layer.crate_servers) self.crates.append(layer) self.clients.append(client) self.node_names.append(self.node_name(i)) self.layer = CascadedLayer( "{0}_{1}_crates".format(self.__class__.__name__, num_servers), *self.crates)
class MonitoringSettingIntegrationTest(unittest.TestCase): JMX_PORT = GLOBAL_PORT_POOL.get() def test_enterprise_setting_disabled(self): jmx_client = JmxTermClient(MonitoringIntegrationTest.JMX_PORT) value, _ = jmx_client.query_jmx('io.crate.monitoring:type=QueryStats', 'SelectQueryFrequency') try: float(value) raise AssertionError('''The JMX monitoring is enabled.''') except: pass
def test_suite(): crate_layer = HdfsCrateLayer('crate', host='localhost', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, settings={ 'psql.port': GLOBAL_PORT_POOL.get(), }) hadoop_layer = HadoopLayer() layer = HadoopAndCrateLayer(crate_layer, hadoop_layer) suite = unittest.makeSuite(HdfsIntegrationTest) suite.layer = layer return suite
def test_suite(): suite = unittest.TestSuite() s1 = unittest.TestSuite(unittest.makeSuite(MonitoringIntegrationTest)) s1.layer = CrateLayer( 'crate-enterprise', crate_home=crate_path(), port=MonitoringIntegrationTest.CRATE_HTTP_PORT, transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(MonitoringIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': True } ) suite.addTest(s1) s2 = unittest.TestSuite(unittest.makeSuite(MonitoringSettingIntegrationTest)) s2.layer = CrateLayer( 'crate', crate_home=crate_path(), port=GLOBAL_PORT_POOL.get(), transport_port=GLOBAL_PORT_POOL.get(), env={ "CRATE_JAVA_OPTS": JMX_OPTS.format(MonitoringSettingIntegrationTest.JMX_PORT) }, settings={ 'stats.enabled': True, 'license.enterprise': False } ) suite.addTest(s2) return suite
def __init__(self, *args, **kwargs): num_servers = kwargs.pop("num_servers", getattr(self, "NUM_SERVERS", self.DEFAULT_NUM_SERVERS)) super(GracefulStopTest, self).__init__(*args, **kwargs) self.crates = [] self.clients = [] self.node_names = [] # auto-discovery with unicast on the same host only works if all nodes are configured with the same port range transport_port_range = GLOBAL_PORT_POOL.get_range(range_size=num_servers) for i in range(num_servers): layer = GracefulStopCrateLayer( self.node_name(i), crate_path(), host='localhost', port=GLOBAL_PORT_POOL.get(), transport_port=transport_port_range, cluster_name=self.__class__.__name__) client = Client(layer.crate_servers) self.crates.append(layer) self.clients.append(client) self.node_names.append(self.node_name(i)) self.layer = CascadedLayer( "{0}_{1}_crates".format(self.__class__.__name__, num_servers), *self.crates )
# with Crate. Your use of the Enterprise Features if governed by the terms # and conditions of your Enterprise or Subscription Agreement with Crate. import os import re import unittest import time import logging from crate.client import connect from testutils.ports import GLOBAL_PORT_POOL from testutils.paths import crate_path from crate.testing.layer import CrateLayer from subprocess import PIPE, Popen from urllib.request import urlretrieve JMX_PORT = GLOBAL_PORT_POOL.get() CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() JMX_PORT_ENTERPRISE_DISABLED = GLOBAL_PORT_POOL.get() JMX_OPTS = ''' -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port={} -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false ''' log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch)
from testutils.ports import GLOBAL_PORT_POOL from testutils.paths import crate_path, project_root from crate.testing.layer import CrateLayer from crate.client import connect from urllib.error import HTTPError from urllib.request import urlretrieve HADOOP_VERSION = '2.8.4' HADOOP_SOURCE = ('http://www-eu.apache.org/dist/hadoop/common/' 'hadoop-{version}/hadoop-{version}.tar.gz'.format( version=HADOOP_VERSION)) CACHE_DIR = os.environ.get( 'XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache', 'crate-tests')) CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() CRATE_TRANSPORT_PORT = GLOBAL_PORT_POOL.get() NN_PORT = '49000' hdfs_repo_libs_path = os.path.join(project_root, 'es', 'es-repository-hdfs', 'build', 'extraLibs') log = logging.getLogger('crate.testing.layer') ch = logging.StreamHandler() ch.setLevel(logging.ERROR) log.addHandler(ch) def add_hadoop_libs(hdfs_repo_libs_path, path_to_dist): hdfs_plugin_location = os.path.join(path_to_dist, 'plugins', 'es-repository-hdfs')
# License for the specific language governing permissions and limitations # under the License. # # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. import unittest import time from crate.client import connect from crate.testing.layer import CrateLayer from testutils.ports import GLOBAL_PORT_POOL from testutils.paths import crate_path CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() CRATE_TRANSPORT_PORT = GLOBAL_PORT_POOL.get() class SigarIntegrationTest(unittest.TestCase): def test_multiple_probe_selects(self): conn = connect('localhost:{}'.format(CRATE_HTTP_PORT)) cur = conn.cursor() cur.execute("SELECT os, load, os FROM sys.nodes") res = cur.fetchall() for row in res: self.assertEqual(row[0], row[2]) def test_sigar_gets_loaded(self): conn = connect('localhost:{}'.format(CRATE_HTTP_PORT))
import zc.customdoctests from crate.testing.layer import CrateLayer import os import shutil import re import tempfile import glob from . import process_test from .java_repl import JavaRepl from testutils.paths import crate_path, project_path from testutils.ports import GLOBAL_PORT_POOL from crate.crash.command import CrateCmd from crate.crash.printer import PrintWrapper, ColorPrinter from crate.client import connect CRATE_HTTP_PORT = GLOBAL_PORT_POOL.get() CRATE_TRANSPORT_PORT = GLOBAL_PORT_POOL.get() class CrateTestCmd(CrateCmd): def __init__(self, **kwargs): super(CrateTestCmd, self).__init__(**kwargs) doctest_print = PrintWrapper() self.logger = ColorPrinter(False, stream=doctest_print, line_end='\n') def stmt(self, stmt): stmt = stmt.replace('\n', ' ') if stmt.startswith('\\'): self.process(stmt) else: self.execute(stmt)
def create_doctest_suite(): crate_layer = ConnectingCrateLayer( 'crate', host='localhost', crate_home=crate_path(), port=CRATE_HTTP_PORT, transport_port=CRATE_TRANSPORT_PORT, env={'JAVA_HOME': os.environ.get('JAVA_HOME', '')}, settings={ 'license.enterprise': 'true', 'lang.js.enabled': 'true', 'es.api.enabled': 'true', 'psql.port': GLOBAL_PORT_POOL.get(), } ) tests = [] for fn in doctest_files('general/blobs.rst', 'interfaces/http.rst',): s = docsuite(fn, parser=bash_parser, setUp=setUpLocations, globs={ 'sh': partial( subprocess.run, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60, shell=True ), 'pretty_print': pretty_print }) s.layer = crate_layer tests.append(s) for fn in doctest_files('general/ddl/create-table.rst', 'general/ddl/generated-columns.rst', 'general/ddl/constraints.rst', 'general/ddl/sharding.rst', 'general/ddl/replication.rst', 'general/ddl/column-policy.rst', 'general/ddl/system-columns.rst', 'general/ddl/alter-table.rst', 'general/ddl/storage.rst', 'general/ddl/fulltext-indices.rst', 'admin/runtime-config.rst', 'general/ddl/show-create-table.rst', 'general/user-defined-functions.rst', 'admin/user-management.rst', 'admin/snapshots.rst', 'admin/privileges.rst', 'admin/ingestion/rules.rst', 'general/dql/index.rst', 'general/dql/refresh.rst', 'admin/optimization.rst', 'general/dql/fulltext.rst', 'general/ddl/data-types.rst', 'general/occ.rst', 'general/information-schema.rst', 'general/ddl/partitioned-tables.rst', 'general/builtins/aggregation.rst', 'general/builtins/arithmetic.rst', 'general/builtins/scalar.rst', 'general/builtins/table-functions.rst', 'admin/system-information.rst', 'general/dql/selects.rst', 'interfaces/postgres.rst', 'general/ddl/views.rst', 'sql/general/value-expressions.rst'): s = docsuite(fn, setUp=setUpLocationsAndQuotes) s.layer = crate_layer tests.append(s) for fn in doctest_files('general/dql/geo.rst',): s = docsuite(fn, setUp=setUpCountries) s.layer = crate_layer tests.append(s) for fn in doctest_files('general/dql/joins.rst', 'general/builtins/subquery-expressions.rst', 'general/builtins/window-functions.rst',): s = docsuite(fn, setUp=setUpEmpDeptAndColourArticlesAndGeo) s.layer = crate_layer tests.append(s) for fn in doctest_files('general/dml.rst',): s = docsuite(fn, setUp=setUpLocationsQuotesAndUserVisits) s.layer = crate_layer tests.append(s) for fn in doctest_files('general/dql/union.rst',): s = docsuite(fn, setUp=setUpPhotosAndCountries) s.layer = crate_layer tests.append(s) # randomize order of tests to make sure they don't depend on each other random.shuffle(tests) suite = unittest.TestSuite() suite.addTests(tests) return suite