def main(): usage = '%prog -c cluster -t test_config -v version' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='ClusterSpecFilePath') parser.add_option('-t', dest='test_config_fname', help='path to test config file', metavar='TestConfigFilePath') parser.add_option('-v', dest='version', help='build version', metavar='Version') options, _ = parser.parse_args() if not options.cluster_spec_fname or not options.test_config_fname \ or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname) installer = GatewayInstaller(cluster_spec, test_config, options) installer.install()
def main(): options, args = get_options() override = args and (arg.split('.') for arg in ' '.join(args).split(',')) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) cm = ClusterManager(cluster_spec, test_config) # Individual nodes cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.configure_internal_settings() cm.set_data_path() cm.set_auth() cm.set_mem_quota() cm.disable_moxi() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() cm.create_buckets() cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): usage = '%prog -c cluster -t test_config -v version' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='ClusterSpecFilePath') parser.add_option('-t', dest='test_config_fname', help='path to test config file', metavar='TestConfigFilePath') parser.add_option('-v', dest='version', help='build version', metavar='Version') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() override = args and [arg.split('.') for arg in ' '.join(args).split(',')] if not options.cluster_spec_fname or not options.test_config_fname \ or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) installer = GatewayInstaller(cluster_spec, test_config, options) installer.install()
def test_moving_working_set_settings(self): for file_name in glob.glob("tests/gsi/plasma/*.test"): test_config = TestConfig() test_config.parse(file_name) if test_config.access_settings.working_set_move_time: self.assertNotEqual(test_config.access_settings.working_set, 100) self.assertEqual(test_config.access_settings.working_set_access, 100)
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) client_installer = ClientInstaller(cluster_spec, test_config, args) client_installer.install()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) cm = ClusterManager(cluster_spec, test_config, args.verbose) # Individual nodes cm.disable_wan() cm.clear_login_history() cm.tune_memory_settings() cm.throttle_cpu() cm.enable_ipv6() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.flush_iptables() cm.configure_internal_settings() cm.set_data_path() cm.set_index_path() cm.set_analytics_path() cm.set_mem_quotas() cm.set_services() cm.rename() cm.set_auth() cm.configure_xdcr_settings() # Cluster cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() cm.enable_audit() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.create_eventing_buckets() cm.create_eventing_metadata_bucket() cm.add_rbac_users() cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.set_x509_certificates() cm.set_cipher_suite() cm.set_min_tls_version() cm.wait_until_healthy() cm.wait_until_warmed_up() cm.tweak_memory() cm.enable_n2n_encryption()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.configure_xdcr_settings() time.sleep(10) # dkao: crutch cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_fts_index_mem_quota() cm.set_auth() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.test_config.cluster.emptybuckets: cm.create_buckets(empty_buckets=True) if cm.remote: cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() if cm.remote: cm.set_index_settings() cm.change_dcp_io_threads() time.sleep(60) cm.set_query_settings() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) cm = ClusterManager(cluster_spec, test_config, args.verbose) # Individual nodes cm.disable_wan() cm.clear_login_history() cm.tune_memory_settings() cm.throttle_cpu() cm.enable_ipv6() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.flush_iptables() cm.configure_internal_settings() cm.set_data_path() cm.set_index_path() cm.set_analytics_path() cm.set_mem_quotas() cm.set_services() cm.rename() cm.set_auth() cm.configure_xdcr_settings() # Cluster cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() cm.enable_audit() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.create_eventing_buckets() cm.create_eventing_metadata_bucket() cm.add_rbac_users() cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.set_x509_certificates() cm.wait_until_healthy() cm.wait_until_warmed_up() cm.tweak_memory()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, args.override) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, args.verbose) as test: test.run()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() time.sleep(10) # dkao: crutch cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.test_config.cluster.emptybuckets: cm.create_buckets(emptyBuckets=True) if cm.remote: cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() if cm.remote: cm.set_index_settings() time.sleep(60) cm.set_query_settings() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose) as test: test.run()
def main(): options, args = get_options() override = args and (arg.split('.') for arg in ' '.join(args).split(',')) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, experiment) as test: test.run()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.remote.disable_wan() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.rename() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_fts_index_mem_quota() cm.set_auth() cm.configure_xdcr_settings() # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.remote: cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.wait_until_warmed_up() cm.wait_until_healthy() if cm.remote: cm.tweak_memory()
class FunctionalTest(unittest.TestCase): def __init__(self, *args, **kwargs): options, args = get_options() self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname, args) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, args) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.test_config) self.remote = RemoteHelper(self.cluster_spec, self.test_config) self.rest = RestHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
class FunctionalTest(unittest.TestCase): def __init__(self, *args, **kwargs): options, _args = get_options() override = \ _args and (arg.split('.') for arg in ' '.join(_args).split(',')) self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, override) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.cluster_spec) self.remote = RemoteHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def main(): options, args = get_options() override = args and (arg.split('.') for arg in itertools.chain(*csv.reader(args))) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_alternative_jemalloc_conf() # NOTE: configure_tcmalloc_aggressive_decommit() should be after # any restarts, as it doesn't survive a restart. cm.configure_tcmalloc_aggressive_decommit() cm.configure_internal_settings() cm.set_data_path() cm.set_auth() cm.set_mem_quota() cm.disable_moxi() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose, experiment) as test: test.run()
def main(): options, args = get_options() override = args and [arg.split('.') for arg in itertools.chain(*csv.reader(args))] cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, override) test_config = TestConfig() test_config.parse(options.test_config_fname, override) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose, experiment) as test: test.run()
def test_stale_update_after(self): test_config = TestConfig() test_config.parse('tests/query_lat_20M.test', []) views_params = test_config.index_settings.params self.assertEqual(views_params, {})
def main(): global couchbaseConnection, version, runStartTime usage = '%prog -v version -c cluster-spec' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-r', '--runStartTime', dest='runStartTime') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) couchbaseConnection = Couchbase.connect(bucket='Daily-Performance', host='172.23.105.177') runStartTime = options.runStartTime version = options.version #bucket = Bucket('couchbase://'+ '172.23.105.177:8091/Daily-Performance') installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) cm.set_index_settings() airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec, options.version) beer_result = do_beer_queries(conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result sabre_result = do_sabre_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec) sys.exit(not (airline_result and beer_result and sabre_result))
def main(): usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('-f', '--filename', dest='filename') parser.add_option('-b','--build',dest='build') parser.add_option('-n','--number',dest='number') options, args = parser.parse_args() test_id=options.number + '_'+options.build data=None try: with open(options.filename) as data_file: data = json.load(data_file) except (OSError, IOError,ValueError) as e: raise e mng_data = manage_test_result(['beer_queries','travel_queries']) mng_data.create_cb_instance(data["couchbase_server"],data["couchbase_query_bucket"]) options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(mng_data,conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(mng_data,conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result #mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result) mng_data.load_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id,options.version) mng_data.show_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id) sys.exit(not (airline_result and beer_result))
def test_fts_configs(self): for file in glob.glob("tests/fts/enduser/tests_dgm/*latency*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_dgm') self.assertEqual(test_config.showfast.sub_category, 'Latency') for file in glob.glob("tests/fts/enduser/tests_dgm/*throughput*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_dgm') self.assertEqual(test_config.showfast.sub_category, 'Throughput') for file in glob.glob("tests/fts/enduser/tests_dgm/*index*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_dgm') self.assertEqual(test_config.showfast.sub_category, 'Index') for file in glob.glob("tests/fts/enduser/tests_nodgm/*latency*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_non_dgm') self.assertEqual(test_config.showfast.sub_category, 'Latency') for file in glob.glob("tests/fts/enduser/tests_nodgm/*throughput*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_non_dgm') self.assertEqual(test_config.showfast.sub_category, 'Throughput') for file in glob.glob("tests/fts/enduser/tests_nodgm/*index*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'end_user_non_dgm') self.assertEqual(test_config.showfast.sub_category, 'Index') for file in glob.glob("tests/fts/multi_node/*latency*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark_3_nodes') self.assertEqual(test_config.showfast.sub_category, 'Latency') for file in glob.glob("tests/fts/multi_node/*throughput*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark_3_nodes') self.assertEqual(test_config.showfast.sub_category, 'Throughput') for file in glob.glob("tests/fts/multi_node/*index*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark_3_nodes') self.assertEqual(test_config.showfast.sub_category, 'Index') for file in glob.glob("tests/fts/rebalance/*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark') self.assertEqual(test_config.showfast.sub_category, 'Rebalance') for file in glob.glob("tests/fts/single_node/*latency*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark') self.assertEqual(test_config.showfast.sub_category, 'Latency') for file in glob.glob("tests/fts/single_node/*throughput*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark') self.assertEqual(test_config.showfast.sub_category, 'Throughput') for file in glob.glob("tests/fts/single_node/*index*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark') self.assertEqual(test_config.showfast.sub_category, 'Index') for file in glob.glob("tests/fts/single_node_kv/*latency*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark_kv') self.assertEqual(test_config.showfast.sub_category, 'Latency') for file in glob.glob("tests/fts/single_node_kv/*throughput*.test"): test_config = TestConfig() test_config.parse(file) self.assertEqual(test_config.showfast.category, 'benchmark_kv') self.assertEqual(test_config.showfast.sub_category, 'Throughput')
def test_soe_backup_repo(self): for file_name in glob.glob("tests/soe/*.test"): test_config = TestConfig() test_config.parse(file_name) self.assertNotEqual(test_config.restore_settings.backup_repo, '')
def test_override(self): test_config = TestConfig() test_config.parse('tests/query_lat_20M_basic.test', override=['cluster.mem_quota.5555']) self.assertEqual(test_config.cluster.mem_quota, 5555)
def test_stale_update_after(self): test_config = TestConfig() test_config.parse('tests/query_lat_20M_basic.test') query_params = test_config.access_settings.query_params self.assertEqual(query_params, {'stale': 'false'})
def test_stale_false(self): test_config = TestConfig() test_config.parse('tests/query_lat_20M_state_false.test', []) views_params = test_config.index_settings.params self.assertEqual(views_params, {'stale': 'false'})