def main(): usage = '%prog -c cluster -t test_config -v version' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='ClusterSpecFilePath') parser.add_option('-t', dest='test_config_fname', help='path to test config file', metavar='TestConfigFilePath') parser.add_option('-v', dest='version', help='build version', metavar='Version') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() override = args and [arg.split('.') for arg in ' '.join(args).split(',')] if not options.cluster_spec_fname or not options.test_config_fname \ or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) installer = GatewayInstaller(cluster_spec, test_config, options) installer.install()
def main(): usage = '%prog -c cluster -t test_config -v version' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='ClusterSpecFilePath') parser.add_option('-t', dest='test_config_fname', help='path to test config file', metavar='TestConfigFilePath') parser.add_option('-v', dest='version', help='build version', metavar='Version') options, _ = parser.parse_args() if not options.cluster_spec_fname or not options.test_config_fname \ or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname) installer = GatewayInstaller(cluster_spec, test_config, options) installer.install()
def main(): usage = '%prog -c cluster -v version [-t toy]' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('-v', dest='version', help='build version', metavar='2.0.0-1976') parser.add_option('-t', dest='toy', help='optional toy build ID', metavar='couchstore') options, _ = parser.parse_args() if not options.cluster_spec_fname or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) installer = CouchbaseInstaller(cluster_spec, options) installer.install()
def main(): options, args = get_options() override = args and (arg.split('.') for arg in ' '.join(args).split(',')) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) cm = ClusterManager(cluster_spec, test_config) # Individual nodes cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.configure_internal_settings() cm.set_data_path() cm.set_auth() cm.set_mem_quota() cm.disable_moxi() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() cm.create_buckets() cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname) remote = RemoteHelper(cluster_spec, verbose=False) remote.collect_info() for hostname in cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) if cluster_spec.backup is not None: logs = os.path.join(cluster_spec.backup, 'logs') if os.path.exists(logs): shutil.make_archive('tools', 'zip', logs) failures = defaultdict(dict) for file_name in glob.iglob('./*.zip'): panic_files, crash_files, storage_corrupted = validate_logs(file_name) if panic_files: failures['panics'][file_name] = panic_files if crash_files: failures['crashes'][file_name] = crash_files if storage_corrupted: failures['storage_corrupted'][file_name] = True remote.collect_index_datafiles() if failures: logger.interrupt( "Following failures found: {}".format(pretty_dict(failures)))
def main(): usage = '%prog -c cluster -v version [-t toy]' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('-v', dest='version', help='build version', metavar='2.0.0-1976') parser.add_option('-t', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, _ = parser.parse_args() if not options.cluster_spec_fname or not options.version: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) installer = CouchbaseInstaller(cluster_spec, options) installer.install()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(fname=args.cluster) if cluster_spec.cloud_infrastructure: if cluster_spec.kubernetes_infrastructure: infra_provider = cluster_spec.infrastructure_settings['provider'] if infra_provider == 'aws': installer = EKSInstaller(cluster_spec, args) elif infra_provider == 'azure': installer = AKSInstaller(cluster_spec, args) elif infra_provider == 'gcp': installer = GKEInstaller(cluster_spec, args) else: raise Exception("{} is not a valid infrastructure provider" .format(infra_provider)) else: installer = CloudInstaller(cluster_spec, args) if args.uninstall: installer.uninstall() else: installer.install() else: installer = CouchbaseInstaller(cluster_spec, args) installer.install() if args.local_copy: installer.download() installer.download_local(args.local_copy_url) if '--remote-copy' in sys.argv: logger.info('Saving a remote copy') installer.download_remote()
def main(): args = get_args() infra_spec = ClusterSpec() infra_spec.parse(fname=args.cluster) if infra_spec.dynamic_infrastructure: infra_provider = infra_spec.infrastructure_settings['provider'] if infra_provider == 'aws': deployer = AWSDeployer(infra_spec, args) elif infra_provider == 'azure': deployer = AzureDeployer(infra_spec, args) elif infra_provider == 'gcp': deployer = GCPDeployer(infra_spec, args) else: raise Exception("{} is not a valid infrastructure provider".format( infra_provider)) try: deployer.deploy() except Exception as ex: with open(infra_spec.generated_cloud_config_path) as f: logger.info("infrastructure dump:\n{}".format( pretty_dict(json.load(f)))) raise ex
def main(): usage = '%prog -c cluster -v version [-t toy]' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('-e', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('-v', dest='version', help='build version', metavar='2.0.0-1976') parser.add_option('-t', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() if not (options.cluster_spec_fname and options.version) and not options.url: parser.error( 'Missing mandatory parameter. Either specify both cluster' ' spec and version, or specify just the URL to be installed') if options.cluster_edition not in ['community', 'enterprise']: parser.error('Cluster edition must be either community or enterprise') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) installer = CouchbaseInstaller(cluster_spec, options) installer.install()
def main(): usage = '%prog -c cluster -v version' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='the path to a cluster specification file', metavar='cluster.spec') parser.add_option('-e', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('-v', dest='version', help='the build version', metavar='2.0.0-1976') parser.add_option('--url', dest='url', default=None, help='the HTTP URL to a package that should be installed.') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() if options.cluster_edition not in ['community', 'enterprise']: # changed to default to enterprise, with no error: options.cluster_edition = 'enterprise' if not (options.cluster_spec_fname and options.version) and not options.url: parser.error('Missing mandatory parameter. Either pecify both cluster ' 'spec and version, or specify the URL to be installed.') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) installer = CouchbaseInstaller(cluster_spec, options) installer.install()
def main(): usage = '%prog -v version -c cluster-spec' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result sys.exit(not (airline_result and beer_result))
def main(): usage = '%prog -v version -c cluster-spec' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result sys.exit(not (airline_result and beer_result))
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster) remote = RemoteHelper(cluster_spec, args.verbose) remote.get_manifest() projects = parse_manifest() fetch(projects)
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) client_installer = ClientInstaller(cluster_spec, test_config, args) client_installer.install()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster) remote = RemoteHelper(cluster_spec, None, args.verbose) remote.get_manifest() projects = parse_manifest() fetch(projects)
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) cm = ClusterManager(cluster_spec, test_config, args.verbose) # Individual nodes cm.disable_wan() cm.clear_login_history() cm.tune_memory_settings() cm.throttle_cpu() cm.enable_ipv6() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.flush_iptables() cm.configure_internal_settings() cm.set_data_path() cm.set_index_path() cm.set_analytics_path() cm.set_mem_quotas() cm.set_services() cm.rename() cm.set_auth() cm.configure_xdcr_settings() # Cluster cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() cm.enable_audit() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.create_eventing_buckets() cm.create_eventing_metadata_bucket() cm.add_rbac_users() cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.set_x509_certificates() cm.set_cipher_suite() cm.set_min_tls_version() cm.wait_until_healthy() cm.wait_until_warmed_up() cm.tweak_memory() cm.enable_n2n_encryption()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(fname=args.cluster) installer = CouchbaseInstaller(cluster_spec, args) installer.install() if args.local_copy: installer.download()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname) remote = RemoteHelper(cluster_spec, verbose=False) logger.info('Recovering system state') for host, version in remote.get_system_backup_version().items(): remote.start_system_state_recovery(host, version)
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) remote = RemoteHelper(cluster_spec, test_config=None, verbose=False) logger.info('Recovering system state') for host, version in remote.get_system_backup_version().items(): remote.start_system_state_recovery(host, version)
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.configure_xdcr_settings() time.sleep(10) # dkao: crutch cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_fts_index_mem_quota() cm.set_auth() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.test_config.cluster.emptybuckets: cm.create_buckets(empty_buckets=True) if cm.remote: cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() if cm.remote: cm.set_index_settings() cm.change_dcp_io_threads() time.sleep(60) cm.set_query_settings() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) remote = RemoteHelper(cluster_spec, test_config=None, verbose=False) remote.collect_info() for hostname in cluster_spec.yield_hostnames(): for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname))
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, override=args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, override=args.override) cm = ClusterManager(cluster_spec, test_config, args.verbose) # Individual nodes cm.disable_wan() cm.clear_login_history() cm.tune_memory_settings() cm.throttle_cpu() cm.enable_ipv6() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.flush_iptables() cm.configure_internal_settings() cm.set_data_path() cm.set_index_path() cm.set_analytics_path() cm.set_mem_quotas() cm.set_services() cm.rename() cm.set_auth() cm.configure_xdcr_settings() # Cluster cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() cm.enable_audit() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.create_eventing_buckets() cm.create_eventing_metadata_bucket() cm.add_rbac_users() cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.set_x509_certificates() cm.wait_until_healthy() cm.wait_until_warmed_up() cm.tweak_memory()
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname, args.override) test_config = TestConfig() test_config.parse(args.test_config_fname, args.override) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, args.verbose) as test: test.run()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose) as test: test.run()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() time.sleep(10) # dkao: crutch cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.test_config.cluster.emptybuckets: cm.create_buckets(emptyBuckets=True) if cm.remote: cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() if cm.remote: cm.set_index_settings() time.sleep(60) cm.set_query_settings() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def __init__(self, *args, **kwargs): options, args = get_options() self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname, args) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, args) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.test_config) self.remote = RemoteHelper(self.cluster_spec, self.test_config) self.rest = RestHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def main(): usage = '%prog -c cluster -v version [-t toy]' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('-e', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('-v', dest='version', help='build version', metavar='2.0.0-1976') parser.add_option('-t', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() if options.cluster_edition not in ['community', 'enterprise']: # changed to default to enterprise, with no error: options.cluster_edition = 'enterprise' if not (options.cluster_spec_fname and options.version) and not options.url: parser.error( 'Missing mandatory parameter. Either pecify both cluster' ' spec and version, or specify just the URL to be installed') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) installer = CouchbaseInstaller(cluster_spec, options) installer.install()
def main(): args = get_args() infra_spec = ClusterSpec() infra_spec.parse(fname=args.cluster) infra_provider = infra_spec.infrastructure_settings['provider'] if infra_spec.cloud_infrastructure: if infra_provider == 'aws': destroyer = AWSDestroyer(infra_spec, args) elif infra_provider == 'azure': destroyer = AzureDestroyer(infra_spec, args) elif infra_provider == 'gcp': destroyer = GCPDestroyer(infra_spec, args) else: raise Exception("{} is not a valid infrastructure provider".format( infra_provider)) destroyer.destroy()
def __init__(self, *args, **kwargs): options, _args = get_options() override = \ _args and (arg.split('.') for arg in ' '.join(_args).split(',')) self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, override) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.cluster_spec) self.remote = RemoteHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def main(): options, args = get_options() override = args and (arg.split('.') for arg in ' '.join(args).split(',')) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, experiment) as test: test.run()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes if cm.remote: cm.remote.disable_wan() cm.tune_logging() cm.restart_with_alternative_num_vbuckets() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.rename() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_fts_index_mem_quota() cm.set_auth() cm.configure_xdcr_settings() # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() cm.rebalance() cm.enable_auto_failover() cm.configure_auto_compaction() if cm.test_config.cluster.num_buckets: cm.create_buckets() if cm.remote: cm.restart_with_alternative_bucket_options() cm.set_index_settings() cm.set_query_settings() cm.wait_until_warmed_up() cm.wait_until_healthy() if cm.remote: cm.tweak_memory()
class FunctionalTest(unittest.TestCase): def __init__(self, *args, **kwargs): options, args = get_options() self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname, args) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, args) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.test_config) self.remote = RemoteHelper(self.cluster_spec, self.test_config) self.rest = RestHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname) remote = RemoteHelper(cluster_spec, verbose=False) remote.collect_info() for hostname in cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) if cluster_spec.backup is not None: logs = os.path.join(cluster_spec.backup, 'logs') if os.path.exists(logs): shutil.make_archive('tools', 'zip', logs)
def main(): args = get_args() cluster_spec = ClusterSpec() cluster_spec.parse(args.cluster_spec_fname) remote = RemoteHelper(cluster_spec, test_config=None, verbose=False) remote.collect_info() for hostname in cluster_spec.servers: for fname in glob.glob('{}/*.zip'.format(hostname)): shutil.move(fname, '{}.zip'.format(hostname)) if cluster_spec.backup is not None: logs = os.path.join(cluster_spec.backup, 'logs') if os.path.exists(logs): shutil.make_archive('tools', 'zip', logs)
def main(): options, args = get_options() override = args and (arg.split('.') for arg in itertools.chain(*csv.reader(args))) cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) test_config = TestConfig() test_config.parse(options.test_config_fname, override) cm = ClusterManager(cluster_spec, test_config, options.verbose) # Individual nodes cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_alternative_jemalloc_conf() # NOTE: configure_tcmalloc_aggressive_decommit() should be after # any restarts, as it doesn't survive a restart. cm.configure_tcmalloc_aggressive_decommit() cm.configure_internal_settings() cm.set_data_path() cm.set_auth() cm.set_mem_quota() cm.disable_moxi() time.sleep(30) # crutch # Cluster if cm.group_number > 1: cm.create_server_groups() cm.add_nodes() if cm.test_config.cluster.num_buckets: cm.create_buckets() cm.restart_with_alternative_bucket_options() cm.wait_until_warmed_up() cm.wait_until_healthy() cm.configure_auto_compaction() cm.enable_auto_failover() cm.change_watermarks() cm.tweak_memory() cm.remote.disable_wan() cm.start_cbq_engine()
def main(): options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose, experiment) as test: test.run()
class FunctionalTest(unittest.TestCase): def __init__(self, *args, **kwargs): options, _args = get_options() override = \ _args and (arg.split('.') for arg in ' '.join(_args).split(',')) self.cluster_spec = ClusterSpec() self.cluster_spec.parse(options.cluster_spec_fname) self.test_config = TestConfig() self.test_config.parse(options.test_config_fname, override) self.target_iterator = TargetIterator(self.cluster_spec, self.test_config) self.memcached = MemcachedHelper(self.cluster_spec) self.remote = RemoteHelper(self.cluster_spec) super(FunctionalTest, self).__init__(*args, **kwargs)
def main(): options, args = get_options() override = args and [arg.split('.') for arg in itertools.chain(*csv.reader(args))] cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, override) test_config = TestConfig() test_config.parse(options.test_config_fname, override) experiment = options.exp_fname and Experiment(options.exp_fname) test_module = test_config.test_case.test_module test_class = test_config.test_case.test_class exec('from {} import {}'.format(test_module, test_class)) with eval(test_class)(cluster_spec, test_config, options.verbose, experiment) as test: test.run()
def main(): usage = '%prog -c cluster.spec' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, args = parser.parse_args() if not options.cluster_spec_fname: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) installer = MongoDBInstaller(cluster_spec, options) installer.install()
def main(): usage = '%prog -c cluster.spec' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') options, _ = parser.parse_args() if not options.cluster_spec_fname: parser.error('Missing mandatory parameter') cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) installer = MongoDBInstaller(cluster_spec, options) installer.install()
def test_cluster_specs(self): for file_name in glob.glob("clusters/*.spec"): cluster_spec = ClusterSpec() cluster_spec.parse(file_name, override=())
### Create two clusters with an XDCR relationship, run a workload against the clusters for example: ### cbc-pillowfight -t 1 -B 10 -U couchbase://NODE_IP/default -m 512 -M 1024 -r 50 ### Move this file to the root of the perfrunner repo and run ./env/bin/python xdcr_test.py -c clusters/Hermes.spec ### The actual cluster spec is irrelevant for this test so long as it is valid. from optparse import OptionParser from perfrunner.settings import ClusterSpec from perfrunner.helpers.monitor import Monitor def get_options(): usage = '%prog -c cluster -t test_config' parser = OptionParser(usage) parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') options, args = parser.parse_args() return options, args options, args = get_options() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname) monitor = Monitor(cluster_spec) monitor.monitor_xdcr_queues('192.168.42.101:8091', 'default')
def main(): usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('-f', '--filename', dest='filename') parser.add_option('-b', '--build', dest='build') parser.add_option('-n', '--number', dest='number') options, args = parser.parse_args() test_id = options.number + '_' + options.build data = None try: with open(options.filename) as data_file: data = json.load(data_file) except (OSError, IOError, ValueError) as e: raise e mng_data = manage_test_result(['beer_queries', 'travel_queries']) mng_data.create_cb_instance(data["couchbase_server"], data["couchbase_query_bucket"]) options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(mng_data, conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(mng_data, conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result #mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result) mng_data.load_data_query_benchmark(data["couchbase_query_bucket"], 'query_benchmark', test_id, options.version) mng_data.show_data_query_benchmark(data["couchbase_query_bucket"], 'query_benchmark', test_id) sys.exit(not (airline_result and beer_result))
def test_cluster_specs(self): for file_name in glob.glob("clusters/*.spec"): cluster_spec = ClusterSpec() cluster_spec.parse(file_name, override=()) self.assertIn(cluster_spec.parameters['Platform'], ('HW', 'Virtual', 'Localhost'))
def main(): usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag' parser = OptionParser(usage) parser.add_option('-v', '--version', dest='version') parser.add_option('-c', dest='cluster_spec_fname', help='path to cluster specification file', metavar='cluster.spec') parser.add_option('--verbose', dest='verbose', action='store_true', help='enable verbose logging') parser.add_option('-o', dest='toy', help='optional toy build ID', metavar='couchstore') parser.add_option('-t', dest='test_config_fname', help='path to test configuration file', metavar='my_test.test') parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise', help='the cluster edition (community or enterprise)') parser.add_option('--url', dest='url', default=None, help='The http URL to a Couchbase RPM that should be' ' installed. This overrides the URL to be installed.') parser.add_option('-f', '--filename', dest='filename') parser.add_option('-b','--build',dest='build') parser.add_option('-n','--number',dest='number') options, args = parser.parse_args() test_id=options.number + '_'+options.build data=None try: with open(options.filename) as data_file: data = json.load(data_file) except (OSError, IOError,ValueError) as e: raise e mng_data = manage_test_result(['beer_queries','travel_queries']) mng_data.create_cb_instance(data["couchbase_server"],data["couchbase_query_bucket"]) options, args = parser.parse_args() cluster_spec = ClusterSpec() cluster_spec.parse(options.cluster_spec_fname, args) test_config = TestConfig() test_config.parse(options.test_config_fname, args) cm = ClusterManager(cluster_spec, test_config, options.verbose) installer = CouchbaseInstaller(cluster_spec, options) if True: installer.install() if cm.remote: cm.tune_logging() cm.restart_with_sfwi() cm.restart_with_alternative_num_vbuckets() cm.restart_with_alternative_num_cpus() cm.restart_with_tcmalloc_aggressive_decommit() cm.disable_moxi() cm.configure_internal_settings() cm.set_data_path() cm.set_services() cm.set_mem_quota() cm.set_index_mem_quota() cm.set_auth() time.sleep(30) """host = cluster_spec.yield_masters().next()""" host_ip = cluster_spec.yield_masters().next().split(':')[0] URL = 'http://' + host_ip + ':8093' logger.info('logging the URL: {}'.format(URL)) conn = urllib3.connection_from_url(URL) rest = RestHelper(cluster_spec) airline_result = do_airline_benchmarks(mng_data,conn, rest, host_ip, installer.remote, cluster_spec) beer_result = do_beer_queries(mng_data,conn, rest, host_ip, installer.remote) print 'beer_result is', beer_result #mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result) mng_data.load_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id,options.version) mng_data.show_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id) sys.exit(not (airline_result and beer_result))
def test_cluster_specs(self): for file_name in glob.glob("clusters/*.spec"): cluster_spec = ClusterSpec() cluster_spec.parse(file_name, override=None)