def main():
usage = '%prog -c cluster -t test_config -v version'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='ClusterSpecFilePath')
parser.add_option('-t', dest='test_config_fname',
help='path to test config file',
metavar='TestConfigFilePath')
parser.add_option('-v', dest='version',
help='build version', metavar='Version')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, args = parser.parse_args()
override = args and [arg.split('.') for arg in ' '.join(args).split(',')]
if not options.cluster_spec_fname or not options.test_config_fname \
or not options.version:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
installer = GatewayInstaller(cluster_spec, test_config, options)
installer.install()
def main():
usage = '%prog -c cluster -t test_config -v version'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='ClusterSpecFilePath')
parser.add_option('-t', dest='test_config_fname',
help='path to test config file',
metavar='TestConfigFilePath')
parser.add_option('-v', dest='version',
help='build version', metavar='Version')
options, _ = parser.parse_args()
if not options.cluster_spec_fname or not options.test_config_fname \
or not options.version:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname)
installer = GatewayInstaller(cluster_spec, test_config, options)
installer.install()
def main():
usage = '%prog -c cluster -v version [-t toy]'
parser = OptionParser(usage)
parser.add_option('-c',
dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('-v',
dest='version',
help='build version',
metavar='2.0.0-1976')
parser.add_option('-t',
dest='toy',
help='optional toy build ID',
metavar='couchstore')
options, _ = parser.parse_args()
if not options.cluster_spec_fname or not options.version:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
installer = CouchbaseInstaller(cluster_spec, options)
installer.install()
def main():
options, args = get_options()
override = args and (arg.split('.') for arg in ' '.join(args).split(','))
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
cm = ClusterManager(cluster_spec, test_config)
# Individual nodes
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_auth()
cm.set_mem_quota()
cm.disable_moxi()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
cm.create_buckets()
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def main():
usage = '%prog -c cluster -t test_config -v version'
parser = OptionParser(usage)
parser.add_option('-c',
dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='ClusterSpecFilePath')
parser.add_option('-t',
dest='test_config_fname',
help='path to test config file',
metavar='TestConfigFilePath')
parser.add_option('-v',
dest='version',
help='build version',
metavar='Version')
options, _ = parser.parse_args()
if not options.cluster_spec_fname or not options.test_config_fname \
or not options.version:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname)
installer = GatewayInstaller(cluster_spec, test_config, options)
installer.install()
def main():
options, args = get_options()
override = args and (arg.split('.') for arg in ' '.join(args).split(','))
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
cm = ClusterManager(cluster_spec, test_config)
# Individual nodes
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_auth()
cm.set_mem_quota()
cm.disable_moxi()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
cm.create_buckets()
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname)
remote = RemoteHelper(cluster_spec, verbose=False)
remote.collect_info()
for hostname in cluster_spec.servers:
for fname in glob.glob('{}/*.zip'.format(hostname)):
shutil.move(fname, '{}.zip'.format(hostname))
if cluster_spec.backup is not None:
logs = os.path.join(cluster_spec.backup, 'logs')
if os.path.exists(logs):
shutil.make_archive('tools', 'zip', logs)
failures = defaultdict(dict)
for file_name in glob.iglob('./*.zip'):
panic_files, crash_files, storage_corrupted = validate_logs(file_name)
if panic_files:
failures['panics'][file_name] = panic_files
if crash_files:
failures['crashes'][file_name] = crash_files
if storage_corrupted:
failures['storage_corrupted'][file_name] = True
remote.collect_index_datafiles()
if failures:
logger.interrupt(
"Following failures found: {}".format(pretty_dict(failures)))
def main():
usage = '%prog -c cluster -v version [-t toy]'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('-v', dest='version',
help='build version', metavar='2.0.0-1976')
parser.add_option('-t', dest='toy',
help='optional toy build ID', metavar='couchstore')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, _ = parser.parse_args()
if not options.cluster_spec_fname or not options.version:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
installer = CouchbaseInstaller(cluster_spec, options)
installer.install()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(fname=args.cluster)
if cluster_spec.cloud_infrastructure:
if cluster_spec.kubernetes_infrastructure:
infra_provider = cluster_spec.infrastructure_settings['provider']
if infra_provider == 'aws':
installer = EKSInstaller(cluster_spec, args)
elif infra_provider == 'azure':
installer = AKSInstaller(cluster_spec, args)
elif infra_provider == 'gcp':
installer = GKEInstaller(cluster_spec, args)
else:
raise Exception("{} is not a valid infrastructure provider"
.format(infra_provider))
else:
installer = CloudInstaller(cluster_spec, args)
if args.uninstall:
installer.uninstall()
else:
installer.install()
else:
installer = CouchbaseInstaller(cluster_spec, args)
installer.install()
if args.local_copy:
installer.download()
installer.download_local(args.local_copy_url)
if '--remote-copy' in sys.argv:
logger.info('Saving a remote copy')
installer.download_remote()
def main():
args = get_args()
infra_spec = ClusterSpec()
infra_spec.parse(fname=args.cluster)
if infra_spec.dynamic_infrastructure:
infra_provider = infra_spec.infrastructure_settings['provider']
if infra_provider == 'aws':
deployer = AWSDeployer(infra_spec, args)
elif infra_provider == 'azure':
deployer = AzureDeployer(infra_spec, args)
elif infra_provider == 'gcp':
deployer = GCPDeployer(infra_spec, args)
else:
raise Exception("{} is not a valid infrastructure provider".format(
infra_provider))
try:
deployer.deploy()
except Exception as ex:
with open(infra_spec.generated_cloud_config_path) as f:
logger.info("infrastructure dump:\n{}".format(
pretty_dict(json.load(f))))
raise ex
def main():
usage = '%prog -c cluster -v version [-t toy]'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('-e', dest='cluster_edition', default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('-v', dest='version',
help='build version', metavar='2.0.0-1976')
parser.add_option('-t', dest='toy',
help='optional toy build ID', metavar='couchstore')
parser.add_option('--url', dest='url', default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, args = parser.parse_args()
if not (options.cluster_spec_fname and options.version) and not options.url:
parser.error(
'Missing mandatory parameter. Either specify both cluster'
' spec and version, or specify just the URL to be installed')
if options.cluster_edition not in ['community', 'enterprise']:
parser.error('Cluster edition must be either community or enterprise')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
installer = CouchbaseInstaller(cluster_spec, options)
installer.install()
def main():
usage = '%prog -c cluster -v version'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='the path to a cluster specification file',
metavar='cluster.spec')
parser.add_option('-e', dest='cluster_edition', default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('-v', dest='version',
help='the build version', metavar='2.0.0-1976')
parser.add_option('--url', dest='url', default=None,
help='the HTTP URL to a package that should be installed.')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, args = parser.parse_args()
if options.cluster_edition not in ['community', 'enterprise']:
# changed to default to enterprise, with no error:
options.cluster_edition = 'enterprise'
if not (options.cluster_spec_fname and options.version) and not options.url:
parser.error('Missing mandatory parameter. Either pecify both cluster '
'spec and version, or specify the URL to be installed.')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
installer = CouchbaseInstaller(cluster_spec, options)
installer.install()
def main():
usage = '%prog -v version -c cluster-spec'
parser = OptionParser(usage)
parser.add_option('-v', '--version', dest='version')
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
parser.add_option('-o', dest='toy',
help='optional toy build ID', metavar='couchstore')
parser.add_option('-t', dest='test_config_fname',
help='path to test configuration file',
metavar='my_test.test')
parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('--url', dest='url', default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
options, args = parser.parse_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
installer = CouchbaseInstaller(cluster_spec, options)
if True:
installer.install()
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_auth()
time.sleep(30)
"""host = cluster_spec.yield_masters().next()"""
host_ip = cluster_spec.yield_masters().next().split(':')[0]
URL = 'http://' + host_ip + ':8093'
logger.info('logging the URL: {}'.format(URL))
conn = urllib3.connection_from_url(URL)
rest = RestHelper(cluster_spec)
airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec)
beer_result = do_beer_queries(conn, rest, host_ip, installer.remote)
print 'beer_result is', beer_result
sys.exit(not (airline_result and beer_result))
def main():
usage = '%prog -v version -c cluster-spec'
parser = OptionParser(usage)
parser.add_option('-v', '--version', dest='version')
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
parser.add_option('-o', dest='toy',
help='optional toy build ID', metavar='couchstore')
parser.add_option('-t', dest='test_config_fname',
help='path to test configuration file',
metavar='my_test.test')
parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('--url', dest='url', default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
options, args = parser.parse_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
installer = CouchbaseInstaller(cluster_spec, options)
if True:
installer.install()
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_auth()
time.sleep(30)
"""host = cluster_spec.yield_masters().next()"""
host_ip = cluster_spec.yield_masters().next().split(':')[0]
URL = 'http://' + host_ip + ':8093'
logger.info('logging the URL: {}'.format(URL))
conn = urllib3.connection_from_url(URL)
rest = RestHelper(cluster_spec)
airline_result = do_airline_benchmarks(conn, rest, host_ip, installer.remote, cluster_spec)
beer_result = do_beer_queries(conn, rest, host_ip, installer.remote)
print 'beer_result is', beer_result
sys.exit(not (airline_result and beer_result))
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster)
remote = RemoteHelper(cluster_spec, args.verbose)
remote.get_manifest()
projects = parse_manifest()
fetch(projects)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname, override=args.override)
test_config = TestConfig()
test_config.parse(args.test_config_fname, override=args.override)
client_installer = ClientInstaller(cluster_spec, test_config, args)
client_installer.install()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster)
remote = RemoteHelper(cluster_spec, None, args.verbose)
remote.get_manifest()
projects = parse_manifest()
fetch(projects)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname, override=args.override)
test_config = TestConfig()
test_config.parse(args.test_config_fname, override=args.override)
cm = ClusterManager(cluster_spec, test_config, args.verbose)
# Individual nodes
cm.disable_wan()
cm.clear_login_history()
cm.tune_memory_settings()
cm.throttle_cpu()
cm.enable_ipv6()
cm.tune_logging()
cm.restart_with_alternative_num_vbuckets()
cm.flush_iptables()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_index_path()
cm.set_analytics_path()
cm.set_mem_quotas()
cm.set_services()
cm.rename()
cm.set_auth()
cm.configure_xdcr_settings()
# Cluster
cm.add_nodes()
cm.rebalance()
cm.enable_auto_failover()
cm.configure_auto_compaction()
cm.enable_audit()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
cm.create_eventing_buckets()
cm.create_eventing_metadata_bucket()
cm.add_rbac_users()
cm.restart_with_alternative_bucket_options()
cm.set_index_settings()
cm.set_query_settings()
cm.set_x509_certificates()
cm.set_cipher_suite()
cm.set_min_tls_version()
cm.wait_until_healthy()
cm.wait_until_warmed_up()
cm.tweak_memory()
cm.enable_n2n_encryption()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(fname=args.cluster)
installer = CouchbaseInstaller(cluster_spec, args)
installer.install()
if args.local_copy:
installer.download()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname)
remote = RemoteHelper(cluster_spec, verbose=False)
logger.info('Recovering system state')
for host, version in remote.get_system_backup_version().items():
remote.start_system_state_recovery(host, version)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname)
remote = RemoteHelper(cluster_spec, verbose=False)
logger.info('Recovering system state')
for host, version in remote.get_system_backup_version().items():
remote.start_system_state_recovery(host, version)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(fname=args.cluster)
installer = CouchbaseInstaller(cluster_spec, args)
installer.install()
if args.local_copy:
installer.download()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
remote = RemoteHelper(cluster_spec, test_config=None, verbose=False)
logger.info('Recovering system state')
for host, version in remote.get_system_backup_version().items():
remote.start_system_state_recovery(host, version)
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
# Individual nodes
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
cm.configure_xdcr_settings()
time.sleep(10) # dkao: crutch
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_fts_index_mem_quota()
cm.set_auth()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
if cm.test_config.cluster.emptybuckets:
cm.create_buckets(empty_buckets=True)
if cm.remote:
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.wait_until_healthy()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
if cm.remote:
cm.set_index_settings()
cm.change_dcp_io_threads()
time.sleep(60)
cm.set_query_settings()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
remote = RemoteHelper(cluster_spec, test_config=None, verbose=False)
remote.collect_info()
for hostname in cluster_spec.yield_hostnames():
for fname in glob.glob('{}/*.zip'.format(hostname)):
shutil.move(fname, '{}.zip'.format(hostname))
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname, override=args.override)
test_config = TestConfig()
test_config.parse(args.test_config_fname, override=args.override)
cm = ClusterManager(cluster_spec, test_config, args.verbose)
# Individual nodes
cm.disable_wan()
cm.clear_login_history()
cm.tune_memory_settings()
cm.throttle_cpu()
cm.enable_ipv6()
cm.tune_logging()
cm.restart_with_alternative_num_vbuckets()
cm.flush_iptables()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_index_path()
cm.set_analytics_path()
cm.set_mem_quotas()
cm.set_services()
cm.rename()
cm.set_auth()
cm.configure_xdcr_settings()
# Cluster
cm.add_nodes()
cm.rebalance()
cm.enable_auto_failover()
cm.configure_auto_compaction()
cm.enable_audit()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
cm.create_eventing_buckets()
cm.create_eventing_metadata_bucket()
cm.add_rbac_users()
cm.restart_with_alternative_bucket_options()
cm.set_index_settings()
cm.set_query_settings()
cm.set_x509_certificates()
cm.wait_until_healthy()
cm.wait_until_warmed_up()
cm.tweak_memory()
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname, args.override)
test_config = TestConfig()
test_config.parse(args.test_config_fname, args.override)
test_module = test_config.test_case.test_module
test_class = test_config.test_case.test_class
exec('from {} import {}'.format(test_module, test_class))
with eval(test_class)(cluster_spec, test_config, args.verbose) as test:
test.run()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
test_module = test_config.test_case.test_module
test_class = test_config.test_case.test_class
exec('from {} import {}'.format(test_module, test_class))
with eval(test_class)(cluster_spec, test_config, options.verbose) as test:
test.run()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
# Individual nodes
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
time.sleep(10) # dkao: crutch
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_auth()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
if cm.test_config.cluster.emptybuckets:
cm.create_buckets(emptyBuckets=True)
if cm.remote:
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.wait_until_healthy()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
if cm.remote:
cm.set_index_settings()
time.sleep(60)
cm.set_query_settings()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def __init__(self, *args, **kwargs):
options, args = get_options()
self.cluster_spec = ClusterSpec()
self.cluster_spec.parse(options.cluster_spec_fname, args)
self.test_config = TestConfig()
self.test_config.parse(options.test_config_fname, args)
self.target_iterator = TargetIterator(self.cluster_spec,
self.test_config)
self.memcached = MemcachedHelper(self.test_config)
self.remote = RemoteHelper(self.cluster_spec, self.test_config)
self.rest = RestHelper(self.cluster_spec)
super(FunctionalTest, self).__init__(*args, **kwargs)
def main():
usage = '%prog -c cluster -v version [-t toy]'
parser = OptionParser(usage)
parser.add_option('-c',
dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('-e',
dest='cluster_edition',
default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('-v',
dest='version',
help='build version',
metavar='2.0.0-1976')
parser.add_option('-t',
dest='toy',
help='optional toy build ID',
metavar='couchstore')
parser.add_option('--url',
dest='url',
default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
parser.add_option('--verbose',
dest='verbose',
action='store_true',
help='enable verbose logging')
options, args = parser.parse_args()
if options.cluster_edition not in ['community', 'enterprise']:
# changed to default to enterprise, with no error:
options.cluster_edition = 'enterprise'
if not (options.cluster_spec_fname
and options.version) and not options.url:
parser.error(
'Missing mandatory parameter. Either pecify both cluster'
' spec and version, or specify just the URL to be installed')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
installer = CouchbaseInstaller(cluster_spec, options)
installer.install()
def main():
args = get_args()
infra_spec = ClusterSpec()
infra_spec.parse(fname=args.cluster)
infra_provider = infra_spec.infrastructure_settings['provider']
if infra_spec.cloud_infrastructure:
if infra_provider == 'aws':
destroyer = AWSDestroyer(infra_spec, args)
elif infra_provider == 'azure':
destroyer = AzureDestroyer(infra_spec, args)
elif infra_provider == 'gcp':
destroyer = GCPDestroyer(infra_spec, args)
else:
raise Exception("{} is not a valid infrastructure provider".format(
infra_provider))
destroyer.destroy()
def __init__(self, *args, **kwargs):
options, _args = get_options()
override = \
_args and (arg.split('.') for arg in ' '.join(_args).split(','))
self.cluster_spec = ClusterSpec()
self.cluster_spec.parse(options.cluster_spec_fname)
self.test_config = TestConfig()
self.test_config.parse(options.test_config_fname, override)
self.target_iterator = TargetIterator(self.cluster_spec,
self.test_config)
self.memcached = MemcachedHelper(self.cluster_spec)
self.remote = RemoteHelper(self.cluster_spec)
super(FunctionalTest, self).__init__(*args, **kwargs)
def main():
options, args = get_options()
override = args and (arg.split('.') for arg in ' '.join(args).split(','))
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
experiment = options.exp_fname and Experiment(options.exp_fname)
test_module = test_config.test_case.test_module
test_class = test_config.test_case.test_class
exec('from {} import {}'.format(test_module, test_class))
with eval(test_class)(cluster_spec, test_config, experiment) as test:
test.run()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
# Individual nodes
if cm.remote:
cm.remote.disable_wan()
cm.tune_logging()
cm.restart_with_alternative_num_vbuckets()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_services()
cm.rename()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_fts_index_mem_quota()
cm.set_auth()
cm.configure_xdcr_settings()
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
cm.rebalance()
cm.enable_auto_failover()
cm.configure_auto_compaction()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
if cm.remote:
cm.restart_with_alternative_bucket_options()
cm.set_index_settings()
cm.set_query_settings()
cm.wait_until_warmed_up()
cm.wait_until_healthy()
if cm.remote:
cm.tweak_memory()
class FunctionalTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
options, args = get_options()
self.cluster_spec = ClusterSpec()
self.cluster_spec.parse(options.cluster_spec_fname, args)
self.test_config = TestConfig()
self.test_config.parse(options.test_config_fname, args)
self.target_iterator = TargetIterator(self.cluster_spec,
self.test_config)
self.memcached = MemcachedHelper(self.test_config)
self.remote = RemoteHelper(self.cluster_spec, self.test_config)
self.rest = RestHelper(self.cluster_spec)
super(FunctionalTest, self).__init__(*args, **kwargs)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname)
remote = RemoteHelper(cluster_spec, verbose=False)
remote.collect_info()
for hostname in cluster_spec.servers:
for fname in glob.glob('{}/*.zip'.format(hostname)):
shutil.move(fname, '{}.zip'.format(hostname))
if cluster_spec.backup is not None:
logs = os.path.join(cluster_spec.backup, 'logs')
if os.path.exists(logs):
shutil.make_archive('tools', 'zip', logs)
def main():
args = get_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(args.cluster_spec_fname)
remote = RemoteHelper(cluster_spec, test_config=None, verbose=False)
remote.collect_info()
for hostname in cluster_spec.servers:
for fname in glob.glob('{}/*.zip'.format(hostname)):
shutil.move(fname, '{}.zip'.format(hostname))
if cluster_spec.backup is not None:
logs = os.path.join(cluster_spec.backup, 'logs')
if os.path.exists(logs):
shutil.make_archive('tools', 'zip', logs)
def main():
options, args = get_options()
override = args and (arg.split('.')
for arg in itertools.chain(*csv.reader(args)))
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
# Individual nodes
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_alternative_jemalloc_conf()
# NOTE: configure_tcmalloc_aggressive_decommit() should be after
# any restarts, as it doesn't survive a restart.
cm.configure_tcmalloc_aggressive_decommit()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_auth()
cm.set_mem_quota()
cm.disable_moxi()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.wait_until_healthy()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def main():
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
experiment = options.exp_fname and Experiment(options.exp_fname)
test_module = test_config.test_case.test_module
test_class = test_config.test_case.test_class
exec('from {} import {}'.format(test_module, test_class))
with eval(test_class)(cluster_spec,
test_config,
options.verbose,
experiment) as test:
test.run()
class FunctionalTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
options, _args = get_options()
override = \
_args and (arg.split('.') for arg in ' '.join(_args).split(','))
self.cluster_spec = ClusterSpec()
self.cluster_spec.parse(options.cluster_spec_fname)
self.test_config = TestConfig()
self.test_config.parse(options.test_config_fname, override)
self.target_iterator = TargetIterator(self.cluster_spec,
self.test_config)
self.memcached = MemcachedHelper(self.cluster_spec)
self.remote = RemoteHelper(self.cluster_spec)
super(FunctionalTest, self).__init__(*args, **kwargs)
def main():
options, args = get_options()
override = args and (arg.split('.') for arg in itertools.chain(*csv.reader(args)))
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
# Individual nodes
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_alternative_jemalloc_conf()
# NOTE: configure_tcmalloc_aggressive_decommit() should be after
# any restarts, as it doesn't survive a restart.
cm.configure_tcmalloc_aggressive_decommit()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_auth()
cm.set_mem_quota()
cm.disable_moxi()
time.sleep(30) # crutch
# Cluster
if cm.group_number > 1:
cm.create_server_groups()
cm.add_nodes()
if cm.test_config.cluster.num_buckets:
cm.create_buckets()
cm.restart_with_alternative_bucket_options()
cm.wait_until_warmed_up()
cm.wait_until_healthy()
cm.configure_auto_compaction()
cm.enable_auto_failover()
cm.change_watermarks()
cm.tweak_memory()
cm.remote.disable_wan()
cm.start_cbq_engine()
def main():
options, args = get_options()
override = args and [arg.split('.') for arg in itertools.chain(*csv.reader(args))]
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, override)
test_config = TestConfig()
test_config.parse(options.test_config_fname, override)
experiment = options.exp_fname and Experiment(options.exp_fname)
test_module = test_config.test_case.test_module
test_class = test_config.test_case.test_class
exec('from {} import {}'.format(test_module, test_class))
with eval(test_class)(cluster_spec,
test_config,
options.verbose,
experiment) as test:
test.run()
def main():
usage = '%prog -c cluster.spec'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, args = parser.parse_args()
if not options.cluster_spec_fname:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
installer = MongoDBInstaller(cluster_spec, options)
installer.install()
def main():
usage = '%prog -c cluster.spec'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
options, _ = parser.parse_args()
if not options.cluster_spec_fname:
parser.error('Missing mandatory parameter')
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
installer = MongoDBInstaller(cluster_spec, options)
installer.install()
def test_cluster_specs(self):
for file_name in glob.glob("clusters/*.spec"):
cluster_spec = ClusterSpec()
cluster_spec.parse(file_name, override=())
### Create two clusters with an XDCR relationship, run a workload against the clusters for example:
### cbc-pillowfight -t 1 -B 10 -U couchbase://NODE_IP/default -m 512 -M 1024 -r 50
### Move this file to the root of the perfrunner repo and run ./env/bin/python xdcr_test.py -c clusters/Hermes.spec
### The actual cluster spec is irrelevant for this test so long as it is valid.
from optparse import OptionParser
from perfrunner.settings import ClusterSpec
from perfrunner.helpers.monitor import Monitor
def get_options():
usage = '%prog -c cluster -t test_config'
parser = OptionParser(usage)
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
options, args = parser.parse_args()
return options, args
options, args = get_options()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname)
monitor = Monitor(cluster_spec)
monitor.monitor_xdcr_queues('192.168.42.101:8091', 'default')
def main():
usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag'
parser = OptionParser(usage)
parser.add_option('-v', '--version', dest='version')
parser.add_option('-c',
dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose',
dest='verbose',
action='store_true',
help='enable verbose logging')
parser.add_option('-o',
dest='toy',
help='optional toy build ID',
metavar='couchstore')
parser.add_option('-t',
dest='test_config_fname',
help='path to test configuration file',
metavar='my_test.test')
parser.add_option('-e',
'--edition',
dest='cluster_edition',
default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('--url',
dest='url',
default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
parser.add_option('-f', '--filename', dest='filename')
parser.add_option('-b', '--build', dest='build')
parser.add_option('-n', '--number', dest='number')
options, args = parser.parse_args()
test_id = options.number + '_' + options.build
data = None
try:
with open(options.filename) as data_file:
data = json.load(data_file)
except (OSError, IOError, ValueError) as e:
raise e
mng_data = manage_test_result(['beer_queries', 'travel_queries'])
mng_data.create_cb_instance(data["couchbase_server"],
data["couchbase_query_bucket"])
options, args = parser.parse_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
installer = CouchbaseInstaller(cluster_spec, options)
if True:
installer.install()
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_auth()
time.sleep(30)
"""host = cluster_spec.yield_masters().next()"""
host_ip = cluster_spec.yield_masters().next().split(':')[0]
URL = 'http://' + host_ip + ':8093'
logger.info('logging the URL: {}'.format(URL))
conn = urllib3.connection_from_url(URL)
rest = RestHelper(cluster_spec)
airline_result = do_airline_benchmarks(mng_data, conn, rest, host_ip,
installer.remote, cluster_spec)
beer_result = do_beer_queries(mng_data, conn, rest, host_ip,
installer.remote)
print 'beer_result is', beer_result
#mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result)
mng_data.load_data_query_benchmark(data["couchbase_query_bucket"],
'query_benchmark', test_id,
options.version)
mng_data.show_data_query_benchmark(data["couchbase_query_bucket"],
'query_benchmark', test_id)
sys.exit(not (airline_result and beer_result))
def test_cluster_specs(self):
for file_name in glob.glob("clusters/*.spec"):
cluster_spec = ClusterSpec()
cluster_spec.parse(file_name, override=())
self.assertIn(cluster_spec.parameters['Platform'],
('HW', 'Virtual', 'Localhost'))
def main():
usage = '%prog -v version -c cluster-spec -f conffile -b build -t tag'
parser = OptionParser(usage)
parser.add_option('-v', '--version', dest='version')
parser.add_option('-c', dest='cluster_spec_fname',
help='path to cluster specification file',
metavar='cluster.spec')
parser.add_option('--verbose', dest='verbose', action='store_true',
help='enable verbose logging')
parser.add_option('-o', dest='toy',
help='optional toy build ID', metavar='couchstore')
parser.add_option('-t', dest='test_config_fname',
help='path to test configuration file',
metavar='my_test.test')
parser.add_option('-e', '--edition', dest='cluster_edition', default='enterprise',
help='the cluster edition (community or enterprise)')
parser.add_option('--url', dest='url', default=None,
help='The http URL to a Couchbase RPM that should be'
' installed. This overrides the URL to be installed.')
parser.add_option('-f', '--filename', dest='filename')
parser.add_option('-b','--build',dest='build')
parser.add_option('-n','--number',dest='number')
options, args = parser.parse_args()
test_id=options.number + '_'+options.build
data=None
try:
with open(options.filename) as data_file:
data = json.load(data_file)
except (OSError, IOError,ValueError) as e:
raise e
mng_data = manage_test_result(['beer_queries','travel_queries'])
mng_data.create_cb_instance(data["couchbase_server"],data["couchbase_query_bucket"])
options, args = parser.parse_args()
cluster_spec = ClusterSpec()
cluster_spec.parse(options.cluster_spec_fname, args)
test_config = TestConfig()
test_config.parse(options.test_config_fname, args)
cm = ClusterManager(cluster_spec, test_config, options.verbose)
installer = CouchbaseInstaller(cluster_spec, options)
if True:
installer.install()
if cm.remote:
cm.tune_logging()
cm.restart_with_sfwi()
cm.restart_with_alternative_num_vbuckets()
cm.restart_with_alternative_num_cpus()
cm.restart_with_tcmalloc_aggressive_decommit()
cm.disable_moxi()
cm.configure_internal_settings()
cm.set_data_path()
cm.set_services()
cm.set_mem_quota()
cm.set_index_mem_quota()
cm.set_auth()
time.sleep(30)
"""host = cluster_spec.yield_masters().next()"""
host_ip = cluster_spec.yield_masters().next().split(':')[0]
URL = 'http://' + host_ip + ':8093'
logger.info('logging the URL: {}'.format(URL))
conn = urllib3.connection_from_url(URL)
rest = RestHelper(cluster_spec)
airline_result = do_airline_benchmarks(mng_data,conn, rest, host_ip, installer.remote, cluster_spec)
beer_result = do_beer_queries(mng_data,conn, rest, host_ip, installer.remote)
print 'beer_result is', beer_result
#mng_data.cb_load_test(data["couchbase_query_bucket"],beer_result)
mng_data.load_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id,options.version)
mng_data.show_data_query_benchmark(data["couchbase_query_bucket"],'query_benchmark',test_id)
sys.exit(not (airline_result and beer_result))
def test_cluster_specs(self):
for file_name in glob.glob("clusters/*.spec"):
cluster_spec = ClusterSpec()
cluster_spec.parse(file_name, override=None)
