def teardown(*args): try: with client(node.http_url) as c: aio.run(c.execute, 'drop table x.demo') aio.run(c.execute, 'drop blob table blobtable') finally: node.stop()
def retrieve_columns(client, schema, table): r = aio.run(client.execute, "select min(version['number']) from sys.nodes") version = parse_version(r['rows'][0][0]) stmt = SELLECT_COLS.format( schema_column_name='table_schema' if version >= (0, 57, 0) else 'schema_name') r = aio.run(client.execute, stmt, (schema, table)) return OrderedDict(r['rows'])
def save_result(result): if not table_created: aio.run(client.execute, BENCHMARK_TABLE) table_created.append(None) stmt, args = to_insert('benchmarks', result.as_dict()) aio.run(client.execute, stmt, args) log.result(result)
def insert_from_sql(src_uri=None, query=None, fetch_size=100, concurrency=25, table=None, hosts=None, output_fmt=None): """Insert data read from another SQL source into table.""" stats = Stats() with clients.client(hosts, concurrency=concurrency) as client: f = partial(aio.measure, stats, client.execute_many) try: aio.run( async_insert_from_sql, src_uri, concurrency, query, fetch_size, table, f ) except clients.SqlException as e: raise SystemExit(str(e)) try: print(format_stats(stats.get(), output_fmt)) except KeyError: if not stats.sampler.values: raise SystemExit('No data read from source') raise
def exec_instructions(self, instructions): filenames = instructions.statement_files filenames = (os.path.join(self.spec_dir, i) for i in filenames) lines = (line for fn in filenames for line in get_lines(fn)) statements = itertools.chain(as_statements(lines), instructions.statements) for stmt in statements: aio.run(self.client.execute, stmt) for data_file in instructions.data_files: inserts = as_bulk_queries(self._to_inserts(data_file), data_file.get('bulk_size', 5000)) concurrency = data_file.get('concurrency', 25) aio.run_many(self.client.execute_many, inserts, concurrency=concurrency) aio.run(self.client.execute, 'refresh table {target}'.format(target=data_file['target']))
def timeit(hosts=None, stmt=None, warmup=30, repeat=None, duration=None, concurrency=1, output_fmt=None, fail_if=None, sample_mode='reservoir'): """Run the given statement a number of times and return the runtime stats Args: fail-if: An expression that causes cr8 to exit with a failure if it evaluates to true. The expression can contain formatting expressions for: - runtime_stats - statement - meta - concurrency - bulk_size For example: --fail-if "{runtime_stats.mean} > 1.34" """ num_lines = 0 log = Logger(output_fmt) with Runner(hosts, concurrency, sample_mode) as runner: try: version_info = aio.run(runner.client.get_server_version) except Exception: result = aio.run(runner.client.execute, 'select version()') version_info = {'hash': None, 'number': result['rows'][0][0]} for line in as_statements(lines_from_stdin(stmt)): runner.warmup(line, warmup) timed_stats = runner.run(line, iterations=repeat, duration=duration) r = Result(version_info=version_info, statement=line, timed_stats=timed_stats, concurrency=concurrency) log.result(r) if fail_if: eval_fail_if(fail_if, r) num_lines += 1 if num_lines == 0: raise SystemExit( 'No SQL statements provided. Use --stmt or provide statements via stdin' )
def __init__(self, spec_dir, benchmark_hosts, result_hosts, log, fail_if, sample_mode): self.benchmark_hosts = benchmark_hosts self.sample_mode = sample_mode self.spec_dir = spec_dir self.client = clients.client(benchmark_hosts) self.result_client = clients.client(result_hosts) self.server_version_info = aio.run(self.client.get_server_version) self.server_version = parse_version(self.server_version_info['number']) self.log = log self.create_result = partial( Result, version_info=self.server_version_info ) if fail_if: self.fail_if = partial(eval_fail_if, fail_if) else: self.fail_if = lambda x: None if result_hosts: self.process_result = _result_to_crate(self.log, self.result_client) else: self.process_result = log.result
def test_reindex(self): crate_v3 = CrateNode(crate_dir=get_crate('3.x.x'), keep_data=True, settings=self.crate_settings) self._to_stop.append(crate_v3) crate_v3.start() with client(crate_v3.http_url) as c: aio.run(c.execute, "create table t (x int)") args = ( (1, ), (2, ), (3, ), ) aio.run(c.execute_many, "insert into t (x) values (?)", args) crate_v3.stop() self._to_stop.remove(crate_v3) crate_v4 = CrateNode(crate_dir=get_crate('4.0.3'), keep_data=True, settings=self.crate_settings) self._to_stop.append(crate_v4) crate_v4.start() reindex(crate_v4.http_url) with client(crate_v4.http_url) as c: result = aio.run( c.execute, "SELECT version FROM information_schema.tables WHERE table_name = 't'" ) version = result['rows'][0][0] self.assertEqual(version, {'upgraded': None, 'created': '4.0.3'}) cnt = aio.run(c.execute, 'SELECT count(*) FROM t')['rows'][0][0] self.assertEqual(cnt, 3)
def exec_instructions(self, instructions): filenames = instructions.statement_files filenames = (os.path.join(self.spec_dir, i) for i in filenames) lines = (line for fn in filenames for line in get_lines(fn)) statements = itertools.chain(as_statements(lines), instructions.statements) for stmt in statements: aio.run(self.client.execute, stmt) for data_file in instructions.data_files: inserts = as_bulk_queries(self._to_inserts(data_file), data_file.get('bulk_size', 5000)) concurrency = data_file.get('concurrency', 25) aio.run_many(self.client.execute_many, inserts, concurrency=concurrency) if self.client.is_cratedb: aio.run(self.client.execute, f"refresh table {data_file['target']}") for data_cmd in instructions.data_cmds: process = subprocess.Popen(data_cmd['cmd'], stdout=subprocess.PIPE, universal_newlines=True) target = data_cmd['target'] dicts = dicts_from_lines(process.stdout) inserts = as_bulk_queries((to_insert(target, d) for d in dicts), data_cmd.get('bulk_size', 5000)) concurrency = data_cmd.get('concurrency', 25) aio.run_many(self.client.execute_many, inserts, concurrency=concurrency) if self.client.is_cratedb: aio.run(self.client.execute, f"refresh table {target}")
def setup(*args): with client(node.http_url) as c: aio.run( c.execute, 'create table x.demo (id int, name string, country string) \ with (number_of_replicas = 0)' ) aio.run(c.execute, 'create table y.demo (name text) with (number_of_replicas = 0)') aio.run(c.execute, 'create blob table blobtable with (number_of_replicas = 0)')
def __init__(self, spec_dir, benchmark_hosts, result_hosts, log): self.benchmark_hosts = benchmark_hosts self.spec_dir = spec_dir self.client = clients.client(benchmark_hosts) self.result_client = clients.client(result_hosts) self.server_version_info = aio.run(self.client.get_server_version) self.server_version = parse_version(self.server_version_info['number']) self.log = log self.create_result = partial( Result, version_info=self.server_version_info ) if result_hosts: self.process_result = _result_to_crate(self.log, self.result_client) else: self.process_result = log.result
def timeit(hosts=None, stmt=None, warmup=30, repeat=None, duration=None, concurrency=1, output_fmt=None, fail_if=None, sample_mode='reservoir'): """Run the given statement a number of times and return the runtime stats Args: fail-if: An expression that causes cr8 to exit with a failure if it evaluates to true. The expression can contain formatting expressions for: - runtime_stats - statement - meta - concurrency - bulk_size For example: --fail-if "{runtime_stats.mean} > 1.34" """ num_lines = 0 log = Logger(output_fmt) with Runner(hosts, concurrency, sample_mode) as runner: version_info = aio.run(runner.client.get_server_version) for line in as_statements(lines_from_stdin(stmt)): runner.warmup(line, warmup) timed_stats = runner.run(line, iterations=repeat, duration=duration) r = Result( version_info=version_info, statement=line, timed_stats=timed_stats, concurrency=concurrency ) log.result(r) if fail_if: eval_fail_if(fail_if, r) num_lines += 1 if num_lines == 0: raise SystemExit( 'No SQL statements provided. Use --stmt or provide statements via stdin')
def reindex(hosts=None): with clients.client(hosts) as client: run(_async_reindex, client)
def update_server_stats(self): """Triggers ANALYZE on the server to update statistics.""" try: aio.run(self.client.execute, 'ANALYZE') except Exception: pass # swallow; CrateDB 4.1.0+ is required to run ANALYZE