def test_dicts_from_iterable_with_empty_lines(self):
d = list(
dicts_from_lines(
['', '{"name": 10}', '\n', '', '{"name": 20}\n'
'\n']))
self.assertEqual(d[0], {"name": 10})
self.assertEqual(d[1], {"name": 20})
def exec_instructions(self, instructions):
filenames = instructions.statement_files
filenames = (os.path.join(self.spec_dir, i) for i in filenames)
lines = (line for fn in filenames for line in get_lines(fn))
statements = itertools.chain(as_statements(lines),
instructions.statements)
for stmt in statements:
aio.run(self.client.execute, stmt)
for data_file in instructions.data_files:
inserts = as_bulk_queries(self._to_inserts(data_file),
data_file.get('bulk_size', 5000))
concurrency = data_file.get('concurrency', 25)
aio.run_many(self.client.execute_many,
inserts,
concurrency=concurrency)
if self.client.is_cratedb:
aio.run(self.client.execute,
f"refresh table {data_file['target']}")
for data_cmd in instructions.data_cmds:
process = subprocess.Popen(data_cmd['cmd'],
stdout=subprocess.PIPE,
universal_newlines=True)
target = data_cmd['target']
dicts = dicts_from_lines(process.stdout)
inserts = as_bulk_queries((to_insert(target, d) for d in dicts),
data_cmd.get('bulk_size', 5000))
concurrency = data_cmd.get('concurrency', 25)
aio.run_many(self.client.execute_many,
inserts,
concurrency=concurrency)
if self.client.is_cratedb:
aio.run(self.client.execute, f"refresh table {target}")
def test_dicts_from_iterable_with_empty_lines(self):
d = list(dicts_from_lines([
'',
'{"name": 10}',
'\n',
'',
'{"name": 20}\n'
'\n'
]))
self.assertEqual(d[0], {"name": 10})
self.assertEqual(d[1], {"name": 20})
def _to_inserts(self, data_spec):
target = data_spec['target']
source = os.path.join(self.spec_dir, data_spec['source'])
dicts = dicts_from_lines(get_lines(source))
return (to_insert(target, d) for d in dicts)
def test_dicts_from_list_of_objects_in_single_line(self):
it = iter(dicts_from_lines(['[{"name": 10}, {"name": 20}]']))
self.assertEqual(next(it), {"name": 10})
self.assertEqual(next(it), {"name": 20})
def test_dicts_from_iterator(self):
d = next(
iter(dicts_from_lines(iter(['{\n', ' "name": 10\n', '}\n']))))
self.assertEqual(d, {"name": 10})
def test_dicts_from_iterator(self):
d = next(iter(dicts_from_lines(iter(['{\n', ' "name": 10\n', '}\n']))))
self.assertEqual(d, {"name": 10})
