Ejemplo n.º 1
0
 def test_dicts_from_iterable_with_empty_lines(self):
     d = list(
         dicts_from_lines(
             ['', '{"name": 10}', '\n', '', '{"name": 20}\n'
              '\n']))
     self.assertEqual(d[0], {"name": 10})
     self.assertEqual(d[1], {"name": 20})
Ejemplo n.º 2
0
    def exec_instructions(self, instructions):
        filenames = instructions.statement_files
        filenames = (os.path.join(self.spec_dir, i) for i in filenames)
        lines = (line for fn in filenames for line in get_lines(fn))
        statements = itertools.chain(as_statements(lines),
                                     instructions.statements)
        for stmt in statements:
            aio.run(self.client.execute, stmt)

        for data_file in instructions.data_files:
            inserts = as_bulk_queries(self._to_inserts(data_file),
                                      data_file.get('bulk_size', 5000))
            concurrency = data_file.get('concurrency', 25)
            aio.run_many(self.client.execute_many,
                         inserts,
                         concurrency=concurrency)
            if self.client.is_cratedb:
                aio.run(self.client.execute,
                        f"refresh table {data_file['target']}")

        for data_cmd in instructions.data_cmds:
            process = subprocess.Popen(data_cmd['cmd'],
                                       stdout=subprocess.PIPE,
                                       universal_newlines=True)
            target = data_cmd['target']
            dicts = dicts_from_lines(process.stdout)
            inserts = as_bulk_queries((to_insert(target, d) for d in dicts),
                                      data_cmd.get('bulk_size', 5000))
            concurrency = data_cmd.get('concurrency', 25)
            aio.run_many(self.client.execute_many,
                         inserts,
                         concurrency=concurrency)
            if self.client.is_cratedb:
                aio.run(self.client.execute, f"refresh table {target}")
Ejemplo n.º 3
0
 def test_dicts_from_iterable_with_empty_lines(self):
     d = list(dicts_from_lines([
         '',
         '{"name": 10}',
         '\n',
         '',
         '{"name": 20}\n'
         '\n'
     ]))
     self.assertEqual(d[0], {"name": 10})
     self.assertEqual(d[1], {"name": 20})
Ejemplo n.º 4
0
 def _to_inserts(self, data_spec):
     target = data_spec['target']
     source = os.path.join(self.spec_dir, data_spec['source'])
     dicts = dicts_from_lines(get_lines(source))
     return (to_insert(target, d) for d in dicts)
Ejemplo n.º 5
0
 def test_dicts_from_list_of_objects_in_single_line(self):
     it = iter(dicts_from_lines(['[{"name": 10}, {"name": 20}]']))
     self.assertEqual(next(it), {"name": 10})
     self.assertEqual(next(it), {"name": 20})
Ejemplo n.º 6
0
 def test_dicts_from_iterator(self):
     d = next(
         iter(dicts_from_lines(iter(['{\n', '    "name": 10\n', '}\n']))))
     self.assertEqual(d, {"name": 10})
Ejemplo n.º 7
0
 def test_dicts_from_iterator(self):
     d = next(iter(dicts_from_lines(iter(['{\n', '    "name": 10\n', '}\n']))))
     self.assertEqual(d, {"name": 10})