def _dump_and_load(self, expected_object_counts): expected_object_counts.update(self.default_objects_counts) models = list(expected_object_counts) self._check_signals_handle_raw(models) output_stream = StringIO() SqlDataDumper(self.domain_name, []).dump(output_stream) self.delete_sql_data() # make sure that there's no data left in the DB objects_remaining = list(get_objects_to_dump(self.domain_name, [])) object_classes = [obj.__class__.__name__ for obj in objects_remaining] counts = Counter(object_classes) self.assertEqual([], objects_remaining, 'Not all data deleted: {}'.format(counts)) dump_output = output_stream.getvalue() dump_lines = [line.strip() for line in dump_output.split('\n') if line.strip()] total_object_count, loaded_model_counts = SqlDataLoader().load_objects(dump_lines) expected_model_counts = _normalize_object_counter(expected_object_counts) actual_model_counts = Counter([json.loads(line)['model'] for line in dump_lines]) expected_total_objects = sum(expected_object_counts.values()) self.assertDictEqual(dict(expected_model_counts), dict(actual_model_counts)) expected_loaded_counts = _normalize_object_counter(expected_object_counts, for_loaded=True) self.assertDictEqual(dict(expected_loaded_counts), dict(loaded_model_counts)) self.assertEqual(expected_total_objects, sum(loaded_model_counts.values())) self.assertEqual(expected_total_objects, total_object_count) return dump_lines
def _load_with_errors(self, chunk_size): output_stream = StringIO() SqlDataDumper(self.domain_name, []).dump(output_stream) self.delete_sql_data() # resave the product to force an error self.products[0].save() actual_model_counts, dump_lines = self._parse_dump_output(output_stream) self.assertEqual(actual_model_counts['products.sqlproduct'], 3) loader = SqlDataLoader() with self.assertRaises(IntegrityError),\ mock.patch("corehq.apps.dump_reload.sql.load.CHUNK_SIZE", chunk_size): # patch the chunk size so that the queue blocks loader.load_objects(dump_lines)
def _dump_and_load(self, expected_dump_counts, load_filter=None, expected_load_counts=None, dumper_fn=None): expected_load_counts = expected_load_counts or expected_dump_counts expected_dump_counts.update(self.default_objects_counts) models = list(expected_dump_counts) self._check_signals_handle_raw(models) output_stream = StringIO() if dumper_fn: dumper_fn(output_stream) else: SqlDataDumper(self.domain_name, [], []).dump(output_stream) self.delete_sql_data() # make sure that there's no data left in the DB objects_remaining = list(get_objects_to_dump(self.domain_name, [], [])) object_classes = [obj.__class__.__name__ for obj in objects_remaining] counts = Counter(object_classes) self.assertEqual([], objects_remaining, 'Not all data deleted: {}'.format(counts)) # Dump actual_model_counts, dump_lines = self._parse_dump_output( output_stream) expected_model_counts = _normalize_object_counter(expected_dump_counts) self.assertDictEqual(dict(expected_model_counts), dict(actual_model_counts)) # Load loader = SqlDataLoader(object_filter=load_filter) loaded_model_counts = loader.load_objects(dump_lines) normalized_expected_loaded_counts = _normalize_object_counter( expected_load_counts, for_loaded=True) self.assertDictEqual(dict(normalized_expected_loaded_counts), dict(loaded_model_counts)) self.assertEqual(sum(expected_load_counts.values()), sum(loaded_model_counts.values())) return dump_lines
def _dump_and_load(self, expected_dump_counts, load_filter=None, expected_load_counts=None, dumper_fn=None): expected_load_counts = expected_load_counts or expected_dump_counts expected_dump_counts.update(self.default_objects_counts) models = list(expected_dump_counts) self._check_signals_handle_raw(models) # Dump output_stream = StringIO() if dumper_fn: dumper_fn(output_stream) else: SqlDataDumper(self.domain_name, [], []).dump(output_stream) output_stream.seek(0) self.delete_sql_data() self._do_load(output_stream, expected_dump_counts, load_filter, expected_load_counts)