def _do_load(self, output_stream, expected_dump_counts, load_filter,
expected_load_counts):
# make sure that there's no data left in the DB
objects_remaining = list(get_objects_to_dump(self.domain_name, [], []))
object_classes = [obj.__class__.__name__ for obj in objects_remaining]
counts = Counter(object_classes)
self.assertEqual([], objects_remaining,
'Not all data deleted: {}'.format(counts))
actual_model_counts, dump_lines = self._parse_dump_output(
output_stream)
expected_model_counts = _normalize_object_counter(expected_dump_counts)
self.assertDictEqual(dict(expected_model_counts),
dict(actual_model_counts))
# Load
loader = SqlDataLoader(object_filter=load_filter)
loaded_model_counts = loader.load_objects(dump_lines)
normalized_expected_loaded_counts = _normalize_object_counter(
expected_load_counts, for_loaded=True)
self.assertDictEqual(dict(normalized_expected_loaded_counts),
dict(loaded_model_counts))
self.assertEqual(sum(expected_load_counts.values()),
sum(loaded_model_counts.values()))
return dump_lines
def _load_with_errors(self, chunk_size):
output_stream = StringIO()
SqlDataDumper(self.domain_name, []).dump(output_stream)
self.delete_sql_data()
# resave the product to force an error
self.products[0].save()
actual_model_counts, dump_lines = self._parse_dump_output(output_stream)
self.assertEqual(actual_model_counts['products.sqlproduct'], 3)
loader = SqlDataLoader()
with self.assertRaises(IntegrityError),\
mock.patch("corehq.apps.dump_reload.sql.load.CHUNK_SIZE", chunk_size):
# patch the chunk size so that the queue blocks
loader.load_objects(dump_lines)
def _dump_and_load(self, expected_object_counts):
expected_object_counts.update(self.default_objects_counts)
models = list(expected_object_counts)
self._check_signals_handle_raw(models)
output_stream = StringIO()
SqlDataDumper(self.domain_name, []).dump(output_stream)
self.delete_sql_data()
# make sure that there's no data left in the DB
objects_remaining = list(get_objects_to_dump(self.domain_name, []))
object_classes = [obj.__class__.__name__ for obj in objects_remaining]
counts = Counter(object_classes)
self.assertEqual([], objects_remaining, 'Not all data deleted: {}'.format(counts))
dump_output = output_stream.getvalue()
dump_lines = [line.strip() for line in dump_output.split('\n') if line.strip()]
total_object_count, loaded_model_counts = SqlDataLoader().load_objects(dump_lines)
expected_model_counts = _normalize_object_counter(expected_object_counts)
actual_model_counts = Counter([json.loads(line)['model'] for line in dump_lines])
expected_total_objects = sum(expected_object_counts.values())
self.assertDictEqual(dict(expected_model_counts), dict(actual_model_counts))
expected_loaded_counts = _normalize_object_counter(expected_object_counts, for_loaded=True)
self.assertDictEqual(dict(expected_loaded_counts), dict(loaded_model_counts))
self.assertEqual(expected_total_objects, sum(loaded_model_counts.values()))
self.assertEqual(expected_total_objects, total_object_count)
return dump_lines
def _dump_and_load(self,
expected_dump_counts,
load_filter=None,
expected_load_counts=None,
dumper_fn=None):
expected_load_counts = expected_load_counts or expected_dump_counts
expected_dump_counts.update(self.default_objects_counts)
models = list(expected_dump_counts)
self._check_signals_handle_raw(models)
output_stream = StringIO()
if dumper_fn:
dumper_fn(output_stream)
else:
SqlDataDumper(self.domain_name, [], []).dump(output_stream)
self.delete_sql_data()
# make sure that there's no data left in the DB
objects_remaining = list(get_objects_to_dump(self.domain_name, [], []))
object_classes = [obj.__class__.__name__ for obj in objects_remaining]
counts = Counter(object_classes)
self.assertEqual([], objects_remaining,
'Not all data deleted: {}'.format(counts))
# Dump
actual_model_counts, dump_lines = self._parse_dump_output(
output_stream)
expected_model_counts = _normalize_object_counter(expected_dump_counts)
self.assertDictEqual(dict(expected_model_counts),
dict(actual_model_counts))
# Load
loader = SqlDataLoader(object_filter=load_filter)
loaded_model_counts = loader.load_objects(dump_lines)
normalized_expected_loaded_counts = _normalize_object_counter(
expected_load_counts, for_loaded=True)
self.assertDictEqual(dict(normalized_expected_loaded_counts),
dict(loaded_model_counts))
self.assertEqual(sum(expected_load_counts.values()),
sum(loaded_model_counts.values()))
return dump_lines
