def test_csv_record(self):
dumper = CsvDumper([], model=self.model)
expected = ';;;;;\n'
self.assertEqual(dumper._csv_record(None), expected)
expected = '"value_a";"value_a";"value_ref";"some a";"some b";"2021-01-02"\n'
self.assertEqual(dumper._csv_record(self.mock_entity), expected)
def test_iterate(self):
dumper = CsvDumper([], model=self.model)
value = [item for item in dumper]
self.assertEqual(value, [])
header = '"any_entity_id";"ref";"reference_bronwaarde";"json_a";"json_b";"incdate_formatted"\n'
line1 = '"value_a";"value_a";"value_ref";"some a";"some b";"2021-01-02"\n'
# header is True
dumper = CsvDumper([self.mock_entity], model=self.model)
value = [item for item in dumper]
self.assertEqual(value, [header, line1])
# header is False
dumper = CsvDumper([self.mock_entity], model=self.model, header=False)
value = [item for item in dumper]
self.assertEqual(value, [line1])
def test_get_field_names(self):
dumper = CsvDumper([], model=self.model)
expected = [
'any_entity_id',
'ref',
'reference_bronwaarde',
'json_a',
'json_b',
'incdate_formatted'
]
self.assertEqual(dumper.field_names, expected)
def _dump_entities_to_table(self, entities):
connection = self.datastore.connection
sql = f"COPY {self.schema}.{self.tmp_collection_name} FROM STDIN DELIMITER '{DELIMITER_CHAR}' CSV"
csv_lines = CsvDumper(entities, model=self.model, ignore_fields=self.suppressed_columns, header=False)
line_nr = 0
with connection.cursor() as cursor:
yield "Export data"
for stream, line_nr in CsvStream(csv_lines, size=STREAM_PER):
cursor.copy_expert(sql, stream, size=BUFFER_PER)
if not line_nr % COMMIT_PER:
connection.commit()
yield f"\n{self.collection_name}: {line_nr:,}"
else:
yield '.'
connection.commit()
yield f"\nExported {line_nr:,} rows\n"
def _dump(catalog_name, collection_name):
"""
Dump all entities in the requested format. Currently only csv
:param catalog_name:
:param collection_name:
:return: Streaming response of all entities in csv format with header
"""
method = request.method
if method == 'GET':
format = request.args.get('format')
exclude_deleted = request.args.get('exclude_deleted') == 'true'
filter = (lambda table: getattr(table, FIELD.DATE_DELETED).is_(None)
) if exclude_deleted else None
entities, model = dump_entities(catalog_name,
collection_name,
filter=filter)
if format == "csv":
result = CsvDumper(entities, model=model)
return WorkerResponse.stream_with_context(result,
mimetype='text/csv')
elif format == "sql":
return Response(sql_entities(catalog_name, collection_name, model),
mimetype='application/sql')
else:
return f"Unrecognised format parameter '{format}'" if format else "Format parameter not set", 400
elif method == 'POST':
content_type = request.content_type
if content_type == 'application/json':
config = json.loads(request.data)
result = dump_to_db(catalog_name, collection_name, config)
return WorkerResponse.stream_with_context(result,
mimetype='text/plain')
else:
return f"Unrecognised content type '{content_type}'", 400
def test_csv_header(self):
dumper = CsvDumper([], model=self.model)
expected = '"any_entity_id";"ref";"reference_bronwaarde";"json_a";"json_b";"incdate_formatted"\n'
self.assertEqual(dumper._csv_header(), expected)
def test_init_model_is_none(self):
dumper = CsvDumper([])
self.assertEqual(dumper.field_specs, {})
self.assertEqual(dumper.field_order, [])
self.assertEqual(dumper.field_names, [])