def test_second_step_optional_columns_data_ok(self):
"""Test validation of optional columns data"""
request = self.layer["request"]
request.form = {
"form.buttons.import": u"Importer",
"form.widgets.column_0": u"parent_identifier",
"form.widgets.column_1": u"identifier",
"form.widgets.column_2": u"title",
"form.widgets.column_3": u"informations",
"form.widgets.decimal_import": u"False",
"form.widgets.allow_empty": u"False",
}
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
csv = StringIO()
lines = [
["", "key1", "Key 1", "infos"],
["key1", "key1.1", "Key 1.1", ""],
["key1.1", "key1.1.1", "Key 1.1.1", ""],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
form = importform.ImportFormSecondStep(self.container, request)
form.updateFieldsFromSchemata()
form.updateWidgets()
data, errors = form.extractData()
self.assertEqual(0, len(errors))
def test_second_step_basic_delimiter(self):
"""Test edge case related to csv delimiter"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u","
csv = StringIO()
lines = [
["", "key1", "Key 1"],
["key1", "key1.1", '"Key 1,1"'],
["key1.1", "key1.1.1", '"Key 1.1.1"'],
]
for line in lines:
csv.write(",".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
exception = None
try:
form.update()
except Exception as e:
exception = e
self.assertIsNone(exception)
def test_second_step_required_columns_nok(self):
"""Test validation of required columns"""
request = self.layer["request"]
request.form = {
"form.buttons.import": u"Importer",
"form.widgets.column_0": u"--NOVALUE--",
"form.widgets.column_1": u"--NOVALUE--",
"form.widgets.column_2": u"--NOVALUE--",
"form.widgets.decimal_import": u"False",
"form.widgets.allow_empty": u"False",
}
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
annotation["source"] = NamedBlobFile(
data=self._csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
form = importform.ImportFormSecondStep(self.container, request)
form.updateFieldsFromSchemata()
form.updateWidgets()
data, errors = form.extractData()
self.assertEqual(1, len(errors))
self.assertEqual(
"The following required columns are missing: identifier",
translate(errors[0].error.message),
)
def test_second_step_import_encoding_form(self):
"""Test importing csv data with special chars in header and content"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = True
annotation["separator"] = u";"
csv = StringIO()
lines = [
[u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")],
["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")],
[
u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"),
u"猫".encode("utf8")
],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
form.update()
exception = None
try:
render = form.render()
except UnicodeDecodeError as e:
exception = e
self.assertIsNone(exception)
self.assertTrue(u"Column {0}".format(u"猫") in render)
def test_second_step_import_single_column(self):
"""Test importing csv data"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
csv = StringIO()
lines = [
["", "key1", "Key 1"],
["", "key2", "Key 2"],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
data = {
"column_0": None,
"column_1": "identifier",
"column_2": None,
"decimal_import": False,
"allow_empty": False,
}
form._import(data)
self.assertEqual(2, len(self.container))
self.assertEqual(["key1", "key2"],
sorted(
[e.identifier for e in self.container.values()]))
self.assertEqual(["key1", "key2"],
sorted([e.title for e in self.container.values()]))
def test_second_step_basic_encoding(self):
"""Ensure that form can be displayed even with special characters"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
csv = StringIO()
lines = [
["null", "key1", "key1.1", "Key 1.1", "informations"],
[
"null",
"",
u"key1 éà$€".encode("utf8"),
u"Key 1 éà$€".encode("utf8"),
u"informations éà$€".encode("utf8"),
],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
exception = None
try:
form.update()
except UnicodeDecodeError as e:
exception = e
self.assertIsNone(exception)
def test_process_csv_replace_slash(self):
"""Test _process_csv with csv data that contains slashes"""
form = importform.ImportFormSecondStep(self.container, {})
_csv = StringIO()
lines = [
["1", "First/level"],
["11", "Second / level"],
["12", "Other / level / in // Tesla"],
]
for line in lines:
_csv.write(";".join(line) + "\n")
_csv.seek(0)
reader = csv.reader(_csv, delimiter=";")
data = {
"column_0": "identifier",
"column_1": "title",
}
mapping = {int(k.replace("column_", "")): v for k, v in data.items()}
result = form._process_csv(reader,
mapping,
"utf-8", {},
decimal_import=True,
replace_slash=True)
expected_result = {
None: {
u'1': (u'First-level', {})
},
u'1': {
u'11': (u'Second - level', {}),
u'12': (u'Other - level - in -- Tesla', {})
}
}
self.assertEqual(expected_result, result)
def test_process_csv_multiple_identifiers(self):
"""Test _process_csv with csv data that contains multiple identifiers"""
form = importform.ImportFormSecondStep(self.container, {})
_csv = StringIO()
lines = [
["1", "First level"],
["11", "Second level"],
["11, 12", "Other level in / Tesla"],
["21, 22", "New sub levels"],
["111", "Yet one"],
]
for line in lines:
_csv.write(";".join(line) + "\n")
_csv.seek(0)
reader = csv.reader(_csv, delimiter=";")
data = {
"column_0": "identifier",
"column_1": "title",
}
mapping = {int(k.replace("column_", "")): v for k, v in data.items()}
result = form._process_csv(reader,
mapping,
"utf-8", {},
decimal_import=True,
replace_slash=True)
expected_result = {
None: {
u'1': (u'First level', {}),
u'2': (u'2', {
u'enabled': False
}),
},
u'1': {
u'11': (u'Second level', {}),
u'12': (u'12', {})
},
u'11': {
u'111': (u'Yet one', {})
},
u'2': {
u'21': (u'New sub levels', {}),
u'22': (u'22', {})
}
}
self.assertEqual(expected_result, result)
def test_second_step_import_encoding(self):
"""Test importing csv data with special chars in header and content"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = True
annotation["separator"] = u";"
csv = StringIO()
lines = [
[u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")],
["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")],
[
u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"),
u"猫".encode("utf8")
],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
data = {
"column_0": "parent_identifier",
"column_1": "identifier",
"column_2": "title",
"decimal_import": False,
"allow_empty": False,
}
form._import(data)
self.assertEqual(1, len(self.container))
self.assertEqual([u"kéy1"],
[e.identifier for e in self.container.values()])
key1 = self.container.get_by("identifier", u"kéy1")
self.assertEqual(1, len(key1))
self.assertEqual([u"kéy1.1"], [e.identifier for e in key1.values()])
key1_1 = key1.get_by("identifier", u"kéy1.1")
self.assertEqual(u"猫", key1_1.title)
def test_second_step_import_basic(self):
"""Test importing csv data"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
annotation["source"] = NamedBlobFile(
data=self._csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
data = {
"column_0": "parent_identifier",
"column_1": "identifier",
"column_2": "title",
"decimal_import": False,
"allow_empty": False,
}
form._import(data)
self.assertEqual(2, len(self.container))
self.assertEqual(["key1", "key2"],
sorted(
[e.identifier for e in self.container.values()]))
key1 = self.container.get_by("identifier", "key1")
self.assertEqual(3, len(key1))
self.assertEqual(
["key1.1", "key1.2", "key1.3"],
sorted([e.identifier for e in key1.values()]),
)
key1_1 = key1.get_by("identifier", "key1.1")
self.assertEqual(1, len(key1_1))
self.assertEqual(["key1.1.1"],
sorted([e.identifier for e in key1_1.values()]))
key2 = self.container.get_by("identifier", "key2")
self.assertEqual(1, len(key2))
self.assertEqual(["key2.1"],
sorted([e.identifier for e in key2.values()]))
def test_second_step_columns_data_format_nok(self):
"""Test validation of columns data format"""
request = self.layer["request"]
request.form = {
"form.buttons.import": u"Importer",
"form.widgets.column_0": u"identifier",
"form.widgets.column_1": u"title",
"form.widgets.decimal_import": u"selected",
"form.widgets.allow_empty": u"False",
}
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
csv = StringIO()
lines = [
["-1", "key1"],
[".10", "key2"],
["-1.1", "key3"],
["-1 11", "Key4"],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
form = importform.ImportFormSecondStep(self.container, request)
form.updateFieldsFromSchemata()
form.updateWidgets()
data, errors = form.extractData()
self.assertEqual(1, len(errors))
self.assertEqual(
"Bad format values: Line 4, col 1: '-1 11'",
translate(errors[0].error.message),
)
def test_second_step_required_columns_data_ok(self):
"""Test validation of required columns data"""
request = self.layer["request"]
request.form = {
"form.buttons.import": u"Importer",
"form.widgets.column_0": u"parent_identifier",
"form.widgets.column_1": u"identifier",
"form.widgets.column_2": u"title",
"form.widgets.decimal_import": u"False",
}
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
annotation["source"] = NamedBlobFile(
data=self._csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
form = importform.ImportFormSecondStep(self.container, request)
form.updateFieldsFromSchemata()
form.updateWidgets()
data, errors = form.extractData()
self.assertEqual(0, len(errors))
def test_second_step_import_decimal_basic(self):
"""Test importing csv data with decimal codes"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
annotations = IAnnotations(self.container)
annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
annotation["has_header"] = False
annotation["separator"] = u";"
csv = StringIO()
lines = [
["100", "Key 1"],
["100.1", "Key 1.1"],
["100.2", "Key 1.2"],
["200", "Key 2"],
["200.1", "Key 2.1"],
["200.10", "Key 2.10"],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
annotation["source"] = NamedBlobFile(
data=csv.read(),
contentType=u"text/csv",
filename=u"test.csv",
)
data = {
"column_0": "identifier",
"column_1": "title",
"decimal_import": True,
"allow_empty": False,
}
form._import(data)
self.assertEqual(2, len(self.container))
self.assertEqual(["1", "2"],
sorted(
[e.identifier for e in self.container.values()]))
code_1 = self.container.get_by("identifier", "1")
self.assertEqual("1", code_1.title)
self.assertEqual(1, len(code_1))
self.assertEqual(["10"], [e.identifier for e in code_1.values()])
code_10 = code_1.get_by("identifier", "10")
self.assertEqual("10", code_10.title)
self.assertEqual(1, len(code_10))
self.assertEqual(["100"], [e.identifier for e in code_10.values()])
code_100 = code_10.get_by("identifier", "100")
self.assertEqual("Key 1", code_100.title)
self.assertEqual(2, len(code_100))
self.assertEqual(
["100.1", "100.2"],
sorted([e.identifier for e in code_100.values()]),
)
self.assertEqual(
["Key 1.1", "Key 1.2"],
sorted([e.title for e in code_100.values()]),
)
code_2 = self.container.get_by("identifier", "2")
self.assertEqual("2", code_2.title)
self.assertEqual(1, len(code_2))
self.assertEqual(["20"], [e.identifier for e in code_2.values()])
code_20 = code_2.get_by("identifier", "20")
self.assertEqual("20", code_20.title)
self.assertEqual(1, len(code_20))
self.assertEqual(["200"], [e.identifier for e in code_20.values()])
code_200 = code_20.get_by("identifier", "200")
self.assertEqual("Key 2", code_200.title)
self.assertEqual(1, len(code_200))
self.assertEqual(["200.1"], [e.identifier for e in code_200.values()])
code_2001 = code_200.get_by("identifier", "200.1")
self.assertEqual("Key 2.1", code_2001.title)
self.assertEqual(1, len(code_2001))
self.assertEqual(["200.10"],
[e.identifier for e in code_2001.values()])
self.assertEqual(["Key 2.10"], [e.title for e in code_2001.values()])
def test_process_data_multilevel(self):
"""Tests _process_data with multi levels data structure"""
form = importform.ImportFormSecondStep(self.container,
self.layer["request"])
data = {
None: {
u"key1": (u"Key 1", {}),
u"key2": (u"Key 2", {
'enabled': False
})
},
u"key1": {
u"key1.1": (u"Key 1.1", {}),
u"key1.2": (u"Key 1.2", {})
},
u"key2": {
u"key2.1": (u"Key 2.1", {
'enabled': False
})
},
u"key1.1": {
u"key1.1.1": (u"Key 1.1.1", {})
},
u"key1.1.1": {
u"key1.1.1.1": (u"Key 1.1.1.1", {})
},
}
expected_results = [
{
"identifier":
u"key1",
"title":
u"Key 1",
"informations":
None,
"enabled":
None,
"_children": [
{
"identifier":
u"key1.1",
"title":
u"Key 1.1",
"informations":
None,
"enabled":
None,
"_children": [
{
"identifier":
u"key1.1.1",
"title":
u"Key 1.1.1",
"informations":
None,
"enabled":
None,
"_children": [
{
"identifier": u"key1.1.1.1",
"title": u"Key 1.1.1.1",
"informations": None,
"enabled": None,
"_children": [],
},
],
},
],
},
{
"identifier": u"key1.2",
"title": u"Key 1.2",
"informations": None,
"enabled": None,
"_children": [],
},
],
},
{
"identifier":
u"key2",
"title":
u"Key 2",
"informations":
None,
"enabled":
False,
"_children": [
{
"identifier": u"key2.1",
"title": u"Key 2.1",
"informations": None,
"enabled": False,
"_children": [],
},
],
},
]
processed_data = form._process_data(data)
self.assertEqual(self._sort_processed_data(processed_data),
expected_results)
