def test_second_step_optional_columns_data_ok(self): """Test validation of optional columns data""" request = self.layer["request"] request.form = { "form.buttons.import": u"Importer", "form.widgets.column_0": u"parent_identifier", "form.widgets.column_1": u"identifier", "form.widgets.column_2": u"title", "form.widgets.column_3": u"informations", "form.widgets.decimal_import": u"False", "form.widgets.allow_empty": u"False", } annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" csv = StringIO() lines = [ ["", "key1", "Key 1", "infos"], ["key1", "key1.1", "Key 1.1", ""], ["key1.1", "key1.1.1", "Key 1.1.1", ""], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) form = importform.ImportFormSecondStep(self.container, request) form.updateFieldsFromSchemata() form.updateWidgets() data, errors = form.extractData() self.assertEqual(0, len(errors))
def test_second_step_basic_delimiter(self): """Test edge case related to csv delimiter""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u"," csv = StringIO() lines = [ ["", "key1", "Key 1"], ["key1", "key1.1", '"Key 1,1"'], ["key1.1", "key1.1.1", '"Key 1.1.1"'], ] for line in lines: csv.write(",".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) exception = None try: form.update() except Exception as e: exception = e self.assertIsNone(exception)
def test_second_step_required_columns_nok(self): """Test validation of required columns""" request = self.layer["request"] request.form = { "form.buttons.import": u"Importer", "form.widgets.column_0": u"--NOVALUE--", "form.widgets.column_1": u"--NOVALUE--", "form.widgets.column_2": u"--NOVALUE--", "form.widgets.decimal_import": u"False", "form.widgets.allow_empty": u"False", } annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" annotation["source"] = NamedBlobFile( data=self._csv.read(), contentType=u"text/csv", filename=u"test.csv", ) form = importform.ImportFormSecondStep(self.container, request) form.updateFieldsFromSchemata() form.updateWidgets() data, errors = form.extractData() self.assertEqual(1, len(errors)) self.assertEqual( "The following required columns are missing: identifier", translate(errors[0].error.message), )
def test_second_step_import_encoding_form(self): """Test importing csv data with special chars in header and content""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = True annotation["separator"] = u";" csv = StringIO() lines = [ [u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")], ["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")], [ u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"), u"猫".encode("utf8") ], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) form.update() exception = None try: render = form.render() except UnicodeDecodeError as e: exception = e self.assertIsNone(exception) self.assertTrue(u"Column {0}".format(u"猫") in render)
def test_second_step_import_single_column(self): """Test importing csv data""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" csv = StringIO() lines = [ ["", "key1", "Key 1"], ["", "key2", "Key 2"], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) data = { "column_0": None, "column_1": "identifier", "column_2": None, "decimal_import": False, "allow_empty": False, } form._import(data) self.assertEqual(2, len(self.container)) self.assertEqual(["key1", "key2"], sorted( [e.identifier for e in self.container.values()])) self.assertEqual(["key1", "key2"], sorted([e.title for e in self.container.values()]))
def test_second_step_basic_encoding(self): """Ensure that form can be displayed even with special characters""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" csv = StringIO() lines = [ ["null", "key1", "key1.1", "Key 1.1", "informations"], [ "null", "", u"key1 éà$€".encode("utf8"), u"Key 1 éà$€".encode("utf8"), u"informations éà$€".encode("utf8"), ], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) exception = None try: form.update() except UnicodeDecodeError as e: exception = e self.assertIsNone(exception)
def test_process_csv_replace_slash(self): """Test _process_csv with csv data that contains slashes""" form = importform.ImportFormSecondStep(self.container, {}) _csv = StringIO() lines = [ ["1", "First/level"], ["11", "Second / level"], ["12", "Other / level / in // Tesla"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "identifier", "column_1": "title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, decimal_import=True, replace_slash=True) expected_result = { None: { u'1': (u'First-level', {}) }, u'1': { u'11': (u'Second - level', {}), u'12': (u'Other - level - in -- Tesla', {}) } } self.assertEqual(expected_result, result)
def test_process_csv_multiple_identifiers(self): """Test _process_csv with csv data that contains multiple identifiers""" form = importform.ImportFormSecondStep(self.container, {}) _csv = StringIO() lines = [ ["1", "First level"], ["11", "Second level"], ["11, 12", "Other level in / Tesla"], ["21, 22", "New sub levels"], ["111", "Yet one"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "identifier", "column_1": "title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, decimal_import=True, replace_slash=True) expected_result = { None: { u'1': (u'First level', {}), u'2': (u'2', { u'enabled': False }), }, u'1': { u'11': (u'Second level', {}), u'12': (u'12', {}) }, u'11': { u'111': (u'Yet one', {}) }, u'2': { u'21': (u'New sub levels', {}), u'22': (u'22', {}) } } self.assertEqual(expected_result, result)
def test_second_step_import_encoding(self): """Test importing csv data with special chars in header and content""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = True annotation["separator"] = u";" csv = StringIO() lines = [ [u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")], ["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")], [ u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"), u"猫".encode("utf8") ], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) data = { "column_0": "parent_identifier", "column_1": "identifier", "column_2": "title", "decimal_import": False, "allow_empty": False, } form._import(data) self.assertEqual(1, len(self.container)) self.assertEqual([u"kéy1"], [e.identifier for e in self.container.values()]) key1 = self.container.get_by("identifier", u"kéy1") self.assertEqual(1, len(key1)) self.assertEqual([u"kéy1.1"], [e.identifier for e in key1.values()]) key1_1 = key1.get_by("identifier", u"kéy1.1") self.assertEqual(u"猫", key1_1.title)
def test_second_step_import_basic(self): """Test importing csv data""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" annotation["source"] = NamedBlobFile( data=self._csv.read(), contentType=u"text/csv", filename=u"test.csv", ) data = { "column_0": "parent_identifier", "column_1": "identifier", "column_2": "title", "decimal_import": False, "allow_empty": False, } form._import(data) self.assertEqual(2, len(self.container)) self.assertEqual(["key1", "key2"], sorted( [e.identifier for e in self.container.values()])) key1 = self.container.get_by("identifier", "key1") self.assertEqual(3, len(key1)) self.assertEqual( ["key1.1", "key1.2", "key1.3"], sorted([e.identifier for e in key1.values()]), ) key1_1 = key1.get_by("identifier", "key1.1") self.assertEqual(1, len(key1_1)) self.assertEqual(["key1.1.1"], sorted([e.identifier for e in key1_1.values()])) key2 = self.container.get_by("identifier", "key2") self.assertEqual(1, len(key2)) self.assertEqual(["key2.1"], sorted([e.identifier for e in key2.values()]))
def test_second_step_columns_data_format_nok(self): """Test validation of columns data format""" request = self.layer["request"] request.form = { "form.buttons.import": u"Importer", "form.widgets.column_0": u"identifier", "form.widgets.column_1": u"title", "form.widgets.decimal_import": u"selected", "form.widgets.allow_empty": u"False", } annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" csv = StringIO() lines = [ ["-1", "key1"], [".10", "key2"], ["-1.1", "key3"], ["-1 11", "Key4"], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) form = importform.ImportFormSecondStep(self.container, request) form.updateFieldsFromSchemata() form.updateWidgets() data, errors = form.extractData() self.assertEqual(1, len(errors)) self.assertEqual( "Bad format values: Line 4, col 1: '-1 11'", translate(errors[0].error.message), )
def test_second_step_required_columns_data_ok(self): """Test validation of required columns data""" request = self.layer["request"] request.form = { "form.buttons.import": u"Importer", "form.widgets.column_0": u"parent_identifier", "form.widgets.column_1": u"identifier", "form.widgets.column_2": u"title", "form.widgets.decimal_import": u"False", } annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" annotation["source"] = NamedBlobFile( data=self._csv.read(), contentType=u"text/csv", filename=u"test.csv", ) form = importform.ImportFormSecondStep(self.container, request) form.updateFieldsFromSchemata() form.updateWidgets() data, errors = form.extractData() self.assertEqual(0, len(errors))
def test_second_step_import_decimal_basic(self): """Test importing csv data with decimal codes""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) annotations = IAnnotations(self.container) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["has_header"] = False annotation["separator"] = u";" csv = StringIO() lines = [ ["100", "Key 1"], ["100.1", "Key 1.1"], ["100.2", "Key 1.2"], ["200", "Key 2"], ["200.1", "Key 2.1"], ["200.10", "Key 2.10"], ] for line in lines: csv.write(";".join(line) + "\n") csv.seek(0) annotation["source"] = NamedBlobFile( data=csv.read(), contentType=u"text/csv", filename=u"test.csv", ) data = { "column_0": "identifier", "column_1": "title", "decimal_import": True, "allow_empty": False, } form._import(data) self.assertEqual(2, len(self.container)) self.assertEqual(["1", "2"], sorted( [e.identifier for e in self.container.values()])) code_1 = self.container.get_by("identifier", "1") self.assertEqual("1", code_1.title) self.assertEqual(1, len(code_1)) self.assertEqual(["10"], [e.identifier for e in code_1.values()]) code_10 = code_1.get_by("identifier", "10") self.assertEqual("10", code_10.title) self.assertEqual(1, len(code_10)) self.assertEqual(["100"], [e.identifier for e in code_10.values()]) code_100 = code_10.get_by("identifier", "100") self.assertEqual("Key 1", code_100.title) self.assertEqual(2, len(code_100)) self.assertEqual( ["100.1", "100.2"], sorted([e.identifier for e in code_100.values()]), ) self.assertEqual( ["Key 1.1", "Key 1.2"], sorted([e.title for e in code_100.values()]), ) code_2 = self.container.get_by("identifier", "2") self.assertEqual("2", code_2.title) self.assertEqual(1, len(code_2)) self.assertEqual(["20"], [e.identifier for e in code_2.values()]) code_20 = code_2.get_by("identifier", "20") self.assertEqual("20", code_20.title) self.assertEqual(1, len(code_20)) self.assertEqual(["200"], [e.identifier for e in code_20.values()]) code_200 = code_20.get_by("identifier", "200") self.assertEqual("Key 2", code_200.title) self.assertEqual(1, len(code_200)) self.assertEqual(["200.1"], [e.identifier for e in code_200.values()]) code_2001 = code_200.get_by("identifier", "200.1") self.assertEqual("Key 2.1", code_2001.title) self.assertEqual(1, len(code_2001)) self.assertEqual(["200.10"], [e.identifier for e in code_2001.values()]) self.assertEqual(["Key 2.10"], [e.title for e in code_2001.values()])
def test_process_data_multilevel(self): """Tests _process_data with multi levels data structure""" form = importform.ImportFormSecondStep(self.container, self.layer["request"]) data = { None: { u"key1": (u"Key 1", {}), u"key2": (u"Key 2", { 'enabled': False }) }, u"key1": { u"key1.1": (u"Key 1.1", {}), u"key1.2": (u"Key 1.2", {}) }, u"key2": { u"key2.1": (u"Key 2.1", { 'enabled': False }) }, u"key1.1": { u"key1.1.1": (u"Key 1.1.1", {}) }, u"key1.1.1": { u"key1.1.1.1": (u"Key 1.1.1.1", {}) }, } expected_results = [ { "identifier": u"key1", "title": u"Key 1", "informations": None, "enabled": None, "_children": [ { "identifier": u"key1.1", "title": u"Key 1.1", "informations": None, "enabled": None, "_children": [ { "identifier": u"key1.1.1", "title": u"Key 1.1.1", "informations": None, "enabled": None, "_children": [ { "identifier": u"key1.1.1.1", "title": u"Key 1.1.1.1", "informations": None, "enabled": None, "_children": [], }, ], }, ], }, { "identifier": u"key1.2", "title": u"Key 1.2", "informations": None, "enabled": None, "_children": [], }, ], }, { "identifier": u"key2", "title": u"Key 2", "informations": None, "enabled": False, "_children": [ { "identifier": u"key2.1", "title": u"Key 2.1", "informations": None, "enabled": False, "_children": [], }, ], }, ] processed_data = form._process_data(data) self.assertEqual(self._sort_processed_data(processed_data), expected_results)