def test_process_csv_replace_slash(self): """Test _process_csv with csv data that contains slashes""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ ["Folder 1/a", "1", "", ""], ["Folder 1/a", "1", "Subfolder 1/a.1 // cor", "11"], ["Folder 1/a", "1", "Subfolder 1/a.2", "12"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "folder_title", "column_2": "subfolder_title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, replace_slash=True) expected_result = { None: { "F0001": (u"Folder 1-a", {}), }, "F0001": { "F0001-01": (u"Subfolder 1-a.1 -- cor", {}), "F0001-02": (u"Subfolder 1-a.2", {}), }, } self.assertEqual(expected_result, result)
def test_process_csv_complex_titles(self): """Test _process_csv for folder_title/subfolder_title""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) reader = csv.reader(self._complex_csv, delimiter=";") data = { "column_2": "folder_title", "column_3": "subfolder_title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None) expected_result = { None: { "F0001": (u"Folder 1", {}), "F0002": (u"Folder 2", {}), }, "F0001": { "F0001-01": (u"Subfolder 1.1", {}), "F0001-02": (u"Subfolder 1.2", {}), "F0001-03": (u"Subfolder 1.3", {}), }, "F0002": { "F0002-01": (u"Subfolder 2.1", {}), "F0002-02": (u"Subfolder 2.2", {}), "F0002-03": (u"Subfolder 2.3", {}), }, } self.assertEqual(expected_result, result)
def test_process_csv_archived_values(self): """Test _process_csv with csv data with archived informations""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ ["", "001", "First", "", "F1", "Folder 1", "1"], [ "001", "001.1, 001.2", "first 1", "F1", "F1.1", "Folder 1.1", " " ], [ "001", "001.2", "first 1", "F1", "F1.2", "Folder 1.2", "archived" ], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_1": "classification_categories", "column_3": "parent_identifier", "column_4": "internal_reference_no", "column_5": "title", "column_6": "archived", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None) expected_result = { None: { u"F1": ( u"Folder 1", { "classification_categories": [u"001"], "archived": True }, ) }, u"F1": { u"F1.1": ( u"Folder 1.1", { "classification_categories": [u"001.1", u"001.2"], "archived": False }, ), u"F1.2": ( u"Folder 1.2", { "classification_categories": [u"001.2"], "archived": True }, ), }, } self.assertEqual(expected_result, result)
def test_process_csv_complex_archived_values(self): """Test _process_csv with csv data with archived informations""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ ["Folder 1", "", "Folder 1.1", "1"], ["Folder 1", "", "Folder 1.2", " "], ["Folder 1", "", "Folder 1.3", "archived"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "folder_title", "column_1": "folder_archived", "column_2": "subfolder_title", "column_3": "subfolder_archived", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups='Administrators') expected_result = { None: { "F0001": ( u"Folder 1", { 'treating_groups': 'Administrators' }, ) }, "F0001": { "F0001-01": ( u"Folder 1.1", { "archived": True, 'treating_groups': 'Administrators' }, ), "F0001-02": ( u"Folder 1.2", { 'treating_groups': 'Administrators' }, ), "F0001-03": ( u"Folder 1.3", { "archived": True, 'treating_groups': 'Administrators' }, ), }, } self.assertEqual(expected_result, result)
def test_process_csv_complex_irn(self): """Test _process_csv for internal_reference_no""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ ["Folder 1", "1", "", ""], ["Folder 1", "1", "Subfolder 1.1", "11"], ["Folder 1", "1", "Subfolder 1.2", "12"], ["Folder 2/a", "2", "", ""], ["Folder 2/a", "2", "Subfolder 2/a.1", "21"], ["Folder 2/a", "2", "Subfolder 2/a.2", "22"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "folder_title", "column_1": "folder_internal_reference_no", "column_2": "subfolder_title", "column_3": "subfolder_internal_reference_no", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None, replace_slash=True) expected_result = { None: { "1": (u"Folder 1", { u'internal_reference_no': u'1' }), "2": (u"Folder 2-a", { u'internal_reference_no': u'2' }), }, "1": { "11": (u"Subfolder 1.1", { u'internal_reference_no': u'11' }), "12": (u"Subfolder 1.2", { u'internal_reference_no': u'12' }), }, "2": { "21": (u"Subfolder 2-a.1", { u'internal_reference_no': u'21' }), "22": (u"Subfolder 2-a.2", { u'internal_reference_no': u'22' }), }, } self.assertEqual(expected_result, result)
def test_replace_newline(self): form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) self.assertEquals(form._replace_newline(u' \n '), u' ') self.assertEquals(form._replace_newline(u'\n'), u'') self.assertEquals(form._replace_newline(u'\n A'), u'A') self.assertEquals(form._replace_newline(u'B \n'), u'B') self.assertEquals(form._replace_newline(u' \n\n '), u' ') self.assertEquals(form._replace_newline(u'.\n '), u'. ') self.assertEquals(form._replace_newline(u' \nL'), u' L') self.assertEquals(form._replace_newline(u'x\nL'), u'x L') self.assertEquals(form._replace_newline(u'x\nLe\nf'), u'x Le f') self.assertEquals(form._replace_newline(u' \n.\n '), u' . ')
def test_process_csv_basic(self): """Test _process_csv with basic csv data""" form = importform.ImportFormSecondStep(self.folders, {}) reader = csv.reader(self._csv, delimiter=";") data = { "column_1": "classification_categories", "column_3": "parent_identifier", "column_4": "internal_reference_no", "column_5": "title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups='Administrators') expected_result = { None: { u"F1": (u"Folder 1", { "classification_categories": [u"001"], 'treating_groups': 'Administrators' }), u"F2": (u"Folder 2", { "classification_categories": [u"002"], 'treating_groups': 'Administrators' }), }, u"F1": { u"F1.1": (u"Folder 1.1", { "classification_categories": [u"001.1"], 'treating_groups': 'Administrators' }), u"F1.2": (u"Folder 1.2", { "classification_categories": [u"001.2"], 'treating_groups': 'Administrators' }), }, u"F2": { u"F2.1A": (u"Folder 2.1 A", { "classification_categories": [u"002.1"], 'treating_groups': 'Administrators' }), u"F2.1B": (u"Folder 2.1 B", { "classification_categories": [u"002.1"], 'treating_groups': 'Administrators' }), }, } self.assertEqual(expected_result, result)
def test_process_csv_empty_values(self): """Test _process_csv with csv data that contains lines without folder""" form = importform.ImportFormSecondStep(self.folders, {}) _csv = StringIO() lines = [ ["", "001", "First", "", "F1", "Folder 1"], ["001", "001.1", "first 1", "F1", "F1.1", "Folder 1/1"], ["", "003", "Third", "", "", ""], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_1": "classification_categories", "column_3": "parent_identifier", "column_4": "internal_reference_no", "column_5": "title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None, replace_slash=True) expected_result = { None: { u"F1": (u"Folder 1", { "classification_categories": [u"001"] }) }, u"F1": { u"F1.1": (u"Folder 1-1", { "classification_categories": [u"001.1"] }), }, } self.assertEqual(expected_result, result)
def test_second_step_import_basic(self): """Test importing csv data""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) form.data = [] annotations = IAnnotations(self.folders) annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict() annotation["separator"] = u";" annotation["has_header"] = False annotation["source"] = NamedBlobFile( data=self._csv.read(), contentType=u"text/csv", filename=u"test.csv", ) data = { "treating_groups": None, "column_1": "classification_categories", "column_3": "parent_identifier", "column_4": "internal_reference_no", "column_5": "title", } self.assertEqual(0, len(self.folders)) form.update() form._import(data)
def test_process_csv_complex_treating_groups_title(self): """Test _process_csv with csv data with treating_groups_title column""" groups = [(t.value, t.title) for t in services_in_charge_vocabulary() if t.value not in ('AuthenticatedUsers', 'Site Administrators')] self.assertListEqual(groups, [('Administrators', 'Administrators'), ('Reviewers', 'Reviewers'), ('group_1', 'My new group 1')]) form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ ["001", "Folder 1", "My new group 1", "001.1", "Folder 1.1"], ["001", "Folder 1", "Reviewers", "001.2", "Folder 1.2"], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "folder_categories", "column_1": "folder_title", "column_2": "treating_groups_title", "column_3": "subfolder_categories", "column_4": "subfolder_title", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None) expected_result = { None: { "F0001": ( u"Folder 1", { 'treating_groups_title': u'My new group 1', 'classification_categories': [u'001'] }, ) }, "F0001": { "F0001-01": ( u"Folder 1.1", { 'treating_groups_title': u'My new group 1', 'classification_categories': [u'001.1'] }, ), "F0001-02": ( u"Folder 1.2", { 'treating_groups_title': u'Reviewers', 'classification_categories': [u'001.2'] }, ), }, } self.assertEqual(expected_result, result) # import nodes form.vocabulary = getUtility( IVocabularyFactory, "collective.classification.vocabularies:tree_id_mapping")( self.folders) form.data = [] form._import_node(form._process_data(result)[0]) # now direct_operation is done # data_to_import = json.loads(form.data[0][1])['data'] # folder_dic = data_to_import[0] # self.assertEqual(folder_dic[u'title'], u'Folder 1') # self.assertEqual(folder_dic[u'treating_groups'], u'group_1') # subfolders = folder_dic[u'__children__'] # self.assertEqual(subfolders[0][u'title'], u'Folder 1.1') # self.assertEqual(subfolders[0][u'treating_groups'], u'group_1') # self.assertEqual(subfolders[1][u'title'], u'Folder 1.2') # self.assertEqual(subfolders[1][u'treating_groups'], u'Reviewers') # # treating_groups_title not found # result[None]['F0001'] = (u"Folder 1", {'treating_groups_title': u'Unknown group title', # 'classification_categories': [u'001']}) # self.assertRaises(Invalid, form._import_node, form._process_data(result)[0]) self.assertIn('folder-1', self.folders) self.assertIn('folder-1.1', self.folders['folder-1']) self.assertIn('folder-1.2', self.folders['folder-1'])
def test_process_csv_complex_informations_values(self): """Test _process_csv with csv data with classification_informations""" form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) _csv = StringIO() lines = [ [ "Folder 1", "", "Room A", "Folder 1.1", '"This is more informations.\n"' ], [ "Folder 1", "", "", "Folder 1.2", '"Yet\nAnother line\nAnd a third"' ], ["Folder 1", "", "", "Folder 1.3", ""], ] for line in lines: _csv.write(";".join(line) + "\n") _csv.seek(0) reader = csv.reader(_csv, delimiter=";") data = { "column_0": "folder_title", "column_1": "folder_archived", "column_2": "folder_informations", "column_3": "subfolder_title", "column_4": "subfolder_informations", } mapping = {int(k.replace("column_", "")): v for k, v in data.items()} result = form._process_csv(reader, mapping, "utf-8", {}, treating_groups=None) expected_result = { None: { "F0001": ( u"Folder 1", { 'classification_informations': u'Room A' }, ) }, "F0001": { "F0001-01": ( u"Folder 1.1", { 'classification_informations': u'This is more informations.' }, ), "F0001-02": ( u"Folder 1.2", { 'classification_informations': u'Yet\r\nAnother line\r\nAnd a third' }, ), "F0001-03": ( u"Folder 1.3", {}, ), }, } self.assertEqual(expected_result, result)
def test_replace_newline_by_crlf(self): form = importform.ImportFormSecondStep(self.folders, self.layer["request"]) self.assertEquals(form._replace_newline_by_crlf(u'\n A'), u'\r\nA') self.assertEquals(form._replace_newline_by_crlf(u'B\n'), u'B\r\n') self.assertEquals(form._replace_newline_by_crlf(u'B\r\nA'), u'B\r\nA')
def test_process_data_archived(self): """Tests _process_data with archived informations""" form = importform.ImportFormSecondStep(self.folders, {}) data = { None: { u"F1": ( u"Folder 1", { "classification_categories": [u"001"], "archived": False }, ), }, u"F1": { u"F1.1": ( u"Folder 1.1", { "classification_categories": [u"001.1"], "archived": True }, ), u"F1.2": ( u"Folder 1.2", { "classification_categories": [u"001.2"], "archived": False }, ), }, } expected_results = [ { "internal_reference_no": u"F1", "title": u"Folder 1", "data": { "classification_categories": [u"001"], "archived": False }, "_children": [ { "internal_reference_no": u"F1.1", "title": u"Folder 1.1", "data": { "classification_categories": [u"001.1"], "archived": True, }, "_children": [], }, { "internal_reference_no": u"F1.2", "title": u"Folder 1.2", "data": { "classification_categories": [u"001.2"], "archived": False, }, "_children": [], }, ], }, ] processed_data = form._process_data(data) self.assertEqual(self._sort_processed_data(processed_data), expected_results)
def test_process_data_basic(self): """Tests _process_data with basic data structure""" form = importform.ImportFormSecondStep(self.folders, {}) data = { None: { u"F1": (u"Folder 1", { "classification_categories": [u"001"] }), u"F2": (u"Folder 2", { "classification_categories": [u"002"] }), }, u"F1": { u"F1.1": (u"Folder 1.1", { "classification_categories": [u"001.1"] }), u"F1.2": (u"Folder 1.2", { "classification_categories": [u"001.2"] }), }, u"F2": { u"F2.1A": (u"Folder 2.1 A", { "classification_categories": [u"002.1"] }), u"F2.1B": (u"Folder 2.1 B", { "classification_categories": [u"002.1"] }), }, } expected_results = [ { "internal_reference_no": u"F1", "title": u"Folder 1", "data": { "classification_categories": [u"001"] }, "_children": [ { "internal_reference_no": u"F1.1", "title": u"Folder 1.1", "data": { "classification_categories": [u"001.1"] }, "_children": [], }, { "internal_reference_no": u"F1.2", "title": u"Folder 1.2", "data": { "classification_categories": [u"001.2"] }, "_children": [], }, ], }, { "internal_reference_no": u"F2", "title": u"Folder 2", "data": { "classification_categories": [u"002"] }, "_children": [ { "internal_reference_no": u"F2.1A", "title": u"Folder 2.1 A", "data": { "classification_categories": [u"002.1"] }, "_children": [], }, { "internal_reference_no": u"F2.1B", "title": u"Folder 2.1 B", "data": { "classification_categories": [u"002.1"] }, "_children": [], }, ], }, ] processed_data = form._process_data(data) self.assertEqual(self._sort_processed_data(processed_data), expected_results)