def test_init_schema(self): client = web_service.app.test_client() schema_filename = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') with open(schema_filename, 'rb') as schema_file: rv = client.post('/api/init-schema/', data={ 'schema': (schema_file, os.path.basename(schema_filename)), }) self.assertEqual(rv.status_code, 200) py_file = os.path.join(self.tempdir, 'schema.py') with open(py_file, 'wb') as file: file.write(rv.data) schema = utils.get_schema(py_file) self.assertEqual(sorted(utils.get_models(schema)), ['Child', 'Parent', 'Quantity']) # invalid schema schema_filename = os.path.join('tests', 'fixtures', 'declarative_schema', 'invalid-schema.csv') with open(schema_filename, 'rb') as schema_file: rv = client.post('/api/init-schema/', data={ 'schema': (schema_file, os.path.basename(schema_filename)), }) self.assertEqual(rv.status_code, 400)
def test_normalize(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() csv_schema, _, csv_models = utils.init_schema(csv_file) py_schema = utils.get_schema(py_file) py_models = list(utils.get_models(py_schema).values()) self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models)) xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx') p_0 = csv_schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') io.WorkbookWriter().run(xl_file_1, [p_0], models=csv_models) xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx') with __main__.App(argv=['normalize', csv_file, 'Parent', xl_file_1, xl_file_2]) as app: app.run() p_0_b = io.WorkbookReader().run(xl_file_2, models=csv_models, ignore_missing_attributes=True)[csv_schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0)) with self.assertRaises(SystemExit): with __main__.App(argv=['normalize', csv_file, 'Parent2', xl_file_1, xl_file_2]) as app: app.run()
def test_gen_template(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') xl_file = os.path.join(self.tempdir, 'file.xlsx') with __main__.App(argv=['gen-template', csv_file, xl_file]) as app: app.run() csv_schema, _, csv_models = utils.init_schema(csv_file) py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() py_schema = utils.get_schema(py_file) py_models = list(utils.get_models(py_schema).values()) self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models)) objs = io.WorkbookReader().run(xl_file, models=csv_models) self.assertEqual(objs, { csv_schema.Parent: [], csv_schema.Child: [], csv_schema.Quantity: [], }) csv_file = os.path.join(self.tempdir, 'file-*.xlsx') with __main__.App(argv=['gen-template', py_file, csv_file]) as app: app.run() objs = io.WorkbookReader().run(csv_file, models=py_models, group_objects_by_model=False) self.assertEqual(objs, None)
def test_init_schema(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() schema = utils.get_schema(py_file) self.assertEqual(sorted(utils.get_models(schema)), ['Child', 'Parent', 'Quantity'])
def test_get_models(self): out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema, _, models = utils.init_schema( 'tests/fixtures/declarative_schema/schema.csv', out_filename=out_filename) self.assertEqual(set(models), set(utils.get_models(schema).values())) self.assertEqual(sorted(utils.get_models(schema).keys()), ['Child', 'Parent', 'Quantity']) schema = utils.get_schema(out_filename) self.assertEqual(sorted(utils.get_models(schema).keys()), ['Child', 'Parent', 'Quantity'])
def test_diff(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() schema = utils.get_schema(py_file) models = list(utils.get_models(schema).values()) xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx') p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0', name='c_0') p_0.children.create(id='c_1', name='c_1') io.WorkbookWriter().run(xl_file_1, [p_0], models=models) xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx') p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0', name='c_0') p_0.children.create(id='c_1', name='c_0') io.WorkbookWriter().run(xl_file_2, [p_0], models=models) xl_file_3 = os.path.join(self.tempdir, 'file3.xlsx') p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0', name='c_0') p_0.children.create(id='c_1', name='c_1') p_0.children.create(id='c_2', name='c_2') io.WorkbookWriter().run(xl_file_3, [p_0], models=models) with __main__.App(argv=['diff', csv_file, 'Parent', xl_file_1, xl_file_1]) as app: app.run() with self.assertRaises(SystemExit): with __main__.App(argv=['diff', csv_file, 'Parent2', xl_file_1, xl_file_1]) as app: app.run() with self.assertRaises(SystemExit): with __main__.App(argv=['diff', csv_file, 'Parent', xl_file_1, xl_file_2]) as app: app.run() with self.assertRaises(SystemExit): with __main__.App(argv=['diff', csv_file, 'Child', xl_file_1, xl_file_3]) as app: app.run() with self.assertRaises(SystemExit): with __main__.App(argv=['diff', csv_file, 'Child', xl_file_3, xl_file_1]) as app: app.run()
def test_init_schema(self): out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema, _, _ = utils.init_schema( 'tests/fixtures/declarative_schema/schema.csv', out_filename=out_filename) p_0 = schema.Parent(id='p_0') c_0 = p_0.children.create(id='c_0') c_1 = p_0.children.create(id='c_1') c_2 = p_0.children.create(id='c_2') c_0._comments = ['A', 'B'] c_1._comments = ['C', 'D'] filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0)) self.assertEqual( p_0_b.children.get_one(id='c_0')._comments, c_0._comments) self.assertEqual( p_0_b.children.get_one(id='c_1')._comments, c_1._comments) self.assertEqual( p_0_b.children.get_one(id='c_2')._comments, c_2._comments) # import module and test schema = utils.get_schema(out_filename) p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0))
def test_convert(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() schema = utils.get_schema(py_file) models = list(utils.get_models(schema).values()) xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx') p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') io.WorkbookWriter().run(xl_file_1, [p_0], models=models) csv_file_2 = os.path.join(self.tempdir, 'file2-*.csv') with __main__.App(argv=['convert', csv_file, xl_file_1, csv_file_2]) as app: app.run() p_0_b = io.WorkbookReader().run(csv_file_2, models=models, ignore_missing_attributes=True)[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0))
def test_init_schema_from_csv_workbook(self): out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema_csv = 'tests/fixtures/declarative_schema/schema*.csv' schema_csv_wb = os.path.join(self.tmp_dirname, 'schema-*.csv') wb = wc_utils.workbook.io.read(schema_csv) wb['!!' + core.SCHEMA_SHEET_NAME] = wb.pop('') wc_utils.workbook.io.write(schema_csv_wb, wb) out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema, _, _ = utils.init_schema(schema_csv_wb, out_filename=out_filename) p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0)) # import module and test schema = utils.get_schema(out_filename) p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0))
def test_validate(self): csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv') py_file = os.path.join(self.tempdir, 'schema.py') with __main__.App(argv=['init-schema', csv_file, py_file]) as app: app.run() schema = utils.get_schema(py_file) models = list(utils.get_models(schema).values()) xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx') p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') io.WorkbookWriter().run(xl_file_1, [p_0], models=models) with __main__.App(argv=['validate', csv_file, xl_file_1]) as app: app.run() xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx') wb = wc_utils.workbook.io.read(xl_file_1) wb['!!Child'][4][0] = 'c_0' wc_utils.workbook.io.write(xl_file_2, wb) with self.assertRaises(SystemExit): with __main__.App(argv=['validate', csv_file, xl_file_2]) as app: app.run()
def test_init_schema_with_inheritance(self): csv_filename = os.path.join(self.tmp_dirname, 'schema.csv') py_filename = os.path.join(self.tmp_dirname, 'schema.py') ws_metadata = [ '!!ObjTables', "type='{}'".format(core.SCHEMA_TABLE_TYPE), "description='Schema'", "objTablesVersion='{}'".format(obj_tables.__version__), ] col_headings = [ '!Name', '!Type', '!Parent', '!Format', '!Description', ] rows = [ '{}\n'.format(','.join(['ClsA1', 'Class', '', 'row', ''])), '{}\n'.format(','.join( ['id_a1', 'Attribute', 'ClsA1', 'String', 'Id-a1'])), '{}\n'.format(','.join(['ClsA2', 'Class', 'ClsA1', 'row', ''])), '{}\n'.format(','.join( ['id_a2', 'Attribute', 'ClsA2', 'String', 'Id-a2'])), '{}\n'.format(','.join(['ClsA30', 'Class', 'ClsA2', 'row', ''])), '{}\n'.format(','.join( ['id_a30', 'Attribute', 'ClsA30', 'String', 'Id-a30'])), '{}\n'.format(','.join(['ClsA31', 'Class', 'ClsA2', 'row', ''])), '{}\n'.format(','.join( ['id_a31', 'Attribute', 'ClsA31', 'String', 'Id-a31'])), '{}\n'.format(','.join(['ClsB1', 'Class', '', 'row', ''])), '{}\n'.format(','.join( ['id_b1', 'Attribute', 'ClsB1', 'String', 'Id-b1'])), '{}\n'.format(','.join(['ClsB2', 'Class', 'ClsB1', 'row', ''])), '{}\n'.format(','.join( ['id_b2', 'Attribute', 'ClsB2', 'String', 'Id-b2'])), '{}\n'.format(','.join(['ClsB30', 'Class', 'ClsB2', 'row', ''])), '{}\n'.format(','.join( ['id_b30', 'Attribute', 'ClsB30', 'String', 'Id-b30'])), '{}\n'.format(','.join(['ClsB31', 'Class', 'ClsB2', 'row', ''])), '{}\n'.format(','.join( ['id_b31', 'Attribute', 'ClsB31', 'String', 'Id-b31'])), ] with open(csv_filename, 'w') as file: file.write('{}\n'.format(' '.join(ws_metadata))) file.write('{}\n'.format(','.join(col_headings))) file.write(''.join(rows)) schema, _, _ = utils.init_schema(csv_filename, out_filename=py_filename) b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3') self.assertEqual(b31.id_b1, 'l1') self.assertEqual(b31.id_b2, 'l2') self.assertEqual(b31.id_b31, 'l3') self.assertIsInstance(b31, schema.ClsB31) self.assertIsInstance(b31, schema.ClsB2) self.assertIsInstance(b31, schema.ClsB1) self.assertIsInstance(b31, obj_tables.Model) self.assertNotIsInstance(b31, schema.ClsA31) self.assertEqual(set(get_subclasses(schema.ClsA1)), set([schema.ClsA2, schema.ClsA30, schema.ClsA31])) self.assertEqual(set(get_subclasses(schema.ClsA2)), set([schema.ClsA30, schema.ClsA31])) self.assertEqual(set(get_subclasses(schema.ClsA30)), set()) # check that Python file generated correctly schema = utils.get_schema(py_filename) b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3') self.assertEqual(b31.id_b1, 'l1') self.assertEqual(b31.id_b2, 'l2') self.assertEqual(b31.id_b31, 'l3') self.assertEqual(set(get_subclasses(schema.ClsA1)), set([schema.ClsA2, schema.ClsA30, schema.ClsA31])) # check that definition works with random order of rows with open(csv_filename, 'w') as file: file.write('{}\n'.format(' '.join(ws_metadata))) file.write('{}\n'.format(','.join(col_headings))) for i_row in numpy.random.permutation(len(rows)): file.write(rows[i_row]) schema, _, _ = utils.init_schema(csv_filename) b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3') self.assertEqual(b31.id_b1, 'l1') self.assertEqual(b31.id_b2, 'l2') self.assertEqual(b31.id_b31, 'l3') self.assertEqual(set(get_subclasses(schema.ClsA1)), set([schema.ClsA2, schema.ClsA30, schema.ClsA31]))
def test_init_schema_from_xlsx(self): out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema_csv = 'tests/fixtures/declarative_schema/schema*.csv' schema_xl = os.path.join(self.tmp_dirname, 'schema.xlsx') wb = wc_utils.workbook.io.read(schema_csv) wb['!!' + core.SCHEMA_SHEET_NAME] = wb.pop('') wc_utils.workbook.io.write(schema_xl, wb) out_filename = os.path.join(self.tmp_dirname, 'schema.py') schema, _, _ = utils.init_schema(schema_xl, out_filename=out_filename) p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0)) # import module and test schema = utils.get_schema(out_filename) p_0 = schema.Parent(id='p_0') p_0.children.create(id='c_0') p_0.children.create(id='c_1') filename = os.path.join(self.tmp_dirname, 'data.xlsx') obj_tables.io.WorkbookWriter().run( filename, [p_0], models=[schema.Parent, schema.Child]) p_0_b = obj_tables.io.WorkbookReader().run( filename, models=[schema.Parent, schema.Child])[schema.Parent][0] self.assertTrue(p_0_b.is_equal(p_0)) # invalid schema schema_xl_2 = os.path.join(self.tmp_dirname, 'schema-invalid.xlsx') wb = wc_utils.workbook.io.read(schema_xl) wb[core.SCHEMA_SHEET_NAME] = wb.pop('!!' + core.SCHEMA_SHEET_NAME) wc_utils.workbook.io.write(schema_xl_2, wb) with self.assertRaisesRegex(ValueError, 'must contain a sheet'): utils.init_schema(schema_xl_2, out_filename=out_filename) wb = wc_utils.workbook.io.read(schema_xl) wb['!!' + core.SCHEMA_SHEET_NAME][0][0] = wb[ '!!' + core.SCHEMA_SHEET_NAME][0][0].replace( "type='{}'".format(core.SCHEMA_TABLE_TYPE), "id='{}'".format('my' + core.SCHEMA_TABLE_TYPE)) wc_utils.workbook.io.write(schema_xl_2, wb) with self.assertRaisesRegex(ValueError, 'type of the schema must be'): utils.init_schema(schema_xl_2, out_filename=out_filename) wb = wc_utils.workbook.io.read(schema_xl) wb['!!' + core.SCHEMA_SHEET_NAME][3][0] = wb[ '!!' + core.SCHEMA_SHEET_NAME][3][0] + '?' wc_utils.workbook.io.write(schema_xl_2, wb) with self.assertRaisesRegex(ValueError, 'names must consist of'): utils.init_schema(schema_xl_2, out_filename=out_filename)