def test_init_schema(self):
client = web_service.app.test_client()
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema', 'schema.csv')
with open(schema_filename, 'rb') as schema_file:
rv = client.post('/api/init-schema/',
data={
'schema': (schema_file,
os.path.basename(schema_filename)),
})
self.assertEqual(rv.status_code, 200)
py_file = os.path.join(self.tempdir, 'schema.py')
with open(py_file, 'wb') as file:
file.write(rv.data)
schema = utils.get_schema(py_file)
self.assertEqual(sorted(utils.get_models(schema)),
['Child', 'Parent', 'Quantity'])
# invalid schema
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema',
'invalid-schema.csv')
with open(schema_filename, 'rb') as schema_file:
rv = client.post('/api/init-schema/',
data={
'schema': (schema_file,
os.path.basename(schema_filename)),
})
self.assertEqual(rv.status_code, 400)
def test_normalize(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
csv_schema, _, csv_models = utils.init_schema(csv_file)
py_schema = utils.get_schema(py_file)
py_models = list(utils.get_models(py_schema).values())
self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models))
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = csv_schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=csv_models)
xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx')
with __main__.App(argv=['normalize', csv_file, 'Parent', xl_file_1, xl_file_2]) as app:
app.run()
p_0_b = io.WorkbookReader().run(xl_file_2,
models=csv_models,
ignore_missing_attributes=True)[csv_schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
with self.assertRaises(SystemExit):
with __main__.App(argv=['normalize', csv_file, 'Parent2', xl_file_1, xl_file_2]) as app:
app.run()
def test_gen_template(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
xl_file = os.path.join(self.tempdir, 'file.xlsx')
with __main__.App(argv=['gen-template', csv_file, xl_file]) as app:
app.run()
csv_schema, _, csv_models = utils.init_schema(csv_file)
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
py_schema = utils.get_schema(py_file)
py_models = list(utils.get_models(py_schema).values())
self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models))
objs = io.WorkbookReader().run(xl_file,
models=csv_models)
self.assertEqual(objs, {
csv_schema.Parent: [],
csv_schema.Child: [],
csv_schema.Quantity: [],
})
csv_file = os.path.join(self.tempdir, 'file-*.xlsx')
with __main__.App(argv=['gen-template', py_file, csv_file]) as app:
app.run()
objs = io.WorkbookReader().run(csv_file, models=py_models,
group_objects_by_model=False)
self.assertEqual(objs, None)
def test_init_schema(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
schema = utils.get_schema(py_file)
self.assertEqual(sorted(utils.get_models(schema)), ['Child', 'Parent', 'Quantity'])
def test_get_models(self):
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema, _, models = utils.init_schema(
'tests/fixtures/declarative_schema/schema.csv',
out_filename=out_filename)
self.assertEqual(set(models), set(utils.get_models(schema).values()))
self.assertEqual(sorted(utils.get_models(schema).keys()),
['Child', 'Parent', 'Quantity'])
schema = utils.get_schema(out_filename)
self.assertEqual(sorted(utils.get_models(schema).keys()),
['Child', 'Parent', 'Quantity'])
def test_diff(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
schema = utils.get_schema(py_file)
models = list(utils.get_models(schema).values())
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0', name='c_0')
p_0.children.create(id='c_1', name='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=models)
xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0', name='c_0')
p_0.children.create(id='c_1', name='c_0')
io.WorkbookWriter().run(xl_file_2, [p_0], models=models)
xl_file_3 = os.path.join(self.tempdir, 'file3.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0', name='c_0')
p_0.children.create(id='c_1', name='c_1')
p_0.children.create(id='c_2', name='c_2')
io.WorkbookWriter().run(xl_file_3, [p_0], models=models)
with __main__.App(argv=['diff', csv_file, 'Parent', xl_file_1, xl_file_1]) as app:
app.run()
with self.assertRaises(SystemExit):
with __main__.App(argv=['diff', csv_file, 'Parent2', xl_file_1, xl_file_1]) as app:
app.run()
with self.assertRaises(SystemExit):
with __main__.App(argv=['diff', csv_file, 'Parent', xl_file_1, xl_file_2]) as app:
app.run()
with self.assertRaises(SystemExit):
with __main__.App(argv=['diff', csv_file, 'Child', xl_file_1, xl_file_3]) as app:
app.run()
with self.assertRaises(SystemExit):
with __main__.App(argv=['diff', csv_file, 'Child', xl_file_3, xl_file_1]) as app:
app.run()
def test_init_schema(self):
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema, _, _ = utils.init_schema(
'tests/fixtures/declarative_schema/schema.csv',
out_filename=out_filename)
p_0 = schema.Parent(id='p_0')
c_0 = p_0.children.create(id='c_0')
c_1 = p_0.children.create(id='c_1')
c_2 = p_0.children.create(id='c_2')
c_0._comments = ['A', 'B']
c_1._comments = ['C', 'D']
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
self.assertEqual(
p_0_b.children.get_one(id='c_0')._comments, c_0._comments)
self.assertEqual(
p_0_b.children.get_one(id='c_1')._comments, c_1._comments)
self.assertEqual(
p_0_b.children.get_one(id='c_2')._comments, c_2._comments)
# import module and test
schema = utils.get_schema(out_filename)
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
def test_convert(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
schema = utils.get_schema(py_file)
models = list(utils.get_models(schema).values())
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=models)
csv_file_2 = os.path.join(self.tempdir, 'file2-*.csv')
with __main__.App(argv=['convert', csv_file, xl_file_1, csv_file_2]) as app:
app.run()
p_0_b = io.WorkbookReader().run(csv_file_2,
models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
def test_init_schema_from_csv_workbook(self):
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema_csv = 'tests/fixtures/declarative_schema/schema*.csv'
schema_csv_wb = os.path.join(self.tmp_dirname, 'schema-*.csv')
wb = wc_utils.workbook.io.read(schema_csv)
wb['!!' + core.SCHEMA_SHEET_NAME] = wb.pop('')
wc_utils.workbook.io.write(schema_csv_wb, wb)
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema, _, _ = utils.init_schema(schema_csv_wb,
out_filename=out_filename)
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# import module and test
schema = utils.get_schema(out_filename)
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
def test_validate(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
schema = utils.get_schema(py_file)
models = list(utils.get_models(schema).values())
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=models)
with __main__.App(argv=['validate', csv_file, xl_file_1]) as app:
app.run()
xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx')
wb = wc_utils.workbook.io.read(xl_file_1)
wb['!!Child'][4][0] = 'c_0'
wc_utils.workbook.io.write(xl_file_2, wb)
with self.assertRaises(SystemExit):
with __main__.App(argv=['validate', csv_file, xl_file_2]) as app:
app.run()
def test_init_schema_with_inheritance(self):
csv_filename = os.path.join(self.tmp_dirname, 'schema.csv')
py_filename = os.path.join(self.tmp_dirname, 'schema.py')
ws_metadata = [
'!!ObjTables',
"type='{}'".format(core.SCHEMA_TABLE_TYPE),
"description='Schema'",
"objTablesVersion='{}'".format(obj_tables.__version__),
]
col_headings = [
'!Name',
'!Type',
'!Parent',
'!Format',
'!Description',
]
rows = [
'{}\n'.format(','.join(['ClsA1', 'Class', '', 'row', ''])),
'{}\n'.format(','.join(
['id_a1', 'Attribute', 'ClsA1', 'String', 'Id-a1'])),
'{}\n'.format(','.join(['ClsA2', 'Class', 'ClsA1', 'row', ''])),
'{}\n'.format(','.join(
['id_a2', 'Attribute', 'ClsA2', 'String', 'Id-a2'])),
'{}\n'.format(','.join(['ClsA30', 'Class', 'ClsA2', 'row', ''])),
'{}\n'.format(','.join(
['id_a30', 'Attribute', 'ClsA30', 'String', 'Id-a30'])),
'{}\n'.format(','.join(['ClsA31', 'Class', 'ClsA2', 'row', ''])),
'{}\n'.format(','.join(
['id_a31', 'Attribute', 'ClsA31', 'String', 'Id-a31'])),
'{}\n'.format(','.join(['ClsB1', 'Class', '', 'row', ''])),
'{}\n'.format(','.join(
['id_b1', 'Attribute', 'ClsB1', 'String', 'Id-b1'])),
'{}\n'.format(','.join(['ClsB2', 'Class', 'ClsB1', 'row', ''])),
'{}\n'.format(','.join(
['id_b2', 'Attribute', 'ClsB2', 'String', 'Id-b2'])),
'{}\n'.format(','.join(['ClsB30', 'Class', 'ClsB2', 'row', ''])),
'{}\n'.format(','.join(
['id_b30', 'Attribute', 'ClsB30', 'String', 'Id-b30'])),
'{}\n'.format(','.join(['ClsB31', 'Class', 'ClsB2', 'row', ''])),
'{}\n'.format(','.join(
['id_b31', 'Attribute', 'ClsB31', 'String', 'Id-b31'])),
]
with open(csv_filename, 'w') as file:
file.write('{}\n'.format(' '.join(ws_metadata)))
file.write('{}\n'.format(','.join(col_headings)))
file.write(''.join(rows))
schema, _, _ = utils.init_schema(csv_filename,
out_filename=py_filename)
b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3')
self.assertEqual(b31.id_b1, 'l1')
self.assertEqual(b31.id_b2, 'l2')
self.assertEqual(b31.id_b31, 'l3')
self.assertIsInstance(b31, schema.ClsB31)
self.assertIsInstance(b31, schema.ClsB2)
self.assertIsInstance(b31, schema.ClsB1)
self.assertIsInstance(b31, obj_tables.Model)
self.assertNotIsInstance(b31, schema.ClsA31)
self.assertEqual(set(get_subclasses(schema.ClsA1)),
set([schema.ClsA2, schema.ClsA30, schema.ClsA31]))
self.assertEqual(set(get_subclasses(schema.ClsA2)),
set([schema.ClsA30, schema.ClsA31]))
self.assertEqual(set(get_subclasses(schema.ClsA30)), set())
# check that Python file generated correctly
schema = utils.get_schema(py_filename)
b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3')
self.assertEqual(b31.id_b1, 'l1')
self.assertEqual(b31.id_b2, 'l2')
self.assertEqual(b31.id_b31, 'l3')
self.assertEqual(set(get_subclasses(schema.ClsA1)),
set([schema.ClsA2, schema.ClsA30, schema.ClsA31]))
# check that definition works with random order of rows
with open(csv_filename, 'w') as file:
file.write('{}\n'.format(' '.join(ws_metadata)))
file.write('{}\n'.format(','.join(col_headings)))
for i_row in numpy.random.permutation(len(rows)):
file.write(rows[i_row])
schema, _, _ = utils.init_schema(csv_filename)
b31 = schema.ClsB31(id_b1='l1', id_b2='l2', id_b31='l3')
self.assertEqual(b31.id_b1, 'l1')
self.assertEqual(b31.id_b2, 'l2')
self.assertEqual(b31.id_b31, 'l3')
self.assertEqual(set(get_subclasses(schema.ClsA1)),
set([schema.ClsA2, schema.ClsA30, schema.ClsA31]))
def test_init_schema_from_xlsx(self):
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema_csv = 'tests/fixtures/declarative_schema/schema*.csv'
schema_xl = os.path.join(self.tmp_dirname, 'schema.xlsx')
wb = wc_utils.workbook.io.read(schema_csv)
wb['!!' + core.SCHEMA_SHEET_NAME] = wb.pop('')
wc_utils.workbook.io.write(schema_xl, wb)
out_filename = os.path.join(self.tmp_dirname, 'schema.py')
schema, _, _ = utils.init_schema(schema_xl, out_filename=out_filename)
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# import module and test
schema = utils.get_schema(out_filename)
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
filename = os.path.join(self.tmp_dirname, 'data.xlsx')
obj_tables.io.WorkbookWriter().run(
filename, [p_0], models=[schema.Parent, schema.Child])
p_0_b = obj_tables.io.WorkbookReader().run(
filename, models=[schema.Parent, schema.Child])[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# invalid schema
schema_xl_2 = os.path.join(self.tmp_dirname, 'schema-invalid.xlsx')
wb = wc_utils.workbook.io.read(schema_xl)
wb[core.SCHEMA_SHEET_NAME] = wb.pop('!!' + core.SCHEMA_SHEET_NAME)
wc_utils.workbook.io.write(schema_xl_2, wb)
with self.assertRaisesRegex(ValueError, 'must contain a sheet'):
utils.init_schema(schema_xl_2, out_filename=out_filename)
wb = wc_utils.workbook.io.read(schema_xl)
wb['!!' + core.SCHEMA_SHEET_NAME][0][0] = wb[
'!!' + core.SCHEMA_SHEET_NAME][0][0].replace(
"type='{}'".format(core.SCHEMA_TABLE_TYPE),
"id='{}'".format('my' + core.SCHEMA_TABLE_TYPE))
wc_utils.workbook.io.write(schema_xl_2, wb)
with self.assertRaisesRegex(ValueError, 'type of the schema must be'):
utils.init_schema(schema_xl_2, out_filename=out_filename)
wb = wc_utils.workbook.io.read(schema_xl)
wb['!!' + core.SCHEMA_SHEET_NAME][3][0] = wb[
'!!' + core.SCHEMA_SHEET_NAME][3][0] + '?'
wc_utils.workbook.io.write(schema_xl_2, wb)
with self.assertRaisesRegex(ValueError, 'names must consist of'):
utils.init_schema(schema_xl_2, out_filename=out_filename)