def test_gen_template(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
xl_file = os.path.join(self.tempdir, 'file.xlsx')
with __main__.App(argv=['gen-template', csv_file, xl_file]) as app:
app.run()
csv_schema, _, csv_models = utils.init_schema(csv_file)
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
py_schema = utils.get_schema(py_file)
py_models = list(utils.get_models(py_schema).values())
self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models))
objs = io.WorkbookReader().run(xl_file,
models=csv_models)
self.assertEqual(objs, {
csv_schema.Parent: [],
csv_schema.Child: [],
csv_schema.Quantity: [],
})
csv_file = os.path.join(self.tempdir, 'file-*.xlsx')
with __main__.App(argv=['gen-template', py_file, csv_file]) as app:
app.run()
objs = io.WorkbookReader().run(csv_file, models=py_models,
group_objects_by_model=False)
self.assertEqual(objs, None)
def test_normalize(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
csv_schema, _, csv_models = utils.init_schema(csv_file)
py_schema = utils.get_schema(py_file)
py_models = list(utils.get_models(py_schema).values())
self.assertEqual(set(model.__name__ for model in csv_models), set(model.__name__ for model in py_models))
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = csv_schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=csv_models)
xl_file_2 = os.path.join(self.tempdir, 'file2.xlsx')
with __main__.App(argv=['normalize', csv_file, 'Parent', xl_file_1, xl_file_2]) as app:
app.run()
p_0_b = io.WorkbookReader().run(xl_file_2,
models=csv_models,
ignore_missing_attributes=True)[csv_schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
with self.assertRaises(SystemExit):
with __main__.App(argv=['normalize', csv_file, 'Parent2', xl_file_1, xl_file_2]) as app:
app.run()
def test_convert(self):
csv_file = os.path.join('tests', 'fixtures', 'declarative_schema', 'schema.csv')
py_file = os.path.join(self.tempdir, 'schema.py')
with __main__.App(argv=['init-schema', csv_file, py_file]) as app:
app.run()
schema = utils.get_schema(py_file)
models = list(utils.get_models(schema).values())
xl_file_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(xl_file_1, [p_0], models=models)
csv_file_2 = os.path.join(self.tempdir, 'file2-*.csv')
with __main__.App(argv=['convert', csv_file, xl_file_1, csv_file_2]) as app:
app.run()
p_0_b = io.WorkbookReader().run(csv_file_2,
models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
def test_gen_template(self):
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema', 'schema.csv')
schema, _, models = utils.init_schema(schema_filename)
self.assertEqual(set(models), set(utils.get_models(schema).values()))
client = web_service.app.test_client()
with open(schema_filename, 'rb') as schema_file:
rv = client.post('/api/gen-template/',
data={
'schema': (schema_file,
os.path.basename(schema_filename)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 200)
workbook_filename = os.path.join(self.tempdir, 'file.xlsx')
with open(workbook_filename, 'wb') as file:
file.write(rv.data)
objs = io.WorkbookReader().run(workbook_filename,
models=models,
group_objects_by_model=False)
self.assertEqual(objs, None)
# invalid schema
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema',
'invalid-schema.csv')
with open(schema_filename, 'rb') as schema_file:
rv = client.post('/api/gen-template/',
data={
'schema': (schema_file,
os.path.basename(schema_filename)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 400)
def test_normalize(self):
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema', 'schema.csv')
schema, _, models = utils.init_schema(schema_filename)
self.assertEqual(set(models), set(utils.get_models(schema).values()))
in_workbook_filename = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(in_workbook_filename, [p_0], models=models)
client = web_service.app.test_client()
# to xlsx
with open(schema_filename, 'rb') as schema_file:
with open(in_workbook_filename, 'rb') as in_workbook_file:
rv = client.post('/api/normalize/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'model':
'Parent',
'workbook':
(in_workbook_file,
os.path.basename(in_workbook_filename)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 200)
out_workbook_file = os.path.join(self.tempdir, 'file2.xlsx')
with open(out_workbook_file, 'wb') as file:
file.write(rv.data)
p_0_b = io.WorkbookReader().run(
out_workbook_file, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# to tsv
with open(schema_filename, 'rb') as schema_file:
with open(in_workbook_filename, 'rb') as in_workbook_file:
rv = client.post('/api/normalize/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'model':
'Parent',
'workbook':
(in_workbook_file,
os.path.basename(in_workbook_filename)),
'format':
'tsv',
})
self.assertEqual(rv.status_code, 200)
out_workbook_file = os.path.join(self.tempdir, '*.tsv')
with zipfile.ZipFile(BytesIO(rv.data)) as zip_file:
zip_file.extractall(self.tempdir)
p_0_b = io.WorkbookReader().run(
out_workbook_file, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# invalid workbook
wb = wc_utils.workbook.io.read(in_workbook_filename)
wb['!!Child2'] = wb.pop('!!Child')
wb['!!Child2'][0][0] = wb['!!Child2'][0][0].replace(
"'Child'", "'Child2'")
wc_utils.workbook.io.write(in_workbook_filename, wb)
with open(schema_filename, 'rb') as schema_file:
with open(in_workbook_filename, 'rb') as in_workbook_file:
rv = client.post('/api/normalize/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'model':
'Parent',
'workbook':
(in_workbook_file,
os.path.basename(in_workbook_filename)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 400)
# invalid schema
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema',
'invalid-schema.csv')
with open(schema_filename, 'rb') as schema_file:
with open(in_workbook_filename, 'rb') as in_workbook_file:
rv = client.post('/api/normalize/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'model':
'Parent',
'workbook':
(in_workbook_file,
os.path.basename(in_workbook_filename)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 400)
def test_convert(self):
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema', 'schema.csv')
schema, _, models = utils.init_schema(schema_filename)
self.assertEqual(set(models), set(utils.get_models(schema).values()))
workbook_filename_1 = os.path.join(self.tempdir, 'file1.xlsx')
p_0 = schema.Parent(id='p_0')
p_0.children.create(id='c_0')
p_0.children.create(id='c_1')
io.WorkbookWriter().run(workbook_filename_1, [p_0], models=models)
# XLSX -> XLSX
client = web_service.app.test_client()
with open(schema_filename, 'rb') as schema_file:
with open(workbook_filename_1, 'rb') as workbook_file:
rv = client.post('/api/convert/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'workbook':
(workbook_file,
os.path.basename(workbook_filename_1)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 200)
workbook_filename_2 = os.path.join(self.tempdir, 'file2.xlsx')
with open(workbook_filename_2, 'wb') as file:
file.write(rv.data)
p_0_b = io.WorkbookReader().run(
workbook_filename_2, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_b.is_equal(p_0))
# XLSX -> multi.csv
client = web_service.app.test_client()
with open(schema_filename, 'rb') as schema_file:
with open(workbook_filename_1, 'rb') as workbook_file:
rv = client.post('/api/convert/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'workbook':
(workbook_file,
os.path.basename(workbook_filename_1)),
'format':
'multi.csv',
})
self.assertEqual(rv.status_code, 200)
workbook_filename_3 = os.path.join(self.tempdir, 'file3.csv')
with open(workbook_filename_3, 'wb') as file:
file.write(rv.data)
p_0_c = io.MultiSeparatedValuesReader().run(
workbook_filename_3, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_c.is_equal(p_0))
# XLSX -> JSON
client = web_service.app.test_client()
with open(schema_filename, 'rb') as schema_file:
with open(workbook_filename_1, 'rb') as workbook_file:
rv = client.post('/api/convert/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'workbook':
(workbook_file,
os.path.basename(workbook_filename_1)),
'format':
'json',
})
self.assertEqual(rv.status_code, 200)
workbook_filename_4 = os.path.join(self.tempdir, 'file4.json')
with open(workbook_filename_4, 'wb') as file:
file.write(rv.data)
p_0_d = io.JsonReader().run(
workbook_filename_4, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_d.is_equal(p_0))
# JSON -> YAML
client = web_service.app.test_client()
with open(schema_filename, 'rb') as schema_file:
with open(workbook_filename_4, 'rb') as workbook_file:
rv = client.post('/api/convert/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'workbook':
(workbook_file,
os.path.basename(workbook_filename_4)),
'format':
'yml',
})
self.assertEqual(rv.status_code, 200)
workbook_filename_5 = os.path.join(self.tempdir, 'file5.yml')
with open(workbook_filename_5, 'wb') as file:
file.write(rv.data)
p_0_e = io.JsonReader().run(
workbook_filename_5, models=models,
ignore_missing_attributes=True)[schema.Parent][0]
self.assertTrue(p_0_e.is_equal(p_0))
# invalid schema
schema_filename = os.path.join('tests', 'fixtures',
'declarative_schema',
'invalid-schema.csv')
with open(schema_filename, 'rb') as schema_file:
with open(workbook_filename_1, 'rb') as workbook_file:
rv = client.post('/api/convert/',
data={
'schema':
(schema_file,
os.path.basename(schema_filename)),
'workbook':
(workbook_file,
os.path.basename(workbook_filename_1)),
'format':
'xlsx',
})
self.assertEqual(rv.status_code, 400)
def test_one_to_many(self):
class Parent(core.Model):
id = core.SlugAttribute()
name = core.StringAttribute()
age = core.IntegerAttribute()
class Meta(core.Model.Meta):
table_format = core.TableFormat.cell
grammar_filename = os.path.join(self.dirname, 'grammar.lark')
with open(grammar_filename, 'w') as file:
file.write('''
?start: parent ("; " parent)*
parent: PARENT__ID ": " PARENT__NAME " (" PARENT__AGE ")"
PARENT__ID: /[a-zA-Z0-9_]+/
PARENT__NAME: /[a-zA-Z0-9_\-][a-zA-Z0-9_\- ]*[a-zA-Z0-9_\-]/
PARENT__AGE: /[0-9]+/
''')
class OneToManyParentGrammarAttribute(
obj_tables.grammar.ToManyGrammarAttribute,
core.OneToManyAttribute):
grammar_path = grammar_filename
def serialize(self, values, encoded=None):
serialized_value = []
for parent in values:
serialized_value.append('{}: {} ({})'.format(
parent.id, parent.name, parent.age))
return '; '.join(serialized_value)
class Transformer(obj_tables.grammar.ToManyGrammarTransformer):
@lark.v_args(inline=True)
def parent(self, *args):
kwargs = {}
for arg in args:
cls_name, _, attr_name = arg.type.partition('__')
kwargs[attr_name.lower()] = arg.value
return self.get_or_create_model_obj(Parent, **kwargs)
class Child(core.Model):
id = core.SlugAttribute()
name = core.StringAttribute()
parents = OneToManyParentGrammarAttribute(Parent,
related_name='child')
class Meta(core.Model.Meta):
attribute_order = ('id', 'name', 'parents')
c_1 = Child(id='c_1', name='c 1')
c_2 = Child(id='c_2', name='c 2')
p_11 = Parent(id='p_11', name='p 11', age=11)
p_12 = Parent(id='p_12', name='p 12', age=12)
p_21 = Parent(id='p_21', name='p 21', age=21)
p_22 = Parent(id='p_22', name='p 22', age=22)
c_1.parents = [p_11, p_12]
c_2.parents = [p_21, p_22]
objects = {
Parent: [p_11, p_12, p_21, p_22],
Child: [c_1, c_2],
}
filename = os.path.join(self.dirname, 'test.xlsx')
io.WorkbookWriter().run(filename,
objects[Child],
models=[Child, Parent])
objects_b = io.WorkbookReader().run(filename,
models=[Child, Parent],
group_objects_by_model=True)
parents = sorted(objects[Parent], key=lambda parent: parent.id)
parents_b = sorted(objects_b[Parent], key=lambda parent: parent.id)
for parent, parent_b in zip(parents, parents_b):
self.assertTrue(parent_b.is_equal(parent))
children = sorted(objects[Child], key=lambda child: child.id)
children_b = sorted(objects_b[Child], key=lambda child: child.id)
for child, child_b in zip(children, children_b):
self.assertTrue(child_b.is_equal(child))
# test deserialization
self.assertEqual(Child.parents.deserialize(None, {}), ([], None))
self.assertEqual(Child.parents.deserialize('', {}), ([], None))
# test Transformer.get_or_create_model_obj
class NoPrimary(core.Model):
name = core.StringAttribute()
def serialize(self):
return self.name
Child.parents.Transformer({}).get_or_create_model_obj(NoPrimary,
name='new')
with self.assertRaisesRegex(
ValueError, 'Insufficient information to make new instance'):
Child.parents.Transformer({}).get_or_create_model_obj(NoPrimary)
with self.assertRaisesRegex(
ValueError, 'Insufficient information to make new instance'):
Child.parents.Transformer({}).get_or_create_model_obj(
Child, _serialized_val='c_new')
def test_many_to_many(self):
class Parent(core.Model):
id = core.SlugAttribute()
name = core.StringAttribute()
age = core.IntegerAttribute()
class Meta(core.Model.Meta):
table_format = core.TableFormat.cell
class ManyToManyParentGrammarAttribute(
obj_tables.grammar.ToManyGrammarAttribute,
core.ManyToManyAttribute):
grammar = '''
?start: parent ("; " parent)*
parent: PARENT__ID ": " PARENT__NAME " (" PARENT__AGE ")"
PARENT__ID: /[a-zA-Z0-9_]+/
PARENT__NAME: /[a-zA-Z0-9_\-][a-zA-Z0-9_\- ]*[a-zA-Z0-9_\-]/
PARENT__AGE: /[a-z0-9]+/
'''
def serialize(self, values, encoded=None):
serialized_value = []
for parent in values:
serialized_value.append('{}: {} ({})'.format(
parent.id, parent.name, parent.age))
return '; '.join(serialized_value)
class Child(core.Model):
id = core.SlugAttribute()
name = core.StringAttribute()
parents = ManyToManyParentGrammarAttribute(Parent,
related_name='children')
class Meta(core.Model.Meta):
attribute_order = ('id', 'name', 'parents')
c_1 = Child(id='c_1', name='c 1')
c_2 = Child(id='c_2', name='c 2')
p_1 = Parent(id='p_1', name='p 1', age=1)
p_2 = Parent(id='p_2', name='p 2', age=2)
p_12 = Parent(id='p_12', name='p 12', age=12)
c_1.parents = [p_1, p_12]
c_2.parents = [p_2, p_12]
objects = {
Parent: [p_1, p_2, p_12],
Child: [c_1, c_2],
}
filename = os.path.join(self.dirname, 'test.xlsx')
io.WorkbookWriter().run(filename,
objects[Child],
models=[Child, Parent])
objects_b = io.WorkbookReader().run(filename,
models=[Child, Parent],
group_objects_by_model=True)
parents = sorted(objects[Parent], key=lambda parent: parent.id)
parents_b = sorted(objects_b[Parent], key=lambda parent: parent.id)
for parent, parent_b in zip(parents, parents_b):
self.assertTrue(parent_b.is_equal(parent))
children = sorted(objects[Child], key=lambda child: child.id)
children_b = sorted(objects_b[Child], key=lambda child: child.id)
for child, child_b in zip(children, children_b):
self.assertTrue(child_b.is_equal(child))
# test parsing error
wb = wc_utils.workbook.io.read(filename)
wb['!!Children'][2][2] = 'old_parent: old name (old)'
filename2 = os.path.join(self.dirname, 'test2.xlsx')
wc_utils.workbook.io.write(filename2, wb)
with self.assertRaisesRegex(ValueError, 'Unable to clean'):
objects_b = io.WorkbookReader().run(filename2,
models=[Child, Parent],
group_objects_by_model=True)