def test_selectrelated(self):
generator = InstanceGenerator(models.RelatedRelated)
dic = {
'value': 'test',
'key': {
'numero': {
'name': 'hello',
},
'another': {
'record': 'r10',
'last_name': 'Mueller',
},
'etl_create': True,
'etl_update': False
}
}
generator.get_instance(dic)
self.assertEqual(models.Numero.objects.count(), 1)
self.assertEqual(models.AnotherModel.objects.count(), 1)
self.assertEqual(models.TwoRelatedAsUnique.objects.count(), 1)
qs = models.RelatedRelated.objects.all()
self.assertEqual(qs.count(), 1)
item = qs[0]
self.assertEqual(item.key.numero.name, 'hello')
self.assertEqual(item.key.another.last_name, 'Mueller')
self.assertEqual(item.value, 'test')
generator.get_instance(dic)
qs = models.RelatedRelated.objects.all()
self.assertEqual(qs.count(), 2)
item = qs[1]
self.assertEqual(item.key.numero.name, 'hello')
self.assertEqual(item.key.another.last_name, 'Mueller')
self.assertEqual(item.value, 'test')
def test_complex_relationships(self):
dics = [
{'record': 40, 'numero': 'uno'},
{'record': 40, 'numero': 'due'},
{'record': 43, 'numero': 'tres',
'related': [
{'record': '10', 'ilosc': 'jedynka zero'},
{'record': '21', 'ilosc': 'dwadziescia jeden'}
]},
{'record': 43, 'numero': 'quattre',
'related': [
{'record': '10', 'ilosc': 'jedynka zero'},
{'record': '21', 'ilosc': 'jeden'},
{'record': '19', 'ilosc': 'jeden'}
]},
{'record': 43, 'zahl': None, 'numero': 'uno'},
{'record': 43, 'zahl': None, 'numero': None},
{'record': 43, 'numero': 'tres',
'related': [
{'record': '10', 'ilosc': 'jedynka zero'},
{'record': '21', 'ilosc': 'dwadziescia jeden'}
]
}
]
for dic in dics:
generator = InstanceGenerator(
models.HashTestModel, persistence='record')
generator.get_instance(dic)
self.assertEqual(models.Polish.objects.count(), 3)
def test_prepare_integer(self):
generator = InstanceGenerator(models.SomeModel)
tests = [
('1', 1), ('', None), (0, 0), (1, 1), ('bla', None)]
for test in tests:
self.assertEqual(
generator.prepare_integer(None, test[0]), test[1])
def test_onetoone(self):
ins = models.Nombre.objects.create(name='un', id=1)
dos = models.Nombre.objects.create(name='dos', id=2)
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
'numero': 'uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due'},
{'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
{'name': 'troix'}, 'numero': 'tre'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': dos, 'numero': 'cinque'},
{'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
'numero': 'test'},
{'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
{'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
'nombre': '2'},
{'record': '7', 'name': 'test', 'numero': '1'}
]
for dic in dics:
generator = InstanceGenerator(
models.TestOnetoOneModel, persistence='record')
generator.get_instance(dic)
res = models.Nombre.objects.all()
self.assertEqual(res.count(), 5)
res = models.TestOnetoOneModel.objects.all()
self.assertEqual(res.count(), 7)
rec = res.filter(record='1')[0]
self.assertEqual(rec.nombre.name, 'dos')
self.assertEqual(rec.nombre.testonetoonemodel, rec)
def test_prepare_integer(self):
generator = InstanceGenerator(models.SomeModel)
tests = [
('1', 1), ('', None), (0, 0), (1, 1), ('bla', None)]
for test in tests:
self.assertEqual(
generator.prepare_integer(None, test[0]), test[1])
def test_prepare_boolean(self):
generator = InstanceGenerator(SomeModel, {})
tests = [('0', False), ('1', True), ('false', False), ('true', True),
('f', False), ('t', True), (1, True), (0, False)]
for test in tests:
self.assertEqual(generator._prepare_boolean(None, test[0]),
test[1])
def test_through(self):
"""Test M2M with through model."""
dic = {
'record':
'1',
'name':
'John',
'lnames': [{
'record': '1:1',
'last_name': 'Doe',
'attribute': 'generic'
}, {
'record': '1:2',
'last_name': 'Carvello',
'attribute': 'more_fancy'
}]
}
generator = InstanceGenerator(SomeModel, dic, persistence='record')
generator.get_instance()
qs = SomeModel.objects.all()
self.assertEqual(qs[0].record, '1')
self.assertEqual(qs[0].name, 'John')
self.assertEqual(qs[0].lnames.all()[0].last_name, 'Doe')
qs = AnotherModel.objects.all()
self.assertEqual(qs.count(), 2)
self.assertEqual(qs[0].record, '1:1')
self.assertEqual(qs[1].record, '1:2')
qs = IntermediateModel.objects.all()
self.assertEqual(qs[0].attribute, 'generic')
self.assertEqual(qs[1].attribute, 'more_fancy')
def test_rel_creation(self):
dics = [
{'record': '1', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'dziesiec',
'persistence': 'record'},
{'record': '20', 'ilosc': 'dwadziescia',
'persistence': 'record'},
],
'numero': 'uno'},
{'record': '2', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'jedynka zero',
'persistence': 'record'},
{'record': '21', 'ilosc': 'dwadziescia jeden',
'persistence': 'record'},
],
'numero': 'tre'}]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = Polish.objects.all()
self.assertEqual(res.count(), 3)
# tests also field truncation
self.assertEqual(res.filter(record='10')[0].ilosc, 'jedynka ze')
res = TestModel.objects.all()
self.assertEqual(res.count(), 2)
self.assertEqual(len(res.filter(record='1')[0].related.values()), 2)
def test_onetoone(self):
ins = Nombre(name='un')
dos = Nombre(name='dos')
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
'numero': 'uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due'},
{'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
{'name': 'troix'}, 'numero': 'tre'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': dos, 'numero': 'cinque'},
{'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
'numero': 'test'},
{'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
{'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
'nombre': '2'},
{'record': '7', 'name': 'test', 'numero': '1'}
]
for dic in dics:
generator = InstanceGenerator(TestOnetoOneModel, dic, persistence='record')
generator.get_instance()
res = Nombre.objects.all()
self.assertEqual(res.count(), 5)
res = TestOnetoOneModel.objects.all()
self.assertEqual(res.count(), 7)
rec1 = res.filter(record='1')[0]
self.assertEqual(rec1.nombre.name, 'dos')
#OneToOneField has the related object as a property
self.assertEqual(rec1.nombre.testonetoonemodel, rec1)
res.delete()
def test_complex_relationships(self):
dics = [{
'record': 40,
'numero': 'uno'
}, {
'record': 40,
'numero': 'due'
}, {
'record':
43,
'numero':
'tres',
'related': [{
'record': '10',
'ilosc': 'jedynka zero'
}, {
'record': '21',
'ilosc': 'dwadziescia jeden'
}]
}, {
'record':
43,
'numero':
'quattre',
'related': [{
'record': '10',
'ilosc': 'jedynka zero'
}, {
'record': '21',
'ilosc': 'jeden'
}, {
'record': '19',
'ilosc': 'jeden'
}]
}, {
'record': 43,
'zahl': None,
'numero': 'uno'
}, {
'record': 43,
'zahl': None,
'numero': None
}, {
'record':
43,
'numero':
'tres',
'related': [{
'record': '10',
'ilosc': 'jedynka zero'
}, {
'record': '21',
'ilosc': 'dwadziescia jeden'
}]
}]
for dic in dics:
generator = InstanceGenerator(models.HashTestModel,
persistence='record')
generator.get_instance(dic)
self.assertEqual(models.Polish.objects.count(), 3)
def test_fk(self):
ins = Nombre(name='un')
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
'numero': 'uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due'},
{'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
{'name': 'troix'}, 'numero': 'tre'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': 'quatre', 'numero': 'cinque'},
{'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
'numero': 'test'},
{'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
{'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
'nombre': '2'},
{'record': '7', 'name': 'test', 'numero': '1'}
]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = Nombre.objects.all()
self.assertEqual(res.count(), 5)
res = TestModel.objects.all()
self.assertEqual(res.count(), 7)
self.assertEqual(res.filter(record='1')[0].nombre.name, 'quatre')
res.delete()
def load(self):
"""
Loads data into database using Django models and error logging.
"""
print('Opening {0} using {1}'.format(self.filename, self.encoding))
with FileReaderLogManager(self.filename,
logname=self.logfilename,
reader_class=self.reader_class,
reader_kwargs=self.reader_kwargs,
encoding=self.encoding) as reader:
reader.log(
'Data extraction started {0}\n\nStart line: '
'{1}\nEnd line: {2}\n'.format(
datetime.now().strftime(
'%Y-%m-%d'), self.slice_begin, self.slice_end))
counter = FeedbackCounter(
feedbacksize=self.feedbacksize, message=self.message)
while self.slice_begin and self.slice_begin > counter.counter:
reader.next()
counter.increment()
while not self.slice_end or self.slice_end >= counter.counter:
try:
csv_dic = reader.next()
except (UnicodeDecodeError, csv.Error):
reader.log(
'Text decoding or CSV error in line {0} '
'=> rejected'.format(counter.counter))
counter.reject()
continue
except StopIteration:
reader.log('End of file.')
break
transformer = self.transformer_class(csv_dic, self.defaults)
if transformer.is_valid():
dic = transformer.cleaned_data
else:
reader.log(
'Validation error in line {0}: {1} '
'=> rejected'.format(
counter.counter, transformer.error))
counter.reject()
continue
# remove keywords conflicting with Django model
# TODO: I think that is done in several places now
# determine the correct one and get rid of the others
if 'id' in dic:
del dic['id']
generator = InstanceGenerator(
self.model_class, dic,
persistence=self.etl_persistence)
try:
generator.get_instance()
except (ValidationError, IntegrityError, DatabaseError), e:
reader.log('Error in line {0}: {1} => rejected'.format(
counter.counter, str(e)))
counter.reject()
continue
else:
counter.use_result(generator.res)
reader.log(counter.finished())
def test_onetoone(self):
ins = models.Nombre.objects.create(name='un', id=1)
dos = models.Nombre.objects.create(name='dos', id=2)
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
'numero': 'uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due'},
{'record': '3', 'name': 'three', 'zahl': 'drei', 'nombre':
{'name': 'troix'}, 'numero': 'tre'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': dos, 'numero': 'cinque'},
{'record': '4', 'name': 'four', 'zahl': 'vier', 'nombre': 1,
'numero': 'test'},
{'record': '5', 'name': 'six', 'zahl': 'sechs', 'numero': 2},
{'record': '6', 'name': 'six', 'zahl': 'sechs', 'numero': '45',
'nombre': '2'},
{'record': '7', 'name': 'test', 'numero': '1'}
]
for dic in dics:
generator = InstanceGenerator(
models.TestOnetoOneModel, persistence='record')
generator.get_instance(dic)
res = models.Nombre.objects.all()
self.assertEqual(res.count(), 5)
res = models.TestOnetoOneModel.objects.all()
self.assertEqual(res.count(), 7)
rec = res.filter(record='1')[0]
self.assertEqual(rec.nombre.name, 'dos')
self.assertEqual(rec.nombre.testonetoonemodel, rec)
def test_full_instance_generator(self):
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'trash': 'rubish'},
]
generator = InstanceGenerator(
TestModel, dics[0], persistence=['record'])
generator.get_instance()
def test_rel_creation(self):
dics = [
{'record': '1', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'dziesiec',
'persistence': 'record'},
{'record': '20', 'ilosc': 'dwadziescia',
'persistence': 'record'},
],
'numero': 'uno'},
{'record': '2', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'jedynka zero',
'persistence': 'record'},
{'record': '21', 'ilosc': 'dwadziescia jeden',
'persistence': 'record'},
],
'numero': 'tre'}]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = Polish.objects.all()
self.assertEqual(res.count(), 3)
# tests also field truncation
self.assertEqual(res.filter(record='10')[0].ilosc, 'jedynka ze')
res = TestModel.objects.all()
self.assertEqual(res.count(), 2)
self.assertEqual(len(res.filter(record='1')[0].related.values()), 2)
def test_prepare_geometry(self):
from django.contrib.gis.geos import GEOSGeometry
example3d_string = ('MULTIPOLYGON (((2 4 5, 2 6 5, 3 3 3, 2 4 5)),'
'((10 10 3, 11 10 4, 10 11 4, 10 10 3)))')
geom = GEOSGeometry(example3d_string)
generator = InstanceGenerator(models.GeometryModel)
generator.get_instance({
'geom2d': geom,
'geom3d': geom,
'name': 'testcase 1'
})
item = models.GeometryModel.objects.filter(name='testcase 1')[0]
self.assertFalse(item.geom2d.hasz)
self.assertTrue(item.geom3d.hasz)
example2d_string = item.geom2d
geom = GEOSGeometry(example2d_string)
generator = InstanceGenerator(models.GeometryModel)
generator.get_instance({
'geom2d': geom,
'geom3d': geom,
'name': 'testcase 2'
})
item = models.GeometryModel.objects.filter(name='testcase 2')[0]
self.assertFalse(item.geom2d.hasz)
generator.get_instance({
'geom2d': None,
'geom3d': None,
'name': 'emptytest'
})
item = models.GeometryModel.objects.filter(name='emptytest')[0]
self.assertFalse(item.geom2d)
self.assertFalse(item.geom3d)
def test_prepare_boolean(self):
generator = InstanceGenerator(models.SomeModel)
tests = [
('0', False), ('1', True), ('false', False), ('true', True),
('f', False), ('t', True), (1, True), (0, False)]
for test in tests:
self.assertEqual(
generator.prepare_boolean(None, test[0]), test[1])
def test_instance_generation(self):
generator = InstanceGenerator(models.TestModelWoFk)
res = generator.get_instance({
'record': 1, 'name': 'test', 'zahl': '1'})
self.assertIsInstance(res, models.TestModelWoFk)
self.assertEqual(res.record, '1')
self.assertEqual(res.name, 'test')
self.assertEqual(res.zahl, '1')
def test_instance_generation(self):
generator = InstanceGenerator(models.TestModelWoFk)
res = generator.get_instance({
'record': 1, 'name': 'test', 'zahl': '1'})
self.assertIsInstance(res, models.TestModelWoFk)
self.assertEqual(res.record, '1')
self.assertEqual(res.name, 'test')
self.assertEqual(res.zahl, '1')
def test_prepare(self):
"""Testing whether result gets properly added to dic."""
generator = InstanceGenerator(models.WellDefinedModel)
res, bk_ref = generator.prepare({
'something': 'thing',
'somenumber': 0
})
self.assertEqual(res['somenumber'], 0)
self.assertEqual(bk_ref, {})
def test_complex_model(self):
models.TestModel.objects.all().delete()
generator = InstanceGenerator(models.TestModel)
generator.get_instance({
'record': 1,
'name': 'eins',
'zahl': 'uno',
'numero': 'test'
})
def test_md5(self):
dics = [{
'record': 43,
'numero': 'due'
}, {
'record': 43,
'numero': 'due'
}, {
'record': 43,
'numero': 'tres'
}]
generator = InstanceGenerator(HashTestModel,
dics[0],
persistence='record')
instance = generator.get_instance()
hashvalue1 = instance.md5
generator = InstanceGenerator(HashTestModel,
dics[1],
persistence='record')
instance = generator.get_instance()
self.assertEqual(hashvalue1, instance.md5)
generator = InstanceGenerator(HashTestModel,
dics[2],
persistence='record')
instance = generator.get_instance()
self.assertNotEqual(hashvalue1, instance.md5)
res = HashTestModel.objects.filter(record=43)
self.assertNotEqual(hashvalue1, res[0].md5)
def test_prepare_geometry(self):
from django.contrib.gis.geos import GEOSGeometry
example3d_string = (
'MULTIPOLYGON (((2 4 5, 2 6 5, 3 3 3, 2 4 5)),'
'((10 10 3, 11 10 4, 10 11 4, 10 10 3)))')
geom = GEOSGeometry(example3d_string)
generator = InstanceGenerator(
GeometryModel, {'geom2d': geom, 'geom3d': geom,
'name': 'testcase 1'})
generator.get_instance()
item = GeometryModel.objects.filter(name='testcase 1')[0]
self.assertFalse(item.geom2d.hasz)
self.assertTrue(item.geom3d.hasz)
example2d_string = item.geom2d
geom = GEOSGeometry(example2d_string)
generator = InstanceGenerator(
GeometryModel, {'geom2d': geom, 'geom3d': geom,
'name': 'testcase 2'})
generator.get_instance()
item = GeometryModel.objects.filter(name='testcase 2')[0]
self.assertFalse(item.geom2d.hasz)
generator = InstanceGenerator(
GeometryModel, {'geom2d': None, 'geom3d': None,
'name': 'emptytest'})
generator.get_instance()
item = GeometryModel.objects.filter(name='emptytest')[0]
self.assertFalse(item.geom2d)
self.assertFalse(item.geom3d)
def load(self):
"""Loads data into database using Django models and error logging."""
print('Opening {0} using {1}'.format(self.filename, self.encoding))
with FileReaderLogManager(self.filename,
logname=self.logfilename,
reader_class=self.reader_class,
encoding=self.encoding) as reader:
reader.log('Data extraction started {0}\n\nStart line: '
'{1}\nEnd line: {2}\n'.format(
datetime.now().strftime('%Y-%m-%d'),
self.slice_begin, self.slice_end))
counter = FeedbackCounter(feedbacksize=self.feedbacksize,
message=self.message)
while self.slice_begin and self.slice_begin > counter.counter:
reader.next()
counter.increment()
while not self.slice_end or self.slice_end >= counter.counter:
try:
csv_dic = reader.next()
except (UnicodeDecodeError, csv.Error):
reader.log('Text decoding or CSV error in line {0} '
'=> rejected'.format(counter.counter))
counter.reject()
continue
except StopIteration:
reader.log('End of file.')
break
transformer = self.transformer_class(csv_dic, self.defaults)
if transformer.is_valid():
dic = transformer.cleaned_data
else:
reader.log('Validation error in line {0}: {1} '
'=> rejected'.format(counter.counter,
transformer.error))
counter.reject()
continue
# remove keywords conflicting with Django model
# TODO: I think that is done in several places now
# determine the correct one and get rid of the others
if 'id' in dic:
del dic['id']
generator = InstanceGenerator(self.model_class,
dic,
persistence=self.etl_persistence)
try:
generator.get_instance()
except (ValidationError, IntegrityError, DatabaseError), e:
reader.log('Error in line {0}: {1} => rejected'.format(
counter.counter, str(e)))
counter.reject()
continue
else:
counter.use_result(generator.res)
reader.log(counter.finished())
def test_fk_with_rel_field_specified(self):
generator = InstanceGenerator(
models.TestModel, persistence=['record'])
instance = generator.get_instance({
'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': 'un',
'numero': 'uno', 'elnumero': 'el uno'})
self.assertEqual(generator.res, 'created')
self.assertEqual(instance.elnumero.rec, 'el uno')
instance = generator.get_instance({
'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due', 'elnumero': 'el dos'})
self.assertEqual(generator.res, 'created')
def test_fk_with_rel_field_specified(self):
generator = InstanceGenerator(
models.TestModel, persistence=['record'])
instance = generator.get_instance({
'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': 'un',
'numero': 'uno', 'elnumero': 'el uno'})
self.assertEqual(generator.res, 'created')
self.assertEqual(instance.elnumero.rec, 'el uno')
instance = generator.get_instance({
'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due', 'elnumero': 'el dos'})
self.assertEqual(generator.res, 'created')
def test_fk_with_rel_field_specified(self):
dics = [
{
'record': '1',
'name': 'one',
'zahl': 'eins',
'nombre': 'un',
'numero': 'uno',
'elnumero': 'el uno'
},
{
'record': '2',
'name': 'two',
'zahl': 'zwei',
'nombre': 'deux',
'numero': 'due',
'elnumero': 'el dos'
},
{
'record': '3',
'name': 'three',
'zahl': 'drei',
'nombre': {
'name': 'troix'
},
'numero': 'tre',
'elnumero': 'el tres'
},
{
'record': '1',
'name': 'one',
'zahl': 'vier',
'nombre': 1,
'numero': 'quattro',
'elnumero': 'el tres'
},
{
'record': '1',
'name': 'one again',
'zahl': 'fuenf',
'nombre': 'quatre',
'numero': 'cinque'
},
]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = TestModel.objects.all()
self.assertEqual(res.count(), 3)
self.assertEqual(res[0].elnumero_id, 'el tres')
res = ElNumero.objects.all()
self.assertEqual(res.count(), 3)
def test_fk_rejection(self):
generator = InstanceGenerator(models.TestModel)
generator.get_instance({
'record': '1', 'name': 'one', 'zahl': 'eins',
'nombre': {
'name': 'un', 'etl_create': False},
'numero': 'quattre'})
self.assertEqual(generator.res, 'created')
with self.assertRaises((ValueError, IntegrityError)):
generator.get_instance({
'record': '2', 'name': 'two', 'zahl': 'eins',
'numero': {
'name': 'uno', 'etl_create': False}})
def test_update(self):
"""
Test record update.
"""
dic = {'record': '100', 'numero': 'cento', 'zahl': 'hundert'}
generator = InstanceGenerator(HashTestModel, dic, persistence='record')
res = generator.get_instance()
self.assertEqual(res.numero.name, 'cento')
self.assertTrue(generator.res['created'])
self.assertFalse(generator.res['updated'])
generator.get_instance()
self.assertFalse(generator.res['created'])
self.assertFalse(generator.res['updated'])
self.assertTrue(generator.res['exists'])
dic = {'record': '100', 'numero': 'hundert', 'zahl': 'hundert'}
generator = InstanceGenerator(HashTestModel, dic, persistence='record')
res = generator.get_instance()
self.assertTrue(generator.res['updated'])
self.assertEqual(res.numero.name, 'hundert')
dic = {'record': '101', 'name': 'test', 'zahl': 'vier'}
generator = InstanceGenerator(
TestModelWoFk, dic, persistence='record')
res = generator.get_instance()
res = generator.get_instance()
self.assertTrue(generator.res['updated'])
def test_fk_model(self):
generator = InstanceGenerator(models.SimpleFkModel)
res = generator.get_instance({
'fk': {
'name': 'test'
},
'name': 'britta'
})
self.assertIsInstance(res, models.SimpleFkModel)
self.assertIsInstance(res.fk, models.Nombre)
self.assertEqual(res.fk.name, 'test')
res = generator.get_instance({'fk': {'name': 'test'}, 'name': 'ulf'})
res = generator.get_instance({'fk': 1, 'name': 'ursula'})
self.assertEqual(res.fk.name, 'test')
def test_fk_model(self):
generator = InstanceGenerator(models.SimpleFkModel)
res = generator.get_instance({
'fk': {
'name': 'test'},
'name': 'britta'})
self.assertIsInstance(res, models.SimpleFkModel)
self.assertIsInstance(res.fk, models.Nombre)
self.assertEqual(res.fk.name, 'test')
res = generator.get_instance({
'fk': {
'name': 'test'},
'name': 'ulf'})
res = generator.get_instance({'fk': 1, 'name': 'ursula'})
self.assertEqual(res.fk.name, 'test')
def test_rel_creation(self):
dics = [
{
'record':
'1',
'name':
'one',
'related': [
{
'record': '10',
'ilosc': 'dziesiec',
'persistence': 'record'
},
{
'record': '20',
'ilosc': 'dwadziescia',
'persistence': 'record'
},
],
'numero':
'uno'
},
{
'record':
'2',
'name':
'one',
'related': [
{
'record': '10',
'ilosc': 'jedynka zero',
'persistence': 'record'
},
{
'record': '21',
'ilosc': 'dwadziescia jeden',
'persistence': 'record'
},
],
'numero':
'tre'
},
]
for dic in dics:
generator = InstanceGenerator(models.TestModel)
generator.get_instance(dic)
res = models.Polish.objects.all()
self.assertEqual(res.count(), 3)
def test_complex_dict(self):
generator = InstanceGenerator(models.ParentModel)
for item in range(0, 2):
instance = generator.get_instance(
{'well_defined': {
'something': 'donkey',
'somenumber': 1
}})
self.assertEqual(instance.well_defined.somenumber, 1)
self.assertEqual(instance.well_defined.something, 'donkey')
qs = models.WellDefinedModel.objects.all()
self.assertEqual(qs.count(), 1)
generator.get_instance(
{'well_defined': {
'something': 'donkey',
'somenumber': 2
}})
qs = models.WellDefinedModel.objects.all()
self.assertEqual(qs.count(), 2)
with self.assertRaises((ValueError, IntegrityError)):
generator.get_instance({
'well_defined': {
'something': 'horse',
'somenumber': 2,
'etl_create': False
}
})
def test_results(self):
dic = {'record': 40, 'numero': 'uno'}
generator = InstanceGenerator(models.HashTestModel,
persistence='record')
generator.get_instance(dic)
self.assertEqual(generator.res, 'created')
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
dic['numero'] = 'due'
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
def test_prepare_date(self):
generator = InstanceGenerator(models.DateTimeModel)
generator.get_instance({
'datetimenotnull': '2014-10-14', 'datetimenull': '2014-10-14'})
self.assertTrue(generator.res, 'created')
with self.assertRaises(ValidationError):
generator.get_instance({
'datetimenotnull': '', 'datetimenull': '2014-10-14'})
generator.get_instance({
'datetimenotnull': '2014-10-14', 'datetimenull': ''})
self.assertEqual(generator.res, 'created')
def test_related_update(self):
"""
Test update of related records if parent record is
unchanged.
"""
dic = {
'record':
'1',
'name':
'John',
'lnames': [{
'record': '1:1',
'last_name': 'Doe',
'attribute': 'generic'
}, {
'record': '1:2',
'last_name': 'Carvello',
'attribute': 'more_fancy'
}]
}
InstanceGenerator(models.SomeModel,
persistence='record').get_instance(dic)
dic = {
'record':
'1',
'name':
'John',
'lnames': [{
'record': '1:1',
'last_name': 'Deer',
'attribute': 'generic'
}, {
'record': '1:2',
'last_name': 'Carvello',
'attribute': 'more_fancy'
}]
}
InstanceGenerator(models.SomeModel,
persistence='record').get_instance(dic)
qs = models.SomeModel.objects.all()
self.assertEqual(qs[0].lnames.all()[0].last_name, 'Deer')
def test_through(self):
"""Test M2M with through model."""
dic = {'record': '1', 'name': 'John', 'lnames': [
{'record': '1:1', 'last_name': 'Doe',
'attribute': 'generic'},
{'record': '1:2', 'last_name': 'Carvello',
'attribute': 'more_fancy'}
]}
generator = InstanceGenerator(SomeModel, dic, persistence='record')
generator.get_instance()
qs = SomeModel.objects.all()
self.assertEqual(qs[0].record, '1')
self.assertEqual(qs[0].name, 'John')
self.assertEqual(qs[0].lnames.all()[0].last_name, 'Doe')
qs = AnotherModel.objects.all()
self.assertEqual(qs.count(), 2)
self.assertEqual(qs[0].record, '1:1')
self.assertEqual(qs[1].record, '1:2')
qs = IntermediateModel.objects.all()
self.assertEqual(qs[0].attribute, 'generic')
self.assertEqual(qs[1].attribute, 'more_fancy')
def test_rel_creation(self):
dics = [
{'record': '1', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'dziesiec',
'persistence': 'record'},
{'record': '20', 'ilosc': 'dwadziescia',
'persistence': 'record'},
],
'numero': 'uno'},
{'record': '2', 'name': 'one', 'related': [
{'record': '10', 'ilosc': 'jedynka zero',
'persistence': 'record'},
{'record': '21', 'ilosc': 'dwadziescia jeden',
'persistence': 'record'},
],
'numero': 'tre'}]
for dic in dics:
generator = InstanceGenerator(models.TestModel)
generator.get_instance(dic)
res = models.Polish.objects.all()
self.assertEqual(res.count(), 3)
def test_fk_rejection(self):
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins',
'nombre': {'name': 'un', 'etl_create': False},
'numero': 'quattre'},
{'record': '2', 'name': 'two', 'zahl': 'eins',
'numero': {'name': 'uno', 'etl_create': False}}
]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
result = generator.res
# nombre example with fk that allows None
# numero example with fk that does not allow None
self.assertEqual(result['rejected'], True)
self.assertEqual(result['created'], False)
self.assertEqual(result['updated'], False)
self.assertEqual(Nombre.objects.all().count(), 0)
res = TestModel.objects.all()
self.assertEqual(res.count(), 1)
self.assertEqual(res[0].nombre, None)
def test_selectrelated(self):
generator = InstanceGenerator(models.RelatedRelated)
dic = {
'value': 'test',
'key': {
'numero': {
'name': 'hello',
},
'another': {
'record': 'r10',
'last_name': 'Mueller',
},
'etl_create': True,
'etl_update': False
}
}
generator.get_instance(dic)
self.assertEqual(models.Numero.objects.count(), 1)
self.assertEqual(models.AnotherModel.objects.count(), 1)
self.assertEqual(models.TwoRelatedAsUnique.objects.count(), 1)
qs = models.RelatedRelated.objects.all()
self.assertEqual(qs.count(), 1)
item = qs[0]
self.assertEqual(item.key.numero.name, 'hello')
self.assertEqual(item.key.another.last_name, 'Mueller')
self.assertEqual(item.value, 'test')
generator.get_instance(dic)
qs = models.RelatedRelated.objects.all()
self.assertEqual(qs.count(), 2)
item = qs[1]
self.assertEqual(item.key.numero.name, 'hello')
self.assertEqual(item.key.another.last_name, 'Mueller')
self.assertEqual(item.value, 'test')
def test_results(self):
dic = {'record': 40, 'numero': 'uno'}
generator = InstanceGenerator(
models.HashTestModel, persistence='record')
generator.get_instance(dic)
self.assertEqual(generator.res, 'created')
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
dic['numero'] = 'due'
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
def test_prepare_date(self):
generator = InstanceGenerator(models.DateTimeModel)
generator.get_instance({
'datetimenotnull': '2014-10-14', 'datetimenull': '2014-10-14'})
self.assertTrue(generator.res, 'created')
with self.assertRaises(ValidationError):
generator.get_instance({
'datetimenotnull': '', 'datetimenull': '2014-10-14'})
generator.get_instance({
'datetimenotnull': '2014-10-14', 'datetimenull': ''})
self.assertEqual(generator.res, 'created')
def test_fk_with_rel_field_specified(self):
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': 'un',
'numero': 'uno', 'elnumero': 'el uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due', 'elnumero': 'el dos'},
{'record': '3', 'name': 'three', 'zahl': 'drei',
'nombre': {'name': 'troix'}, 'numero': 'tre',
'elnumero': 'el tres'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro', 'elnumero': 'el tres'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': 'quatre', 'numero': 'cinque'},
]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = TestModel.objects.all()
self.assertEqual(res.count(), 3)
self.assertEqual(res[0].elnumero_id, 'el tres')
res = ElNumero.objects.all()
self.assertEqual(res.count(), 3)
def test_fk(self):
ins = Nombre(name='un')
dics = [
{'record': '1', 'name': 'one', 'zahl': 'eins', 'nombre': ins,
'numero': 'uno'},
{'record': '2', 'name': 'two', 'zahl': 'zwei', 'nombre': 'deux',
'numero': 'due'},
{'record': '3', 'name': 'three', 'zahl': 'drei',
'nombre': {'name': 'troix'}, 'numero': 'tre'},
{'record': '1', 'name': 'one', 'zahl': 'vier', 'nombre': 1,
'numero': 'quattro'},
{'record': '1', 'name': 'one again', 'zahl': 'fuenf',
'nombre': 'quatre', 'numero': 'cinque'}
]
for dic in dics:
generator = InstanceGenerator(TestModel, dic, persistence='record')
generator.get_instance()
res = Nombre.objects.all()
self.assertEqual(res.count(), 4)
res = TestModel.objects.all()
self.assertEqual(res.count(), 3)
self.assertEqual(res.filter(record='1')[0].nombre.name, 'quatre')
res.delete()
def test_fk_rejection(self):
generator = InstanceGenerator(models.TestModel)
generator.get_instance({
'record': '1', 'name': 'one', 'zahl': 'eins',
'nombre': {
'name': 'un', 'etl_create': False},
'numero': 'quattre'})
self.assertEqual(generator.res, 'created')
with self.assertRaises((ValueError, IntegrityError)):
generator.get_instance({
'record': '2', 'name': 'two', 'zahl': 'eins',
'numero': {
'name': 'uno', 'etl_create': False}})
def test_signals(self):
"""Test whether auto_now=True fields get updated as well."""
dic = {'record': '1110'}
generator = InstanceGenerator(
TestModelWoFk, dic, persistence='record')
generator.get_instance()
date_1 = TestModelWoFk.objects.filter(record='1110')[0].date
dic = {'record': '1110', 'name': 'test'}
generator = InstanceGenerator(
TestModelWoFk, dic, persistence='record')
generator.get_instance()
qs = TestModelWoFk.objects.filter(record='1110')
self.assertLess(date_1, qs[0].date)
def test_prepare_date(self):
generator = InstanceGenerator(
DateTimeModel,
{'datetimenotnull': '2014-10-14',
'datetimenull': '2014-10-14'})
generator.get_instance()
self.assertTrue(generator.res['created'])
generator = InstanceGenerator(
DateTimeModel,
{'datetimenotnull': '',
'datetimenull': '2014-10-14'})
self.assertRaises(ValidationError, generator.get_instance)
generator = InstanceGenerator(
DateTimeModel,
{'datetimenotnull': '2014-10-14',
'datetimenull': ''})
generator.get_instance()
self.assertTrue(generator.res['created'])
def test_update(self):
"""
Test record update.
"""
dic = {'record': '100', 'numero': 'cento', 'zahl': 'hundert'}
generator = InstanceGenerator(models.HashTestModel,
persistence='record')
res = generator.get_instance(dic)
self.assertEqual(res.numero.name, 'cento')
self.assertEqual(generator.res, 'created')
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
dic = {'record': '100', 'numero': 'hundert', 'zahl': 'hundert'}
res = generator.get_instance(dic)
self.assertTrue(generator.res, 'updated')
self.assertEqual(res.numero.name, 'hundert')
dic = {'record': '101', 'name': 'test', 'zahl': 'vier'}
res = generator.get_instance(dic)
self.assertTrue(generator.res, 'updated')
def test_md5(self):
dics = [{'record': 43, 'numero': 'due'},
{'record': 43, 'numero': 'due'},
{'record': 43, 'numero': 'tres'}]
generator = InstanceGenerator(
HashTestModel, dics[0], persistence='record')
instance = generator.get_instance()
hashvalue1 = instance.md5
generator = InstanceGenerator(
HashTestModel, dics[1], persistence='record')
instance = generator.get_instance()
self.assertEqual(hashvalue1, instance.md5)
generator = InstanceGenerator(
HashTestModel, dics[2], persistence='record')
instance = generator.get_instance()
self.assertNotEqual(hashvalue1, instance.md5)
res = HashTestModel.objects.filter(record=43)
self.assertNotEqual(hashvalue1, res[0].md5)
def test_update(self):
"""
Test record update.
"""
dic = {'record': '100', 'numero': 'cento', 'zahl': 'hundert'}
generator = InstanceGenerator(
models.HashTestModel, persistence='record')
res = generator.get_instance(dic)
self.assertEqual(res.numero.name, 'cento')
self.assertEqual(generator.res, 'created')
generator.get_instance(dic)
self.assertEqual(generator.res, 'updated')
dic = {'record': '100', 'numero': 'hundert', 'zahl': 'hundert'}
res = generator.get_instance(dic)
self.assertTrue(generator.res, 'updated')
self.assertEqual(res.numero.name, 'hundert')
dic = {'record': '101', 'name': 'test', 'zahl': 'vier'}
res = generator.get_instance(dic)
self.assertTrue(generator.res, 'updated')
def test_complex_dict(self):
generator = InstanceGenerator(models.ParentModel)
for item in range(0, 2):
instance = generator.get_instance({
'well_defined': {
'something': 'donkey', 'somenumber': 1}})
self.assertEqual(instance.well_defined.somenumber, 1)
self.assertEqual(instance.well_defined.something, 'donkey')
qs = models.WellDefinedModel.objects.all()
self.assertEqual(qs.count(), 1)
generator.get_instance({
'well_defined': {
'something': 'donkey', 'somenumber': 2}})
qs = models.WellDefinedModel.objects.all()
self.assertEqual(qs.count(), 2)
with self.assertRaises((ValueError, IntegrityError)):
generator.get_instance({
'well_defined': {
'something': 'horse', 'somenumber': 2,
'etl_create': False}})
def test_related_update(self):
"""
Test update of related records if parent record is
unchanged.
"""
dic = {'record': '1', 'name': 'John', 'lnames': [
{'record': '1:1', 'last_name': 'Doe',
'attribute': 'generic'},
{'record': '1:2', 'last_name': 'Carvello',
'attribute': 'more_fancy'}
]}
generator = InstanceGenerator(SomeModel, dic, persistence='record')
generator.get_instance()
dic = {'record': '1', 'name': 'John', 'lnames': [
{'record': '1:1', 'last_name': 'Deer',
'attribute': 'generic'},
{'record': '1:2', 'last_name': 'Carvello',
'attribute': 'more_fancy'}
]}
generator = InstanceGenerator(SomeModel, dic, persistence='record')
generator.get_instance()
qs = SomeModel.objects.all()
self.assertEqual(qs[0].lnames.all()[0].last_name, 'Deer')
def test_complex_model(self):
models.TestModel.objects.all().delete()
generator = InstanceGenerator(models.TestModel)
generator.get_instance({
'record': 1, 'name': 'eins', 'zahl': 'uno', 'numero': 'test'})
def test_prepare(self):
"""Testing whether result gets properly added to dic."""
generator = InstanceGenerator(models.WellDefinedModel)
res = generator.prepare({'something': 'thing', 'somenumber': 0})
self.assertEqual(res['somenumber'], 0)
def test_prepare_string(self):
generator = InstanceGenerator(models.TestModel)
res = generator.prepare_text(CharField(max_length=4), 'test')
self.assertEqual(res, 'test')
res = generator.prepare_text(CharField(max_length=3), 'test')
self.assertEqual(res, 'tes')
def test_rejection_by_db_integrity(self):
dic = {'record': '30', 'date': '2014-01-01'}
generator = InstanceGenerator(models.TestModel)
with self.assertRaises(IntegrityError):
generator.get_instance(dic)
def test_rejection_by_missing_fk(self):
dic = {'record': '30', 'date': '2014-01-01', 'numero': 2}
generator = InstanceGenerator(models.TestModel)
with self.assertRaises(ValueError):
generator.get_instance(dic)
def test_rejection_by_field_validation(self):
dic = {'record': '30', 'date': '3333', 'numero': 'uno'}
generator = InstanceGenerator(models.TestModel)
with self.assertRaises(ValidationError):
generator.get_instance(dic)