def test_save_osm_data_with_non_existing_media_file(self):
"""
Test that saving osm data with a non existing media file
fails silenty and does not throw an IOError
"""
osm_fixtures_dir = os.path.realpath(os.path.join(
os.path.dirname(__file__), '..', 'fixtures', 'osm'))
# publish form
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_form_to_project(xlsform_path=xlsform_path)
self.xform.save()
# make submission with osm data
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
media_file = open(os.path.join(osm_fixtures_dir,
'OSMWay234134797.osm'))
self._make_submission(submission_path, media_file=media_file)
# save osm data with a non existing file
submission = Instance.objects.first()
attachment = submission.attachments.first()
attachment.media_file = os.path.join(
settings.PROJECT_ROOT, "test_media", "noFile.osm")
attachment.save()
try:
save_osm_data(submission.id)
except IOError:
self.fail("IOError was raised")
def test_save_osm_data_with_non_existing_media_file(self):
"""
Test that saving osm data with a non existing media file
fails silenty and does not throw an IOError
"""
osm_fixtures_dir = os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'osm'))
# publish form
xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx')
self._publish_xls_form_to_project(xlsform_path=xlsform_path)
self.xform.save()
# make submission with osm data
submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml')
media_file = open(os.path.join(osm_fixtures_dir,
'OSMWay234134797.osm'))
self._make_submission(submission_path, media_file=media_file)
# save osm data with a non existing file
submission = Instance.objects.first()
attachment = submission.attachments.first()
attachment.media_file = os.path.join(settings.PROJECT_ROOT,
"test_media", "noFile.osm")
attachment.save()
try:
save_osm_data(submission.id)
except IOError:
self.fail("IOError was raised")
def handle(self, *args, **kwargs):
xforms = XForm.objects.filter(instances_with_osm=True)
# username
if args:
xforms = xforms.filter(user__username=args[0])
for xform in queryset_iterator(xforms):
attachments = Attachment.objects.filter(
extension=Attachment.OSM,
instance__xform=xform).distinct('instance')
count = attachments.count()
c = 0
for a in queryset_iterator(attachments):
pk = a.instance.parsed_instance.pk
save_osm_data(pk)
c += 1
if c % 1000 == 0:
self.stdout.write(
"%s:%s: Processed %s of %s." %
(xform.user.username, xform.id_string, c, count))
self.stdout.write("%s:%s: processed %s of %s submissions." %
(xform.user.username, xform.id_string, c, count))
def handle(self, *args, **kwargs):
xforms = XForm.objects.filter(instances_with_osm=True)
# username
if args:
xforms = xforms.filter(user__username=args[0])
for xform in queryset_iterator(xforms):
attachments = Attachment.objects.filter(
extension=Attachment.OSM,
instance__xform=xform
).distinct('instance')
count = attachments.count()
c = 0
for a in queryset_iterator(attachments):
pk = a.instance.parsed_instance.pk
save_osm_data(pk)
c += 1
if c % 1000 == 0:
self.stdout.write("%s:%s: Processed %s of %s." %
(xform.user.username,
xform.id_string, c, count))
self.stdout.write("%s:%s: processed %s of %s submissions." %
(xform.user.username, xform.id_string, c, count))
def test_save_osm_data_transaction_atomic(self):
"""
Test that an IntegrityError within save_osm_data does not cause
a TransactionManagementError, which arises because of new queries
while inside a transaction.atomic() block
"""
# first publish a form and make a submission with OSM data
self._publish_osm_with_submission()
# make sure we have a submission
submission = Instance.objects.first()
self.assertNotEqual(submission, None)
# mock the save method on OsmData and cause it to raise an
# IntegrityError on its first call only, so that we get into the
# catch inside save_osm_data
with patch('onadata.libs.utils.osm.OsmData.save') as mock:
def _side_effect(*args):
"""
We want to raise an IntegrityError only on the first call
of our mock
"""
def __second_side_effect(*args):
return None
# change the side effect so that the next time we do not
# raise an IntegrityError
mock.side_effect = __second_side_effect
# we need to manually rollback the atomic transaction
# merely raising IntegrityError is not enough
# doing this means that a TransactionManagementError is raised
# if we do new queries inside the transaction.atomic block
transaction.set_rollback(True)
raise IntegrityError
# excplicity use an atomic block
with transaction.atomic():
mock.side_effect = _side_effect
try:
save_osm_data(submission.id)
except TransactionManagementError:
self.fail("TransactionManagementError was raised")
def test_save_osm_data_transaction_atomic(self):
"""
Test that an IntegrityError within save_osm_data does not cause
a TransactionManagementError, which arises because of new queries
while inside a transaction.atomic() block
"""
# first publish a form and make a submission with OSM data
self._publish_osm_with_submission()
# make sure we have a submission
submission = Instance.objects.first()
self.assertNotEqual(submission, None)
# mock the save method on OsmData and cause it to raise an
# IntegrityError on its first call only, so that we get into the
# catch inside save_osm_data
with patch('onadata.libs.utils.osm.OsmData.save') as mock:
def _side_effect(*args):
"""
We want to raise an IntegrityError only on the first call
of our mock
"""
def __second_side_effect(*args):
return None
# change the side effect so that the next time we do not
# raise an IntegrityError
mock.side_effect = __second_side_effect
# we need to manually rollback the atomic transaction
# merely raising IntegrityError is not enough
# doing this means that a TransactionManagementError is raised
# if we do new queries inside the transaction.atomic block
transaction.set_rollback(True)
raise IntegrityError
# excplicity use an atomic block
with transaction.atomic():
mock.side_effect = _side_effect
try:
save_osm_data(submission.id)
except TransactionManagementError:
self.fail("TransactionManagementError was raised")