def test_exports_outdated_only_considers_successful_exports(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create a bad export
export = Export.objects.create(
xform=self.xform, export_type=Export.XLS_EXPORT,
internal_status=Export.FAILED)
self.assertTrue(
Export.exports_outdated(self.xform, export.export_type))
def generate_export(export_type, extension, username, id_string,
export_id = None, filter_query=None,
group_delimiter='/',
split_select_multiples=True):
"""
Create appropriate export object given the export type
"""
from odk_viewer.models import Export
xform = XForm.objects.get(user__username=username, id_string=id_string)
df_builder = _df_builder_for_export_type(
export_type, username, id_string, group_delimiter,
split_select_multiples, filter_query)
if hasattr(df_builder, 'get_exceeds_xls_limits')\
and df_builder.get_exceeds_xls_limits():
extension = 'xlsx'
temp_file = NamedTemporaryFile(suffix=("." + extension))
df_builder.export_to(temp_file.name)
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if(export_id):
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query == None:
export.save()
return export
def should_create_new_export(xform, export_type):
"""Determine whether or not a new export file for this
xform and data type should be generated.
This legacy logic is really ugly: the exports_outdated() really
should *not* be a classmethod of Export, but doing it incrementally
like this for now, until can do a more thorough overhaul later.
"""
from odk_viewer.models import Export # ugly!
return Export.exports_outdated(xform, export_type=export_type)
def test_last_submission_time_empty(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
# set time of last submission to None
export.time_of_last_submission = None
export.save()
self.assertTrue(Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
def should_create_new_export(xform, export_type):
"""Determine whether or not a new export file for this
xform and data type should be generated.
This legacy logic is really ugly: the exports_outdated() really
should *not* be a classmethod of Export, but doing it incrementally
like this for now, until can do a more thorough overhaul later.
"""
from odk_viewer.models import Export # ugly!
return Export.exports_outdated(xform, export_type=export_type)
def test_last_submission_time_on_export(self):
self._publish_transportation_form()
self._submit_transport_instance()
# create export
xls_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
self.xform.id_string)
num_exports = Export.objects.filter(xform=self.xform,
export_type=Export.XLS_EXPORT).count()
# check that our function knows there are no more submissions
self.assertFalse(Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
# force new last submission date on xform
last_submission = self.xform.surveys.order_by('-date_created')[0]
last_submission.date_created += datetime.timedelta(hours=1)
last_submission.save()
# check that our function knows data has changed
self.assertTrue(Export.exports_outdated(xform=self.xform,
export_type=Export.XLS_EXPORT))
# check that requesting list url will generate a new export
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.XLS_EXPORT
})
response = self.client.get(export_list_url)
self.assertEqual(Export.objects.filter(xform=self.xform,
export_type=Export.XLS_EXPORT).count(), num_exports + 1)
# make sure another export type causes auto-generation
num_exports = Export.objects.filter(xform=self.xform,
export_type=Export.CSV_EXPORT).count()
export_list_url = reverse(export_list, kwargs={
'username': self.user.username,
'id_string': self.xform.id_string,
'export_type': Export.CSV_EXPORT
})
response = self.client.get(export_list_url)
self.assertEqual(Export.objects.filter(xform=self.xform,
export_type=Export.CSV_EXPORT).count(), num_exports + 1)
def should_create_new_export(xform, export_type):
from odk_viewer.models import Export
if Export.objects.filter(xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def generate_export(export_type, extension, username, id_string,
export_id=None, filter_query=None, group_delimiter='/',
split_select_multiples=True):
"""
Create appropriate export object given the export type
"""
from odk_viewer.models import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
}
xform = XForm.objects.get(user__username=username, id_string=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(
temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query == None:
export.save()
return export
def should_create_new_export(xform, export_type):
from odk_viewer.models import Export
if Export.objects.filter(xform=xform, export_type=export_type).count() == 0\
or Export.exports_outdated(xform, export_type=export_type):
return True
return False
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True):
"""
Create appropriate export object given the export type
"""
from odk_viewer.models import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
}
xform = XForm.objects.get(user__username=username, id_string=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query == None:
export.save()
return export