def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
"""
Create appropriate export object given the export type
"""
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username,
id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename = '{}_ANALYSER_{}'.format(
id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
return export
def generate_export(export_type, extension, username, id_string,
export_id=None, filter_query=None, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
}
xform = XForm.objects.get(
user__username__iexact=username, id_string__iexact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(
temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
return export
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
start=None,
end=None,
remove_group_name=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__iexact=id_string)
records = ParsedInstance.query_data(xform,
query=filter_query,
start=start,
end=end)
export_builder = ExportBuilder()
export_builder.TRUNCATE_GROUP_TITLE = remove_group_name
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.set_survey(xform.data_dictionary().survey)
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
try:
func.__call__(temp_file.name,
records,
username,
id_string,
filter_query,
start=start,
end=end)
except NoRecordsFoundError:
pass
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None and start is None and end is None:
export.save()
return export
def generate_export(export_type, extension, username, id_string,
export_id=None, filter_query=None, group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
sync_to_gsuit=False, user=None):
"""
Create appropriate export object given the export type
"""
time.sleep(5)
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.ANALYSER_EXPORT: 'to_analyser_export'
}
xform = XForm.objects.get(
user__username__iexact=username, id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
__version__ = "0"
try:
__version__ = filter_query['$and'][0]['__version__']
except Exception as e:
print(str(e))
if __version__:
survey = build_survey_from_history(xform, __version__)
if not survey:
export_builder.set_survey(xform.data_dictionary().survey)
else:
export_builder.set_survey(survey)
else:
export_builder.set_survey(xform.data_dictionary().survey)
prefix = slugify('{}_export__{}__{}'.format(export_type, username, id_string))
temp_file = NamedTemporaryFile(prefix=prefix, suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(
temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (
id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
if export_type == Export.ANALYSER_EXPORT:
# Analyser exports should be distinguished by more than just their file extension.
basename= '{}_ANALYSER_{}'.format(id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(
username,
'exports',
id_string,
export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
print sync_to_gsuit, 'file_url--------->', temp_file, filter_query
try:
if sync_to_gsuit == True and '__version__' not in filter_query['$and'][0]:
if not os.path.exists("media/forms/"):
os.makedirs("media/forms/")
temporarylocation="media/forms/submissions_{}.xls".format(id_string)
import shutil
shutil.copy(temp_file.name, temporarylocation)
fxf_form = FieldSightXF.objects.get(pk=filter_query['$and'][0]['fs_project_uuid'])
upload_to_drive(temporarylocation, str(fxf_form.id) + '_' +id_string, None, fxf_form.project, user)
os.remove(temporarylocation)
except Exception as e:
print e.__dict__
# get or create export object
temp_file.seek(0)
export_filename = storage.save(
file_path,
File(temp_file, file_path))
dir_name, basename = os.path.split(export_filename)
temp_file.close()
if export_id:
export = Export.objects.get(id=export_id)
else:
fsxf = filter_query.values()[0]
# print("fsxf", fsxf)
export = Export(xform=xform, export_type=export_type, fsxf_id=fsxf)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# dont persist exports that have a filter
if filter_query is None:
export.save()
export.save()
return export
def generate_export(export_type,
extension,
username,
id_string,
export_id=None,
filter_query=None,
group_delimiter='/',
split_select_multiples=True,
binary_select_multiples=False,
show_label=False):
"""
Create appropriate export object given the export type
"""
# TODO resolve circular import
from onadata.apps.viewer.models.export import Export
export_type_func_map = {
Export.XLS_EXPORT: 'to_xls_export',
Export.CSV_EXPORT: 'to_flat_csv_export',
Export.CSV_ZIP_EXPORT: 'to_zipped_csv',
Export.SAV_ZIP_EXPORT: 'to_zipped_sav',
Export.DB_EXPORT: 'to_postgres_sav',
}
xform = XForm.objects.get(user__username__iexact=username,
id_string__exact=id_string)
# query mongo for the cursor
records = query_mongo(username, id_string, filter_query)
export_builder = ExportBuilder()
export_builder.GROUP_DELIMITER = group_delimiter
export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples
export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples
export_builder.SHOW_LABEL = show_label
export_builder.set_survey(xform.data_dictionary().survey)
print "xform.data_dictionary().survey"
print xform.data_dictionary().survey
export_builder.FORM_OWNER = xform.user.username
temp_file = NamedTemporaryFile(suffix=("." + extension))
# get the export function by export type
func = getattr(export_builder, export_type_func_map[export_type])
func.__call__(temp_file.name, records, username, id_string, filter_query)
# generate filename
basename = "%s_%s" % (id_string,
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
filename = basename + "." + extension
# check filename is unique
while not Export.is_filename_unique(xform, filename):
filename = increment_index_in_filename(filename)
file_path = os.path.join(username, 'exports', id_string, export_type,
filename)
# TODO: if s3 storage, make private - how will we protect local storage??
storage = get_storage_class()()
# seek to the beginning as required by storage classes
temp_file.seek(0)
export_filename = storage.save(file_path, File(temp_file, file_path))
temp_file.close()
dir_name, basename = os.path.split(export_filename)
# get or create export object
if export_id:
export = Export.objects.get(id=export_id)
else:
export = Export(xform=xform, export_type=export_type)
export.filedir = dir_name
export.filename = basename
export.internal_status = Export.SUCCESSFUL
# start: original code commented out
# dont persist exports that have a filter
# if filter_query is None:
# export.save()
# end: original code commented out
export.save()
return export