def to_flat_csv_export(
self, path, data, username, id_string, filter_query):
from odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(
username, id_string, filter_query, self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES)
csv_builder.export_to(path)
def to_flat_csv_export(self, path, data, username, id_string,
filter_query):
from odk_viewer.pandas_mongo_bridge import CSVDataFrameBuilder
csv_builder = CSVDataFrameBuilder(username, id_string, filter_query,
self.GROUP_DELIMITER,
self.SPLIT_SELECT_MULTIPLES)
csv_builder.export_to(path)
def get_csv_data(xform):
csv_dataframe_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string, '')
buff = StringIO.StringIO()
# might raise NoRecordsFoundError
csv_dataframe_builder.export_to(buff)
return buff
def csv_export(request, username, id_string):
owner = get_object_or_404(User, username=username)
xform = get_object_or_404(XForm, id_string=id_string, user=owner)
if not has_permission(xform, owner, request):
return HttpResponseForbidden(_(u"Not shared."))
query = request.GET.get("query")
csv_dataframe_builder = CSVDataFrameBuilder(username, id_string, query)
try:
temp_file = NamedTemporaryFile(suffix=".csv")
csv_dataframe_builder.export_to(temp_file)
if request.GET.get("raw"):
id_string = None
response = response_with_mimetype_and_name("application/csv", id_string, extension="csv")
temp_file.seek(0)
response.write(temp_file.read())
temp_file.seek(0, os.SEEK_END)
response["Content-Length"] = temp_file.tell()
temp_file.close()
return response
except NoRecordsFoundError:
return HttpResponse(_("No records found to export"))
def get_csv_data(xform, force_last=False):
def getbuff():
return StringIO.StringIO()
def get_headers_from(csv_data):
csv_data.seek(0)
header_row = csv_data.readline()
csv_data.read()
return header_row.split(',')
def get_csv_data_manual(xform,
only_last=False,
with_header=True,
headers_to_use=None):
# TODO: find out a better way to handle this
# when form has only one submission, CSVDFB is empty.
# we still want to create the BB ds with row 1
# so we extract is and CSV it.
pifilter = ParsedInstance.objects.filter(instance__xform=xform) \
.order_by('-instance__date_modified')
if pifilter.count() == 0:
raise NoRecordsFoundError
else:
# we should only do it for count == 1 but eh.
csv_buf = getbuff()
if only_last:
pifilter = [pifilter[0]]
rows = [pi.to_dict_for_mongo() for pi in pifilter]
if headers_to_use is None:
headers_to_use = [
key for key in rows[0].keys() if not key.startswith('_')
]
w = unicodecsv.DictWriter(csv_buf,
fieldnames=headers_to_use,
extrasaction='ignore',
lineterminator='\n',
encoding='utf-8')
if with_header:
w.writeheader()
w.writerows(rows)
csv_buf.flush()
if not csv_buf.len:
raise NoRecordsFoundError
return csv_buf.getvalue()
# setup an IO stream
buff = getbuff()
# prepare/generate a standard CSV export.
# note that it omits the current submission (if called from rest)
csv_dataframe_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string)
try:
csv_dataframe_builder.export_to(buff)
if force_last:
# requested to add last submission to the buffer
buff.write(
get_csv_data_manual(xform,
only_last=True,
with_header=False,
headers_to_use=get_headers_from(buff)))
except NoRecordsFoundError:
# verify that we don't have a single submission before giving up
get_csv_data_manual(xform, with_header=True)
if buff.len:
# rewrite CSV header so that meta fields (starting with _ or meta)
# are prefixed to ensure that the dataset will be joinable to
# another formhub dataset
prefix = (u'%(id_string)s_%(id)s' % {
'id_string': xform.id_string,
'id': xform.id
})
new_buff = getbuff()
buff.seek(0)
reader = unicodecsv.reader(buff, encoding='utf-8')
writer = unicodecsv.writer(new_buff, encoding='utf-8')
is_header = True
for row in reader:
if is_header:
is_header = False
for idx, col in enumerate(row):
if col.startswith('_') or col.startswith('meta_')\
or col.startswith('meta/'):
row[idx] = (u'%(prefix)s%(col)s' % {
'prefix': prefix,
'col': col
})
writer.writerow(row)
return new_buff.getvalue()
else:
raise NoRecordsFoundError
def get_csv_data(xform, force_last=False):
def getbuff():
return StringIO.StringIO()
def get_headers_from(csv_data):
csv_data.seek(0)
header_row = csv_data.readline()
csv_data.read()
return header_row.split(',')
def get_csv_data_manual(xform,
only_last=False, with_header=True,
headers_to_use=None):
# TODO: find out a better way to handle this
# when form has only one submission, CSVDFB is empty.
# we still want to create the BB ds with row 1
# so we extract is and CSV it.
pifilter = ParsedInstance.objects.filter(instance__xform=xform) \
.order_by('-instance__date_modified')
if pifilter.count() == 0:
raise NoRecordsFoundError
else:
# we should only do it for count == 1 but eh.
csv_buf = getbuff()
if only_last:
pifilter = [pifilter[0]]
rows = [pi.to_dict_for_mongo() for pi in pifilter]
if headers_to_use is None:
headers_to_use = [key for key in rows[0].keys()
if not key.startswith('_')]
w = csv.DictWriter(csv_buf, fieldnames=headers_to_use,
extrasaction='ignore', lineterminator='\n')
if with_header:
w.writeheader()
w.writerows(rows)
csv_buf.flush()
if not csv_buf.len:
raise NoRecordsFoundError
return csv_buf.getvalue()
# setup an IO stream
buff = getbuff()
# prepare/generate a standard CSV export.
# note that it omits the current submission (if called from rest)
csv_dataframe_builder = CSVDataFrameBuilder(xform.user.username,
xform.id_string)
try:
csv_dataframe_builder.export_to(buff)
if force_last:
# requested to add last submission to the buffer
buff.write(get_csv_data_manual(xform,
only_last=True, with_header=False,
headers_to_use=
get_headers_from(buff)))
except NoRecordsFoundError:
# verify that we don't have a single submission before giving up
get_csv_data_manual(xform, with_header=True)
if buff.len:
# rewrite CSV header so that meta fields (starting with _ or meta)
# are prefixed to ensure that the dataset will be joinable to
# another formhub dataset
prefix = (u'%(id_string)s_%(id)s'
% {'id_string': xform.id_string, 'id': xform.id})
new_buff = getbuff()
buff.seek(0)
reader = csv.reader(buff)
writer = csv.writer(new_buff)
is_header = True
for row in reader:
if is_header:
is_header = False
for idx, col in enumerate(row):
if col.startswith('_') or col.startswith('meta_') \
or col.startswith('meta/'):
row[idx] = (u'%(prefix)s%(col)s'
% {'prefix': prefix, 'col': col})
writer.writerow(row)
return new_buff.getvalue()
else:
raise NoRecordsFoundError