def get(self, request, domain, app_id):
case_metadata = self.app.get_case_metadata()
language = request.GET.get("lang", "en")
headers = [("All Case Properties", ("case_type", "case_property"))]
headers += list(
(case_type.name, tuple(CASE_SUMMARY_EXPORT_HEADER_NAMES)) for case_type in case_metadata.case_types
)
data = list(
("All Case Properties", self.get_case_property_rows(case_type)) for case_type in case_metadata.case_types
)
data += list(
(case_type.name, self.get_case_questions_rows(case_type, language))
for case_type in case_metadata.case_types
)
export_string = StringIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
u"{app_name} v.{app_version} - Case Summary ({lang})".format(
app_name=self.app.name, app_version=self.app.version, lang=language
),
)
def download_bulk_app_translations(request, domain, app_id):
lang = request.GET.get('lang')
skip_blacklisted = request.GET.get('skipbl', 'false') == 'true'
app = get_app(domain, app_id)
# if there is a lang selected, assume that user wants a single sheet
is_single_sheet = bool(lang)
headers = get_bulk_app_sheet_headers(
app,
single_sheet=is_single_sheet,
lang=lang,
eligible_for_transifex_only=skip_blacklisted)
if is_single_sheet:
sheets = get_bulk_app_single_sheet_by_name(app, lang, skip_blacklisted)
else:
sheets = get_bulk_app_sheets_by_name(
app, eligible_for_transifex_only=skip_blacklisted)
temp = io.BytesIO()
data = [(k, v) for k, v in sheets.items()]
export_raw(headers, data, temp)
filename = '{app_name} v.{app_version} - App Translations{lang}{transifex_only}'.format(
app_name=app.name,
app_version=app.version,
lang=' ' + lang if is_single_sheet else '',
transifex_only=' (Transifex only)' if skip_blacklisted else '',
)
return export_response(temp, Format.XLS_2007, filename)
def export_data_source(request, domain, config_id):
config = get_document_or_404(DataSourceConfiguration, domain, config_id)
adapter = IndicatorSqlAdapter(config)
q = adapter.get_query_object()
table = adapter.get_table()
try:
params = process_url_params(request.GET, table.columns)
except UserQueryError as e:
return HttpResponse(e.message, status=400)
q = q.filter_by(**params.keyword_filters)
for sql_filter in params.sql_filters:
q = q.filter(sql_filter)
# build export
def get_table(q):
yield table.columns.keys()
for row in q:
yield row
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmpfile:
try:
tables = [[config.table_id, get_table(q)]]
export_from_tables(tables, tmpfile, params.format)
except exc.DataError:
msg = _("There was a problem executing your query, please make "
"sure your parameters are valid.")
return HttpResponse(msg, status=400)
return export_response(Temp(path), params.format, config.display_name)
def get(self, request, domain, app_id):
language = request.GET.get("lang", "en")
modules = list(self.app.get_modules())
headers = [("All Forms", ("module_name", "form_name", "comment"))]
headers += [
(self._get_form_sheet_name(module, form, language), tuple(FORM_SUMMARY_EXPORT_HEADER_NAMES))
for module in modules
for form in module.get_forms()
]
data = list(
("All Forms", self.get_all_forms_row(module, form, language))
for module in modules
for form in module.get_forms()
)
data += list(
(self._get_form_sheet_name(module, form, language), self._get_form_row(form, language))
for module in modules
for form in module.get_forms()
)
export_string = StringIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
u"{app_name} v.{app_version} - Form Summary ({lang})".format(
app_name=self.app.name, app_version=self.app.version, lang=language
),
)
def download_data(self, format="", previous_export=None, filter=None):
"""
If there is data, return an HTTPResponse with the appropriate data.
If there is not data returns None.
"""
from couchexport.shortcuts import export_response
from couchexport.export import get_writer, get_schema_new, format_tables, create_intermediate_tables
if not format:
format = self.default_format or Format.XLS_2007
from couchexport.export import ExportConfiguration
database = get_db()
config = ExportConfiguration(database, self.index, previous_export, util.intersect_filters(self.filter, filter))
# get and checkpoint the latest schema
updated_schema = get_schema_new(config)
export_schema_checkpoint = ExportSchema(
seq=config.current_seq, schema=updated_schema, index=config.schema_index
)
export_schema_checkpoint.save()
# transform docs onto output and save
writer = get_writer(format)
# open the doc and the headers
formatted_headers = self.get_table_headers()
tmp = StringIO()
writer.open(formatted_headers, tmp)
for doc in config.get_docs():
writer.write(self.trim(format_tables(create_intermediate_tables(doc, updated_schema), separator=".")))
writer.close()
return export_response(tmp, format, self.name)
def _export_pis(report, report_slug, non_data_cols=3):
context = report.get_data(include_urls=False)
# THESE FUNCTIONS ARE TOTAL HACKS.
# they rely on knowing the first two or three values are clinic, year, month,
# and then the rest are fractional
def _transform_headings(headings, non_data_cols):
ret = headings[:non_data_cols]
for h in headings[non_data_cols:]:
ret.append("%s num" % h)
ret.append("%s denom" % h)
ret.append("%s pct" % h)
return ret
def _transform_rows(rows, non_data_cols):
return [_transform_values(r, non_data_cols) for r in rows]
def _transform_values(values, non_data_cols):
ret = values[:non_data_cols]
for v in values[non_data_cols:]:
if v != "N/A":
for special in "(/)":
v = v.replace(special, " ")
pct, num, denom = v.split()
ret.extend([num, denom, pct])
else:
ret.extend(["N/A"] * 3)
return ret
temp = StringIO()
export_raw((("data", _transform_headings(context["headings"], non_data_cols)),),
(("data", _transform_rows(context["rows"], non_data_cols)),), temp)
return export_response(temp, "xlsx", report_slug)
def export_all_form_metadata(req, domain):
"""
Export metadata for _all_ forms in a domain.
"""
format = req.GET.get("format", Format.XLS_2007)
headers = ("domain", "instanceID", "received_on", "type",
"timeStart", "timeEnd", "deviceID", "username",
"userID", "xmlns", "version")
def _form_data_to_row(formdata):
def _key_to_val(formdata, key):
if key == "type": return xmlns_to_name(domain, formdata.xmlns, app_id=None)
else: return getattr(formdata, key)
return [_key_to_val(formdata, key) for key in headers]
fd, path = tempfile.mkstemp()
data = (_form_data_to_row(f) for f in stream_qs(
HQFormData.objects.filter(domain=domain).order_by('received_on')
))
with os.fdopen(fd, 'w') as temp:
export_raw((("forms", headers),), (("forms", data),), temp)
return export_response(open(path), format, "%s_forms" % domain)
def get(self, request, domain, app_id):
language = request.GET.get('lang', 'en')
modules = list(self.app.get_modules())
case_meta = self.app.get_case_metadata()
headers = [(_('All Forms'),
('module_name', 'form_name', 'comment', 'module_display_condition', 'form_display_condition'))]
headers += [
(self._get_form_sheet_name(module, form, language), tuple(FORM_SUMMARY_EXPORT_HEADER_NAMES))
for module in modules for form in module.get_forms()
]
data = list((
_('All Forms'),
self.get_all_forms_row(module, form, language)
) for module in modules for form in module.get_forms())
data += list(
(self._get_form_sheet_name(module, form, language), self._get_form_row(form, language, case_meta))
for module in modules for form in module.get_forms()
)
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
'{app_name} v.{app_version} - Form Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language
),
)
def export_all_form_metadata(req, domain):
"""
Export metadata for _all_ forms in a domain.
"""
format = req.GET.get("format", Format.XLS_2007)
headers = ("domain", "instanceID", "received_on", "type", "timeStart",
"timeEnd", "deviceID", "username", "userID", "xmlns", "version")
def _form_data_to_row(formdata):
def _key_to_val(formdata, key):
if key == "type":
return xmlns_to_name(domain, formdata.xmlns, app_id=None)
else:
return getattr(formdata, key)
return [_key_to_val(formdata, key) for key in headers]
fd, path = tempfile.mkstemp()
data = (_form_data_to_row(f) for f in stream_qs(
HQFormData.objects.filter(domain=domain).order_by('received_on')))
with os.fdopen(fd, 'w') as temp:
export_raw((("forms", headers), ), (("forms", data), ), temp)
return export_response(open(path), format, "%s_forms" % domain)
def get(self, request, domain, app_id):
case_metadata = self.app.get_case_metadata()
language = request.GET.get('lang', 'en')
headers = [(_('All Case Properties'), ('case_type', 'case_property', 'description')),
(_('Case Types'), ('type', 'relationships', 'opened_by', 'closed_by'))]
headers += list((
case_type.name,
tuple(CASE_SUMMARY_EXPORT_HEADER_NAMES)
)for case_type in case_metadata.case_types)
data = [(
_('All Case Properties'),
self.get_case_property_rows(case_type)
) for case_type in case_metadata.case_types]
data += [self.get_case_type_rows(case_metadata.case_types, language)]
data += [(
case_type.name,
self.get_case_questions_rows(case_type, language)
) for case_type in case_metadata.case_types]
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
'{app_name} v.{app_version} - Case Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language
),
)
def get(self, request, domain, app_id):
language = request.GET.get('lang', 'en')
modules = list(self.app.get_modules())
case_meta = self.app.get_case_metadata()
headers = [(_('All Forms'),
('module_name', 'form_name', 'comment',
'module_display_condition', 'form_display_condition'))]
headers += [(self._get_form_sheet_name(form, language),
tuple(FORM_SUMMARY_EXPORT_HEADER_NAMES))
for module in modules for form in module.get_forms()]
data = list(
(_('All Forms'), self.get_all_forms_row(module, form, language))
for module in modules for form in module.get_forms())
data += list((self._get_form_sheet_name(form, language),
self._get_form_row(form, language, case_meta))
for module in modules for form in module.get_forms())
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
'{app_name} v.{app_version} - Form Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language),
)
def get(self, request, domain, app_id):
case_metadata = self.app.get_case_metadata()
language = request.GET.get('lang', 'en')
headers = [(_('All Case Properties'), ('case_type', 'case_property',
'description')),
(_('Case Types'), ('type', 'relationships', 'opened_by',
'closed_by'))]
headers += list(
(case_type.name, tuple(CASE_SUMMARY_EXPORT_HEADER_NAMES))
for case_type in case_metadata.case_types)
data = list(
(_('All Case Properties'), self.get_case_property_rows(case_type))
for case_type in case_metadata.case_types)
data += list(
(_('Case Types'), self.get_case_type_rows(case_type, language))
for case_type in case_metadata.case_types)
data += list(
(case_type.name, self.get_case_questions_rows(case_type, language))
for case_type in case_metadata.case_types)
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
u'{app_name} v.{app_version} - Case Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language),
)
def download_bulk_ui_translations(request, domain, app_id):
app = get_app(domain, app_id)
temp = build_ui_translation_download_file(app)
filename = '{app_name} v.{app_version} - CommCare Translations'.format(
app_name=app.name,
app_version=app.version)
return export_response(temp, Format.XLS_2007, filename)
def export_data_source(request, domain, config_id):
config = get_document_or_404(DataSourceConfiguration, domain, config_id)
adapter = IndicatorSqlAdapter(config)
q = adapter.get_query_object()
table = adapter.get_table()
try:
params = process_url_params(request.GET, table.columns)
except UserQueryError as e:
return HttpResponse(e.message, status=400)
q = q.filter_by(**params.keyword_filters)
for sql_filter in params.sql_filters:
q = q.filter(sql_filter)
# build export
def get_table(q):
yield table.columns.keys()
for row in q:
yield row
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmpfile:
try:
tables = [[config.table_id, get_table(q)]]
export_from_tables(tables, tmpfile, params.format)
except exc.DataError:
msg = _("There was a problem executing your query, please make "
"sure your parameters are valid.")
return HttpResponse(msg, status=400)
return export_response(Temp(path), params.format, config.display_name)
def download_data(self, format="", previous_export=None, filter=None, limit=0):
"""
If there is data, return an HTTPResponse with the appropriate data.
If there is not data returns None.
"""
from couchexport.shortcuts import export_response
files = self.get_export_files(format, previous_export, filter, limit=limit)
return export_response(files.file, files.format, self.name)
def download_data(self, format="", previous_export=None, filter=None, limit=0):
"""
If there is data, return an HTTPResponse with the appropriate data.
If there is not data returns None.
"""
from couchexport.shortcuts import export_response
files = self.get_export_files(format, previous_export, filter, limit=limit)
return export_response(files.file, files.format, self.name)
def export_response(self):
"""
Intention: Not to be overridden in general.
Returns the tabular export of the data, if available.
"""
temp = StringIO()
export_from_tables(self.export_table, temp, self.export_format)
return export_response(temp, self.export_format, self.export_name)
def export_response(self):
"""
Intention: Not to be overridden in general.
Returns the tabular export of the data, if available.
"""
temp = StringIO()
export_from_tables(self.export_table, temp, self.export_format)
return export_response(temp, self.export_format, self.export_name)
def export_all_form_metadata(req, domain):
"""
Export metadata for _all_ forms in a domain.
"""
format = req.GET.get("format", Format.XLS_2007)
tmp_path = save_metadata_export_to_tempfile(domain, format=format)
return export_response(open(tmp_path), format, "%s_forms" % domain)
def download_bulk_app_translations(request, domain, app_id):
app = get_app(domain, app_id)
headers = expected_bulk_app_sheet_headers(app)
rows = expected_bulk_app_sheet_rows(app)
temp = StringIO()
data = [(k, v) for k, v in rows.iteritems()]
export_raw(headers, data, temp)
return export_response(temp, Format.XLS_2007, "bulk_app_translations")
def export_all_form_metadata(req, domain):
"""
Export metadata for _all_ forms in a domain.
"""
format = req.GET.get("format", Format.XLS_2007)
tmp_path = save_metadata_export_to_tempfile(domain, format=format)
return export_response(open(tmp_path), format, "%s_forms" % domain)
def download_bulk_app_translations(request, domain, app_id):
app = get_app(domain, app_id)
headers = expected_bulk_app_sheet_headers(app)
rows = expected_bulk_app_sheet_rows(app)
temp = io.BytesIO()
data = [(k, v) for k, v in six.iteritems(rows)]
export_raw(headers, data, temp)
return export_response(temp, Format.XLS_2007, "bulk_app_translations")
def download_item_lists(request, domain):
data_types = FixtureDataType.by_domain(domain)
data_type_schemas = []
max_fields = 0
max_groups = 0
max_users = 0
mmax_groups = 0
mmax_users = 0
data_tables = []
for data_type in data_types:
type_schema = [data_type.name, data_type.tag]
fields = [field for field in data_type.fields]
type_id = data_type.get_id
data_table_of_type = []
for item_row in FixtureDataItem.by_data_type(domain, type_id):
group_len = len(item_row.get_groups())
max_groups = group_len if group_len>max_groups else max_groups
user_len = len(item_row.get_users())
max_users = user_len if user_len>max_users else max_users
for item_row in FixtureDataItem.by_data_type(domain, type_id):
groups = [group.name for group in item_row.get_groups()] + ["" for x in range(0,max_groups-len(item_row.get_groups()))]
users = [user.raw_username for user in item_row.get_users()] + ["" for x in range(0, max_users-len(item_row.get_users()))]
data_row = tuple([str(_id_from_doc(item_row)),"N"]+
[item_row.fields[field] for field in fields]+
groups + users)
data_table_of_type.append(data_row)
type_schema.extend(fields)
data_type_schemas.append(tuple(type_schema))
if max_fields<len(type_schema):
max_fields = len(type_schema)
data_tables.append((data_type.tag,tuple(data_table_of_type)))
mmax_users = max_users if max_users>mmax_users else mmax_users
mmax_groups = max_groups if max_groups>mmax_groups else mmax_groups
max_users = 0
max_groups = 0
type_headers = ["name", "tag"] + ["field %d" % x for x in range(1, max_fields - 1)]
type_headers = ("types", tuple(type_headers))
table_headers = [type_headers]
for type_schema in data_type_schemas:
item_header = (type_schema[1], tuple(["UID", DELETE_HEADER] +
["field: " + x for x in type_schema[2:]] +
["group %d" % x for x in range(1, mmax_groups + 1)] +
["user %d" % x for x in range(1, mmax_users + 1)]))
table_headers.append(item_header)
table_headers = tuple(table_headers)
type_rows = ("types", tuple(data_type_schemas))
data_tables = tuple([type_rows]+data_tables)
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as temp:
export_raw((table_headers), (data_tables), temp)
format = Format.XLS_2007
return export_response(open(path), format, "%s_fixtures" % domain)
def export_data_source(request, domain, config_id):
config, _ = get_datasource_config_or_404(config_id, domain)
adapter = IndicatorSqlAdapter(config)
q = adapter.get_query_object()
table = adapter.get_table()
try:
params = process_url_params(request.GET, table.columns)
allowed_formats = [
Format.CSV,
Format.HTML,
Format.XLS,
Format.XLS_2007,
]
if params.format not in allowed_formats:
msg = ugettext_lazy('format must be one of the following: {}').format(', '.join(allowed_formats))
return HttpResponse(msg, status=400)
except UserQueryError as e:
return HttpResponse(e.message, status=400)
q = q.filter_by(**params.keyword_filters)
for sql_filter in params.sql_filters:
q = q.filter(sql_filter)
# xls format has limit of 65536 rows
# First row is taken up by headers
if params.format == Format.XLS and q.count() >= 65535:
keyword_params = dict(**request.GET)
# use default format
if 'format' in keyword_params:
del keyword_params['format']
return HttpResponseRedirect(
'%s?%s' % (
reverse('export_configurable_data_source', args=[domain, config._id]),
urlencode(keyword_params)
)
)
# build export
def get_table(q):
yield table.columns.keys()
for row in q:
yield row
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmpfile:
try:
tables = [[config.table_id, get_table(q)]]
export_from_tables(tables, tmpfile, params.format)
except exc.DataError:
msg = ugettext_lazy(
"There was a problem executing your query, "
"please make sure your parameters are valid."
)
return HttpResponse(msg, status=400)
return export_response(Temp(path), params.format, config.display_name)
def download_bulk_app_translations(request, domain, app_id):
app = get_app(domain, app_id)
headers = get_bulk_app_sheet_headers(app)
rows = get_bulk_app_sheet_rows(app)
temp = io.BytesIO()
data = [(k, v) for k, v in six.iteritems(rows)]
export_raw(headers, data, temp)
filename = '{app_name} v.{app_version} - App Translations'.format(
app_name=app.name, app_version=app.version)
return export_response(temp, Format.XLS_2007, filename)
def export_response(self):
if self.export_too_large:
# Frontend should check size with export_size_check_response()
# Before hitting this endpoint, but we check the size again here
# in case the user modifies the url manually.
return HttpResponseBadRequest()
temp = StringIO()
export_from_tables(self.export_table, temp, Format.XLS_2007)
return export_response(temp, Format.XLS_2007, self.title)
def export_response(self):
if self.export_too_large:
# Frontend should check size with export_size_check_response()
# Before hitting this endpoint, but we check the size again here
# in case the user modifies the url manually.
return HttpResponseBadRequest()
temp = StringIO()
export_from_tables(self.export_table, temp, Format.XLS_2007)
return export_response(temp, Format.XLS_2007, self.title)
def download_bulk_app_translations(request, domain, app_id):
app = get_app(domain, app_id)
headers = expected_bulk_app_sheet_headers(app)
rows = expected_bulk_app_sheet_rows(app)
temp = io.BytesIO()
data = [(k, v) for k, v in six.iteritems(rows)]
export_raw(headers, data, temp)
filename = '{app_name} v.{app_version} - App Translations'.format(
app_name=app.name,
app_version=app.version)
return export_response(temp, Format.XLS_2007, filename)
def export_data_source(request, domain, config_id):
config, _ = get_datasource_config_or_404(config_id, domain)
adapter = IndicatorSqlAdapter(config)
q = adapter.get_query_object()
table = adapter.get_table()
try:
params = process_url_params(request.GET, table.columns)
allowed_formats = [
Format.CSV,
Format.HTML,
Format.XLS,
Format.XLS_2007,
]
if params.format not in allowed_formats:
msg = ugettext_lazy(
'format must be one of the following: {}').format(
', '.join(allowed_formats))
return HttpResponse(msg, status=400)
except UserQueryError as e:
return HttpResponse(e.message, status=400)
q = q.filter_by(**params.keyword_filters)
for sql_filter in params.sql_filters:
q = q.filter(sql_filter)
# xls format has limit of 65536 rows
# First row is taken up by headers
if params.format == Format.XLS and q.count() >= 65535:
keyword_params = dict(**request.GET)
# use default format
if 'format' in keyword_params:
del keyword_params['format']
return HttpResponseRedirect(
'%s?%s' %
(reverse('export_configurable_data_source',
args=[domain, config._id]), urlencode(keyword_params)))
# build export
def get_table(q):
yield table.columns.keys()
for row in q:
yield row
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as tmpfile:
try:
tables = [[config.table_id, get_table(q)]]
export_from_tables(tables, tmpfile, params.format)
except exc.DataError:
msg = ugettext_lazy("There was a problem executing your query, "
"please make sure your parameters are valid.")
return HttpResponse(msg, status=400)
return export_response(Temp(path), params.format, config.display_name)
def download_multimedia_paths(request, domain, app_id):
from corehq.apps.hqmedia.view_helpers import download_multimedia_paths_rows
app = get_app(domain, app_id)
headers = ((_("Paths"), (_("Old Path"), _("New Path"), _("Usages"))), )
rows = download_multimedia_paths_rows(app)
temp = io.BytesIO()
export_raw(headers, rows, temp)
filename = '{app_name} v.{app_version} - App Multimedia Paths'.format(
app_name=app.name, app_version=app.version)
return export_response(temp, Format.XLS_2007, filename)
def export_response(self):
"""
Intention: Not to be overridden in general.
Returns the tabular export of the data, if available.
"""
if self.exportable_all:
export_all_rows_task.delay(self.__class__, self.__getstate__())
return HttpResponse()
else:
temp = StringIO()
export_from_tables(self.export_table, temp, self.export_format)
return export_response(temp, self.export_format, self.export_name)
def export_response(self):
"""
Intention: Not to be overridden in general.
Returns the tabular export of the data, if available.
"""
if self.exportable_all:
export_all_rows_task.delay(self.__class__, self.__getstate__())
return HttpResponse()
else:
temp = io.BytesIO()
export_from_tables(self.export_table, temp, self.export_format)
return export_response(temp, self.export_format, self.export_name)
def download_bulk_multimedia_translations(request, domain, app_id):
lang = request.GET.get('lang')
app = get_app(domain, app_id)
headers = get_bulk_multimedia_sheet_headers(lang)
rows = get_bulk_multimedia_sheet_rows(lang, app)
temp = io.BytesIO()
export_raw(headers, [(headers[0][0], rows)], temp)
filename = '{app_name} v.{app_version} - Multimedia Translations {lang}'.format(
app_name=app.name, app_version=app.version, lang=lang)
return export_response(temp, Format.XLS_2007, filename)
def download_multimedia_paths(request, domain, app_id):
from corehq.apps.hqmedia.view_helpers import download_multimedia_paths_rows
app = get_app(domain, app_id)
headers = ((_("Paths"), (_("Old Path"), _("New Path"), _("Usages"))),)
rows = download_multimedia_paths_rows(app, only_missing=request.GET.get('only_missing', False))
temp = io.BytesIO()
export_raw(headers, rows, temp)
filename = '{app_name} v.{app_version} - App Multimedia Paths'.format(
app_name=app.name,
app_version=app.version)
return export_response(temp, Format.XLS_2007, filename)
def get(self, request, domain):
headers = get_conditional_alert_headers(self.domain)
(translated_rows,
untranslated_rows) = get_conditional_alert_rows(self.domain)
temp = io.BytesIO()
export_raw(headers, [
(TranslatedConditionalAlertUploader.sheet_name, translated_rows),
(UntranslatedConditionalAlertUploader.sheet_name,
untranslated_rows),
], temp)
filename = 'Conditional Alerts - {domain}'.format(domain=domain)
return export_response(temp, Format.XLS_2007, filename)
def download_bulk_app_translations(request, domain, app_id):
lang = request.GET.get('lang')
app = get_app(domain, app_id)
headers = get_bulk_app_sheet_headers(app, lang=lang)
sheets = get_bulk_app_single_sheet_by_name(app, lang) if lang else get_bulk_app_sheets_by_name(app)
temp = io.BytesIO()
data = [(k, v) for k, v in six.iteritems(sheets)]
export_raw(headers, data, temp)
filename = '{app_name} v.{app_version} - App Translations{lang}'.format(
app_name=app.name,
app_version=app.version,
lang=' ' + lang if lang else '')
return export_response(temp, Format.XLS_2007, filename)
def get(self, request, domain):
title = _("Conditional Alerts")
headers = ((title, (_('id'), _('name'), _('case_type'))),)
rows = [(
rule.pk,
rule.name,
rule.case_type,
) for rule in self.get_conditional_alerts_queryset()]
temp = io.BytesIO()
export_raw(headers, [(title, rows)], temp)
filename = '{title} - {domain}'.format(
domain=domain,
title=title)
return export_response(temp, Format.XLS_2007, filename)
def domain_list_download(request):
domains = Domain.get_all()
properties = ("name", "city", "country", "region", "project_type", "customer_type", "is_test?")
def _row(domain):
def _prop(domain, prop):
if prop.endswith("?"):
return yesno(getattr(domain, prop[:-1], ""))
return getattr(domain, prop, "")
return (_prop(domain, prop) for prop in properties)
temp = StringIO()
headers = (("domains", properties),)
data = (("domains", (_row(domain) for domain in domains)),)
export_raw(headers, data, temp)
return export_response(temp, Format.XLS_2007, "domains")
def domain_list_download(request):
domains = Domain.get_all()
properties = ("name", "city", "countries", "region", "project_type",
"customer_type", "is_test?")
def _row(domain):
def _prop(domain, prop):
if prop.endswith("?"):
return yesno(getattr(domain, prop[:-1], ""))
return getattr(domain, prop, "")
return (_prop(domain, prop) for prop in properties)
temp = StringIO()
headers = (("domains", properties),)
data = (("domains", (_row(domain) for domain in domains)),)
export_raw(headers, data, temp)
return export_response(temp, Format.XLS_2007, "domains")
def export_all_form_metadata(req, domain):
"""
Export metadata for _all_ forms in a domain.
"""
format = req.GET.get("format", Format.XLS_2007)
headers = ("domain", "instanceID", "received_on", "type",
"timeStart", "timeEnd", "deviceID", "username",
"userID", "xmlns", "version")
def _form_data_to_row(formdata):
def _key_to_val(formdata, key):
if key == "type": return xmlns_to_name(domain, formdata.xmlns, app_id=None)
else: return getattr(formdata, key)
return [_key_to_val(formdata, key) for key in headers]
temp = StringIO()
data = (_form_data_to_row(f) for f in HQFormData.objects.filter(domain=domain))
export_raw((("forms", headers),), (("forms", data),), temp)
return export_response(temp, format, "%s_forms" % domain)
def export_case_transactions(request, domain, case_id):
case = get_document_or_404(CommCareCase, domain, case_id)
products_by_id = dict(SQLProduct.objects.filter(domain=domain).values_list('product_id', 'name'))
headers = [
_('case id'),
_('case name'),
_('section'),
_('date'),
_('product_id'),
_('product_name'),
_('transaction amount'),
_('type'),
_('ending balance'),
]
def _make_row(transaction):
return [
transaction.case_id,
case.name,
transaction.section_id,
transaction.report.date if transaction.report_id else '',
transaction.product_id,
products_by_id.get(transaction.product_id, _('unknown product')),
transaction.quantity,
transaction.type,
transaction.stock_on_hand,
]
query_set = StockTransaction.objects.select_related('report')\
.filter(case_id=case_id).order_by('section_id', 'report__date')
formatted_table = [
[
'stock transactions',
[headers] + [_make_row(txn) for txn in query_set]
]
]
tmp = StringIO()
export_from_tables(formatted_table, tmp, 'xlsx')
return export_response(tmp, 'xlsx', '{}-stock-transactions'.format(case.name))
def download_sms_translations(request, domain):
tdoc = StandaloneTranslationDoc.get_obj(domain, "sms")
columns = ["property"] + tdoc.langs + ["default"]
msg_ids = sorted(_MESSAGES.keys())
rows = []
for msg_id in msg_ids:
rows.append([msg_id])
for lang in tdoc.langs:
for row in rows:
row.append(tdoc.translations[lang].get(row[0], ""))
for row in rows:
row.append(_MESSAGES.get(row[0]))
temp = StringIO()
headers = (("translations", tuple(columns)),)
data = (("translations", tuple(rows)),)
export_raw(headers, data, temp)
return export_response(temp, Format.XLS_2007, "translations")
def export_data_source(request, domain, config_id):
format = request.GET.get('format', Format.UNZIPPED_CSV)
config = get_document_or_404(DataSourceConfiguration, domain, config_id)
table = get_indicator_table(config)
q = Session.query(table)
column_headers = [col['name'] for col in q.column_descriptions]
# apply filtering if any
filter_values = {key: value for key, value in request.GET.items() if key != 'format'}
for key in filter_values:
if key not in column_headers:
return HttpResponse('Invalid filter parameter: {}'.format(key), status=400)
q = q.filter_by(**filter_values)
# build export
def get_table(q):
yield column_headers
for row in q:
yield row
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'wb') as temp:
export_from_tables([[config.table_id, get_table(q)]], temp, format)
return export_response(Temp(path), format, config.display_name)
def global_report(request, template="hqadmin/global.html", as_export=False):
def _flot_format(result):
return int(
datetime(year=result['key'][0], month=result['key'][1],
day=1).strftime("%s")) * 1000
def _export_format(result):
return datetime(year=result['key'][0], month=result['key'][1],
day=1).strftime("%Y-%m-%d")
context = get_hqadmin_base_context(request)
def _metric(name):
counts = []
for result in get_db().view("hqadmin/%ss_over_time" % name,
group_level=2):
if not result or not result.has_key('key') or not result.has_key(
'value'):
continue
if result['key'][0] and int(result['key'][0]) >= 2010 and \
(int(result['key'][0]) < datetime.utcnow().year or
(int(result['key'][0]) == datetime.utcnow().year and
int(result['key'][1]) <= datetime.utcnow().month)):
counts.append([
_export_format(result)
if as_export else _flot_format(result), result['value']
])
context['%s_counts' % name] = counts
counts_int = deepcopy(counts)
for i in range(1, len(counts_int)):
if isinstance(counts_int[i][1], int):
counts_int[i][1] += counts_int[i - 1][1]
context['%s_counts_int' % name] = counts_int
standard_metrics = ["case", "form", "user"]
for m in standard_metrics:
_metric(m)
def _active_metric(name):
dates = {}
for result in get_db().view("hqadmin/%ss_over_time" % name,
group=True):
if not result or not result.has_key('key') or not result.has_key(
'value'):
continue
if result['key'][0] and int(result['key'][0]) >= 2010 and\
(int(result['key'][0]) < datetime.utcnow().year or
(int(result['key'][0]) == datetime.utcnow().year and
int(result['key'][1]) <= datetime.utcnow().month)):
date = _export_format(result) if as_export else _flot_format(
result)
if not date in dates:
dates[date] = set([result['key'][2]])
else:
dates[date].update([result['key'][2]])
datelist = [[date, dates[date]] for date in sorted(dates.keys())]
domainlist = [[x[0], len(x[1])] for x in datelist]
domainlist_int = deepcopy(datelist)
for i in range(1, len(domainlist_int)):
domainlist_int[i][1] = list(
set(domainlist_int[i - 1][1]).union(domainlist_int[i][1]))
domainlist_int = [[x[0], len(x[1])] for x in domainlist_int]
context['%s_counts' % name] = domainlist
context['%s_counts_int' % name] = domainlist_int
active_metrics = ["active_domain", "active_user"]
for a in active_metrics:
_active_metric(a)
if as_export:
all_metrics = standard_metrics + active_metrics
format = request.GET.get("format", "xls")
tables = []
for metric_name in all_metrics:
table = context.get('%s_counts' % metric_name, [])
table = [["%s" % item[0], "%d" % item[1]] for item in table]
table.reverse()
table.append(["date", "%s count" % metric_name])
table.reverse()
table_int = context.get('%s_counts_int' % metric_name, [])
table_int = [["%s" % item[0], "%d" % item[1]]
for item in table_int]
table_int.reverse()
table_int.append(["date", "%s count, cumulative" % metric_name])
table_int.reverse()
tables.append(["%s counts" % metric_name, table])
tables.append(["%s cumulative" % metric_name, table_int])
temp = StringIO()
export_from_tables(tables, temp, format)
return export_response(temp, format, "GlobalReport")
context['hide_filters'] = True
return render(request, template, context)
def get(self, request, domain, app_id):
language = request.GET.get('lang', 'en')
headers = [(self.app.name, tuple(APP_SUMMARY_EXPORT_HEADER_NAMES))]
data = [
(self.app.name,
[AppSummaryRow(
app=self.app.name,
comments=self.app.comment,
)])
]
for module in self.app.get_modules():
try:
case_list_filter = module.case_details.short.filter
except AttributeError:
case_list_filter = None
data += [(self.app.name, [
AppSummaryRow(app=self.app.name,
module=_get_translated_module_name(
self.app, module.unique_id, language),
display_filter=module.module_filter,
case_type=module.case_type,
case_list_filter=case_list_filter,
case_actions=module.case_details.short.filter
if hasattr(module, 'case_details') else None,
filter=module.module_filter,
module_type='advanced' if isinstance(
module, AdvancedModule) else 'standard',
comments=module.comment,
parent_module=(_get_translated_module_name(
self.app, module.root_module_id, language)
if module.root_module_id else ''))
])]
for form in module.get_forms():
post_form_workflow = form.post_form_workflow
if form.post_form_workflow == WORKFLOW_FORM:
post_form_workflow = u"form:\n{}".format(u"\n".join([
u"{form}: {xpath} [{datums}]".format(
form=_get_translated_form_name(
self.app, link.form_id, language),
xpath=link.xpath,
datums=u", ".join(
u"{}: {}".format(datum.name, datum.xpath)
for datum in link.datums))
for link in form.form_links
]))
data += [(self.app.name, [
AppSummaryRow(
app=self.app.name,
module=_get_translated_module_name(
self.app, module.unique_id, language),
form=_get_translated_form_name(self.app,
form.get_unique_id(),
language),
display_filter=form.form_filter,
case_type=form.get_case_type(),
case_actions=self._get_form_actions(form),
filter=form.form_filter,
module_type='advanced' if isinstance(
module, AdvancedModule) else 'standard',
comments=form.comment,
end_of_form_navigation=post_form_workflow,
)
])]
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
u'{app_name} v.{app_version} - App Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language),
)
def download_bulk_ui_translations(request, domain, app_id):
app = get_app(domain, app_id)
temp = build_ui_translation_download_file(app)
return export_response(temp, Format.XLS_2007, "translations")
def global_report(request, template="hqadmin/global.html", as_export=False):
def _flot_format(result):
return int(datetime(year=result["key"][0], month=result["key"][1], day=1).strftime("%s")) * 1000
def _export_format(result):
return datetime(year=result["key"][0], month=result["key"][1], day=1).strftime("%Y-%m-%d")
context = get_hqadmin_base_context(request)
def _metric(name):
counts = []
for result in get_db().view("hqadmin/%ss_over_time" % name, group_level=2):
if not result or not result.has_key("key") or not result.has_key("value"):
continue
if (
result["key"][0]
and int(result["key"][0]) >= 2010
and (
int(result["key"][0]) < datetime.utcnow().year
or (
int(result["key"][0]) == datetime.utcnow().year
and int(result["key"][1]) <= datetime.utcnow().month
)
)
):
counts.append([_export_format(result) if as_export else _flot_format(result), result["value"]])
context["%s_counts" % name] = counts
counts_int = deepcopy(counts)
for i in range(1, len(counts_int)):
if isinstance(counts_int[i][1], int):
counts_int[i][1] += counts_int[i - 1][1]
context["%s_counts_int" % name] = counts_int
standard_metrics = ["case", "form", "user"]
for m in standard_metrics:
_metric(m)
def _active_metric(name):
dates = {}
for result in get_db().view("hqadmin/%ss_over_time" % name, group=True):
if not result or not result.has_key("key") or not result.has_key("value"):
continue
if (
result["key"][0]
and int(result["key"][0]) >= 2010
and (
int(result["key"][0]) < datetime.utcnow().year
or (
int(result["key"][0]) == datetime.utcnow().year
and int(result["key"][1]) <= datetime.utcnow().month
)
)
):
date = _export_format(result) if as_export else _flot_format(result)
if not date in dates:
dates[date] = set([result["key"][2]])
else:
dates[date].update([result["key"][2]])
datelist = [[date, dates[date]] for date in sorted(dates.keys())]
domainlist = [[x[0], len(x[1])] for x in datelist]
domainlist_int = deepcopy(datelist)
for i in range(1, len(domainlist_int)):
domainlist_int[i][1] = list(set(domainlist_int[i - 1][1]).union(domainlist_int[i][1]))
domainlist_int = [[x[0], len(x[1])] for x in domainlist_int]
context["%s_counts" % name] = domainlist
context["%s_counts_int" % name] = domainlist_int
active_metrics = ["active_domain", "active_user"]
for a in active_metrics:
_active_metric(a)
if as_export:
all_metrics = standard_metrics + active_metrics
format = request.GET.get("format", "xls")
tables = []
for metric_name in all_metrics:
table = context.get("%s_counts" % metric_name, [])
table = [["%s" % item[0], "%d" % item[1]] for item in table]
table.reverse()
table.append(["date", "%s count" % metric_name])
table.reverse()
table_int = context.get("%s_counts_int" % metric_name, [])
table_int = [["%s" % item[0], "%d" % item[1]] for item in table_int]
table_int.reverse()
table_int.append(["date", "%s count, cumulative" % metric_name])
table_int.reverse()
tables.append(["%s counts" % metric_name, table])
tables.append(["%s cumulative" % metric_name, table_int])
temp = StringIO()
export_from_tables(tables, temp, format)
return export_response(temp, format, "GlobalReport")
context["hide_filters"] = True
return render(request, template, context)
def download_bulk_ui_translations(request, domain, app_id):
app = get_app(domain, app_id)
temp = build_ui_translation_download_file(app)
return export_response(temp, Format.XLS_2007, "translations")
def get(self, request, domain, app_id):
language = request.GET.get('lang', 'en')
headers = [(self.app.name, tuple(APP_SUMMARY_EXPORT_HEADER_NAMES))]
data = [(self.app.name, [
AppSummaryRow(
app=self.app.name,
comments=self.app.comment,
)
])]
for module in self.app.get_modules():
try:
case_list_filter = module.case_details.short.filter
except AttributeError:
case_list_filter = None
data += [
(self.app.name, [
AppSummaryRow(
app=self.app.name,
module=_get_translated_module_name(self.app, module.unique_id, language),
display_filter=module.module_filter,
case_type=module.case_type,
case_list_filter=case_list_filter,
case_actions=module.case_details.short.filter if hasattr(module, 'case_details') else None,
filter=module.module_filter,
module_type='advanced' if isinstance(module, AdvancedModule) else 'standard',
comments=module.comment,
parent_module=(_get_translated_module_name(self.app, module.root_module_id, language)
if module.root_module_id else '')
)
])
]
for form in module.get_forms():
post_form_workflow = form.post_form_workflow
if form.post_form_workflow == WORKFLOW_FORM:
post_form_workflow = "form:\n{}".format(
"\n".join(
["{form}: {xpath} [{datums}]".format(
form=_get_translated_form_name(self.app, link.form_id, language),
xpath=link.xpath,
datums=", ".join(
"{}: {}".format(
datum.name, datum.xpath
) for datum in link.datums)
) for link in form.form_links]
)
)
data += [
(self.app.name, [
AppSummaryRow(
app=self.app.name,
module=_get_translated_module_name(self.app, module.unique_id, language),
form=_get_translated_form_name(self.app, form.get_unique_id(), language),
display_filter=form.form_filter,
case_type=form.get_case_type(),
case_actions=self._get_form_actions(form),
filter=form.form_filter,
module_type='advanced' if isinstance(module, AdvancedModule) else 'standard',
comments=form.comment,
end_of_form_navigation=post_form_workflow,
)
])
]
export_string = io.BytesIO()
export_raw(tuple(headers), data, export_string, Format.XLS_2007),
return export_response(
export_string,
Format.XLS_2007,
'{app_name} v.{app_version} - App Summary ({lang})'.format(
app_name=self.app.name,
app_version=self.app.version,
lang=language
),
)
def _export_response_direct(self):
temp = io.BytesIO()
export_from_tables(self.export_table, temp, self.export_format)
return export_response(temp, self.export_format, self.export_name)
def to_export(self, format, location):
export_file = BytesIO()
excel_data = self.get_excel_data(location)
export_from_tables(excel_data, export_file, format)
return export_response(export_file, format, self.title)
def to_export(self, format, location):
export_file = BytesIO()
excel_data = self.get_excel_data(location)
export_from_tables(excel_data, export_file, format)
return export_response(export_file, format, self.title)
