def _get_data_source_properties_from_form(form, form_xml):
properties = OrderedDict()
questions = form_xml.get_questions([])
for question in questions:
properties[question['value']] = DataSourceProperty(
type="question",
id=question['value'],
column_id=get_column_name(question['value'].strip("/")),
text=question['label'],
source=question,
)
for prop in FORM_METADATA_PROPERTIES:
properties[prop[0]] = DataSourceProperty(
type="meta",
id=prop[0],
column_id=get_column_name(prop[0].strip("/")),
text=prop[0],
source=prop,
)
if form.get_app().auto_gps_capture:
properties['location'] = DataSourceProperty(
type="meta",
id='location',
column_id=get_column_name('location'),
text='location',
source=(['location', '#text'], 'Text'),
)
return properties
def test_column_trickery(self):
real = get_column_name("its/a/trap")
trap1 = get_column_name("its/a_trap")
trap2 = get_column_name("its_a/trap")
trap3 = get_column_name("its_a_trap")
trap4 = get_column_name("its/a/path/that/also/happens/to/be/a/bunch/longer/than/sixty/three/characters")
trap4_expected = 'ppens_to_be_a_bunch_longer_than_sixty_three_characters_6174b354'
self.assertNotEqual(real, trap1)
self.assertNotEqual(real, trap2)
self.assertNotEqual(real, trap3)
self.assertEqual(trap4, trap4_expected)
def _get_data_source_properties_from_case(cls, case_properties):
property_map = {
'closed': _('Case Closed'),
'user_id': _('User ID Last Updating Case'),
'owner_name': _('Case Owner'),
'mobile worker': _('Mobile Worker Last Updating Case'),
}
static_case_props = [
"closed",
"modified_on",
"name",
"opened_on",
"owner_id",
"user_id",
]
properties = OrderedDict()
for property in case_properties:
properties[property] = DataSourceProperty(
type='case_property',
id=property,
column_id=get_column_name(property),
text=property_map.get(property, property.replace('_', ' ')),
source=property,
is_non_numeric=property in static_case_props,
)
properties['computed/owner_name'] = cls._get_owner_name_pseudo_property()
properties['computed/user_name'] = cls._get_user_name_pseudo_property()
return properties
def _get_filter_and_agg_indicator(self):
"""
Return the data source indicator that will be used for filtering, or aggregating on this question (but not
display)
"""
column_id = get_column_name(self._property.strip("/"))
return make_form_question_indicator(self._question_source, column_id)
def _convert_closed_filter_to_choice_list(cls, filters):
for f in filters:
if f['field'] == get_column_name('closed') and f['type'] == 'dynamic_choice_list':
f['type'] = 'choice_list'
f['choices'] = [
{'value': 'True'},
{'value': 'False'}
]
def test_column_length_with_suffix(self):
long_path = ("its/a/path/that/also/happens/to/be/a/bunch/longer/than/"
"sixty/three/characters")
column_name = get_column_name(long_path, suffix="decimal")
self.assertEqual(len(column_name), 63)
self.assertEqual(
column_name,
'_be_a_bunch_longer_than_sixty_three_characters_6174b354_decimal',
)
def _get_choice_indicator(self):
"""
Return the data source indicator that will be used for displaying this question in the report (but not
filtering or aggregation)
"""
# column_id must match that used in _get_filter_and_agg_indicator() because the data will come from the
# same data source column
column_id = get_column_name(self._property.strip("/"))
return make_multiselect_question_indicator(self._question_source, column_id)
def _get_user_name_pseudo_property():
# user_name is a special pseudo case property for which
# the report builder will create a related_doc indicator based on the
# user_id of the case
return DataSourceProperty(
type='case_property',
id='computed/user_name',
column_id=get_column_name('computed/user_name'),
text='mobile worker',
source='computed/user_name',
)
def _get_owner_name_pseudo_property():
# owner_name is a special pseudo-case property for which
# the report builder will create a related_doc indicator based
# on the owner_id of the case.
return DataSourceProperty(
type='case_property',
id='computed/owner_name',
column_id=get_column_name('computed/owner_name'),
text='owner_name (computed)',
source='computed/owner_name'
)
def _export_item_to_ucr_indicator(export_item):
"""
Converts an ExportItem (from exports module) to a UCR indicator definition.
:param export_item:
:return: a dict ready to be inserted into a UCR data source
"""
if isinstance(export_item, CaseIndexItem):
# dereference indices
inner_expression = {
"type": "nested",
# this pulls out the entire CommCareCaseIndex object
"argument_expression": {
"type": "array_index",
"array_expression": {
# filter indices down to those with the correct case type
"type": "filter_items",
"items_expression": {
"type": "property_name",
"property_name": "indices"
},
"filter_expression": {
"type": "boolean_expression",
"expression": {
"type": "property_name",
"property_name": "referenced_type"
},
"operator": "eq",
"property_value": export_item.case_type
}
},
# return the first (assumes only 0 or 1)
"index_expression": {
"type": "constant",
"constant": 0
}
},
# this pulls out the referenced case ID from the object
"value_expression": {
"type": "property_name",
"property_name": "referenced_id"
}
}
else:
inner_expression = {
"type": "property_path",
'property_path': [p.name for p in export_item.path],
}
return {
"type": "expression",
"column_id": get_column_name(export_item.readable_path, add_hash=False),
"display_name": export_item.readable_path,
"datatype": export_item.datatype or 'string',
"expression": inner_expression,
}
def _get_data_source_properties_from_form(xform):
properties = OrderedDict()
questions = xform.get_questions([])
for question in questions:
properties[question['value']] = DataSourceProperty(
type="question",
id=question['value'],
column_id=get_column_name(question['value'].strip("/")),
text=question['label'],
source=question,
)
for prop in FORM_METADATA_PROPERTIES:
properties[prop[0]] = DataSourceProperty(
type="meta",
id=prop[0],
column_id=get_column_name(prop[0].strip("/")),
text=prop[0],
source=prop,
)
return properties
def _get_data_source_properties_from_case(cls, case_properties):
properties = OrderedDict()
for property in case_properties:
properties[property] = DataSourceProperty(
type='case_property',
id=property,
column_id=get_column_name(property),
text=property,
source=property
)
properties['computed/owner_name'] = cls._get_owner_name_pseudo_property()
return properties
def _get_user_name_pseudo_property():
# user_name is a special pseudo case property for which
# the report builder will create a related_doc indicator based on the
# user_id of the case
return DataSourceProperty(
type='case_property',
id='computed/user_name',
column_id=get_column_name('computed/user_name'),
text=_('Mobile Worker Last Updating Case'),
source='computed/user_name',
is_non_numeric=True,
)
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
# ui_aggregation parameter is never used because we need not infer the data type
# self._question_source is a tuple of (identifier, datatype)
identifier = self._meta_property_spec[0]
if isinstance(identifier, six.string_types):
soft_assert_type_text(identifier)
identifier = [identifier]
identifier = "/".join(identifier)
column_id = get_column_name(identifier.strip("/"))
return make_form_meta_block_indicator(
self._meta_property_spec, column_id, root_doc=is_multiselect_chart_report
)
def _deduplicate_columns_if_necessary(config):
try:
config.validate()
except DuplicateColumnIdError as e:
# deduplicate any columns by adding a hash of the full display_name/path
for indicator in config.configured_indicators:
if indicator['column_id'] in e.columns:
indicator['column_id'] = get_column_name(indicator['display_name'], add_hash=True)
# clear indicators cache, which is awkward with properties
DataSourceConfiguration.indicators.fget.reset_cache(config)
config.validate()
return config
def _get_data_source_properties_from_form(form, form_xml):
property_map = {
'username': _('User Name'),
'userID': _('User ID'),
'timeStart': _('Date Form Started'),
'timeEnd': _('Date Form Completed'),
}
properties = OrderedDict()
questions = form_xml.get_questions([])
for prop in FORM_METADATA_PROPERTIES:
properties[prop[0]] = DataSourceProperty(
type="meta",
id=prop[0],
column_id=get_column_name(prop[0].strip("/")),
text=property_map.get(prop[0], prop[0]),
source=prop,
is_non_numeric=True,
)
for question in questions:
properties[question['value']] = DataSourceProperty(
type="question",
id=question['value'],
column_id=get_column_name(question['value'].strip("/")),
text=question['label'],
source=question,
is_non_numeric=question['type'] not in ("DataBindOnly", "Int", "Double", "Long"),
)
if form.get_app().auto_gps_capture:
properties['location'] = DataSourceProperty(
type="meta",
id='location',
column_id=get_column_name('location'),
text='location',
source=(['location', '#text'], 'Text'),
is_non_numeric=True,
)
return properties
def get_indicator(self, aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property)
expression = {
'type': 'property_name',
'property_name': 'owner_id',
'datatype': 'string',
}
if is_multiselect_chart_report:
expression = {"type": "root_doc", "expression": expression}
return {
'datatype': 'string',
'type': 'expression',
'column_id': column_id,
'expression': expression
}
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property)
expression = {
'type': 'property_name',
'property_name': 'owner_id',
'datatype': 'string',
}
if is_multiselect_chart_report:
expression = {"type": "root_doc", "expression": expression}
return {
'datatype': 'string',
'type': 'expression',
'column_id': column_id,
'expression': expression
}
def _export_column_to_ucr_indicator(export_column):
"""
Converts an ExportColumn (from exports module) to a UCR indicator definition.
:param export_column:
:return: a dict ready to be inserted into a UCR data source
"""
return {
"type": "expression",
"column_id": get_column_name(export_column.label, add_hash=False),
"display_name": export_column.label,
"datatype": export_column.item.datatype or 'string',
"expression": {
"type": "property_path",
'property_path': [p.name for p in export_column.item.path],
}
}
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
"""
Return the report config snippet for the data source indicator for this form question column option
"""
assert ui_aggregation in UI_AGGREGATIONS, (
'"{}" is not an aggregation recognised by the Report Builder interface.'.format(ui_aggregation)
)
if ui_aggregation in (UI_AGG_SUM, UI_AGG_AVERAGE):
data_type = "decimal"
else:
data_type = None # use the default
column_id = get_column_name(self._property.strip("/"), suffix=data_type)
return make_form_question_indicator(
self._question_source, column_id, data_type, root_doc=is_multiselect_chart_report
)
def get_form_data_source(app, form):
xform = XForm(form.source)
form_name = form.default_name()
questions = xform.get_questions([])
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type="XFormInstance",
table_id=_clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns),
configured_indicators=[
make_form_question_indicator(q, column_id=get_column_name(q["value"])) for q in questions
]
+ [make_form_meta_block_indicator(field) for field in FORM_METADATA_PROPERTIES],
)
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
"""
Return the report config snippet for the data source indicator for this form question column option
"""
assert ui_aggregation in UI_AGGREGATIONS, (
'"{}" is not an aggregation recognised by the Report Builder interface.'.format(ui_aggregation)
)
if ui_aggregation in (UI_AGG_SUM, UI_AGG_AVERAGE):
data_type = "decimal"
else:
data_type = None # use the default
column_id = get_column_name(self._property.strip("/"), suffix=data_type)
return make_form_question_indicator(
self._question_source, column_id, data_type, root_doc=is_multiselect_chart_report
)
def get_indicator(self, aggregation, is_multiselect_chart_report=False):
if aggregation in ("Sum", "Avg"):
data_type = "decimal"
elif aggregation in ("sum", "avg"):
raise Exception(
"I think this should be Sum or Avg, where did you find this?..."
)
else:
data_type = None # use the default
column_id = get_column_name(self._property.strip("/"))
if data_type:
column_id += ("_" + data_type)
return make_form_question_indicator(
self._question_source,
column_id,
data_type,
root_doc=is_multiselect_chart_report)
def get_form_data_source(app, form):
xform = XForm(form.source)
form_name = form.default_name()
questions = xform.get_questions([])
return DataSourceConfiguration(
domain=app.domain,
referenced_doc_type='XFormInstance',
table_id=_clean_table_name(app.domain, form_name),
display_name=form_name,
configured_filter=make_form_data_source_filter(xform.data_node.tag_xmlns),
configured_indicators=[
make_form_question_indicator(q, column_id=get_column_name(q['value']))
for q in questions
] + [
make_form_meta_block_indicator(field)
for field in FORM_METADATA_PROPERTIES
],
)
def data_source_properties(self):
"""
A dictionary containing the various properties that may be used as indicators
or columns in the data source or report.
Keys are strings that uniquely identify properties.
Values are dicts representing the properties, ex:
>> self.data_source_properties
{
"/data/question1": {
"type": "question",
"id": "/data/question1",
"text": "Enter the child's name",
"column_id": "data--question1",
"source": {
'repeat': None,
'group': None,
'value': '/data/question1',
'label': 'question1',
'tag': 'input',
'type': 'Text'
}
},
"meta/deviceID": {
"type": "meta",
"id": "meta/deviceID",
"text": "deviceID",
"column_id": "meta--deviceID",
"source": ("deviceID", "string")
}
}
"id" is used as the value in selects/select2s in the form. Uniquely identifies questions.
"column_id" is used as the column name for this indicator. There are bugs
with slashes which requires this to be different from "id"
"text" will be used as the visible text in selects/select2s
"type" is "question", "case_property", or "meta"
For questions, "source" is the dict returned by Xform.get_questions, for
case properties and form metadata it is simply the name of the property.
"""
if self.source_type == 'case':
ret = OrderedDict((cp, {
'type': 'case_property',
'id': cp,
'column_id': get_column_name(cp),
'text': cp,
'source': cp
}) for cp in self.case_properties)
ret['computed/owner_name'] = {
'type': 'case_property',
'id': 'computed/owner_name',
'column_id': get_column_name('computed/owner_name'),
'text': 'owner_name (computed)',
'source': 'computed/owner_name'
}
return ret
# Note that owner_name is a special pseudo-case property.
# The report builder will create a related_doc indicator based
# on the owner_id of the case.
if self.source_type == 'form':
ret = OrderedDict()
questions = self.source_xform.get_questions([])
ret.update((q['value'], {
"type": "question",
"id": q['value'],
"column_id": get_column_name(q['value'].strip("/")),
'text': q['label'],
"source": q,
}) for q in questions)
ret.update((p[0], {
"type": "meta",
"id": p[0],
"column_id": get_column_name(p[0].strip("/")),
'text': p[0],
"source": p,
}) for p in FORM_METADATA_PROPERTIES)
return ret
def test_column_unicode(self):
name = get_column_name("Zouti_pou_travay_t\u00e8")
name_expected = 'Zouti_pou_travay_t\xe8_488e6086'
self.assertEqual(name, name_expected)
def get_indicator(self, aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property)
return make_owner_name_indicator(column_id)
def _convert_closed_filter_to_choice_list(cls, filters):
for f in filters:
if f['field'] == get_column_name(
'closed') and f['type'] == 'dynamic_choice_list':
f['type'] = 'choice_list'
f['choices'] = [{'value': 'True'}, {'value': 'False'}]
def test_column_name_suffix(self):
self.assertEqual(get_column_name("its/a/path", suffix="string"),
"its_a_path_dab095d5_string")
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property, suffix=self._get_datatype(ui_aggregation))
return make_case_property_indicator(self._property, column_id, datatype=self._get_datatype(ui_aggregation))
def data_source_properties(self):
"""
A dictionary containing the various properties that may be used as indicators
or columns in the data source or report.
Keys are strings that uniquely identify properties.
Values are dicts representing the properties, ex:
>> self.data_source_properties
{
"/data/question1": {
"type": "question",
"id": "/data/question1",
"text": "Enter the child's name",
"column_id": "data--question1",
"source": {
'repeat': None,
'group': None,
'value': '/data/question1',
'label': 'question1',
'tag': 'input',
'type': 'Text'
}
},
"meta/deviceID": {
"type": "meta",
"id": "meta/deviceID",
"text": "deviceID",
"column_id": "meta--deviceID",
"source": ("deviceID", "string")
}
}
"id" is used as the value in selects/select2s in the form. Uniquely identifies questions.
"column_id" is used as the column name for this indicator. There are bugs
with slashes which requires this to be different from "id"
"text" will be used as the visible text in selects/select2s
"type" is "question", "case_property", or "meta"
For questions, "source" is the dict returned by Xform.get_questions, for
case properties and form metadata it is simply the name of the property.
"""
if self.source_type == 'case':
return {
cp: {
'type': 'case_property',
'id': cp,
'column_id': get_column_name(cp),
'text': cp,
'source': cp
} for cp in self.case_properties
}
if self.source_type == 'form':
ret = {}
questions = self.source_xform.get_questions([])
ret.update({
q['value']: {
"type": "question",
"id": q['value'],
"column_id": get_column_name(q['value'].strip("/")),
'text': q['label'],
"source": q,
} for q in questions
})
ret.update({
p[0]: {
"type": "meta",
"id": p[0],
"column_id": get_column_name(p[0].strip("/")),
'text': p[0],
"source": p,
} for p in FORM_METADATA_PROPERTIES
})
return ret
def test_column_unicode(self):
name = get_column_name("Zouti_pou_travay_t\u00e8")
name_expected = 'Zouti_pou_travay_t\xe8_488e6086'
self.assertEqual(name, name_expected)
def test_column_name_suffix(self):
self.assertEqual(
get_column_name("its/a/path", suffix="string"),
"its_a_path_dab095d5_string"
)
def _get_indicator(self, ui_aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property, suffix=self._get_datatype(ui_aggregation))
return make_case_property_indicator(self._property, column_id, datatype=self._get_datatype(ui_aggregation))
def get_indicator(self, aggregation, is_multiselect_chart_report=False):
column_id = get_column_name(self._property) + "_" + self._get_datatype(aggregation)
return make_case_property_indicator(self._property, column_id, datatype=self._get_datatype(aggregation))
