def _split_gps_fields(cls, record, gps_fields):
updated_gps_fields = {}
for key, value in record.iteritems():
if key in gps_fields and isinstance(value, basestring):
gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key)
gps_parts = dict([(xpath, None) for xpath in gps_xpaths])
# hack, check if its a list and grab the object within that
parts = value.split(' ')
# TODO: check whether or not we can have a gps recording
# from ODKCollect that has less than four components,
# for now we are assuming that this is not the case.
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
updated_gps_fields.update(gps_parts)
# check for repeats within record i.e. in value
elif type(value) == list:
for list_item in value:
if type(list_item) == dict:
cls._split_gps_fields(list_item, gps_fields)
record.update(updated_gps_fields)
def _split_gps_fields(cls, record, gps_fields):
updated_gps_fields = {}
for key, value in record.iteritems():
if key in gps_fields and isinstance(value, basestring):
gps_xpaths = DataDictionary.get_additional_geopoint_xpaths(key)
gps_parts = dict([(xpath, None) for xpath in gps_xpaths])
# hack, check if its a list and grab the object within that
parts = value.split(' ')
# TODO: check whether or not we can have a gps recording
# from ODKCollect that has less than four components,
# for now we are assuming that this is not the case.
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
updated_gps_fields.update(gps_parts)
# check for repeats within record i.e. in value
elif type(value) == list:
for list_item in value:
if type(list_item) == dict:
cls._split_gps_fields(list_item, gps_fields)
record.update(updated_gps_fields)
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath()
for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/',
remove_group_name=False):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter,
remove_group_name)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter,
remove_group_name)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
child.get_abbreviated_xpath(), field_delimiter, dd,
remove_group_name)
_label = \
dd.get_label(child_xpath, elem=child) or _title
current_section['elements'].append({
'label':
_label,
'title':
_title,
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
choices = self._get_select_mulitples_choices(
child, dd, field_delimiter, remove_group_name)
for choice in choices:
if choice not in current_section['elements']:
current_section['elements'].append(choice)
choices_xpaths = [c['xpath'] for c in choices]
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), choices_xpaths)
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
for xpath in xpaths:
_title = ExportBuilder.format_field_title(
xpath, field_delimiter, dd, remove_group_name)
current_section['elements'].append({
'label':
_title,
'title':
_title,
'xpath':
xpath,
'type':
'decimal'
})
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def process_tableau_data(data, xform):
"""
Streamlines the row header fields
with the column header fields for the same form.
Handles Flattenning repeat data for tableau
"""
def get_xpath(key, nested_key):
val = nested_key.split('/')
nested_key_diff = val[len(key.split('/')):]
xpaths = key + f'[{index}]/' + '/'.join(nested_key_diff)
return xpaths
def get_updated_data_dict(key, value, data_dict):
"""
Generates key, value pairs for select multiple question types.
Defining the new xpaths from the
question name(key) and the choice name(value)
in accordance with how we generate the tableau schema.
"""
if isinstance(value, str) and data_dict:
choices = value.split(" ")
for choice in choices:
xpaths = f'{key}/{choice}'
data_dict[xpaths] = choice
elif isinstance(value, list):
try:
for item in value:
for (nested_key, nested_val) in item.items():
xpath = get_xpath(key, nested_key)
data_dict[xpath] = nested_val
except AttributeError:
data_dict[key] = value
return data_dict
def get_ordered_repeat_value(key, item, index):
"""
Return Ordered Dict of repeats in the order in which they appear in
the XForm.
"""
children = xform.get_child_elements(key, split_select_multiples=False)
item_list = OrderedDict()
data = {}
for elem in children:
if not question_types_to_exclude(elem.type):
new_xpath = elem.get_abbreviated_xpath()
item_list[new_xpath] = item.get(new_xpath, DEFAULT_NA_REP)
# Loop through repeat data and flatten it
# given the key "children/details" and nested_key/
# abbreviated xpath "children/details/immunization/polio_1",
# generate ["children", index, "immunization/polio_1"]
for (nested_key, nested_val) in item_list.items():
qstn_type = xform.get_element(nested_key).type
xpaths = get_xpath(key, nested_key)
if qstn_type == MULTIPLE_SELECT_TYPE:
data = get_updated_data_dict(xpaths, nested_val, data)
elif qstn_type == REPEAT_SELECT_TYPE:
data = get_updated_data_dict(xpaths, nested_val, data)
else:
data[xpaths] = nested_val
return data
result = []
if data:
headers = xform.get_headers()
tableau_headers = remove_metadata_fields(headers)
for row in data:
diff = set(tableau_headers).difference(set(row))
flat_dict = dict.fromkeys(diff, None)
for (key, value) in row.items():
if isinstance(value, list) and key not in [
ATTACHMENTS, NOTES, GEOLOCATION
]:
for index, item in enumerate(value, start=1):
# order repeat according to xform order
item = get_ordered_repeat_value(key, item, index)
flat_dict.update(item)
else:
try:
qstn_type = xform.get_element(key).type
if qstn_type == MULTIPLE_SELECT_TYPE:
flat_dict = get_updated_data_dict(
key, value, flat_dict)
if qstn_type == 'geopoint':
parts = value.split(' ')
gps_xpaths = \
DataDictionary.get_additional_geopoint_xpaths(
key)
gps_parts = dict([(xpath, None)
for xpath in gps_xpaths])
if len(parts) == 4:
gps_parts = dict(zip(gps_xpaths, parts))
flat_dict.update(gps_parts)
else:
flat_dict[key] = value
except AttributeError:
flat_dict[key] = value
result.append(flat_dict)
return result
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
select_one,
select_label_map,
field_delimiter='/'):
print "survey_element.children"
print survey_element.children
print "current_section"
print current_section
for child in survey_element.children:
print "survey_element.child"
print child
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, select_one,
select_label_map, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, select_one,
select_label_map, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
# print('child bind type: ' ,child.bind.get(u"type"))
if child.bind.get(
u"type"
) == SINGLE_SELECT_BIND_TYPE and self.SHOW_LABEL:
# print ('get select 1 type question :D ')
for c in child.children:
_xpath = c.get_abbreviated_xpath()
if not self.SHOW_LABEL:
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
else:
_title = c.label
_title_custom = c.label
select_label_map[_xpath] = _title_custom
# _xpath = c.get_abbreviated_xpath()
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string',
'label': _title
}
# if choice not in current_section['elements']:current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_one,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type"),
'label':
child.label
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
_title_custom = c.label
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string',
'label': _title_custom
}
if self.SHOW_LABEL:
select_label_map[_xpath] = _title_custom
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)