def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = _encode_for_mongo(field)
self.assertEqual(encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % \
{"dollar": base64.b64encode("$"), \
"dot": base64.b64encode(".")}))
decoded = _decode_from_mongo(encoded)
self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = elem.get_abbreviated_xpath()
return names
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \
elem.get_abbreviated_xpath()
return names
def get_form_submissions_grouped_by_field(xform, field, name=None):
"""Number of submissions grouped by field"""
query = {}
mongo_field = _encode_for_mongo(field)
query[ParsedInstance.USERFORM_ID] =\
u'%s_%s' % (xform.user.username, xform.id_string)
query[mongo_field] = {"$exists": True}
# check if requested field a datetime str
record = xform_instances.find_one(query, {mongo_field: 1})
if not record:
raise ValueError(_(u"Field '%s' does not exist." % field))
group = {"count": {"$sum": 1}}
group["_id"] = _get_id_for_type(record, mongo_field)
field_name = field if name is None else name
pipeline = [
{
"$group": group
},
{
"$sort": {"_id": 1}
},
{
"$project": {
field_name: "$_id",
"count": 1
}
}
]
kargs = {
'query': query,
'pipeline': pipeline
}
records = ParsedInstance.mongo_aggregate(**kargs)
# delete mongodb's _id field from records
# TODO: is there an elegant way to do this? should we remove the field?
for record in records:
del record['_id']
return records
try:
created_on = datetime.strptime(val,
DATETIME_FORMAT)
except ValueError, e:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0, minute=0,
second=0, microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {"$gte": start_time,
"$lte": end_time}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count":cursor.count()}]
cursor.skip(max(start,0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort_key = sort.keys()[0]
#todo: encode sort key if it has dots
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(_encode_for_mongo(sort_key), sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(), field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
current_section['elements'].extend(
[{
'title': ExportBuilder.format_field_title(
c.get_abbreviated_xpath(),
field_delimiter),
'xpath': c.get_abbreviated_xpath(),
'type': 'string'
}
for c in child.children])
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath() for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name,gps_fields,
child.get_abbreviated_xpath(), xpaths)
# create start and end times for the entire day
start_time = created_on.replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {
"$gte": start_time,
"$lte": end_time
}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(_encode_for_mongo(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort_key = sort.keys()[0]
#todo: encode sort key if it has dots
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(_encode_for_mongo(sort_key), sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
c.get_abbreviated_xpath(), field_delimiter),
'xpath':
c.get_abbreviated_xpath(),
'type':
'string'
} for c in child.children])
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
