def test_api_decode_from_mongo(self): field = "$section1.group01.question1" encoded = _encode_for_mongo(field) self.assertEqual(encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % \ {"dollar": base64.b64encode("$"), \ "dot": base64.b64encode(".")})) decoded = _decode_from_mongo(encoded) self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self): """ Return a dictionary of fieldnames as saved in mongodb with corresponding xform field names e.g {"Q1Lg==1": "Q1.1"} """ names = {} for elem in self.get_survey_elements(): names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = elem.get_abbreviated_xpath() return names
def get_mongo_field_names_dict(self): """ Return a dictionary of fieldnames as saved in mongodb with corresponding xform field names e.g {"Q1Lg==1": "Q1.1"} """ names = {} for elem in self.get_survey_elements(): names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \ elem.get_abbreviated_xpath() return names
def get_form_submissions_grouped_by_field(xform, field, name=None): """Number of submissions grouped by field""" query = {} mongo_field = _encode_for_mongo(field) query[ParsedInstance.USERFORM_ID] =\ u'%s_%s' % (xform.user.username, xform.id_string) query[mongo_field] = {"$exists": True} # check if requested field a datetime str record = xform_instances.find_one(query, {mongo_field: 1}) if not record: raise ValueError(_(u"Field '%s' does not exist." % field)) group = {"count": {"$sum": 1}} group["_id"] = _get_id_for_type(record, mongo_field) field_name = field if name is None else name pipeline = [ { "$group": group }, { "$sort": {"_id": 1} }, { "$project": { field_name: "$_id", "count": 1 } } ] kargs = { 'query': query, 'pipeline': pipeline } records = ParsedInstance.mongo_aggregate(**kargs) # delete mongodb's _id field from records # TODO: is there an elegant way to do this? should we remove the field? for record in records: del record['_id'] return records
try: created_on = datetime.strptime(val, DATETIME_FORMAT) except ValueError, e: pass else: # create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = {"$gte": start_time, "$lte": end_time} # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count":cursor.count()}] cursor.skip(max(start,0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort_key = sort.keys()[0] #todo: encode sort key if it has dots sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(_encode_for_mongo(sort_key), sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def build_sections( current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': []} self.sections.append(section) build_sections( section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections( current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: _encode_for_mongo(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: current_section['elements'].extend( [{ 'title': ExportBuilder.format_field_title( c.get_abbreviated_xpath(), field_delimiter), 'xpath': c.get_abbreviated_xpath(), 'type': 'string' } for c in child.children]) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [c.get_abbreviated_xpath() for c in child.children]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend( [ { 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths ]) _append_xpaths_to_section( current_section_name,gps_fields, child.get_abbreviated_xpath(), xpaths)
# create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = { "$gte": start_time, "$lte": end_time } # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count": cursor.count()}] cursor.skip(max(start, 0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort_key = sort.keys()[0] #todo: encode sort key if it has dots sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(_encode_for_mongo(sort_key), sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def build_sections(current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': [] } self.sections.append(section) build_sections(section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections(current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: _encode_for_mongo(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: current_section['elements'].extend([{ 'title': ExportBuilder.format_field_title( c.get_abbreviated_xpath(), field_delimiter), 'xpath': c.get_abbreviated_xpath(), 'type': 'string' } for c in child.children]) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [ c.get_abbreviated_xpath() for c in child.children ]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend([{ 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths]) _append_xpaths_to_section( current_section_name, gps_fields, child.get_abbreviated_xpath(), xpaths)