def _get_mongo_cursor(cls, query, fields): """ Returns a Mongo cursor based on the query. :param query: JSON string :param fields: Array string :return: pymongo Cursor """ fields_to_select = {cls.USERFORM_ID: 0} # fields must be a string array i.e. '["name", "age"]' if isinstance(fields, string_types): fields = json.loads(fields, object_hook=json_util.object_hook) fields = fields if fields else [] # TODO: current mongo (3.4 of this writing) # cannot mix including and excluding fields in a single query if type(fields) == list and len(fields) > 0: fields_to_select = dict([(MongoHelper.encode(field), 1) for field in fields]) return xform_instances.find( query, fields_to_select, max_time_ms=settings.MONGO_DB_MAX_TIME_MS, )
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( MongoHelper.encode('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( MongoHelper.encode('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual(encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0, limit=DEFAULT_LIMIT, count=False): query = MongoHelper.to_safe_dict(query) if query else {} query[cls.ACCOUNT] = username # TODO find better method # check for the created_on key in query and turn its values into dates if type(query) == dict and cls.CREATED_ON in query: if type(query[cls.CREATED_ON]) is dict: for op, val in query[cls.CREATED_ON].iteritems(): try: query[cls.CREATED_ON][op] = datetime.strptime( val, DATETIME_FORMAT) except ValueError: pass elif isinstance(query[cls.CREATED_ON], basestring): val = query[cls.CREATED_ON] try: created_on = datetime.strptime(val, DATETIME_FORMAT) except ValueError: pass else: # create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = { "$gte": start_time, "$lte": end_time } # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(MongoHelper.encode(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count": cursor.count()}] cursor.skip(max(start, 0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort = MongoHelper.to_safe_dict(sort, reading=True) sort_key = sort.keys()[0] sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(sort_key, sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def test_api_decode_from_mongo(self): field = "$section1.group01.question1" encoded = MongoHelper.encode(field) self.assertEqual(encoded, ( "%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % { "dollar": base64.b64encode("$"), "dot": base64.b64encode(".")})) decoded = MongoHelper.decode(encoded) self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self): """ Return a dictionary of fieldnames as saved in mongodb with corresponding xform field names e.g {"Q1Lg==1": "Q1.1"} """ names = {} for elem in self.get_survey_elements(): names[MongoHelper.encode(text_type(elem.get_abbreviated_xpath()))] = \ elem.get_abbreviated_xpath() return names
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( MongoHelper.encode('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( MongoHelper.encode('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual( encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def test_api_decode_from_mongo(self): field = "$section1.group01.question1" encoded = MongoHelper.encode(field) self.assertEqual( encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % { "dollar": base64.b64encode("$"), "dot": base64.b64encode(".") })) decoded = MongoHelper.decode(encoded) self.assertEqual(field, decoded)
def test_decode_fields_names_encoded_for_mongo(self): encoded_fields = \ { 'tel/tel.office': 'tel/{0}'.format( MongoHelper.encode('tel.office')) } row = \ { 'name': 'Abe', 'age': 35, 'tel/{0}'.format( MongoHelper.encode('tel.office')): '123-456-789' } new_row = ExportBuilder.decode_mongo_encoded_fields( row, encoded_fields) expected_row = \ { 'name': 'Abe', 'age': 35, 'tel/tel.office': '123-456-789' } self.assertEqual(new_row, expected_row)
def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string=None): """ Returns a Mongo cursor based on the query. :param query: JSON string :param fields: Array string :param hide_deleted: boolean :param username: string :param id_string: string :return: pymongo Cursor """ fields_to_select = {cls.USERFORM_ID: 0} # TODO: give more detailed error messages to 3rd parties # using the API when json.loads fails if isinstance(query, basestring): query = json.loads(query, object_hook=json_util.object_hook) query = query if query else {} query = MongoHelper.to_safe_dict(query, reading=True) if username and id_string: query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string) # check if query contains and _id and if its a valid ObjectID if '_uuid' in query and ObjectId.is_valid(query['_uuid']): query['_uuid'] = ObjectId(query['_uuid']) if hide_deleted: # display only active elements # join existing query with deleted_at_query on an $and query = {"$and": [query, {"_deleted_at": None}]} # fields must be a string array i.e. '["name", "age"]' if isinstance(fields, basestring): fields = json.loads(fields, object_hook=json_util.object_hook) fields = fields if fields else [] # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query if type(fields) == list and len(fields) > 0: fields_to_select = dict([(MongoHelper.encode(field), 1) for field in fields]) return xform_instances.find(query, fields_to_select)
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0, limit=DEFAULT_LIMIT, count=False): query = MongoHelper.to_safe_dict(query) if query else {} query[cls.ACCOUNT] = username # TODO find better method # check for the created_on key in query and turn its values into dates if type(query) == dict and cls.CREATED_ON in query: if type(query[cls.CREATED_ON]) is dict: for op, val in query[cls.CREATED_ON].iteritems(): try: query[cls.CREATED_ON][op] = datetime.strptime( val, DATETIME_FORMAT) except ValueError: pass elif isinstance(query[cls.CREATED_ON], basestring): val = query[cls.CREATED_ON] try: created_on = datetime.strptime(val, DATETIME_FORMAT) except ValueError: pass else: # create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = {"$gte": start_time, "$lte": end_time} # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(MongoHelper.encode(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count": cursor.count()}] cursor.skip(max(start, 0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort = MongoHelper.to_safe_dict(sort, reading=True) sort_key = sort.keys()[0] sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(sort_key, sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string=None): """ Returns a Mongo cursor based on the query. :param query: JSON string :param fields: Array string :param hide_deleted: boolean :param username: string :param id_string: string :return: pymongo Cursor """ fields_to_select = {cls.USERFORM_ID: 0} # TODO: give more detailed error messages to 3rd parties # using the API when json.loads fails if isinstance(query, basestring): query = json.loads(query, object_hook=json_util.object_hook) query = query if query else {} query = MongoHelper.to_safe_dict(query, reading=True) if username and id_string: query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string) # check if query contains and _id and if its a valid ObjectID if '_uuid' in query and ObjectId.is_valid(query['_uuid']): query['_uuid'] = ObjectId(query['_uuid']) if hide_deleted: # display only active elements # join existing query with deleted_at_query on an $and query = {"$and": [query, {"_deleted_at": None}]} # fields must be a string array i.e. '["name", "age"]' if isinstance(fields, basestring): fields = json.loads(fields, object_hook=json_util.object_hook) fields = fields if fields else [] # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query if type(fields) == list and len(fields) > 0: fields_to_select = dict( [(MongoHelper.encode(field), 1) for field in fields]) return xform_instances.find(query, fields_to_select)
def build_sections(current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': [] } self.sections.append(section) build_sections(section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections(current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if MongoHelper.is_attribute_invalid(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: MongoHelper.encode(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( _xpath, field_delimiter) choice = { 'title': _title, 'xpath': _xpath, 'type': 'string' } if choice not in current_section['elements']: current_section['elements'].append(choice) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [ c.get_abbreviated_xpath() for c in child.children ]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend([{ 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths]) _append_xpaths_to_section( current_section_name, gps_fields, child.get_abbreviated_xpath(), xpaths)
def build_sections( current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': []} self.sections.append(section) build_sections( section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections( current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if MongoHelper.is_attribute_invalid(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: MongoHelper.encode(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( _xpath, field_delimiter) choice = { 'title': _title, 'xpath': _xpath, 'type': 'string' } if choice not in current_section['elements']: current_section['elements'].append(choice) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [c.get_abbreviated_xpath() for c in child.children]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend( [ { 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths ]) _append_xpaths_to_section( current_section_name, gps_fields, child.get_abbreviated_xpath(), xpaths)