def query_mongo(cls, username, query=None, fields=None, sort=None, start=0, limit=DEFAULT_LIMIT, count=False): query = dict_for_mongo(query) if query else {} query[cls.ACCOUNT] = username # TODO find better method # check for the created_on key in query and turn its values into dates if type(query) == dict and cls.CREATED_ON in query: if type(query[cls.CREATED_ON]) is dict: for op, val in query[cls.CREATED_ON].iteritems(): try: query[cls.CREATED_ON][op] = datetime.strptime( val, DATETIME_FORMAT) except ValueError: pass elif isinstance(query[cls.CREATED_ON], basestring): val = query[cls.CREATED_ON] try: created_on = datetime.strptime(val, DATETIME_FORMAT) except ValueError: pass else: # create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = { "$gte": start_time, "$lte": end_time } # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count": cursor.count()}] cursor.skip(max(start, 0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort_key = sort.keys()[0] # TODO: encode sort key if it has dots sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(_encode_for_mongo(sort_key), sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields = { "childrens_survey": { "tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")), "tel/tel.mobile": "tel/{0}".format(_encode_for_mongo("tel.mobile")), } } encoded_fields = export_builder.encoded_fields self.assertTrue("childrens_survey" in encoded_fields) self.assertEqual(encoded_fields["childrens_survey"], expected_encoded_fields["childrens_survey"])
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0, limit=DEFAULT_LIMIT, count=False): query = dict_for_mongo(query) if query else {} query[cls.ACCOUNT] = username # TODO find better method # check for the created_on key in query and turn its values into dates if type(query) == dict and cls.CREATED_ON in query: if type(query[cls.CREATED_ON]) is dict: for op, val in query[cls.CREATED_ON].iteritems(): try: query[cls.CREATED_ON][op] = datetime.strptime( val, DATETIME_FORMAT) except ValueError: pass elif isinstance(query[cls.CREATED_ON], basestring): val = query[cls.CREATED_ON] try: created_on = datetime.strptime(val, DATETIME_FORMAT) except ValueError: pass else: # create start and end times for the entire day start_time = created_on.replace(hour=0, minute=0, second=0, microsecond=0) end_time = start_time + timedelta(days=1) query[cls.CREATED_ON] = {"$gte": start_time, "$lte": end_time} # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query fields_to_select = None if type(fields) == list and len(fields) > 0: fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields]) cursor = audit.find(query, fields_to_select) if count: return [{"count": cursor.count()}] cursor.skip(max(start, 0)).limit(limit) if type(sort) == dict and len(sort) == 1: sort_key = sort.keys()[0] # TODO: encode sort key if it has dots sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(_encode_for_mongo(sort_key), sort_dir) # set batch size for cursor iteration cursor.batch_size = cls.DEFAULT_BATCHSIZE return cursor
def test_api_decode_from_mongo(self): field = "$section1.group01.question1" encoded = _encode_for_mongo(field) self.assertEqual(encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % \ {"dollar": base64.b64encode("$"), \ "dot": base64.b64encode(".")})) decoded = _decode_from_mongo(encoded) self.assertEqual(field, decoded)
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( _encode_for_mongo('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual(encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def test_api_decode_from_mongo(self): field = "$section1.group01.question1" encoded = _encode_for_mongo(field) self.assertEqual(encoded, ( "%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % { "dollar": base64.b64encode("$"), "dot": base64.b64encode(".")})) decoded = _decode_from_mongo(encoded) self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self): """ Return a dictionary of fieldnames as saved in mongodb with corresponding xform field names e.g {"Q1Lg==1": "Q1.1"} """ names = {} for elem in self.get_survey_elements(): names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \ elem.get_abbreviated_xpath() return names
def get_mongo_field_names_dict(self): """ Return a dictionary of fieldnames as saved in mongodb with corresponding xform field names e.g {"Q1Lg==1": "Q1.1"} """ names = {} for elem in self.get_survey_elements(): names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \ elem.get_abbreviated_xpath() return names
def test_generation_of_mongo_encoded_fields_works(self): survey = self._create_childrens_survey() export_builder = ExportBuilder() export_builder.set_survey(survey) expected_encoded_fields =\ { 'childrens_survey': { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')), 'tel/tel.mobile': 'tel/{0}'.format( _encode_for_mongo('tel.mobile')), } } encoded_fields = export_builder.encoded_fields self.assertTrue('childrens_survey' in encoded_fields) self.assertEqual( encoded_fields['childrens_survey'], expected_encoded_fields['childrens_survey'])
def test_decode_fields_names_encoded_for_mongo(self): encoded_fields = \ { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')) } row = \ { 'name': 'Abe', 'age': 35, 'tel/{0}'.format( _encode_for_mongo('tel.office')): '123-456-789' } new_row = ExportBuilder.decode_mongo_encoded_fields( row, encoded_fields) expected_row = \ { 'name': 'Abe', 'age': 35, 'tel/tel.office': '123-456-789' } self.assertEqual(new_row, expected_row)
def test_decode_fields_names_encoded_for_mongo(self): encoded_fields = \ { 'tel/tel.office': 'tel/{0}'.format( _encode_for_mongo('tel.office')) } row = \ { 'name': 'Abe', 'age': 35, 'tel/{0}'.format( _encode_for_mongo('tel.office')): '123-456-789' } new_row = ExportBuilder.decode_mongo_encoded_fields( row, encoded_fields) expected_row = \ { 'name': 'Abe', 'age': 35, 'tel/tel.office': '123-456-789' } self.assertEqual(new_row, expected_row)
def build_sections( current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': []} self.sections.append(section) build_sections( section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections( current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: _encode_for_mongo(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( _xpath, field_delimiter) choice = { 'title': _title, 'xpath': _xpath, 'type': 'string' } if choice not in current_section['elements']: current_section['elements'].append(choice) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [c.get_abbreviated_xpath() for c in child.children]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend( [ { 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths ]) _append_xpaths_to_section( current_section_name, gps_fields, child.get_abbreviated_xpath(), xpaths)
def build_sections(current_section, survey_element, sections, select_multiples, gps_fields, encoded_fields, field_delimiter='/'): for child in survey_element.children: current_section_name = current_section['name'] # if a section, recurs if isinstance(child, Section): # if its repeating, build a new section if isinstance(child, RepeatingSection): # section_name in recursive call changes section = { 'name': child.get_abbreviated_xpath(), 'elements': [] } self.sections.append(section) build_sections(section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) else: # its a group, recurs using the same section build_sections(current_section, child, sections, select_multiples, gps_fields, encoded_fields, field_delimiter) elif isinstance(child, Question) and child.bind.get(u"type")\ not in QUESTION_TYPES_TO_EXCLUDE: # add to survey_sections if isinstance(child, Question): child_xpath = child.get_abbreviated_xpath() current_section['elements'].append({ 'title': ExportBuilder.format_field_title( child.get_abbreviated_xpath(), field_delimiter), 'xpath': child_xpath, 'type': child.bind.get(u"type") }) if _is_invalid_for_mongo(child_xpath): if current_section_name not in encoded_fields: encoded_fields[current_section_name] = {} encoded_fields[current_section_name].update( {child_xpath: _encode_for_mongo(child_xpath)}) # if its a select multiple, make columns out of its choices if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\ and self.SPLIT_SELECT_MULTIPLES: for c in child.children: _xpath = c.get_abbreviated_xpath() _title = ExportBuilder.format_field_title( _xpath, field_delimiter) choice = { 'title': _title, 'xpath': _xpath, 'type': 'string' } if choice not in current_section['elements']: current_section['elements'].append(choice) _append_xpaths_to_section( current_section_name, select_multiples, child.get_abbreviated_xpath(), [ c.get_abbreviated_xpath() for c in child.children ]) # split gps fields within this section if child.bind.get(u"type") == GEOPOINT_BIND_TYPE: # add columns for geopoint components xpaths = DataDictionary.get_additional_geopoint_xpaths( child.get_abbreviated_xpath()) current_section['elements'].extend([{ 'title': ExportBuilder.format_field_title( xpath, field_delimiter), 'xpath': xpath, 'type': 'decimal' } for xpath in xpaths]) _append_xpaths_to_section( current_section_name, gps_fields, child.get_abbreviated_xpath(), xpaths)
def query_mongo(username, id_string, query, fields, sort, start=0, limit=DEFAULT_LIMIT, count=False, hide_deleted=True, fs_uuid=None, fs_project_uuid=None, site_id=None): USERFORM_ID = u'_userform_id' STATUS = u'_status' DEFAULT_BATCHSIZE = 1000 fields_to_select = {USERFORM_ID: 0} # TODO: give more detailed error messages to 3rd parties # using the API when json.loads fails if isinstance(query, basestring): query = json.loads(query, object_hook=json_util.object_hook) query = query if query else {} query = dict_for_mongo(query) query[USERFORM_ID] = u'%s_%s' % (username, id_string) # check if query contains and _id and if its a valid ObjectID # if '_uuid' in query and ObjectId.is_valid(query['_uuid']): # query['_uuid'] = ObjectId(query['_uuid']) query.pop('_userform_id') if fs_uuid is not None: query = { "$and": [ query, { "$or": [{ "_uuid": fs_uuid }, { "fs_uuid": fs_uuid }, { "_uuid": str(fs_uuid) }, { "fs_uuid": str(fs_uuid) }] } ] } # query['_uuid'] = { '$in': [fs_uuid, str(fs_uuid)] } #fs_uuid if fs_project_uuid is not None: if site_id is None: query['fs_project_uuid'] = { '$in': [fs_project_uuid, str(fs_project_uuid)] } #fs_project_uuid elif site_id and count: query['fs_project_uuid'] = { '$in': [fs_project_uuid, str(fs_project_uuid)] } #query = query.update({'fs_project_uuid': {'$in': [fs_project_uuid, str(fs_project_uuid)] }, 'fs_site': { '$in': [site_id, str(site_id)] }}) elif site_id: query['fs_project_uuid'] = { '$in': [fs_project_uuid, str(fs_project_uuid)] } # if hide_deleted: # # display only active elements # # join existing query with deleted_at_query on an $and # query = {"$and": [query, {"_deleted_at": None}]} # fields must be a string array i.e. '["name", "age"]' if isinstance(fields, basestring): fields = json.loads(fields, object_hook=json_util.object_hook) fields = fields if fields else [] # TODO: current mongo (2.0.4 of this writing) # cant mix including and excluding fields in a single query if type(fields) == list and len(fields) > 0: fields_to_select = dict([(_encode_for_mongo(field), 1) for field in fields]) if isinstance(sort, basestring): sort = json.loads(sort, object_hook=json_util.object_hook) sort = sort if sort else {} # if fs_uuid is not None: # cursor = xform_instances.find({"$or":[ {"_uuid":fs_uuid}, {"fs_uuid":fs_uuid}, {"_uuid":str(fs_uuid)}, {"fs_uuid":str(fs_uuid)}]}, fields_to_select) # else: # cursor = xform_instances.find({"$or":[ {"fs_project_uuid":fs_project_uuid}, {"fs_project_uuid": str(fs_project_uuid)}]}, fields_to_select) cursor = xform_instances.find(query, fields_to_select) if count: return [{"count": cursor.count()}] if start < 0 or limit < 0: raise ValueError(_("Invalid start/limit params")) cursor.skip(start).limit(limit) if type(sort) == dict and len(sort) == 1: sort_key = sort.keys()[0] # TODO: encode sort key if it has dots sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc cursor.sort(_encode_for_mongo(sort_key), sort_dir) # set batch size cursor.batch_size = DEFAULT_BATCHSIZE return cursor
def test_decode_fields_names_encoded_for_mongo(self): encoded_fields = {"tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office"))} row = {"name": "Abe", "age": 35, "tel/{0}".format(_encode_for_mongo("tel.office")): "123-456-789"} new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields) expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"} self.assertEqual(new_row, expected_row)