def _get_mongo_cursor(cls, query, fields):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, string_types):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (3.4 of this writing)
# cannot mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
return xform_instances.find(
query,
fields_to_select,
max_time_ms=settings.MONGO_DB_MAX_TIME_MS,
)
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
MongoHelper.encode('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def query_mongo(cls,
username,
query=None,
fields=None,
sort=None,
start=0,
limit=DEFAULT_LIMIT,
count=False):
query = MongoHelper.to_safe_dict(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {
"$gte": start_time,
"$lte": end_time
}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = MongoHelper.encode(field)
self.assertEqual(encoded, (
"%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % {
"dollar": base64.b64encode("$"),
"dot": base64.b64encode(".")}))
decoded = MongoHelper.decode(encoded)
self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[MongoHelper.encode(text_type(elem.get_abbreviated_xpath()))] = \
elem.get_abbreviated_xpath()
return names
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
MongoHelper.encode('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(
encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = MongoHelper.encode(field)
self.assertEqual(
encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % {
"dollar": base64.b64encode("$"),
"dot": base64.b64encode(".")
}))
decoded = MongoHelper.decode(encoded)
self.assertEqual(field, decoded)
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
MongoHelper.encode('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
MongoHelper.encode('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def _get_mongo_cursor(cls,
query,
fields,
hide_deleted,
username=None,
id_string=None):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:param hide_deleted: boolean
:param username: string
:param id_string: string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, basestring):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = MongoHelper.to_safe_dict(query, reading=True)
if username and id_string:
query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string)
# check if query contains and _id and if its a valid ObjectID
if '_uuid' in query and ObjectId.is_valid(query['_uuid']):
query['_uuid'] = ObjectId(query['_uuid'])
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
return xform_instances.find(query, fields_to_select)
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0,
limit=DEFAULT_LIMIT, count=False):
query = MongoHelper.to_safe_dict(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0, minute=0,
second=0, microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {"$gte": start_time,
"$lte": end_time}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string=None):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:param hide_deleted: boolean
:param username: string
:param id_string: string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, basestring):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = MongoHelper.to_safe_dict(query, reading=True)
if username and id_string:
query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string)
# check if query contains and _id and if its a valid ObjectID
if '_uuid' in query and ObjectId.is_valid(query['_uuid']):
query['_uuid'] = ObjectId(query['_uuid'])
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict(
[(MongoHelper.encode(field), 1) for field in fields])
return xform_instances.find(query, fields_to_select)
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath()
for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)