def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = MongoHelper.encode(field)
self.assertEqual(encoded, (
"%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % {
"dollar": base64.b64encode("$"),
"dot": base64.b64encode(".")}))
decoded = MongoHelper.decode(encoded)
self.assertEqual(field, decoded)
def query_mongo(cls,
username,
query=None,
fields=None,
sort=None,
start=0,
limit=DEFAULT_LIMIT,
count=False):
query = MongoHelper.to_safe_dict(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {
"$gte": start_time,
"$lte": end_time
}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = MongoHelper.encode(field)
self.assertEqual(
encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % {
"dollar": base64.b64encode("$"),
"dot": base64.b64encode(".")
}))
decoded = MongoHelper.decode(encoded)
self.assertEqual(field, decoded)
def _get_mongo_cursor(cls, query, fields):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, string_types):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (3.4 of this writing)
# cannot mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
return xform_instances.find(
query,
fields_to_select,
max_time_ms=settings.MONGO_DB_MAX_TIME_MS,
)
def mongo_aggregate(cls, query, pipeline, hide_deleted=True):
"""Perform mongo aggregate queries
query - is a dict which is to be passed to $match, a pipeline operator
pipeline - list of dicts or dict of mongodb pipeline operators,
http://docs.mongodb.org/manual/reference/operator/aggregation-pipeline
"""
if isinstance(query, basestring):
query = json.loads(
query, object_hook=json_util.object_hook) if query else {}
if not (isinstance(pipeline, dict) or isinstance(pipeline, list)):
raise Exception(_(u"Invalid pipeline! %s" % pipeline))
if not isinstance(query, dict):
raise Exception(_(u"Invalid query! %s" % query))
query = MongoHelper.to_safe_dict(query)
if hide_deleted:
# display only active elements
deleted_at_query = {
"$or": [{
"_deleted_at": {
"$exists": False
}
}, {
"_deleted_at": None
}]
}
# join existing query with deleted_at_query on an $and
query = {"$and": [query, deleted_at_query]}
k = [{'$match': query}]
if isinstance(pipeline, list):
k.extend(pipeline)
else:
k.append(pipeline)
results = xform_instances.aggregate(k)
return results['result']
def to_dict_for_mongo(self):
d = self.to_dict()
data = {
UUID:
self.instance.uuid,
ID:
self.instance.id,
BAMBOO_DATASET_ID:
self.instance.xform.bamboo_dataset,
self.USERFORM_ID:
u'%s_%s' %
(self.instance.xform.user.username, self.instance.xform.id_string),
ATTACHMENTS:
_get_attachments_from_instance(self.instance),
self.STATUS:
self.instance.status,
GEOLOCATION: [self.lat, self.lng],
SUBMISSION_TIME:
self.instance.date_created.strftime(MONGO_STRFTIME),
TAGS:
list(self.instance.tags.names()),
NOTES:
self.get_notes(),
VALIDATION_STATUS:
self.instance.get_validation_status(),
SUBMITTED_BY:
self.instance.user.username if self.instance.user else None
}
if isinstance(self.instance.deleted_at, datetime.datetime):
data[DELETEDAT] = self.instance.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return MongoHelper.to_safe_dict(d)
def to_representation(self, obj):
if not hasattr(obj, 'xform'):
return super(DataInstanceSerializer, self).to_representation(obj)
request = self.context.get('request')
query_params = (request and request.query_params) or {}
query = {
ParsedInstance.USERFORM_ID:
u'%s_%s' % (obj.xform.user.username, obj.xform.id_string),
u'_id':
obj.pk
}
query_kwargs = {
'query': json.dumps(query),
'fields': query_params.get('fields'),
'sort': query_params.get('sort')
}
cursor = ParsedInstance.query_mongo_minimal(**query_kwargs)
records = list(record for record in cursor)
returned_dict = {}
if (len(records)):
returned_dict = records[0]
#break code fix
# returned_dict = (len(records) and records[0]) or records
return MongoHelper.to_readable_dict(returned_dict)
def to_dict_for_mongo(self):
d = self.to_dict()
data = {
UUID: self.instance.uuid,
ID: self.instance.id,
BAMBOO_DATASET_ID: self.instance.xform.bamboo_dataset,
self.USERFORM_ID: u'%s_%s' % (
self.instance.xform.user.username,
self.instance.xform.id_string),
ATTACHMENTS: _get_attachments_from_instance(self.instance),
self.STATUS: self.instance.status,
GEOLOCATION: [self.lat, self.lng],
SUBMISSION_TIME: self.instance.date_created.strftime(
MONGO_STRFTIME),
TAGS: list(self.instance.tags.names()),
NOTES: self.get_notes(),
VALIDATION_STATUS: self.instance.get_validation_status(),
SUBMITTED_BY: self.instance.user.username
if self.instance.user else None
}
if isinstance(self.instance.deleted_at, datetime.datetime):
data[DELETEDAT] = self.instance.deleted_at.strftime(MONGO_STRFTIME)
d.update(data)
return MongoHelper.to_safe_dict(d)
def query_mongo(username, id_string, query=None):
query = json.loads(query, object_hook=json_util.object_hook)\
if query else {}
query = MongoHelper.to_safe_dict(query)
query[USERFORM_ID] = '{0}_{1}'.format(username, id_string)
return xform_instances.find(query,
max_time_ms=settings.MONGO_DB_MAX_TIME_MS)
def mongo_aggregate(cls, query, pipeline, hide_deleted=True):
"""Perform mongo aggregate queries
query - is a dict which is to be passed to $match, a pipeline operator
pipeline - list of dicts or dict of mongodb pipeline operators,
http://docs.mongodb.org/manual/reference/operator/aggregation-pipeline
"""
if isinstance(query, basestring):
query = json.loads(
query, object_hook=json_util.object_hook) if query else {}
if not (isinstance(pipeline, dict) or isinstance(pipeline, list)):
raise Exception(_(u"Invalid pipeline! %s" % pipeline))
if not isinstance(query, dict):
raise Exception(_(u"Invalid query! %s" % query))
query = MongoHelper.to_safe_dict(query)
if hide_deleted:
# display only active elements
deleted_at_query = {
"$or": [{"_deleted_at": {"$exists": False}},
{"_deleted_at": None}]}
# join existing query with deleted_at_query on an $and
query = {"$and": [query, deleted_at_query]}
k = [{'$match': query}]
if isinstance(pipeline, list):
k.extend(pipeline)
else:
k.append(pipeline)
results = xform_instances.aggregate(k)
return results['result']
def _get_mongo_cursor(cls,
query,
fields,
hide_deleted,
username=None,
id_string=None):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:param hide_deleted: boolean
:param username: string
:param id_string: string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, basestring):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = MongoHelper.to_safe_dict(query, reading=True)
if username and id_string:
query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string)
# check if query contains and _id and if its a valid ObjectID
if '_uuid' in query and ObjectId.is_valid(query['_uuid']):
query['_uuid'] = ObjectId(query['_uuid'])
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
return xform_instances.find(query, fields_to_select)
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0,
limit=DEFAULT_LIMIT, count=False):
query = MongoHelper.to_safe_dict(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0, minute=0,
second=0, microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {"$gte": start_time,
"$lte": end_time}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(MongoHelper.encode(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
MongoHelper.encode('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def query_mongo(username, id_string, query=None, hide_deleted=True):
query = json.loads(query, object_hook=json_util.object_hook)\
if query else {}
query = MongoHelper.to_safe_dict(query)
query[USERFORM_ID] = u'{0}_{1}'.format(username, id_string)
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
return xform_instances.find(query)
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
MongoHelper.encode('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(
encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def query_mongo(username, id_string, query=None, hide_deleted=True):
query = json.loads(query, object_hook=json_util.object_hook)\
if query else {}
query = MongoHelper.to_safe_dict(query)
query[USERFORM_ID] = u'{0}_{1}'.format(username, id_string)
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
return xform_instances.find(query)
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[MongoHelper.encode(text_type(elem.get_abbreviated_xpath()))] = \
elem.get_abbreviated_xpath()
return names
def _get_mongo_cursor(cls, query, fields, hide_deleted, username=None, id_string=None):
"""
Returns a Mongo cursor based on the query.
:param query: JSON string
:param fields: Array string
:param hide_deleted: boolean
:param username: string
:param id_string: string
:return: pymongo Cursor
"""
fields_to_select = {cls.USERFORM_ID: 0}
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, basestring):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = MongoHelper.to_safe_dict(query, reading=True)
if username and id_string:
query[cls.USERFORM_ID] = u'%s_%s' % (username, id_string)
# check if query contains and _id and if its a valid ObjectID
if '_uuid' in query and ObjectId.is_valid(query['_uuid']):
query['_uuid'] = ObjectId(query['_uuid'])
if hide_deleted:
# display only active elements
# join existing query with deleted_at_query on an $and
query = {"$and": [query, {"_deleted_at": None}]}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict(
[(MongoHelper.encode(field), 1) for field in fields])
return xform_instances.find(query, fields_to_select)
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
MongoHelper.encode('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
MongoHelper.encode('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
MongoHelper.encode('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def to_representation(self, obj):
request = self.context.get('request')
if not isinstance(obj, XForm):
return super(DataListSerializer, self).to_representation(obj)
query_params = (request and request.query_params) or {}
query = {
ParsedInstance.USERFORM_ID:
u'%s_%s' % (obj.user.username, obj.id_string)
}
limit = query_params.get('limit', False)
start = query_params.get('start', False)
count = query_params.get('count', False)
try:
query.update(json.loads(query_params.get('query', '{}')))
except ValueError:
raise ParseError(
_("Invalid query: %(query)s" %
{'query': query_params.get('query')}))
query_kwargs = {
'query': json.dumps(query),
'fields': query_params.get('fields'),
'sort': query_params.get('sort')
}
# if we want the count, we don't kwow to paginate the records.
# start and limit are useless then.
if count:
query_kwargs['count'] = True
else:
if limit:
query_kwargs['limit'] = int(limit)
if start:
query_kwargs['start'] = int(start)
cursor = ParsedInstance.query_mongo_minimal(**query_kwargs)
# if we want the count, we only need the first index of the list.
if count:
return cursor[0]
else:
return [MongoHelper.to_readable_dict(record) for record in cursor]
def to_representation(self, obj):
request = self.context.get('request')
if not isinstance(obj, XForm):
return super(DataListSerializer, self).to_representation(obj)
query_params = (request and request.query_params) or {}
query = {
ParsedInstance.USERFORM_ID:
u'%s_%s' % (obj.user.username, obj.id_string)
}
limit = query_params.get('limit', False)
start = query_params.get('start', False)
count = query_params.get('count', False)
try:
query.update(json.loads(query_params.get('query', '{}')))
except ValueError:
raise ParseError(_("Invalid query: %(query)s"
% {'query': query_params.get('query')}))
query_kwargs = {
'query': json.dumps(query),
'fields': query_params.get('fields'),
'sort': query_params.get('sort')
}
# if we want the count, we don't kwow to paginate the records.
# start and limit are useless then.
if count:
query_kwargs['count'] = True
else:
if limit:
query_kwargs['limit'] = int(limit)
if start:
query_kwargs['start'] = int(start)
cursor = ParsedInstance.query_mongo_minimal(**query_kwargs)
# if we want the count, we only need the first index of the list.
if count:
return cursor[0]
else:
return [MongoHelper.to_readable_dict(record) for record in cursor]
def _get_mongo_cursor_query(cls, query, username=None, id_string=None):
"""
Returns the query to get a Mongo cursor.
:param query: JSON string
:param username: string
:param id_string: string
:return: dict
"""
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, string_types):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = MongoHelper.to_safe_dict(query, reading=True)
if username and id_string:
query.update(cls.get_base_query(username, id_string))
return query
def mongo_aggregate(cls, query, pipeline):
"""Perform mongo aggregate queries
query - is a dict which is to be passed to $match, a pipeline operator
pipeline - list of dicts or dict of mongodb pipeline operators,
http://docs.mongodb.org/manual/reference/operator/aggregation-pipeline
"""
if isinstance(query, string_types):
query = json.loads(
query, object_hook=json_util.object_hook) if query else {}
if not (isinstance(pipeline, dict) or isinstance(pipeline, list)):
raise Exception(t("Invalid pipeline! %s" % pipeline))
if not isinstance(query, dict):
raise Exception(t("Invalid query! %s" % query))
query = MongoHelper.to_safe_dict(query)
k = [{'$match': query}]
if isinstance(pipeline, list):
k.extend(pipeline)
else:
k.append(pipeline)
results = xform_instances.aggregate(k)
return results['result']
def _get_paginated_and_sorted_cursor(cls, cursor, start, limit, sort):
"""
Applies pagination and sorting on mongo cursor.
:param mongo_cursor: pymongo.cursor.Cursor
:param start: integer
:param limit: integer
:param sort: dict
:return: pymongo.cursor.Cursor
"""
cursor.skip(start).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def to_representation(self, obj):
if not hasattr(obj, 'xform'):
return super(DataInstanceSerializer, self).to_representation(obj)
request = self.context.get('request')
query_params = (request and request.query_params) or {}
query = {
ParsedInstance.USERFORM_ID:
u'%s_%s' % (obj.xform.user.username, obj.xform.id_string),
u'_id': obj.pk
}
query_kwargs = {
'query': json.dumps(query),
'fields': query_params.get('fields'),
'sort': query_params.get('sort')
}
cursor = ParsedInstance.query_mongo_minimal(**query_kwargs)
records = list(record for record in cursor)
returned_dict = (len(records) and records[0]) or records
return MongoHelper.to_readable_dict(returned_dict)
def _get_paginated_and_sorted_cursor(cls, cursor, start, limit, sort):
"""
Applies pagination and sorting on mongo cursor.
:param cursor: pymongo.cursor.Cursor
:param start: integer
:param limit: integer
:param sort: dict
:return: pymongo.cursor.Cursor
"""
cursor.skip(start).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort = MongoHelper.to_safe_dict(sort, reading=True)
sort_key = sort.keys()[0]
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(sort_key, sort_dir)
# set batch size
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def decode_mongo_encoded_section_names(cls, data):
return dict([(MongoHelper.decode(k), v) for k, v in data.iteritems()])
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if MongoHelper.is_attribute_invalid(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: MongoHelper.encode(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath()
for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def decode_mongo_encoded_section_names(cls, data):
return dict([(MongoHelper.decode(k), v) for k, v in data.iteritems()])
