def query_mongo(cls,
username,
query=None,
fields=None,
sort=None,
start=0,
limit=DEFAULT_LIMIT,
count=False):
query = dict_for_mongo(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0,
minute=0,
second=0,
microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {
"$gte": start_time,
"$lte": end_time
}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(_encode_for_mongo(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort_key = sort.keys()[0]
# TODO: encode sort key if it has dots
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(_encode_for_mongo(sort_key), sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields = {
"childrens_survey": {
"tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office")),
"tel/tel.mobile": "tel/{0}".format(_encode_for_mongo("tel.mobile")),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue("childrens_survey" in encoded_fields)
self.assertEqual(encoded_fields["childrens_survey"], expected_encoded_fields["childrens_survey"])
def query_mongo(cls, username, query=None, fields=None, sort=None, start=0,
limit=DEFAULT_LIMIT, count=False):
query = dict_for_mongo(query) if query else {}
query[cls.ACCOUNT] = username
# TODO find better method
# check for the created_on key in query and turn its values into dates
if type(query) == dict and cls.CREATED_ON in query:
if type(query[cls.CREATED_ON]) is dict:
for op, val in query[cls.CREATED_ON].iteritems():
try:
query[cls.CREATED_ON][op] = datetime.strptime(
val, DATETIME_FORMAT)
except ValueError:
pass
elif isinstance(query[cls.CREATED_ON], basestring):
val = query[cls.CREATED_ON]
try:
created_on = datetime.strptime(val, DATETIME_FORMAT)
except ValueError:
pass
else:
# create start and end times for the entire day
start_time = created_on.replace(hour=0, minute=0,
second=0, microsecond=0)
end_time = start_time + timedelta(days=1)
query[cls.CREATED_ON] = {"$gte": start_time,
"$lte": end_time}
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
fields_to_select = None
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(_encode_for_mongo(field), 1)
for field in fields])
cursor = audit.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
cursor.skip(max(start, 0)).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort_key = sort.keys()[0]
# TODO: encode sort key if it has dots
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(_encode_for_mongo(sort_key), sort_dir)
# set batch size for cursor iteration
cursor.batch_size = cls.DEFAULT_BATCHSIZE
return cursor
def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = _encode_for_mongo(field)
self.assertEqual(encoded, ("%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % \
{"dollar": base64.b64encode("$"), \
"dot": base64.b64encode(".")}))
decoded = _decode_from_mongo(encoded)
self.assertEqual(field, decoded)
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
_encode_for_mongo('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
_encode_for_mongo('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def test_api_decode_from_mongo(self):
field = "$section1.group01.question1"
encoded = _encode_for_mongo(field)
self.assertEqual(encoded, (
"%(dollar)ssection1%(dot)sgroup01%(dot)squestion1" % {
"dollar": base64.b64encode("$"),
"dot": base64.b64encode(".")}))
decoded = _decode_from_mongo(encoded)
self.assertEqual(field, decoded)
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \
elem.get_abbreviated_xpath()
return names
def get_mongo_field_names_dict(self):
"""
Return a dictionary of fieldnames as saved in mongodb with
corresponding xform field names e.g {"Q1Lg==1": "Q1.1"}
"""
names = {}
for elem in self.get_survey_elements():
names[_encode_for_mongo(unicode(elem.get_abbreviated_xpath()))] = \
elem.get_abbreviated_xpath()
return names
def test_generation_of_mongo_encoded_fields_works(self):
survey = self._create_childrens_survey()
export_builder = ExportBuilder()
export_builder.set_survey(survey)
expected_encoded_fields =\
{
'childrens_survey':
{
'tel/tel.office': 'tel/{0}'.format(
_encode_for_mongo('tel.office')),
'tel/tel.mobile': 'tel/{0}'.format(
_encode_for_mongo('tel.mobile')),
}
}
encoded_fields = export_builder.encoded_fields
self.assertTrue('childrens_survey' in encoded_fields)
self.assertEqual(
encoded_fields['childrens_survey'],
expected_encoded_fields['childrens_survey'])
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
_encode_for_mongo('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
_encode_for_mongo('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = \
{
'tel/tel.office': 'tel/{0}'.format(
_encode_for_mongo('tel.office'))
}
row = \
{
'name': 'Abe',
'age': 35,
'tel/{0}'.format(
_encode_for_mongo('tel.office')): '123-456-789'
}
new_row = ExportBuilder.decode_mongo_encoded_fields(
row, encoded_fields)
expected_row = \
{
'name': 'Abe',
'age': 35,
'tel/tel.office': '123-456-789'
}
self.assertEqual(new_row, expected_row)
def build_sections(
current_section, survey_element, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []}
self.sections.append(section)
build_sections(
section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(
current_section, child, sections, select_multiples,
gps_fields, encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title': ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath': child_xpath,
'type': child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(),
[c.get_abbreviated_xpath()
for c in child.children])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend(
[
{
'title': ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath': xpath,
'type': 'decimal'
}
for xpath in xpaths
])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def build_sections(current_section,
survey_element,
sections,
select_multiples,
gps_fields,
encoded_fields,
field_delimiter='/'):
for child in survey_element.children:
current_section_name = current_section['name']
# if a section, recurs
if isinstance(child, Section):
# if its repeating, build a new section
if isinstance(child, RepeatingSection):
# section_name in recursive call changes
section = {
'name': child.get_abbreviated_xpath(),
'elements': []
}
self.sections.append(section)
build_sections(section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
else:
# its a group, recurs using the same section
build_sections(current_section, child, sections,
select_multiples, gps_fields,
encoded_fields, field_delimiter)
elif isinstance(child, Question) and child.bind.get(u"type")\
not in QUESTION_TYPES_TO_EXCLUDE:
# add to survey_sections
if isinstance(child, Question):
child_xpath = child.get_abbreviated_xpath()
current_section['elements'].append({
'title':
ExportBuilder.format_field_title(
child.get_abbreviated_xpath(),
field_delimiter),
'xpath':
child_xpath,
'type':
child.bind.get(u"type")
})
if _is_invalid_for_mongo(child_xpath):
if current_section_name not in encoded_fields:
encoded_fields[current_section_name] = {}
encoded_fields[current_section_name].update(
{child_xpath: _encode_for_mongo(child_xpath)})
# if its a select multiple, make columns out of its choices
if child.bind.get(u"type") == MULTIPLE_SELECT_BIND_TYPE\
and self.SPLIT_SELECT_MULTIPLES:
for c in child.children:
_xpath = c.get_abbreviated_xpath()
_title = ExportBuilder.format_field_title(
_xpath, field_delimiter)
choice = {
'title': _title,
'xpath': _xpath,
'type': 'string'
}
if choice not in current_section['elements']:
current_section['elements'].append(choice)
_append_xpaths_to_section(
current_section_name, select_multiples,
child.get_abbreviated_xpath(), [
c.get_abbreviated_xpath()
for c in child.children
])
# split gps fields within this section
if child.bind.get(u"type") == GEOPOINT_BIND_TYPE:
# add columns for geopoint components
xpaths = DataDictionary.get_additional_geopoint_xpaths(
child.get_abbreviated_xpath())
current_section['elements'].extend([{
'title':
ExportBuilder.format_field_title(
xpath, field_delimiter),
'xpath':
xpath,
'type':
'decimal'
} for xpath in xpaths])
_append_xpaths_to_section(
current_section_name, gps_fields,
child.get_abbreviated_xpath(), xpaths)
def query_mongo(username,
id_string,
query,
fields,
sort,
start=0,
limit=DEFAULT_LIMIT,
count=False,
hide_deleted=True,
fs_uuid=None,
fs_project_uuid=None,
site_id=None):
USERFORM_ID = u'_userform_id'
STATUS = u'_status'
DEFAULT_BATCHSIZE = 1000
fields_to_select = {USERFORM_ID: 0}
# TODO: give more detailed error messages to 3rd parties
# using the API when json.loads fails
if isinstance(query, basestring):
query = json.loads(query, object_hook=json_util.object_hook)
query = query if query else {}
query = dict_for_mongo(query)
query[USERFORM_ID] = u'%s_%s' % (username, id_string)
# check if query contains and _id and if its a valid ObjectID
# if '_uuid' in query and ObjectId.is_valid(query['_uuid']):
# query['_uuid'] = ObjectId(query['_uuid'])
query.pop('_userform_id')
if fs_uuid is not None:
query = {
"$and": [
query, {
"$or": [{
"_uuid": fs_uuid
}, {
"fs_uuid": fs_uuid
}, {
"_uuid": str(fs_uuid)
}, {
"fs_uuid": str(fs_uuid)
}]
}
]
}
# query['_uuid'] = { '$in': [fs_uuid, str(fs_uuid)] } #fs_uuid
if fs_project_uuid is not None:
if site_id is None:
query['fs_project_uuid'] = {
'$in': [fs_project_uuid, str(fs_project_uuid)]
} #fs_project_uuid
elif site_id and count:
query['fs_project_uuid'] = {
'$in': [fs_project_uuid, str(fs_project_uuid)]
}
#query = query.update({'fs_project_uuid': {'$in': [fs_project_uuid, str(fs_project_uuid)] }, 'fs_site': { '$in': [site_id, str(site_id)] }})
elif site_id:
query['fs_project_uuid'] = {
'$in': [fs_project_uuid, str(fs_project_uuid)]
}
# if hide_deleted:
# # display only active elements
# # join existing query with deleted_at_query on an $and
# query = {"$and": [query, {"_deleted_at": None}]}
# fields must be a string array i.e. '["name", "age"]'
if isinstance(fields, basestring):
fields = json.loads(fields, object_hook=json_util.object_hook)
fields = fields if fields else []
# TODO: current mongo (2.0.4 of this writing)
# cant mix including and excluding fields in a single query
if type(fields) == list and len(fields) > 0:
fields_to_select = dict([(_encode_for_mongo(field), 1)
for field in fields])
if isinstance(sort, basestring):
sort = json.loads(sort, object_hook=json_util.object_hook)
sort = sort if sort else {}
# if fs_uuid is not None:
# cursor = xform_instances.find({"$or":[ {"_uuid":fs_uuid}, {"fs_uuid":fs_uuid}, {"_uuid":str(fs_uuid)}, {"fs_uuid":str(fs_uuid)}]}, fields_to_select)
# else:
# cursor = xform_instances.find({"$or":[ {"fs_project_uuid":fs_project_uuid}, {"fs_project_uuid": str(fs_project_uuid)}]}, fields_to_select)
cursor = xform_instances.find(query, fields_to_select)
if count:
return [{"count": cursor.count()}]
if start < 0 or limit < 0:
raise ValueError(_("Invalid start/limit params"))
cursor.skip(start).limit(limit)
if type(sort) == dict and len(sort) == 1:
sort_key = sort.keys()[0]
# TODO: encode sort key if it has dots
sort_dir = int(sort[sort_key]) # -1 for desc, 1 for asc
cursor.sort(_encode_for_mongo(sort_key), sort_dir)
# set batch size
cursor.batch_size = DEFAULT_BATCHSIZE
return cursor
def test_decode_fields_names_encoded_for_mongo(self):
encoded_fields = {"tel/tel.office": "tel/{0}".format(_encode_for_mongo("tel.office"))}
row = {"name": "Abe", "age": 35, "tel/{0}".format(_encode_for_mongo("tel.office")): "123-456-789"}
new_row = ExportBuilder.decode_mongo_encoded_fields(row, encoded_fields)
expected_row = {"name": "Abe", "age": 35, "tel/tel.office": "123-456-789"}
self.assertEqual(new_row, expected_row)