def __init__(self, dbm, entity_type=None, location=None, aggregation_paths=None,
geometry=None, centroid=None, gr_id=None, id=None, short_code=None, is_datasender=True):
"""
Construct a new entity.
Note: _couch_document is used for 'protected' factory methods and
should not be passed in standard construction.
If _couch_document is passed, the other args are ignored
:param entity_type: may be a string (flat type) or sequence (hierarchical type)
:param location: hierarchical list of location names
:pram aggregation_paths: hierarchical list of aggregation path
:pram geometry: hierarchical list of aggregation path
:pram short_code: code for the entity
"""
assert isinstance(dbm, DatabaseManager)
assert entity_type is None or is_sequence(entity_type) or is_string(entity_type)
assert location is None or is_sequence(location)
assert aggregation_paths is None or isinstance(aggregation_paths, dict)
assert geometry is None or isinstance(geometry, dict)
assert centroid is None or isinstance(centroid, list)
assert gr_id is None or is_string(gr_id)
DataObject.__init__(self, dbm)
# Are we being constructed from an existing doc, in which case all the work is
# in _set_document?
if entity_type is None:
return
# Not made from existing doc, so create a new one
self._create_new_entity_doc(aggregation_paths, centroid, entity_type, geometry, gr_id, id, location, short_code,
is_datasender)
def __init__(self, dbm, name=None, label=None, form_code=None, fields=None, entity_type=None, type=None,
language="eng", state=attributes.ACTIVE_STATE):
assert isinstance(dbm, DatabaseManager)
assert name is None or is_not_empty(name)
assert fields is None or is_sequence(fields)
assert form_code is None or (is_string(form_code) and is_not_empty(form_code))
assert type is None or is_not_empty(type)
assert entity_type is None or is_sequence(entity_type)
DataObject.__init__(self, dbm)
self._form_fields = []
self.errors = []
# Are we being constructed from scratch or existing doc?
if name is None:
return
# Not made from existing doc, so build ourselves up
self._form_fields = fields
self.validate_fields()
doc = FormModelDocument()
doc.name = name
doc.add_label(language, label)
doc.form_code = form_code
doc.entity_type = entity_type
doc.type = type
doc.state = state
doc.active_languages = language
DataObject._set_document(self, doc)
def __init__(self, dbm, entity_type=None, location=None, aggregation_paths=None,
geometry=None, centroid=None, gr_id=None, id=None, short_code=None):
"""
Construct a new entity.
Note: _couch_document is used for 'protected' factory methods and
should not be passed in standard construction.
If _couch_document is passed, the other args are ignored
entity_type may be a string (flat type) or sequence (hierarchical type)
"""
assert isinstance(dbm, DatabaseManager)
assert entity_type is None or is_sequence(entity_type) or is_string(entity_type)
assert location is None or is_sequence(location)
assert aggregation_paths is None or isinstance(aggregation_paths, dict)
assert geometry is None or isinstance(geometry, dict)
assert centroid is None or isinstance(centroid, list)
assert gr_id is None or is_string(gr_id)
DataObject.__init__(self, dbm)
# Are we being constructed from an existing doc, in which case all the work is
# in _set_document?
if entity_type is None:
return
# Not made from existing doc, so create a new one
doc = EntityDocument(id)
self._set_document(doc)
# add aggregation paths
if is_string(entity_type):
entity_type = [entity_type]
doc.entity_type = entity_type
if location is not None:
doc.location = location
if geometry is not None:
doc.geometry = geometry
if centroid is not None:
doc.centroid = centroid
if gr_id is not None:
doc.gr_id = gr_id
if short_code is not None:
doc.short_code = short_code
if aggregation_paths is not None:
reserved_names = (attributes.TYPE_PATH, attributes.GEO_PATH)
for name in aggregation_paths.keys():
if name in reserved_names:
raise ValueError(u'Attempted to add an aggregation path with a reserved name')
self.set_aggregation_path(name, aggregation_paths[name])
def adapt_submissions_for_template(questions, submissions):
assert is_sequence(questions)
assert is_sequence(submissions)
for s in submissions:
assert type(s) is Submission and s._doc is not None
formatted_list = []
for each in submissions:
case_insensitive_dict = {key.lower(): value for key, value in each.values.items()}
formatted_list.append(
[each.uuid, each.destination, each.source, each.created, each.errors,
"Success" if each.status else "Error"] +
["Yes" if is_submission_deleted(each.data_record) else "No"] + [
get_according_value(case_insensitive_dict, q) for q in questions])
return [tuple(each) for each in formatted_list]
def add_data(self, entity=None, data=(), event_time=None, submission=None, multiple_records=False):
"""
Add a new datarecord to this Entity and return a UUID for the datarecord.
Arguments:
data: a sequence of ordered tuples, (label, value, type)
event_time: the time at which the event occured rather than
when it was reported
submission_id: an id to a 'submission' document in the
submission log from which this data came
"""
assert is_sequence(data)
assert event_time is None or isinstance(event_time, datetime)
assert self.id is not None, u"id should never be none, even if haven't been saved,an entity should have a UUID."
if event_time is None:
event_time = utcnow()
for (label, value) in data:
if is_empty(label):
raise ValueError(u'Empty label')
if multiple_records:
data_list = []
for (label, value) in data:
data_record = DataRecordDocument(
event_time=event_time,
data=[(label, value)],
submission=submission
)
data_list.append(data_record)
return self._dbm._save_documents(data_list)
else:
data_record_doc = DataRecordDocument(
event_time=event_time,
data=data,
submission=submission
)
return self._dbm._save_document(data_record_doc)
def set_aggregation_path(self, name, path):
assert self._doc is not None
assert is_string(name) and is_not_empty(name)
assert is_sequence(path) and is_not_empty(path)
assert isinstance(self._doc[attributes.AGG_PATHS], dict)
self._doc[attributes.AGG_PATHS][name] = list(path)
def set_aggregation_path(self, name, path):
assert self._doc is not None
assert is_string(name) and is_not_empty(name)
assert is_sequence(path) and is_not_empty(path)
assert isinstance(self._doc[attributes.AGG_PATHS], dict)
self._doc[attributes.AGG_PATHS][name] = list(path)
def get_analysis_statistics(self):
if not self._raw_values: return []
field_header = self.header_class(
self.form_model).header_list[self.leading_part_length:]
result = self._init_statistics_result()
for row in self._raw_values:
for idx, question_values in enumerate(
row[self.leading_part_length:]):
question_name = field_header[idx]
if isinstance(self.form_model.get_field_by_name(question_name),
SelectField) and question_values:
result[question_name]['total'] += 1
if is_sequence(question_values):
for each in question_values:
result[question_name]['choices'][each] += 1
else:
result[question_name]['choices'][question_values] += 1
list_result = []
for key, value in result.items():
row = [key, value['type'], value['total'], []]
sorted_value = sorted(value['choices'].items(),
key=lambda t: (t[1] * -1, t[0]))
for option, count in sorted_value:
row[-1].append([option, count])
list_result.append(row)
return list_result
def get_by_short_code(dbm, short_code, entity_type):
assert is_string(short_code)
assert is_sequence(entity_type)
rows = dbm.load_all_rows_in_view("by_short_codes", key=[entity_type, short_code], reduce=False)
if is_empty(rows):
raise DataObjectNotFound("Entity", "Unique Identification Number (ID)", short_code)
doc_id = rows[0].id
return Entity.get(dbm, doc_id)
def create_registration_form(manager, entity_name):
if is_sequence(entity_name):
entity_name = entity_name[0]
prefix = entity_name.lower()[:3]
form_code = _generate_form_code(manager, prefix)
form_model = _create_registration_form(manager, entity_name, form_code, [entity_name])
form_model.save()
return form_model
def create_registration_form(manager, entity_name):
if is_sequence(entity_name):
entity_name = entity_name[0]
form_code = _get_generated_form_code(entity_name, manager)
form_model = _create_registration_form(manager, entity_name, form_code,
[entity_name])
form_model.save()
return form_model
def exists(func, list, need_flatten=False):
assert callable(func)
assert is_sequence(list)
sequence = flatten(list) if need_flatten else list
for i in sequence:
if func(i): return True
return False
def _to_string(self, errors):
if is_string(errors):
return errors
if isinstance(errors, dict):
return sequence_to_str(errors.values())
if is_sequence(errors):
return sequence_to_str(errors)
return None
def get_submissions(questions, submissions):
assert is_sequence(questions)
assert is_sequence(submissions)
for s in submissions:
assert isinstance(s, dict) and s.get('values') is not None
formatted_list = []
for each in submissions:
submission_source = TEST_FLAG if each.get('test') else each.get(
'source')
formatted_list.append([
each.get('destination'), submission_source,
each.get('created'),
each.get('status'),
each.get('voided'),
each.get('error_message')
] + [each.get('values').get(q[0].lower()) for q in questions])
return [tuple(each) for each in formatted_list]
def flatten(less_than_two_layers_list):
"""
Only two layers list is supported.
"""
result = []
for i in less_than_two_layers_list:
if is_sequence(i): result.extend(i)
else: result.append(i)
return result
def get_form_model_by_entity_type(dbm, entity_type):
assert isinstance(dbm, DatabaseManager)
assert is_sequence(entity_type)
rows = dbm.view.registration_form_model_by_entity_type(key=entity_type,
include_docs=True)
if len(rows):
doc = EntityFormModelDocument.wrap(rows[0]['doc'])
return EntityFormModel.new_from_doc(dbm, doc)
return None
def get_by_short_code(dbm, short_code, entity_type):
"""
Finds Entity with a given short code
:param dbm: DatabaseManager
:param short_code: short code of the Entity
:param entity_type: hierarchical list of entity types
"""
assert is_string(short_code)
assert is_sequence(entity_type)
return by_short_code(dbm, short_code.lower(), entity_type)
def generate_short_code(dbm, entity_type):
# todo: couchdb cannot guarantee uniqueness, so short codes should
# be assigned from an application using the mangrove
# library. ideally, we would put a short code onto an entity using
# the add_data method and then there should be a view that can
# easily find all entities with a particular short code.
assert is_sequence(entity_type)
count = get_entity_count_for_type(dbm, entity_type=entity_type)
assert count >= 0
return _make_short_code(entity_type, count + 1)
def generate_short_code(dbm, entity_type):
# todo: couchdb cannot guarantee uniqueness, so short codes should
# be assigned from an application using the mangrove
# library. ideally, we would put a short code onto an entity using
# the add_data method and then there should be a view that can
# easily find all entities with a particular short code.
assert is_sequence(entity_type)
count = get_entity_count_for_type(dbm, entity_type=entity_type)
assert count >= 0
return _make_short_code(entity_type, count + 1)
def _to_str(value, form_field=None):
if value is None:
return u"--"
if is_sequence(value):
return sequence_to_str(value)
if isinstance(value, datetime):
date_format = DateField.FORMAT_DATE_DICTIONARY.get(
form_field.date_format) if form_field else DEFAULT_DATE_FORMAT
return format_date(value, date_format)
return value
def list_form_models_by_code(dbm, codes):
assert isinstance(dbm, DatabaseManager)
assert is_sequence(codes)
rows = dbm.load_all_rows_in_view('questionnaire', keys=codes)
def _row_to_form_model(row):
doc = FormModelDocument.wrap(row['value'])
return FormModel.new_from_doc(dbm, doc)
return map(_row_to_form_model, rows)
def get_by_short_code(dbm, short_code, entity_type):
assert is_string(short_code)
assert is_sequence(entity_type)
rows = dbm.load_all_rows_in_view("by_short_codes",
key=[entity_type, short_code],
reduce=False)
if is_empty(rows):
raise DataObjectNotFound("Entity", "Unique Identification Number (ID)",
short_code)
doc_id = rows[0].id
return Entity.get(dbm, doc_id)
def __init__(self,
dbm,
name=None,
label=None,
form_code=None,
fields=None,
entity_type=None,
type=None,
language="eng",
state=attributes.ACTIVE_STATE):
assert isinstance(dbm, DatabaseManager)
assert name is None or is_not_empty(name)
assert fields is None or is_sequence(fields)
assert form_code is None or (is_string(form_code)
and is_not_empty(form_code))
assert type is None or is_not_empty(type)
assert entity_type is None or is_sequence(entity_type)
DataObject.__init__(self, dbm)
self._form_fields = []
self.errors = []
# Are we being constructed from scratch or existing doc?
if name is None:
return
# Not made from existing doc, so build ourselves up
self._form_fields = fields
self.validate_fields()
doc = FormModelDocument()
doc.name = name
doc.add_label(language, label)
doc.form_code = form_code
doc.entity_type = entity_type
doc.type = type
doc.state = state
doc.active_languages = language
DataObject._set_document(self, doc)
def add_path(self, nodes):
"""""Adds a path to the tree.
If the first item in the path is AggregationTree.root_id, this will be added at root.
Otherwise, the method attempts to find the first (depth first search) occurrence of the first element
in the sequence, and adds the path there.
The 'nodes' list can contain tuples of the form '(node_name, dict)' to associate data
with the nodes.
e.g. add_path('a', ('b', {size: 10}), 'c')
Raises a 'ValueError' if the path does NOT start with root and the first item cannot be found.
"""""
assert is_sequence(nodes) and is_not_empty(nodes)
first = (nodes[0][0] if is_sequence(nodes[0]) else nodes[0])
if not first in self.graph:
raise ValueError('First item in path: %s not in Tree.' % first)
# iterate, add nodes, and pull out path
path = []
for n in nodes:
data = None
name = None
if is_sequence(n):
name = n[0]
data = n[1]
else:
name = n
self._add_node(name, data)
path.append(name)
# add the path
self.graph.add_path(path)
def add_path(self, nodes):
"""""Adds a path to the tree.
If the first item in the path is AggregationTree.root_id, this will be added at root.
Otherwise, the method attempts to find the first (depth first search) occurrence of the first element
in the sequence, and adds the path there.
The 'nodes' list can contain tuples of the form '(node_name, dict)' to associate data
with the nodes.
e.g. add_path('a', ('b', {size: 10}), 'c')
Raises a 'ValueError' if the path does NOT start with root and the first item cannot be found.
""" ""
assert is_sequence(nodes) and is_not_empty(nodes)
first = (nodes[0][0] if is_sequence(nodes[0]) else nodes[0])
if not first in self.graph:
raise ValueError('First item in path: %s not in Tree.' % first)
# iterate, add nodes, and pull out path
path = []
for n in nodes:
data = None
name = None
if is_sequence(n):
name = n[0]
data = n[1]
else:
name = n
self._add_node(name, data)
path.append(name)
# add the path
self.graph.add_path(path)
def define_type(dbm, entity_type):
"""
Add this entity type to the tree of all entity types and save it
to the database. entity_type may be a string or a list of
strings.
"""
assert is_not_empty(entity_type)
assert is_sequence(entity_type)
type_path = ([entity_type] if is_string(entity_type) else entity_type)
type_path = [item.strip() for item in type_path]
if entity_type_already_defined(dbm, type_path):
raise EntityTypeAlreadyDefined(u"Type: %s is already defined" % u'.'.join(entity_type))
# now make the new one
entity_tree = _get_entity_type_tree(dbm)
entity_tree.add_path([atree.AggregationTree.root_id] + entity_type)
entity_tree.save()
def define_type(dbm, entity_type):
"""
Add this entity type to the tree of all entity types and save it
to the database. entity_type may be a string or a list of
strings.
"""
assert is_not_empty(entity_type)
assert is_sequence(entity_type)
type_path = ([entity_type] if is_string(entity_type) else entity_type)
type_path = [item.strip() for item in type_path]
if entity_type_already_defined(dbm, type_path):
raise EntityTypeAlreadyDefined(u"Type: %s is already defined" %
u'.'.join(entity_type))
# now make the new one
entity_tree = _get_entity_type_tree(dbm)
entity_tree.add_path([atree.AggregationTree.root_id] + entity_type)
entity_tree.save()
def define_type(dbm, entity_type):
"""
Add this entity type to the tree of all entity types and save it
to the database. entity_type may be a string or a list of
strings.
"""
assert is_not_empty(entity_type)
assert is_sequence(entity_type)
assert isinstance(dbm, DatabaseManager)
if entity_type_already_defined(dbm, entity_type):
raise EntityTypeAlreadyDefined(u"Type: %s is already defined" %
u'.'.join(entity_type))
entity_tree = AggregationTree.get(dbm,
ENTITY_TYPE_TREE_ID,
get_or_create=True)
entity_tree.add_path([AggregationTree.root_id] + entity_type)
entity_tree.save()
def add_data(self,
data=(),
event_time=None,
submission=None,
multiple_records=False):
"""
Add a new datarecord to this Entity and return a UUID for the datarecord.
Arguments:
data: a sequence of ordered tuples, (label, value, type)
where type is a DataDictType
event_time: the time at which the event occured rather than
when it was reported
submission_id: an id to a 'submission' document in the
submission log from which this data came
"""
assert is_sequence(data)
assert event_time is None or isinstance(event_time, datetime)
assert self.id is not None, u"id should never be none, even if haven't been saved,an entity should have a UUID."
# TODO: should we have a flag that says that this has been
# saved at least once to avoid adding data records for an
# Entity that may never be saved? Should docs just be saved on
# init?
if event_time is None:
event_time = utcnow()
for (label, value, dd_type) in data:
if not isinstance(dd_type, DataDictType) or is_empty(label):
raise ValueError(
u'Data must be of the form (label, value, DataDictType).')
self.update_latest_data(data=data)
if multiple_records:
data_list = []
for (label, value, dd_type) in data:
data_record = DataRecordDocument(entity_doc=self._doc,
event_time=event_time,
data=[(label, value, dd_type)
],
submission=submission)
data_list.append(data_record)
return self._dbm._save_documents(data_list)
else:
data_record_doc = DataRecordDocument(entity_doc=self._doc,
event_time=event_time,
data=data,
submission=submission)
return self._dbm._save_document(data_record_doc)
def __init__(self,
dbm,
name=None,
label=None,
form_code=None,
fields=None,
language="en",
is_registration_model=False,
validators=None,
enforce_unique_labels=True,
entity_type=None):
super(EntityFormModel,
self).__init__(dbm, name, label, form_code, fields, language,
is_registration_model, validators,
enforce_unique_labels)
assert entity_type is None or is_sequence(entity_type)
if self._doc:
self._doc.entity_type = entity_type
def get_many(self, ids, object_class):
"""
Get many data objects at once.
Returns a (possibly empty) list of retrieved objects
"""
assert issubclass(object_class, DataObject)
assert is_sequence(ids)
objs = []
rows = self.database.view("_all_docs", keys=ids, include_docs=True)
for row in rows:
if "error" in row:
continue
obj = object_class.new_from_doc(self, object_class.__document_class__.wrap(row.get("doc")))
objs.append(obj)
return objs
def get_many(self, ids, object_class):
"""
Get many data objects at once.
Returns a (possibly empty) list of retrieved objects
"""
assert issubclass(object_class, DataObject)
assert is_sequence(ids)
objs = []
rows = self.database.view('_all_docs', keys=ids, include_docs=True)
for row in rows:
if 'error' in row:
continue
obj = object_class.new_from_doc(
self, object_class.__document_class__.wrap(row.get('doc')))
objs.append(obj)
return objs
def add_data(self, data=(), event_time=None, submission=None, multiple_records=False):
"""
Add a new datarecord to this Entity and return a UUID for the datarecord.
Arguments:
data: a sequence of ordered tuples, (label, value, type)
where type is a DataDictType
event_time: the time at which the event occured rather than
when it was reported
submission_id: an id to a 'submission' document in the
submission log from which this data came
"""
assert is_sequence(data)
assert event_time is None or isinstance(event_time, datetime)
assert self.id is not None, u"id should never be none, even if haven't been saved,an entity should have a UUID."
# TODO: should we have a flag that says that this has been
# saved at least once to avoid adding data records for an
# Entity that may never be saved? Should docs just be saved on
# init?
if event_time is None:
event_time = utcnow()
for (label, value, dd_type) in data:
if not isinstance(dd_type, DataDictType) or is_empty(label):
raise ValueError(u'Data must be of the form (label, value, DataDictType).')
self.update_latest_data(data=data)
if multiple_records:
data_list = []
for (label, value, dd_type) in data:
data_record = DataRecordDocument(
entity_doc=self._doc,
event_time=event_time,
data=[(label, value, dd_type)],
submission=submission
)
data_list.append(data_record)
return self._dbm._save_documents(data_list)
else:
data_record_doc = DataRecordDocument(
entity_doc=self._doc,
event_time=event_time,
data=data,
submission=submission
)
return self._dbm._save_document(data_record_doc)
def _save_documents(self, documents, modified=None):
assert is_sequence(documents)
assert modified is None or isinstance(modified, datetime)
for doc in documents:
assert isinstance(doc, DocumentBase)
doc.modified = modified if modified is not None else dates.utcnow()
# TODO: what should we return here? this is what is available from db.update()...
# The return value of this method is a list containing a tuple for every
# element in the `documents` sequence. Each tuple is of the form
# ``(success, docid, rev_or_exc)``, where ``success`` is a boolean
# indicating whether the update succeeded, ``docid`` is the ID of the
# document, and ``rev_or_exc`` is either the new document revision, or
# an exception instance (e.g. `ResourceConflict`) if the update failed.
# Fix up rev, 'cause bulk update seems not to do that
results = self.database.update(documents)
for x in range(len(results)):
if results[x][0]:
documents[x]._data["_rev"] = results[x][2]
return results
def _save_documents(self, documents, modified=None):
assert is_sequence(documents)
assert modified is None or isinstance(modified, datetime)
for doc in documents:
assert isinstance(doc, DocumentBase)
doc.modified = (modified
if modified is not None else dates.utcnow())
# TODO: what should we return here? this is what is available from db.update()...
# The return value of this method is a list containing a tuple for every
# element in the `documents` sequence. Each tuple is of the form
# ``(success, docid, rev_or_exc)``, where ``success`` is a boolean
# indicating whether the update succeeded, ``docid`` is the ID of the
# document, and ``rev_or_exc`` is either the new document revision, or
# an exception instance (e.g. `ResourceConflict`) if the update failed.
# Fix up rev, 'cause bulk update seems not to do that
results = self.database.update(documents)
for x in range(len(results)):
if results[x][0]:
documents[x]._data['_rev'] = results[x][2]
return results
def __init__(self,
dbm,
name=None,
label=None,
form_code=None,
fields=None,
language="en",
is_registration_model=False,
validators=None,
enforce_unique_labels=True):
if not validators: validators = [MandatoryValidator()]
assert isinstance(dbm, DatabaseManager)
assert name is None or is_not_empty(name)
assert fields is None or is_sequence(fields)
assert form_code is None or (is_string(form_code)
and is_not_empty(form_code))
# assert type is None or is_not_empty(type)
DataObject.__init__(self, dbm)
self.xform_model = None
self._old_doc = None
self._snapshots = {}
self._form_fields = []
self.errors = []
self.validators = validators
self._enforce_unique_labels = enforce_unique_labels
self._validation_exception = []
# Are we being constructed from scratch or existing doc?
if name is None:
return
# Not made from existing doc, so build ourselves up
self._validate_fields(fields)
self._form_fields = fields
self._set_doc(form_code, is_registration_model, label, language, name)
def entity_type_as_sequence(entity_type):
if not is_sequence(entity_type):
entity_type = [entity_type.lower()]
return entity_type
def validate(self, value):
# todo call all validators of the child fields
Field.validate(self, value)
if is_sequence(value) or value is None:
return value
return [value]
def filter(self, submission_logs):
assert self.entity_question_code
assert is_sequence(self.subject_ids)
return filter(lambda x: self._with_subject(x.values), submission_logs)
def _stringify(item):
if is_sequence(item):
return sequence_to_str(item)
return unicode(item)
def validate(self, value):
Field.validate(self, value)
if is_sequence(value) or value is None:
return value
return [value]
def validate(self, value):
if is_sequence(value) or value is None:
return value
return [value]
