def setUp(self):
self.couch_db = FakeCouchDb()
self.batches = [
list(range(0, 3)),
list(range(3, 6)),
list(range(6, 8)),
]
self.all_items = list(itertools.chain(*self.batches))
self.itr = self.get_iterator()
def setUp(self):
self.db = FormQuestionSchema.get_db()
self.fakedb = FakeCouchDb()
FormQuestionSchema.set_db(self.fakedb)
self.domain = 'test'
self.app_id = '123'
self.xmlns = 'this_xmlns'
self.schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id, xmlns=self.xmlns)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
MockIndicatorsSql.set_db(self.fakedb)
rebuild_table(self.engine, None, MockIndicatorsSql)
rebuild_table(self.engine, None, MockIndicatorsSqlWithFlatFields)
def setUp(self):
views = {
"all_docs/by_doc_type": self._get_view_results(4, chuck_size=2)
}
docs = [self._get_row(ident)['doc'] for ident in range(4)]
self.db = FakeCouchDb(views=views, docs={
doc['_id']: doc for doc in docs
})
Bar.set_db(self.db)
self.processor_slug = uuid.uuid4().hex
class TestLocationParentIdExpression(SimpleTestCase):
def setUp(self):
# we have to set the fake database before any other calls
self.domain = "test-loc-parent-id"
self.evaluation_context = EvaluationContext({"domain": self.domain})
self.orig_db = Location.get_db()
self.database = FakeCouchDb()
Location.set_db(self.database)
self.parent = self._make_location(_id=uuid.uuid4().hex)
self.child = self._make_location(_id=uuid.uuid4().hex, lineage=[self.parent._id])
self.grandchild = self._make_location(_id=uuid.uuid4().hex, lineage=[self.child._id, self.parent._id])
self.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {"type": "property_name", "property_name": "location_id"},
}
self.expression = ExpressionFactory.from_spec(self.expression_spec)
def tearDown(self):
Location.set_db(self.orig_db)
def test_location_parent_id(self):
self.assertEqual(self.parent._id, self.expression({"location_id": self.child._id}, self.evaluation_context))
self.assertEqual(self.child._id, self.expression({"location_id": self.grandchild._id}, self.evaluation_context))
def test_location_parent_missing(self):
self.assertEqual(None, self.expression({"location_id": "bad-id"}, self.evaluation_context))
def test_location_parent_bad_domain(self):
self.assertEqual(
None, self.expression({"location_id": self.child._id}, EvaluationContext({"domain": "bad-domain"}))
)
def test_location_parents_chained(self):
expression = ExpressionFactory.from_spec(
{
"type": "location_parent_id",
"location_id_expression": {
"type": "location_parent_id",
"location_id_expression": {"type": "property_name", "property_name": "location_id"},
},
}
)
self.assertEqual(self.parent._id, expression({"location_id": self.grandchild._id}, self.evaluation_context))
def _make_location(self, **kwargs):
kwargs["domain"] = self.domain
loc = Location(**kwargs)
self.database.save_doc(loc.to_json())
return loc
def setUp(self):
# we have to set the fake database before any other calls
self.orig_db = CommCareCase.get_db()
self.database = FakeCouchDb()
CommCareCase.set_db(self.database)
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {"type": "property_name", "property_name": "related_property"},
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec(
{
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {"type": "property_name", "property_name": "related_property"},
},
}
)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
create_update_indicator_table(MockIndicatorsSql, None)
def setUpClass(cls):
super(TestCouchDomainFormChangeProvider, cls).setUpClass()
cls.domains = ['d1', 'd2', 'd3']
cls.form_ids = {
(domain, doc_type):
['{}-{}-{}'.format(doc_type, domain, i) for i in range(3)]
for domain in cls.domains
for doc_type in ['XFormInstance', 'XFormArchived']
}
cls.couch_db = FakeCouchDb(
views={
"by_domain_doc_type_date/view": [(
{
'startkey': list(domain_doc_type),
'endkey': list(domain_doc_type) + [{}],
'include_docs': True,
'limit': 1000,
'reduce': False
}, [
cls._get_row(form_id, domain_doc_type[0],
domain_doc_type[1])
for form_id in form_ids
]) for domain_doc_type, form_ids in cls.form_ids.items()]
})
XFormInstance.set_db(cls.couch_db)
def setUp(self):
self.database = FakeCouchDb()
self.case_orig_db = CommCareCase.get_db()
self.form_orig_db = XFormInstance.get_db()
self.user_orig_db = CommCareUser.get_db()
CommCareCase.set_db(self.database)
XFormInstance.set_db(self.database)
CommCareUser.set_db(self.database)
def setUp(self):
self.db = FormQuestionSchema.get_db()
self.fakedb = FakeCouchDb()
FormQuestionSchema.set_db(self.fakedb)
self.domain = 'test'
self.app_id = '123'
self.xmlns = 'this_xmlns'
self.schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id, xmlns=self.xmlns)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
MockIndicatorsSql.set_db(self.fakedb)
rebuild_table(self.engine, None, MockIndicatorsSql)
rebuild_table(self.engine, None, MockIndicatorsSqlWithFlatFields)
def setUp(self):
self.couch_db = FakeCouchDb()
self.batches = [
list(range(0, 3)),
list(range(3, 6)),
list(range(6, 8)),
]
self.all_items = list(itertools.chain(*self.batches))
self.itr = self.get_iterator()
def setUp(self):
# we have to set the fake database before any other calls
self.domain = 'test-loc-parent-id'
self.evaluation_context = EvaluationContext({"domain": self.domain})
self.orig_db = Location.get_db()
self.database = FakeCouchDb()
Location.set_db(self.database)
self.parent = self._make_location(_id=uuid.uuid4().hex)
self.child = self._make_location(_id=uuid.uuid4().hex,
lineage=[self.parent._id])
self.grandchild = self._make_location(
_id=uuid.uuid4().hex, lineage=[self.child._id, self.parent._id])
self.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {
"type": "property_name",
"property_name": "location_id",
}
}
self.expression = ExpressionFactory.from_spec(self.expression_spec)
def setUp(self):
# we have to set the fake database before any other calls
self.orig_db = CommCareCase.get_db()
self.database = FakeCouchDb()
CommCareCase.set_db(self.database)
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type":
"related_doc",
"related_doc_type":
"CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
def setUp(self):
self._fake_couch = FakeCouchDb()
self._fake_couch.dbname = 'test_commcarehq'
with trap_extra_setup(KafkaUnavailableError):
self.consumer = KafkaConsumer(
topics.CASE,
group_id='test-consumer',
bootstrap_servers=[settings.KAFKA_URL],
consumer_timeout_ms=100,
)
self.pillow = get_change_feed_pillow_for_db(
'fake-changefeed-pillow-id', self._fake_couch)
self.original_process_change = self.pillow.process_change
def setUp(self):
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
self.database = FakeCouchDb()
RelatedDocExpressionSpec.db_lookup = lambda _, type: self.database
def setUp(self):
self._fake_couch = FakeCouchDb()
# use a 'real' db name here so that we don't cause other
# tests down the line to fail.
# Specifically KafkaChangeFeedTest.test_multiple_topics_with_partial_checkpoint
self._fake_couch.dbname = 'test_commcarehq'
with trap_extra_setup(KafkaUnavailableError):
self.consumer = KafkaConsumer(
topics.CASE,
group_id='test-consumer',
bootstrap_servers=[settings.KAFKA_URL],
consumer_timeout_ms=100,
)
self.pillow = get_change_feed_pillow_for_db(
'fake-changefeed-pillow-id', self._fake_couch)
def __init__(self, views=None, docs=None):
from fakecouch import FakeCouchDb
self.views = views
self.docs = docs
self.db = FakeCouchDb(views=views, docs=docs)
@classmethod
def _get_db(*args):
return self.db
self.patches = [
mock.patch('dimagi.ext.couchdbkit.Document.get_db', new=_get_db),
mock.patch('dimagi.ext.couchdbkit.SafeSaveDocument.get_db', new=_get_db),
mock.patch('dimagi.utils.couch.undo.UndoableDocument.get_db', new=_get_db),
]
def setUp(self):
# we have to set the fake database before any other calls
self.domain = "test-loc-parent-id"
self.evaluation_context = EvaluationContext({"domain": self.domain})
self.orig_db = Location.get_db()
self.database = FakeCouchDb()
Location.set_db(self.database)
self.parent = self._make_location(_id=uuid.uuid4().hex)
self.child = self._make_location(_id=uuid.uuid4().hex, lineage=[self.parent._id])
self.grandchild = self._make_location(_id=uuid.uuid4().hex, lineage=[self.child._id, self.parent._id])
self.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {"type": "property_name", "property_name": "location_id"},
}
self.expression = ExpressionFactory.from_spec(self.expression_spec)
def setUp(self):
super(CouchPillowRetryProcessingTest, self).setUp()
self._fake_couch = FakeCouchDb()
self._fake_couch.dbname = 'test_commcarehq'
self.consumer = KafkaConsumer(
topics.CASE,
client_id='test-consumer',
bootstrap_servers=settings.KAFKA_BROKERS,
consumer_timeout_ms=100,
enable_auto_commit=False,
)
try:
next(self.consumer)
except StopIteration:
pass
self.pillow = get_change_feed_pillow_for_db('fake-changefeed-pillow-id', self._fake_couch)
self.original_process_change = self.pillow.process_change
def setUp(self):
self.database = FakeCouchDb()
self.user_orig_db = CommCareUser.get_db()
CommCareUser.set_db(self.database)
self.factory = RequestFactory()
domain = Domain.get_or_create_with_name(DOMAIN)
domain.is_active = True
domain.save()
self.domain = domain
user = WebUser.get_by_username('test')
if not user:
user = WebUser.create(domain.name, 'test', 'passwordtest', None,
None)
user.is_authenticated = True
user.is_superuser = True
user.is_authenticated = True
user.is_active = True
self.user = user
def mock_out_couch(views=None, docs=None):
"""
Mock out calls to couch so you can use SimpleTestCase
@mock_out_couch()
class TestMyStuff(SimpleTestCase):
...
You can optionally pass default return values for specific views and doc
gets. See the FakeCouchDb docstring for more specifics.
"""
from fakecouch import FakeCouchDb
db = FakeCouchDb(views=views, docs=docs)
def _get_db(*args):
return db
return mock.patch('dimagi.ext.couchdbkit.Document.get_db', new=_get_db)
def setUp(self):
super(ChangeFeedPillowTest, self).setUp()
self._fake_couch = FakeCouchDb()
# use a 'real' db name here so that we don't cause other
# tests down the line to fail.
# Specifically KafkaChangeFeedTest.test_multiple_topics_with_partial_checkpoint
self._fake_couch.dbname = 'test_commcarehq'
self.consumer = KafkaConsumer(
topics.CASE_SQL,
bootstrap_servers=settings.KAFKA_BROKERS,
consumer_timeout_ms=100,
enable_auto_commit=False,
)
try:
# This initialized the consumer listening from the latest offset
next(self.consumer)
except StopIteration:
pass
self.pillow = get_change_feed_pillow_for_db('fake-changefeed-pillow-id', self._fake_couch)
def setUp(self):
self.database = FakeCouchDb()
self.case_orig_db = CommCareCase.get_db()
self.form_orig_db = XFormInstance.get_db()
self.user_orig_db = CommCareUser.get_db()
CommCareCase.set_db(self.database)
XFormInstance.set_db(self.database)
CommCareUser.set_db(self.database)
self.factory = RequestFactory()
domain = Domain.get_or_create_with_name('up-nrhm')
domain.is_active = True
domain.save()
self.domain = domain
user = WebUser.get_by_username('test')
if not user:
user = WebUser.create(domain.name, 'test', 'passwordtest')
user.is_authenticated = True
user.is_superuser = True
user.is_authenticated = True
user.is_active = True
self.user = user
def setUp(self):
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
self.database = FakeCouchDb()
RelatedDocExpressionSpec.db_lookup = lambda _, type: self.database
class BaseCouchDocProcessorTest(SimpleTestCase):
processor_class = None
@staticmethod
def _get_row(ident, doc_type="Bar"):
doc_id_prefix = '{}-'.format(doc_type.lower())
doc_id = '{}{}'.format(doc_id_prefix, ident)
return {
'id': doc_id,
'key': [doc_type, doc_id], 'value': None, 'doc': {'_id': doc_id, 'doc_type': doc_type}
}
def _get_view_results(self, total, chuck_size, doc_type="Bar"):
doc_id_prefix = '{}-'.format(doc_type.lower())
results = [(
{"endkey": [doc_type, {}], "group_level": 1, "reduce": True, "startkey": [doc_type]},
[{"key": doc_type, "value": total}]
)]
for chunk in chunked(list(range(total)), chuck_size):
chunk_rows = [self._get_row(ident, doc_type=doc_type) for ident in chunk]
if chunk[0] == 0:
results.append((
{
'startkey': [doc_type], 'endkey': [doc_type, {}], 'reduce': False,
'limit': chuck_size, 'include_docs': True
},
chunk_rows
))
else:
previous = '{}{}'.format(doc_id_prefix, chunk[0] - 1)
results.append((
{
'endkey': [doc_type, {}], 'skip': 1, 'startkey_docid': previous, 'reduce': False,
'startkey': [doc_type, previous], 'limit': chuck_size, 'include_docs': True
},
chunk_rows
))
return results
def setUp(self):
views = {
"all_docs/by_doc_type": self._get_view_results(4, chuck_size=2)
}
docs = [self._get_row(ident)['doc'] for ident in range(4)]
self.db = FakeCouchDb(views=views, docs={
doc['_id']: doc for doc in docs
})
Bar.set_db(self.db)
self.processor_slug = uuid.uuid4().hex
def tearDown(self):
self.db.reset()
def _get_processor(self, chunk_size=2, ignore_docs=None, skip_docs=None, reset=False, doc_types=None):
doc_types = doc_types or [Bar]
doc_processor = DemoProcessor()
doc_provider = CouchDocumentProvider(self.processor_slug, doc_types)
processor = self.processor_class(
doc_provider,
doc_processor,
chunk_size=chunk_size,
reset=reset
)
processor.document_iterator.couch_db = self.db
if ignore_docs:
doc_processor.ignore_docs = ignore_docs
if skip_docs:
doc_processor.skip_docs = skip_docs
return doc_processor, processor
class TestResumableFunctionIterator(SimpleTestCase):
def setUp(self):
self.couch_db = FakeCouchDb()
self.batches = [
list(range(0, 3)),
list(range(3, 6)),
list(range(6, 8)),
]
self.all_items = list(itertools.chain(*self.batches))
self.itr = self.get_iterator()
def tearDown(self):
self.couch_db.reset()
def get_iterator(self, missing_items=None):
def data_provider(batch_number):
try:
return self.batches[batch_number]
except IndexError:
return []
def item_getter(item_id):
if missing_items and item_id in missing_items:
return None
return int(item_id)
itr = ResumableFunctionIterator('test', data_provider,
TestArgsProvider(), item_getter)
itr.couch_db = self.couch_db
return itr
def test_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
def test_resume_iteration(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual([item for item in self.itr], self.all_items[3:])
def test_resume_iteration_after_complete_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
# resume iteration
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), [])
def test_discard_state(self):
self.assertEqual(list(self.itr), self.all_items)
self.itr.discard_state()
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), self.all_items)
def test_iteration_with_iterator_detail(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
self.assertEqual(self.itr.get_iterator_detail('progress'), None)
self.itr.set_iterator_detail('progress', {"visited": 6})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'),
{"visited": 6})
self.itr.set_iterator_detail('progress', {"visited": "six"})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'),
{"visited": "six"})
self.assertEqual([item for item in self.itr], self.all_items[3:])
def test_iteration_with_retry(self):
itr = iter(self.itr)
item = next(itr)
self.itr.retry(str(item))
self.assertEqual(item, 0)
self.assertEqual([0] + [d for d in itr], self.all_items + [0])
def test_iteration_complete_after_retry(self):
itr = iter(self.itr)
self.itr.retry(str(next(itr)))
list(itr)
self.itr = self.get_iterator()
self.assertEqual([item for item in self.itr], [])
def test_iteration_with_max_retry(self):
itr = iter(self.itr)
item = next(itr)
ids = [item]
self.assertEqual(item, 0)
self.itr.retry(str(item))
retries = 1
for item in itr:
ids.append(item)
if item == 0:
if retries < 3:
self.itr.retry(str(item))
retries += 1
else:
break
self.assertEqual(item, 0)
with self.assertRaises(TooManyRetries):
self.itr.retry(str(item))
self.assertEqual(ids, self.all_items + [0, 0, 0])
self.assertEqual(list(itr), [])
self.assertEqual(list(self.get_iterator()), [])
def test_iteration_with_missing_retry_doc(self):
iterator = self.get_iterator(missing_items=["0"])
itr = iter(iterator)
item = next(itr)
self.assertEqual(item, 0)
iterator.retry(str(item))
self.assertEqual([0] + [d for d in itr], self.all_items)
class FluffTest(TestCase):
@classmethod
def setUpClass(cls):
super(FluffTest, cls).setUpClass()
# hack - force disconnecting the signals because ctable doesn't play nice with mocks
cls.previous_signal_receivers = indicator_document_updated.receivers
indicator_document_updated.receivers = []
cls.engine = connection_manager.get_engine(DEFAULT_ENGINE_ID)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
MockIndicatorsSql.set_db(self.fakedb)
rebuild_table(self.engine, None, MockIndicatorsSql)
rebuild_table(self.engine, None, MockIndicatorsSqlWithFlatFields)
@classmethod
def tearDownClass(cls):
indicator_document_updated.receivers = cls.previous_signal_receivers
super(FluffTest, cls).tearDownClass()
def tearDown(self):
with self.engine.begin() as connection:
MockIndicatorsSql()._table.drop(connection, checkfirst=True)
MockIndicatorsSqlWithFlatFields()._table.drop(connection, checkfirst=True)
self.engine.dispose()
def test_calculator_base_classes(self):
# Base0
self.assertEqual(Base0._fluff_emitters, set([
'base_0_emitter',
]))
self.assertEqual(Base0._fluff_filters, set([
'base_0_filter',
]))
# Base1
self.assertEqual(Base1._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
]))
self.assertEqual(Base1._fluff_filters, set([
'base_0_filter',
'base_1_filter',
]))
# Base2
self.assertEqual(Base2._fluff_emitters, set([
'base_0_emitter',
'base_2_emitter',
]))
self.assertEqual(Base2._fluff_filters, set([
'base_0_filter',
'base_2_filter',
]))
# Base2
self.assertEqual(Base3._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
'base_2_emitter',
'base_3_emitter',
]))
self.assertEqual(Base3._fluff_filters, set([
'base_0_filter',
'base_1_filter',
'base_2_filter',
'base_3_filter',
]))
def test_indicator_classes(self):
self.assertEqual(list(Indicators1._calculators), ['base0'])
self.assertEqual(list(Indicators2._calculators), ['base1', 'base2'])
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(11, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=1), indicator["value_week"]["group_no_val"][0])
def test_calculator_calculate(self):
calc = ValueCalculator(WEEK)
calc.fluff = MockIndicators
values = calc.calculate(MockDoc.wrap(dict(actions=[dict(date="2012-09-23", x=2),
dict(date="2012-09-24", x=3)])))
self.assertEqual(len(list(values)), 8)
self.assertEqual(values['null_value'], [dict(date=None, value=2, group_by=None)])
self.assertEqual(values['date_value'], [
dict(date=date(2012, 9, 23), value=2, group_by=None),
dict(date=date(2012, 9, 24), value=3, group_by=None)])
self.assertEqual(values['date'], [
dict(date=date(2012, 9, 23), value=1, group_by=None),
dict(date=date(2012, 9, 24), value=1, group_by=None)])
self.assertEqual(values['null'], [dict(date=None, value=1, group_by=None)])
self.assertEqual(values['group_list'], [dict(date=date(2013, 1, 1), group_by=['abc', 'xyz'], value=3)])
self.assertEqual(values['group_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=2)])
self.assertEqual(values['group_no_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=1)])
self.assertEqual(values['group_null'], [dict(date=None, group_by=['abc', 'xyz'], value=1)])
def test_calculator_get_result(self):
key = ['a', 'b']
now = datetime.utcnow().date()
start = json_format_date(now - WEEK)
end = json_format_date(now)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
self.fakedb.add_view('fluff/generic', [
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null', None]},
[{"key": None, "value": {"sum": 3}}]
),
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null_value', None]},
[{"key": None, "value": {"max": 8}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date', end],
'reduce': True},
[{"key": None, "value": {"count": 7}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date_value', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date_value', end],
'reduce': True},
[{"key": None, "value": {"sum": 11}}]
)
])
value = cls.get_result('value_week', key, reduce=True)
self.assertEqual(value['null'], 3)
self.assertEqual(value['date'], 7)
self.assertEqual(value['date_value'], 11)
self.assertEqual(value['null_value'], 8)
def test_indicator_diff_new(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
doc = cls(
domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 2, 23), 1]],
null=[],
date_value=[],
null_value=[[None, 3]]
)
)
diff = doc.diff(None)
self.maxDiff = None
expected = dict(domains=['mock'],
database=cls.Meta.app_label,
doc_type=classname,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 23), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=3, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertItemsEqual(expected.pop('all_indicators'), diff.pop('all_indicators'))
self.assertItemsEqual(expected.pop('indicator_changes'), diff.pop('indicator_changes'))
self.assertEqual(expected, diff)
def test_indicator_diff_same(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
doc = cls(domain="mock",
owner_id="123",
value_week=dict(
date=[date(2012, 2, 23)],
null=[],
date_value=[],
null_value=[[None, 3]]
))
another = cls(doc.to_json())
diff = doc.diff(another)
self.assertIsNone(diff)
def all_indicators(self):
return [
dict(calculator='value_week',
emitter='date_value',
emitter_type='date',
reduce_type='sum'),
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count'),
dict(calculator='value_week',
emitter='null',
emitter_type='null',
reduce_type='sum'),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max'),
]
def test_indicator_diff(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
current = cls(domain="mock",
owner_id="123",
value_week=dict(date=[[date(2012, 2, 23), 1]],
null=[],
date_value=[[date(2012, 2, 23), 3]],
null_value=[]))
new = cls(domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 2, 24), 1]],
null=[[None, 1]],
date_value=[[date(2012, 2, 23), 4]],
null_value=[[None, 2]]))
diff = new.diff(current)
self.assertIsNotNone(diff)
self.maxDiff = None
expected = dict(domains=['mock'],
database=cls.Meta.app_label,
doc_type=cls.__name__,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date_value',
emitter_type='date',
reduce_type='sum',
values=[dict(date=date(2012, 2, 23), value=4, group_by=None)]),
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 24), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null',
emitter_type='null',
reduce_type='sum',
values=[dict(date=None, value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=2, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertItemsEqual(expected.pop('all_indicators'), diff.pop('all_indicators'))
self.assertItemsEqual(expected.pop('indicator_changes'), diff.pop('indicator_changes'))
self.assertEqual(expected, diff)
def test_indicator_diff_dict(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
current = cls(domain="mock",
owner_id="123",
value_week=dict(
date=[dict(date=date(2012, 2, 23), value=1, group_by=None)],
date_value=[[date(2012, 2, 24), 1]],
group_list=[],
null_value=[dict(date=None, value=1, group_by='abc')],
))
new = cls(domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 2, 24), 1]],
date_value=[dict(date=date(2012, 2, 20), value=2, group_by=None)],
group_list=[dict(date=date(2013, 1, 1), value=3, group_by=['abc', '123'])],
null_value=[dict(date=None, value=1, group_by='abc')],
))
diff = new.diff(current)
self.assertIsNotNone(diff)
self.maxDiff = None
expected = dict(domains=['mock'],
database=cls.Meta.app_label,
doc_type=cls.__name__,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date_value',
emitter_type='date',
reduce_type='sum',
values=[dict(date=date(2012, 2, 20), value=2, group_by=None)]),
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 24), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='group_list',
emitter_type='date',
reduce_type='sum',
values=[dict(date=date(2013, 1, 1), value=3, group_by=['abc', '123'])]),
],
all_indicators=[
dict(calculator='value_week',
emitter='date_value',
emitter_type='date',
reduce_type='sum'),
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count'),
dict(calculator='value_week',
emitter='group_list',
emitter_type='date',
reduce_type='sum'),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max'),
])
self.assertItemsEqual(expected.pop('all_indicators'), diff.pop('all_indicators'))
self.assertItemsEqual(expected.pop('indicator_changes'), diff.pop('indicator_changes'))
self.assertEqual(expected, diff)
def test_flat_field_types(self):
str_field = fluff.FlatField(lambda case: "hello")
self.assertEqual(str_field.calculate('bar'), "hello")
unicode_field = fluff.FlatField(lambda case: "unicode!!")
self.assertEqual(unicode_field.calculate('bar'), "unicode!!")
num_field = fluff.FlatField(lambda case: 432123141)
self.assertRaises(AssertionError, num_field.calculate, 'bar')
def test_save_to_sql(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner"
)
current = MockIndicatorsSql(_id='234')
current.calculate(MockDoc.wrap(doc))
current.save_to_sql(current.diff(None), self.engine)
expected = [
('123', date(1, 1, 1), 'mock', 'test_owner', None, None, None, None, None, None, 1, 2),
('123', date(2013, 1, 1), 'abc', '123', None, None, None, 1, None, 2, None, None),
('123', date(2012, 9, 24), 'mock', 'test_owner', 1, 3, None, None, None, None, None, None),
('123', date(2012, 9, 23), 'mock', 'test_owner', 1, 2, None, None, None, None, None, None),
('123', date(1, 1, 1), 'abc', 'xyz', None, None, None, None, 1, None, None, None),
('123', date(2013, 1, 1), 'abc', 'xyz', None, None, 3, None, None, None, None, None),
]
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([current._table]))
self.assertItemsEqual(rows, expected)
def test_save_to_sql_update(self):
self.test_save_to_sql()
actions = [dict(date="2012-09-23", x=5)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner"
)
current = MockIndicatorsSql(_id='234')
current.calculate(MockDoc.wrap(doc))
current.save_to_sql(current.diff(None), self.engine)
expected = [
('123', date(1, 1, 1), 'mock', 'test_owner', None, None, None, None, None, None, 1, 2),
('123', date(2013, 1, 1), 'abc', '123', None, None, None, 1, None, 2, None, None),
('123', date(2012, 9, 23), 'mock', 'test_owner', 1, 5, None, None, None, None, None, None),
('123', date(1, 1, 1), 'abc', 'xyz', None, None, None, None, 1, None, None, None),
('123', date(2013, 1, 1), 'abc', 'xyz', None, None, 3, None, None, None, None, None),
]
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([current._table]))
self.assertItemsEqual(rows, expected)
def test_save_to_sql_flat_fields(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
opened_on="2012-09-23",
closed_on="2013-09-23",
get_id="123",
domain="mock",
owner_id="test_owner"
)
current = MockIndicatorsSqlWithFlatFields(_id='234')
current.calculate(MockDoc.wrap(doc))
current.save_to_sql(current.diff(None), self.engine)
expected = [
('123', date(2012, 9, 24), '2012-09-23', '2013-09-23', 'mock', 'test_owner', 1, 3, None, None, None, None, None, None),
('123', date(2013, 1, 1), '2012-09-23', '2013-09-23', 'abc', '123', None, None, None, 1, None, 2, None, None),
('123', date(1, 1, 1), '2012-09-23', '2013-09-23', 'abc', 'xyz', None, None, None, None, 1, None, None, None),
('123', date(2012, 9, 23), '2012-09-23', '2013-09-23', 'mock', 'test_owner', 1, 2, None, None, None, None, None, None),
('123', date(1, 1, 1), '2012-09-23', '2013-09-23', 'mock', 'test_owner', None, None, None, None, None, None, 1, 2),
('123', date(2013, 1, 1), '2012-09-23', '2013-09-23', 'abc', 'xyz', None, None, 3, None, None, None, None, None),
]
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([current._table]))
self.assertItemsEqual(rows, expected)
def test_deleting_on_doc_type_change(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNone(indicator)
def test_deleting_on_doc_type_change_sql(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 6)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 0)
class TestResumableFunctionIterator(SimpleTestCase):
def setUp(self):
self.couch_db = FakeCouchDb()
self.batches = [
list(range(0, 3)),
list(range(3, 6)),
list(range(6, 8)),
]
self.all_items = list(itertools.chain(*self.batches))
self.itr = self.get_iterator()
def tearDown(self):
self.couch_db.reset()
def get_iterator(self, missing_items=None):
def data_provider(batch_number):
try:
return self.batches[batch_number]
except IndexError:
return []
itr = ResumableFunctionIterator('test', data_provider, TestArgsProvider())
itr.couch_db = self.couch_db
return itr
def test_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
def test_resume_iteration(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual([item for item in self.itr], self.all_items[3:])
def test_resume_iteration_after_exhaustion(self):
itr = iter(self.itr)
self.assertEqual(list(itr), self.all_items)
# resume iteration
self.batches.append([8, 9])
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), [8, 9])
def test_resume_iteration_after_legacy_completion(self):
itr = iter(self.itr)
self.assertEqual(list(itr), self.all_items)
state = self.itr.state
state.complete = True
state.args = state.kwargs = None
self.itr._save_state()
# attempt to resume yields no new items
self.batches.append([8, 9])
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), [])
def test_resume_iteration_after_complete_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
# resume iteration
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), [])
def test_discard_state(self):
self.assertEqual(list(self.itr), self.all_items)
self.itr.discard_state()
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), self.all_items)
def test_iteration_with_iterator_detail(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
self.assertEqual(self.itr.get_iterator_detail('progress'), None)
self.itr.set_iterator_detail('progress', {"visited": 6})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": 6})
self.itr.set_iterator_detail('progress', {"visited": "six"})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": "six"})
self.assertEqual([item for item in self.itr], self.all_items[3:])
class DocJoinExpressionTest(SimpleTestCase):
def setUp(self):
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
self.database = FakeCouchDb()
RelatedDocExpressionSpec.db_lookup = lambda _, type: self.database
def test_simple_lookup(self):
related_id = 'related-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_related_doc_not_found(self):
self.assertEqual(None, self.expression({'parent_id': 'some-missing-id'}))
def test_cross_domain_lookups(self):
related_id = 'cross-domain-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'wrong-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual(None, self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup(self):
related_id = 'nested-id-1'
related_id_2 = 'nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'test-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual('bar', self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup_cross_domains(self):
related_id = 'cross-nested-id-1'
related_id_2 = 'cross-nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'wrong-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual(None, self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_caching(self):
self.test_simple_lookup()
my_doc = self.database.get('my-id')
self.database.mock_docs.clear()
self.assertEqual({}, self.database.mock_docs)
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
same_expression = ExpressionFactory.from_spec(self.spec)
self.assertEqual('foo', same_expression(my_doc, EvaluationContext(my_doc, 0)))
class FluffTest(TestCase):
@classmethod
def setUpClass(cls):
super(FluffTest, cls).setUpClass()
# hack - force disconnecting the signals because ctable doesn't play nice with mocks
cls.previous_signal_receivers = indicator_document_updated.receivers
indicator_document_updated.receivers = []
cls.engine = connection_manager.get_engine('default')
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
MockIndicatorsSql.set_db(self.fakedb)
rebuild_table(self.engine, None, MockIndicatorsSql)
rebuild_table(self.engine, None, MockIndicatorsSqlWithFlatFields)
@classmethod
def tearDownClass(cls):
indicator_document_updated.receivers = cls.previous_signal_receivers
super(FluffTest, cls).tearDownClass()
def tearDown(self):
with self.engine.begin() as connection:
MockIndicatorsSql()._table.drop(connection, checkfirst=True)
MockIndicatorsSqlWithFlatFields()._table.drop(connection, checkfirst=True)
self.engine.dispose()
def test_calculator_base_classes(self):
# Base0
self.assertEqual(Base0._fluff_emitters, set([
'base_0_emitter',
]))
self.assertEqual(Base0._fluff_filters, set([
'base_0_filter',
]))
# Base1
self.assertEqual(Base1._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
]))
self.assertEqual(Base1._fluff_filters, set([
'base_0_filter',
'base_1_filter',
]))
# Base2
self.assertEqual(Base2._fluff_emitters, set([
'base_0_emitter',
'base_2_emitter',
]))
self.assertEqual(Base2._fluff_filters, set([
'base_0_filter',
'base_2_filter',
]))
# Base2
self.assertEqual(Base3._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
'base_2_emitter',
'base_3_emitter',
]))
self.assertEqual(Base3._fluff_filters, set([
'base_0_filter',
'base_1_filter',
'base_2_filter',
'base_3_filter',
]))
def test_indicator_classes(self):
self.assertEquals(Indicators1._calculators.keys(), ['base0'])
self.assertEquals(Indicators2._calculators.keys(), ['base1', 'base2'])
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(11, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=1), indicator["value_week"]["group_no_val"][0])
def test_calculator_calculate(self):
calc = ValueCalculator(WEEK)
calc.fluff = MockIndicators
values = calc.calculate(MockDoc.wrap(dict(actions=[dict(date="2012-09-23", x=2),
dict(date="2012-09-24", x=3)])))
self.assertEquals(len(values.keys()), 8)
self.assertEquals(values['null_value'], [dict(date=None, value=2, group_by=None)])
self.assertEquals(values['date_value'], [
dict(date=date(2012, 9, 23), value=2, group_by=None),
dict(date=date(2012, 9, 24), value=3, group_by=None)])
self.assertEquals(values['date'], [
dict(date=date(2012, 9, 23), value=1, group_by=None),
dict(date=date(2012, 9, 24), value=1, group_by=None)])
self.assertEquals(values['null'], [dict(date=None, value=1, group_by=None)])
self.assertEquals(values['group_list'], [dict(date=date(2013, 1, 1), group_by=['abc', 'xyz'], value=3)])
self.assertEquals(values['group_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=2)])
self.assertEquals(values['group_no_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=1)])
self.assertEquals(values['group_null'], [dict(date=None, group_by=['abc', 'xyz'], value=1)])
def test_calculator_get_result(self):
key = ['a', 'b']
now = datetime.utcnow().date()
start = json_format_date(now - WEEK)
end = json_format_date(now)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
self.fakedb.add_view('fluff/generic', [
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null', None], 'wrap_doc': True},
[{"key": None, "value": {"sum": 3}}]
),
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null_value', None], 'wrap_doc': True},
[{"key": None, "value": {"max": 8}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date', end],
'reduce': True, 'wrap_doc': True},
[{"key": None, "value": {"count": 7}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date_value', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date_value', end],
'reduce': True, 'wrap_doc': True},
[{"key": None, "value": {"sum": 11}}]
)
])
value = cls.get_result('value_week', key, reduce=True)
self.assertEqual(value['null'], 3)
self.assertEqual(value['date'], 7)
self.assertEqual(value['date_value'], 11)
self.assertEqual(value['null_value'], 8)
def test_indicator_diff_new(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
doc = cls(
domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 02, 23), 1]],
null=[],
date_value=[],
null_value=[[None, 3]]
)
)
diff = doc.diff(None)
self.maxDiff = None
expected = dict(domains=['mock'],
database=cls.Meta.app_label,
doc_type=classname,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 23), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=3, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertEqual(expected, diff)
class DocJoinExpressionTest(SimpleTestCase):
def setUp(self):
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
self.database = FakeCouchDb()
RelatedDocExpressionSpec.db_lookup = lambda _, type: self.database
def test_simple_lookup(self):
related_id = 'related-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_related_doc_not_found(self):
self.assertEqual(None, self.expression({'parent_id': 'some-missing-id'}))
def test_cross_domain_lookups(self):
related_id = 'cross-domain-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'wrong-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual(None, self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup(self):
related_id = 'nested-id-1'
related_id_2 = 'nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'test-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual('bar', self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup_cross_domains(self):
related_id = 'cross-nested-id-1'
related_id_2 = 'cross-nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'wrong-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual(None, self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_caching(self):
self.test_simple_lookup()
my_doc = self.database.get('my-id')
self.database.mock_docs.clear()
self.assertEqual({}, self.database.mock_docs)
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
same_expression = ExpressionFactory.from_spec(self.spec)
self.assertEqual('foo', same_expression(my_doc, EvaluationContext(my_doc, 0)))
def setUp(self):
self.orig_db = CommCareCase.get_db()
self.database = FakeCouchDb()
CommCareCase.set_db(self.database)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
class Test(TestCase):
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
def test_calculator_base_classes(self):
# Base0
self.assertEqual(Base0._fluff_emitters, set([
'base_0_emitter',
]))
self.assertEqual(Base0._fluff_filters, set([
'base_0_filter',
]))
# Base1
self.assertEqual(Base1._fluff_emitters,
set([
'base_0_emitter',
'base_1_emitter',
]))
self.assertEqual(Base1._fluff_filters,
set([
'base_0_filter',
'base_1_filter',
]))
# Base2
self.assertEqual(Base2._fluff_emitters,
set([
'base_0_emitter',
'base_2_emitter',
]))
self.assertEqual(Base2._fluff_filters,
set([
'base_0_filter',
'base_2_filter',
]))
# Base2
self.assertEqual(
Base3._fluff_emitters,
set([
'base_0_emitter',
'base_1_emitter',
'base_2_emitter',
'base_3_emitter',
]))
self.assertEqual(
Base3._fluff_filters,
set([
'base_0_filter',
'base_1_filter',
'base_2_filter',
'base_3_filter',
]))
def test_indicator_classes(self):
self.assertEquals(Indicators1._calculators.keys(), ['base0'])
self.assertEquals(Indicators2._calculators.keys(), ['base1', 'base2'])
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
self.fakedb.mock_docs["123"] = dict(actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner")
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.processor({'changes': [], 'id': '123', 'seq': 1})
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(9, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual(
{
'date': "2012-09-23",
'value': 1,
'group_by': None
}, indicator["value_week"]["date"][0])
self.assertEqual(
{
'date': "2012-09-24",
'value': 1,
'group_by': None
}, indicator["value_week"]["date"][1])
self.assertEqual({
'date': None,
'value': 1,
'group_by': None
}, indicator["value_week"]["null"][0])
self.assertEqual(
{
'date': "2012-09-23",
'value': 2,
'group_by': None
}, indicator["value_week"]["date_value"][0])
self.assertEqual(
{
'date': "2012-09-24",
'value': 3,
'group_by': None
}, indicator["value_week"]["date_value"][1])
self.assertEqual({
'date': None,
'value': 2,
'group_by': None
}, indicator["value_week"]["null_value"][0])
self.assertEqual(
dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3),
indicator["value_week"]["group_list"][0])
self.assertEqual(
dict(date='2013-01-01', group_by=['abc'], value=2),
indicator["value_week"]["group_val"][0])
self.assertEqual(
dict(date='2013-01-01', group_by=['abc'], value=1),
indicator["value_week"]["group_no_val"][0])
def test_calculator_calculate(self):
calc = ValueCalculator(WEEK)
values = calc.calculate(
MockDoc.wrap(
dict(actions=[
dict(date="2012-09-23", x=2),
dict(date="2012-09-24", x=3)
])))
self.assertEquals(len(values.keys()), 8)
self.assertEquals(values['null_value'],
[dict(date=None, value=2, group_by=None)])
self.assertEquals(values['date_value'], [
dict(date=date(2012, 9, 23), value=2, group_by=None),
dict(date=date(2012, 9, 24), value=3, group_by=None)
])
self.assertEquals(values['date'], [
dict(date=date(2012, 9, 23), value=1, group_by=None),
dict(date=date(2012, 9, 24), value=1, group_by=None)
])
self.assertEquals(values['null'],
[dict(date=None, value=1, group_by=None)])
self.assertEquals(
values['group_list'],
[dict(date=date(2013, 1, 1), group_by=['abc', 'xyz'], value=3)])
self.assertEquals(
values['group_val'],
[dict(date=date(2013, 1, 1), group_by=['abc'], value=2)])
self.assertEquals(
values['group_no_val'],
[dict(date=date(2013, 1, 1), group_by=['abc'], value=1)])
self.assertEquals(values['group_null'],
[dict(date=None, group_by=['abc'], value=1)])
def test_calculator_get_result(self):
key = ['a', 'b']
now = datetime.utcnow().date()
start = json_format_date(now - WEEK)
end = json_format_date(now)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
self.fakedb.add_view('fluff/generic', [
({
'reduce': True,
'key': [classname, 'a', 'b', 'value_week', 'null', None]
}, [{
"key": None,
"value": {
"sum": 3
}
}]),
({
'reduce': True,
'key':
[classname, 'a', 'b', 'value_week', 'null_value', None]
}, [{
"key": None,
"value": {
"max": 8
}
}]),
({
'startkey':
[classname, 'a', 'b', 'value_week', 'date', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date', end],
'reduce': True
}, [{
"key": None,
"value": {
"count": 7
}
}]),
({
'startkey':
[classname, 'a', 'b', 'value_week', 'date_value', start],
'endkey':
[classname, 'a', 'b', 'value_week', 'date_value', end],
'reduce':
True
}, [{
"key": None,
"value": {
"sum": 11
}
}])
])
value = cls.get_result('value_week', key, reduce=True)
self.assertEqual(value['null'], 3)
self.assertEqual(value['date'], 7)
self.assertEqual(value['date_value'], 11)
self.assertEqual(value['null_value'], 8)
def test_indicator_diff_new(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
doc = cls(domain="mock",
owner_id="123",
value_week=dict(date=[[date(2012, 02, 23), 1]],
null=[],
date_value=[],
null_value=[[None, 3]]))
diff = doc.diff(None)
self.maxDiff = None
expected = dict(
domains=['test'],
database=cls.Meta.app_label,
doc_type=classname,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={
'domain': 'integer',
'owner_id': 'string'
},
indicator_changes=[
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[
dict(date=date(2012, 2, 23),
value=1,
group_by=None)
]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=3, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertEqual(expected, diff)
class DocJoinExpressionTest(SimpleTestCase):
def setUp(self):
# we have to set the fake database before any other calls
self.orig_db = CommCareCase.get_db()
self.database = FakeCouchDb()
CommCareCase.set_db(self.database)
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
def tearDown(self):
CommCareCase.set_db(self.orig_db)
def test_simple_lookup(self):
related_id = 'related-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_related_doc_not_found(self):
self.assertEqual(None, self.expression({'parent_id': 'some-missing-id'}))
def test_cross_domain_lookups(self):
related_id = 'cross-domain-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'wrong-domain',
'related_property': 'foo'
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc
}
self.assertEqual(None, self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup(self):
related_id = 'nested-id-1'
related_id_2 = 'nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'test-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual('bar', self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup_cross_domains(self):
related_id = 'cross-nested-id-1'
related_id_2 = 'cross-nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'wrong-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual(None, self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_fail_on_bad_doc_type(self):
spec = {
"type": "related_doc",
"related_doc_type": "BadDocument",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
with self.assertRaises(BadSpecError):
ExpressionFactory.from_spec(spec)
def test_caching(self):
self.test_simple_lookup()
my_doc = self.database.get('my-id')
self.database.mock_docs.clear()
self.assertEqual({}, self.database.mock_docs)
self.assertEqual('foo', self.expression(my_doc, EvaluationContext(my_doc, 0)))
same_expression = ExpressionFactory.from_spec(self.spec)
self.assertEqual('foo', same_expression(my_doc, EvaluationContext(my_doc, 0)))
class TestGetOrCreateSchema(SimpleTestCase):
def setUp(self):
self.db = FormQuestionSchema.get_db()
self.fakedb = FakeCouchDb()
FormQuestionSchema.set_db(self.fakedb)
self.domain = 'test'
self.app_id = '123'
self.xmlns = 'this_xmlns'
self.schema = FormQuestionSchema(domain=self.domain,
app_id=self.app_id,
xmlns=self.xmlns)
def tearDown(self):
FormQuestionSchema.set_db(self.db)
def test_required_props(self):
with self.assertRaises(BadValueError):
schema = FormQuestionSchema(app_id=self.app_id, xmlns=self.xmlns)
schema.save()
# with self.assertRaises(BadValueError):
# schema = FormQuestionSchema(domain=self.domain, xmlns=self.xmlns)
# schema.save()
with self.assertRaises(BadValueError):
schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id)
schema.save()
def test_unique_key(self):
self.schema.save()
dupe_schema = FormQuestionSchema(domain=self.domain,
app_id=self.app_id,
xmlns=self.xmlns)
with self.assertRaises(ResourceConflict):
dupe_schema.save()
def test_get_existing(self):
self.schema.save()
schema = FormQuestionSchema.get_or_create(self.domain, self.app_id,
self.xmlns)
self.assertIsNotNone(schema)
self.assertEqual(schema._rev, self.schema._rev)
def test_get_new(self):
self.schema.save()
schema = FormQuestionSchema.get_or_create('new_domain', self.app_id,
self.xmlns)
self.assertIsNotNone(schema)
def test_migrate_old(self):
self.schema._id = '123'
self.schema.last_processed_version = 12
self.schema.save()
second_schema = FormQuestionSchema(domain=self.domain,
app_id=self.app_id,
xmlns=self.xmlns,
_id='1234')
second_schema.save()
self.assertEqual(len(self.fakedb.mock_docs), 2)
self.fakedb.add_view(
'form_question_schema/by_xmlns',
[({
'key': [self.domain, self.app_id, self.xmlns],
'include_docs': True
}, [self.schema.to_json(),
second_schema.to_json()])])
schema = FormQuestionSchema.get_or_create(self.domain, self.app_id,
self.xmlns)
self.assertEqual(schema.last_processed_version,
self.schema.last_processed_version)
self.assertNotEqual(schema.get_id, self.schema.get_id)
self.assertNotEqual(schema.get_id, second_schema.get_id)
self.assertEqual(len(self.fakedb.mock_docs), 1)
self.assertTrue(schema.get_id in self.fakedb.mock_docs)
def dao(self):
return CouchDocumentStore(FakeCouchDb())
class TestGetOrCreateSchema(SimpleTestCase):
def setUp(self):
self.db = FormQuestionSchema.get_db()
self.fakedb = FakeCouchDb()
FormQuestionSchema.set_db(self.fakedb)
self.domain = 'test'
self.app_id = '123'
self.xmlns = 'this_xmlns'
self.schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id, xmlns=self.xmlns)
def tearDown(self):
FormQuestionSchema.set_db(self.db)
def test_required_props(self):
with self.assertRaises(BadValueError):
schema = FormQuestionSchema(app_id=self.app_id, xmlns=self.xmlns)
schema.save()
# with self.assertRaises(BadValueError):
# schema = FormQuestionSchema(domain=self.domain, xmlns=self.xmlns)
# schema.save()
with self.assertRaises(BadValueError):
schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id)
schema.save()
def test_unique_key(self):
self.schema.save()
dupe_schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id, xmlns=self.xmlns)
with self.assertRaises(ResourceConflict):
dupe_schema.save()
def test_get_existing(self):
self.schema.save()
schema = FormQuestionSchema.get_or_create(self.domain, self.app_id, self.xmlns)
self.assertIsNotNone(schema)
self.assertEqual(schema._rev, self.schema._rev)
def test_get_new(self):
self.schema.save()
schema = FormQuestionSchema.get_or_create('new_domain', self.app_id, self.xmlns)
self.assertIsNotNone(schema)
def test_migrate_old(self):
self.schema._id = '123'
self.schema.last_processed_version = 12
self.schema.save()
second_schema = FormQuestionSchema(domain=self.domain, app_id=self.app_id, xmlns=self.xmlns, _id='1234')
second_schema.save()
self.assertEqual(len(self.fakedb.mock_docs), 2)
self.fakedb.add_view(
'form_question_schema/by_xmlns',
[(
{'key': [self.domain, self.app_id, self.xmlns], 'include_docs': True},
[
self.schema.to_json(), second_schema.to_json()
]
)]
)
schema = FormQuestionSchema.get_or_create(self.domain, self.app_id, self.xmlns)
self.assertEqual(schema.last_processed_version, self.schema.last_processed_version)
self.assertNotEqual(schema.get_id, self.schema.get_id)
self.assertNotEqual(schema.get_id, second_schema.get_id)
self.assertEqual(len(self.fakedb.mock_docs), 1)
self.assertTrue(schema.get_id in self.fakedb.mock_docs)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
def setUp(self):
self.db = Dhis2Repeater.get_db()
self.fakedb = FakeCouchDb()
Dhis2Repeater.set_db(self.fakedb)
class RelatedDocExpressionTest(SimpleTestCase):
def setUp(self):
# we have to set the fake database before any other calls
self.orig_db = CommCareCase.get_db()
self.database = FakeCouchDb()
CommCareCase.set_db(self.database)
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec({
"type":
"related_doc",
"related_doc_type":
"CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
})
def tearDown(self):
CommCareCase.set_db(self.orig_db)
def test_simple_lookup(self):
related_id = 'related-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {'domain': 'test-domain', 'related_property': 'foo'}
self.database.mock_docs = {'my-id': my_doc, related_id: related_doc}
self.assertEqual('foo',
self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_related_doc_not_found(self):
doc = {'parent_id': 'some-missing-id', 'domain': 'whatever'}
self.assertEqual(None, self.expression(doc, EvaluationContext(doc, 0)))
def test_cross_domain_lookups(self):
related_id = 'cross-domain-id'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {'domain': 'wrong-domain', 'related_property': 'foo'}
self.database.mock_docs = {'my-id': my_doc, related_id: related_doc}
self.assertEqual(None,
self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup(self):
related_id = 'nested-id-1'
related_id_2 = 'nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'test-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual(
'bar', self.nested_expression(my_doc, EvaluationContext(my_doc,
0)))
def test_nested_lookup_cross_domains(self):
related_id = 'cross-nested-id-1'
related_id_2 = 'cross-nested-id-2'
my_doc = {
'domain': 'test-domain',
'parent_id': related_id,
}
related_doc = {
'domain': 'test-domain',
'parent_id': related_id_2,
'related_property': 'foo',
}
related_doc_2 = {
'domain': 'wrong-domain',
'related_property': 'bar',
}
self.database.mock_docs = {
'my-id': my_doc,
related_id: related_doc,
related_id_2: related_doc_2
}
self.assertEqual(
None, self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_fail_on_bad_doc_type(self):
spec = {
"type": "related_doc",
"related_doc_type": "BadDocument",
"doc_id_expression": {
"type": "property_name",
"property_name": "parent_id"
},
"value_expression": {
"type": "property_name",
"property_name": "related_property"
}
}
with self.assertRaises(BadSpecError):
ExpressionFactory.from_spec(spec)
def test_caching(self):
self.test_simple_lookup()
my_doc = self.database.get('my-id')
context = EvaluationContext(my_doc, 0)
self.assertEqual('foo', self.expression(my_doc, context))
my_doc = self.database.get('my-id')
self.database.mock_docs.clear()
self.assertEqual('foo', self.expression(my_doc, context))
class DocJoinExpressionTest(SimpleTestCase):
def setUp(self):
self.spec = {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {"type": "property_name", "property_name": "related_property"},
}
self.expression = ExpressionFactory.from_spec(self.spec)
self.nested_expression = ExpressionFactory.from_spec(
{
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {
"type": "related_doc",
"related_doc_type": "CommCareCase",
"doc_id_expression": {"type": "property_name", "property_name": "parent_id"},
"value_expression": {"type": "property_name", "property_name": "related_property"},
},
}
)
self.database = FakeCouchDb()
RelatedDocExpressionSpec.db_lookup = lambda _, type: self.database
def test_simple_lookup(self):
related_id = "related-id"
my_doc = {"domain": "test-domain", "parent_id": related_id}
related_doc = {"domain": "test-domain", "related_property": "foo"}
self.database.mock_docs = {"my-id": my_doc, related_id: related_doc}
self.assertEqual("foo", self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_related_doc_not_found(self):
self.assertEqual(None, self.expression({"parent_id": "some-missing-id"}))
def test_cross_domain_lookups(self):
related_id = "cross-domain-id"
my_doc = {"domain": "test-domain", "parent_id": related_id}
related_doc = {"domain": "wrong-domain", "related_property": "foo"}
self.database.mock_docs = {"my-id": my_doc, related_id: related_doc}
self.assertEqual(None, self.expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup(self):
related_id = "nested-id-1"
related_id_2 = "nested-id-2"
my_doc = {"domain": "test-domain", "parent_id": related_id}
related_doc = {"domain": "test-domain", "parent_id": related_id_2, "related_property": "foo"}
related_doc_2 = {"domain": "test-domain", "related_property": "bar"}
self.database.mock_docs = {"my-id": my_doc, related_id: related_doc, related_id_2: related_doc_2}
self.assertEqual("bar", self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_nested_lookup_cross_domains(self):
related_id = "cross-nested-id-1"
related_id_2 = "cross-nested-id-2"
my_doc = {"domain": "test-domain", "parent_id": related_id}
related_doc = {"domain": "test-domain", "parent_id": related_id_2, "related_property": "foo"}
related_doc_2 = {"domain": "wrong-domain", "related_property": "bar"}
self.database.mock_docs = {"my-id": my_doc, related_id: related_doc, related_id_2: related_doc_2}
self.assertEqual(None, self.nested_expression(my_doc, EvaluationContext(my_doc, 0)))
def test_caching(self):
self.test_simple_lookup()
my_doc = self.database.get("my-id")
self.database.mock_docs.clear()
self.assertEqual({}, self.database.mock_docs)
self.assertEqual("foo", self.expression(my_doc, EvaluationContext(my_doc, 0)))
same_expression = ExpressionFactory.from_spec(self.spec)
self.assertEqual("foo", same_expression(my_doc, EvaluationContext(my_doc, 0)))
class Test(TestCase):
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
def test_calculator_base_classes(self):
# Base0
self.assertEqual(Base0._fluff_emitters, set([
'base_0_emitter',
]))
self.assertEqual(Base0._fluff_filters, set([
'base_0_filter',
]))
# Base1
self.assertEqual(Base1._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
]))
self.assertEqual(Base1._fluff_filters, set([
'base_0_filter',
'base_1_filter',
]))
# Base2
self.assertEqual(Base2._fluff_emitters, set([
'base_0_emitter',
'base_2_emitter',
]))
self.assertEqual(Base2._fluff_filters, set([
'base_0_filter',
'base_2_filter',
]))
# Base2
self.assertEqual(Base3._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
'base_2_emitter',
'base_3_emitter',
]))
self.assertEqual(Base3._fluff_filters, set([
'base_0_filter',
'base_1_filter',
'base_2_filter',
'base_3_filter',
]))
def test_indicator_classes(self):
self.assertEquals(Indicators1._calculators.keys(), ['base0'])
self.assertEquals(Indicators2._calculators.keys(), ['base1', 'base2'])
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
self.fakedb.mock_docs["123"] = dict(actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner")
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.processor({'changes': [], 'id': '123', 'seq': 1})
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(9, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc'], value=1), indicator["value_week"]["group_no_val"][0])
def test_calculator_calculate(self):
calc = ValueCalculator(WEEK)
values = calc.calculate(MockDoc.wrap(dict(actions=[dict(date="2012-09-23", x=2),
dict(date="2012-09-24", x=3)])))
self.assertEquals(len(values.keys()), 8)
self.assertEquals(values['null_value'], [dict(date=None, value=2, group_by=None)])
self.assertEquals(values['date_value'], [
dict(date=date(2012, 9, 23), value=2, group_by=None),
dict(date=date(2012, 9, 24), value=3, group_by=None)])
self.assertEquals(values['date'], [
dict(date=date(2012, 9, 23), value=1, group_by=None),
dict(date=date(2012, 9, 24), value=1, group_by=None)])
self.assertEquals(values['null'], [dict(date=None, value=1, group_by=None)])
self.assertEquals(values['group_list'], [dict(date=date(2013, 1, 1), group_by=['abc', 'xyz'], value=3)])
self.assertEquals(values['group_val'], [dict(date=date(2013, 1, 1), group_by=['abc'], value=2)])
self.assertEquals(values['group_no_val'], [dict(date=date(2013, 1, 1), group_by=['abc'], value=1)])
self.assertEquals(values['group_null'], [dict(date=None, group_by=['abc'], value=1)])
def test_calculator_get_result(self):
key = ['a', 'b']
now = datetime.utcnow().date()
start = json_format_date(now - WEEK)
end = json_format_date(now)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
self.fakedb.add_view('fluff/generic', [
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null', None]},
[{"key": None, "value": {"sum": 3}}]
),
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null_value', None]},
[{"key": None, "value": {"max": 8}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date', end],
'reduce': True},
[{"key": None, "value": {"count": 7}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date_value', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date_value', end],
'reduce': True},
[{"key": None, "value": {"sum": 11}}]
)
])
value = cls.get_result('value_week', key, reduce=True)
self.assertEqual(value['null'], 3)
self.assertEqual(value['date'], 7)
self.assertEqual(value['date_value'], 11)
self.assertEqual(value['null_value'], 8)
def test_indicator_diff_new(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
doc = cls(
domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 02, 23), 1]],
null=[],
date_value=[],
null_value=[[None, 3]]
)
)
diff = doc.diff(None)
self.maxDiff = None
expected = dict(domains=['test'],
database=cls.Meta.app_label,
doc_type=classname,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 23), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=3, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertEqual(expected, diff)
class FluffTest(TestCase):
@classmethod
def setUpClass(cls):
# hack - force disconnecting the signals because ctable doesn't play nice with mocks
cls.previous_signal_receivers = indicator_document_updated.receivers
indicator_document_updated.receivers = []
cls.engine = sqlalchemy.create_engine(settings.SQL_REPORTING_DATABASE_URL)
def setUp(self):
self.fakedb = FakeCouchDb()
MockIndicators.set_db(self.fakedb)
MockIndicatorsWithGetters.set_db(self.fakedb)
MockDoc.set_db(self.fakedb)
MockIndicatorsSql.set_db(self.fakedb)
rebuild_table(self.engine, None, MockIndicatorsSql)
rebuild_table(self.engine, None, MockIndicatorsSqlWithFlatFields)
@classmethod
def tearDownClass(cls):
indicator_document_updated.receivers = cls.previous_signal_receivers
def tearDown(self):
with self.engine.begin() as connection:
MockIndicatorsSql()._table.drop(connection, checkfirst=True)
MockIndicatorsSqlWithFlatFields()._table.drop(connection, checkfirst=True)
self.engine.dispose()
def test_calculator_base_classes(self):
# Base0
self.assertEqual(Base0._fluff_emitters, set([
'base_0_emitter',
]))
self.assertEqual(Base0._fluff_filters, set([
'base_0_filter',
]))
# Base1
self.assertEqual(Base1._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
]))
self.assertEqual(Base1._fluff_filters, set([
'base_0_filter',
'base_1_filter',
]))
# Base2
self.assertEqual(Base2._fluff_emitters, set([
'base_0_emitter',
'base_2_emitter',
]))
self.assertEqual(Base2._fluff_filters, set([
'base_0_filter',
'base_2_filter',
]))
# Base2
self.assertEqual(Base3._fluff_emitters, set([
'base_0_emitter',
'base_1_emitter',
'base_2_emitter',
'base_3_emitter',
]))
self.assertEqual(Base3._fluff_filters, set([
'base_0_filter',
'base_1_filter',
'base_2_filter',
'base_3_filter',
]))
def test_indicator_classes(self):
self.assertEquals(Indicators1._calculators.keys(), ['base0'])
self.assertEquals(Indicators2._calculators.keys(), ['base1', 'base2'])
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(11, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=1), indicator["value_week"]["group_no_val"][0])
def test_calculator_calculate(self):
calc = ValueCalculator(WEEK)
calc.fluff = MockIndicators
values = calc.calculate(MockDoc.wrap(dict(actions=[dict(date="2012-09-23", x=2),
dict(date="2012-09-24", x=3)])))
self.assertEquals(len(values.keys()), 8)
self.assertEquals(values['null_value'], [dict(date=None, value=2, group_by=None)])
self.assertEquals(values['date_value'], [
dict(date=date(2012, 9, 23), value=2, group_by=None),
dict(date=date(2012, 9, 24), value=3, group_by=None)])
self.assertEquals(values['date'], [
dict(date=date(2012, 9, 23), value=1, group_by=None),
dict(date=date(2012, 9, 24), value=1, group_by=None)])
self.assertEquals(values['null'], [dict(date=None, value=1, group_by=None)])
self.assertEquals(values['group_list'], [dict(date=date(2013, 1, 1), group_by=['abc', 'xyz'], value=3)])
self.assertEquals(values['group_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=2)])
self.assertEquals(values['group_no_val'], [dict(date=date(2013, 1, 1), group_by=['abc', '123'], value=1)])
self.assertEquals(values['group_null'], [dict(date=None, group_by=['abc', 'xyz'], value=1)])
def test_calculator_get_result(self):
key = ['a', 'b']
now = datetime.utcnow().date()
start = json_format_date(now - WEEK)
end = json_format_date(now)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
self.fakedb.add_view('fluff/generic', [
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null', None], 'wrap_doc': True},
[{"key": None, "value": {"sum": 3}}]
),
(
{'reduce': True, 'key': [classname, 'a', 'b', 'value_week', 'null_value', None], 'wrap_doc': True},
[{"key": None, "value": {"max": 8}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date', end],
'reduce': True, 'wrap_doc': True},
[{"key": None, "value": {"count": 7}}]
),
(
{'startkey': [classname, 'a', 'b', 'value_week', 'date_value', start],
'endkey': [classname, 'a', 'b', 'value_week', 'date_value', end],
'reduce': True, 'wrap_doc': True},
[{"key": None, "value": {"sum": 11}}]
)
])
value = cls.get_result('value_week', key, reduce=True)
self.assertEqual(value['null'], 3)
self.assertEqual(value['date'], 7)
self.assertEqual(value['date_value'], 11)
self.assertEqual(value['null_value'], 8)
def test_indicator_diff_new(self):
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
doc = cls(
domain="mock",
owner_id="123",
value_week=dict(
date=[[date(2012, 02, 23), 1]],
null=[],
date_value=[],
null_value=[[None, 3]]
)
)
diff = doc.diff(None)
self.maxDiff = None
expected = dict(domains=['mock'],
database=cls.Meta.app_label,
doc_type=classname,
group_values=['mock', '123'],
group_names=['domain', 'owner_id'],
group_type_map={'domain': 'integer', 'owner_id': 'string'},
indicator_changes=[
dict(calculator='value_week',
emitter='date',
emitter_type='date',
reduce_type='count',
values=[dict(date=date(2012, 2, 23), value=1, group_by=None)]),
dict(calculator='value_week',
emitter='null_value',
emitter_type='null',
reduce_type='max',
values=[dict(date=None, value=3, group_by=None)])
],
all_indicators=self.all_indicators())
self.assertEqual(expected, diff)
class TestResumableFunctionIterator(SimpleTestCase):
def setUp(self):
self.couch_db = FakeCouchDb()
self.batches = [
list(range(0, 3)),
list(range(3, 6)),
list(range(6, 8)),
]
self.all_items = list(itertools.chain(*self.batches))
self.itr = self.get_iterator()
def tearDown(self):
self.couch_db.reset()
def get_iterator(self, missing_items=None):
def data_provider(batch_number):
try:
return self.batches[batch_number]
except IndexError:
return []
def item_getter(item_id):
if missing_items and item_id in missing_items:
return None
return int(item_id)
itr = ResumableFunctionIterator('test', data_provider, TestArgsProvider(), item_getter)
itr.couch_db = self.couch_db
return itr
def test_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
def test_resume_iteration(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual([item for item in self.itr], self.all_items[3:])
def test_resume_iteration_after_complete_iteration(self):
self.assertEqual(list(self.itr), self.all_items)
# resume iteration
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), [])
def test_discard_state(self):
self.assertEqual(list(self.itr), self.all_items)
self.itr.discard_state()
self.itr = self.get_iterator()
self.assertEqual(list(self.itr), self.all_items)
def test_iteration_with_iterator_detail(self):
itr = iter(self.itr)
self.assertEqual([next(itr) for i in range(6)], self.all_items[:6])
self.assertEqual(self.itr.get_iterator_detail('progress'), None)
self.itr.set_iterator_detail('progress', {"visited": 6})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": 6})
self.itr.set_iterator_detail('progress', {"visited": "six"})
# stop/resume iteration
self.itr = self.get_iterator()
self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": "six"})
self.assertEqual([item for item in self.itr], self.all_items[3:])
def test_iteration_with_retry(self):
itr = iter(self.itr)
item = next(itr)
self.itr.retry(str(item))
self.assertEqual(item, 0)
self.assertEqual([0] + [d for d in itr],
self.all_items + [0])
def test_iteration_complete_after_retry(self):
itr = iter(self.itr)
self.itr.retry(str(next(itr)))
list(itr)
self.itr = self.get_iterator()
self.assertEqual([item for item in self.itr], [])
def test_iteration_with_max_retry(self):
itr = iter(self.itr)
item = next(itr)
ids = [item]
self.assertEqual(item, 0)
self.itr.retry(str(item))
retries = 1
for item in itr:
ids.append(item)
if item == 0:
if retries < 3:
self.itr.retry(str(item))
retries += 1
else:
break
self.assertEqual(item, 0)
with self.assertRaises(TooManyRetries):
self.itr.retry(str(item))
self.assertEqual(ids, self.all_items + [0, 0, 0])
self.assertEqual(list(itr), [])
self.assertEqual(list(self.get_iterator()), [])
def test_iteration_with_missing_retry_doc(self):
iterator = self.get_iterator(missing_items=["0"])
itr = iter(iterator)
item = next(itr)
self.assertEqual(item, 0)
iterator.retry(str(item))
self.assertEqual([0] + [d for d in itr], self.all_items)
