def test_deleting_on_doc_type_change(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc')
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(
change_from_couch_row({
'changes': [],
'id': '123',
'seq': 1,
'doc': doc
}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(
change_from_couch_row({
'changes': [],
'id': '123',
'seq': 1,
'doc': doc
}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNone(indicator)
def test_deleting_on_doc_type_change_sql(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc')
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()()
pillow.process_change(
change_from_couch_row({
'changes': [],
'id': '123',
'seq': 1,
'doc': doc
}))
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 6)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()()
pillow.process_change(
change_from_couch_row({
'changes': [],
'id': '123',
'seq': 1,
'doc': doc
}))
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 0)
def test_deleting_on_doc_type_change_sql(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()(chunk_size=0, checkpoint=mock_checkpoint())
pillow.processor(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}),
PillowRuntimeContext())
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 6)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicatorsSql]:
pillow = cls.pillow()(chunk_size=0, checkpoint=mock_checkpoint())
pillow.processor(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}),
PillowRuntimeContext())
with self.engine.begin() as connection:
rows = connection.execute(sqlalchemy.select([cls._table]))
self.assertEqual(rows.rowcount, 0)
def test_deleting_on_doc_type_change(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()(chunk_size=0, checkpoint=mock_checkpoint())
pillow.processor(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}),
PillowRuntimeContext())
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
doc['doc_type'] = 'MockArchive'
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()(chunk_size=0, checkpoint=mock_checkpoint())
pillow.processor(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}),
PillowRuntimeContext())
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNone(indicator)
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(11, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=1), indicator["value_week"]["group_no_val"][0])
def change_object(self):
change = change_from_couch_row(
self.change if self.change else {'id': self.doc_id})
if self.change_metadata:
change.metadata = ChangeMeta.wrap(self.change_metadata)
change.document = None
return change
def test_indicator_calculation(self):
actions = [dict(date="2012-09-23", x=2), dict(date="2012-09-24", x=3)]
doc = dict(
actions=actions,
get_id="123",
domain="mock",
owner_id="test_owner",
doc_type='MockDoc'
)
for cls in [MockIndicators, MockIndicatorsWithGetters]:
classname = cls.__name__
pillow = cls.pillow()()
pillow.process_change(change_from_couch_row({'changes': [], 'id': '123', 'seq': 1, 'doc': doc}))
indicator = self.fakedb.mock_docs.get("%s-123" % classname, None)
self.assertIsNotNone(indicator)
self.assertEqual(11, len(indicator))
self.assertEqual(8, len(indicator['value_week']))
self.assertIn("value_week", indicator)
self.assertIn("date", indicator["value_week"])
self.assertIn("null", indicator["value_week"])
self.assertIn("date_value", indicator["value_week"])
self.assertIn("null_value", indicator["value_week"])
self.assertEqual({'date': "2012-09-23", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][0])
self.assertEqual({'date': "2012-09-24", 'value': 1, 'group_by': None}, indicator["value_week"]["date"][1])
self.assertEqual({'date': None, 'value': 1, 'group_by': None}, indicator["value_week"]["null"][0])
self.assertEqual({'date': "2012-09-23", 'value': 2, 'group_by': None}, indicator["value_week"]["date_value"][0])
self.assertEqual({'date': "2012-09-24", 'value': 3, 'group_by': None}, indicator["value_week"]["date_value"][1])
self.assertEqual({'date': None, 'value': 2, 'group_by': None}, indicator["value_week"]["null_value"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', 'xyz'], value=3), indicator["value_week"]["group_list"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=2), indicator["value_week"]["group_val"][0])
self.assertEqual(dict(date='2013-01-01', group_by=['abc', '123'], value=1), indicator["value_week"]["group_no_val"][0])
def test_deleted_doc(self):
id = 'test_doc'
change_dict = {'id': id, 'seq': 54321}
error = create_error(change_from_couch_row(change_dict))
error.save()
# this used to error out
process_pillow_retry(error)
with self.assertRaises(PillowError.DoesNotExist):
PillowError.objects.get(id=error.id)
def process_row(self, row, count):
if count >= self.start_num:
retries = 0
while retries < MAX_TRIES:
try:
if not self.custom_filter(row):
break
if not isinstance(row, Change):
assert isinstance(row, dict)
row = change_from_couch_row(row)
self.pillow.process_change(row)
break
except Exception, ex:
retries += 1
self.log("\tException sending single item %s, %s, retrying..." % (row['id'], ex))
time.sleep(RETRY_DELAY + retries * RETRY_TIME_DELAY_FACTOR)
def test_convert_to_and_from_couch_row(self):
couch_row = {
'id': 'an-id',
'doc': {'a': 'document'},
'seq': '21',
'deleted': False
}
ways_to_make_change_object = [
change_from_couch_row(couch_row),
]
for change in ways_to_make_change_object:
self.assertTrue(isinstance(change, Change))
self.assertEqual('an-id', change.id)
self.assertEqual(couch_row['doc'], change.document)
self.assertEqual('21', change.sequence_id)
self.assertEqual(False, change.deleted)
for key in couch_row:
self.assertEqual(couch_row[key], change[key])
def test_convert_to_and_from_couch_row(self):
couch_row = {
'id': 'an-id',
'doc': {'a': 'document'},
'seq': '21',
'deleted': False
}
ways_to_make_change_object = [
change_from_couch_row(couch_row),
]
for change in ways_to_make_change_object:
self.assertTrue(isinstance(change, Change))
self.assertEqual('an-id', change.id)
self.assertEqual(couch_row['doc'], change.document)
self.assertEqual('21', change.sequence_id)
self.assertEqual(False, change.deleted)
for key in couch_row:
self.assertEqual(couch_row[key], change[key])
def change_object(self):
return change_from_couch_row(json.loads(self.change) if self.change else {'id': self.doc_id})
def _change_from_doc(doc_dict):
doc_dict['domain'] = INDICATOR_TEST_DOMAIN
return change_from_couch_row({'id': doc_dict['_id'], 'doc': doc_dict})
def change_object(self):
change = change_from_couch_row(self.change if self.change else {'id': self.doc_id})
if self.change_metadata:
change.metadata = ChangeMeta.wrap(self.change_metadata)
change.document = None
return change
def _change_from_doc(doc_dict):
doc_dict['domain'] = INDICATOR_TEST_DOMAIN
return change_from_couch_row({
'id': doc_dict['_id'], 'doc': doc_dict
})
def change_object(self):
return change_from_couch_row(
json.loads(self.change) if self.change else {'id': self.doc_id})