def test_refresh_doc_meta(self):
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values('id','submission_json')
raw_indicator_list = [k for k,v in json\
.loads(source_submissions_data[0]['submission_json']).iteritems()]
mr.refresh_doc_meta()
source_indicator_id_for_this_doc = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'indicator',
source_object_code__in = raw_indicator_list
).values_list('id',flat=True)
)
## -3 because we dont want campaign_col,indicator_col and unique_col
## showing up as indicators to map!
self.assertEqual(len(source_indicator_id_for_this_doc)\
,len(raw_indicator_list) -3 )
def test_refresh_doc_meta(self):
self.set_up()
mr = MasterRefresh(self.user.id ,self.document.id)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values('id','submission_json')
raw_indicator_list = [k for k,v in json\
.loads(source_submissions_data[0]['submission_json']).iteritems()]
mr.refresh_doc_meta()
source_indicator_id_for_this_doc = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'indicator',
source_object_code__in = raw_indicator_list
).values_list('id',flat=True)
)
## -3 because we dont want campaign_col,indicator_col and unique_col
## showing up as indicators to map!
self.assertEqual(len(source_indicator_id_for_this_doc)\
,len(raw_indicator_list) -3 )
def test_refresh_doc_meta(self):
self.set_up()
mr = MasterRefresh(self.user.id ,self.document.id)
## load this w read_csv(self.test_file_location) remove DocTransform
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values('id','submission_json')
raw_indicator_list = [k for k,v in json\
.loads(source_submissions_data[0]['submission_json']).iteritems()]
mr.refresh_doc_meta()
source_indicator_id_for_this_doc = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'indicator',
source_object_code__in = raw_indicator_list
).values_list('id',flat=True)
)
## should be more specific here.. but this proves with a high degree ##
## of certainty that the source_object_map rows have been created ##
self.assertEqual(len(source_indicator_id_for_this_doc)\
,len(raw_indicator_list))
def test_submission_detail_refresh(self,):
self.set_up()
mr = MasterRefresh(self.user.id ,self.document.id)
mr.refresh_doc_meta()
## FIXME replace source_submission_data with read_csv(self.test_file)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id',flat=True)
mr.refresh_submission_details()
submission_details = SourceSubmissionDetail.objects\
.filter(document_id = self.document.id)
self.assertEqual(len(source_submissions_data)\
,len(submission_details))
def test_submission_detail_refresh(self, ):
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
mr.refresh_doc_meta()
## FIXME replace source_submission_data with read_csv(self.test_file)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id',flat=True)
# = read_csv(self.test_file)
## fake the submission_data
mr.refresh_submission_details()
submission_details = SourceSubmission.objects\
.filter(document_id = self.document.id)
self.assertEqual(len(source_submissions_data)\
,len(submission_details))
def test_submission_to_datapoint(self):
'''
This simulates a new document being processed with source_object_map
records that have been sucessfullly mapped to master_ids but from a
different document.
A few things this method checks:
1. there is a record in doc_object_map even for
mappings that existed before this document was ingested.
2. WHen the submission detail is refreshed, the reiogn/campaign ids
that we mapped should exist in that row.
2. DocDataPoint records are created if the necessary mapping exists
3. There are no zero or null values allowed in doc_datapoint
4. The doc_datapoint from #3 is merged into datpaoint.
'''
self.set_up()
mr = MasterRefresh(self.user.id ,self.document.id)
submission_qs = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id','submission_json')[0]
ss_id, first_submission = submission_qs[0],json.loads(submission_qs[1])
location_code = first_submission[self.location_code_input_column]
campaign_code = first_submission[self.campaign_code_input_column]
raw_indicator_list = [k for k,v in first_submission.iteritems()]
indicator_code = raw_indicator_list[-1]
map_location_id = Location.objects.all()[0].id
som_id_r = SourceObjectMap.objects.create(
content_type = 'location',
source_object_code = location_code,
master_object_id = map_location_id,
mapped_by_id = self.user.id
)
map_campaign_id = Campaign.objects.all()[0].id
som_id_c = SourceObjectMap.objects.create(
content_type = 'campaign',
source_object_code = campaign_code,
master_object_id = map_campaign_id,
mapped_by_id = self.user.id
)
map_indicator_id = Indicator.objects.all()[0].id
som_id_i = SourceObjectMap.objects.create(
content_type = 'indicator',
source_object_code = indicator_code,
master_object_id = map_indicator_id,
mapped_by_id = self.user.id
)
mr.refresh_doc_meta()
doc_som_id_for_location_code = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'location',
source_object_code = location_code,
).values_list('id',flat=True)
).values_list('id',flat=True)
## Test Case 1 ##
self.assertEqual(len(doc_som_id_for_location_code),1) # 1
mr_with_new_meta = mr = MasterRefresh(self.user.id ,self.document.id)
first_submission_detail = SourceSubmissionDetail.objects\
.get(source_submission_id = ss_id)
## Test Case 2 ##
self.assertEqual(first_submission_detail.location_id, map_location_id)
self.assertEqual(first_submission_detail.campaign_id, map_campaign_id)
mr.submissions_to_doc_datapoints()
doc_dp_ids = DocDataPoint.objects.filter(document_id =
self.document.id)
## Test Case #3
self.assertEqual(1,len(doc_dp_ids))
mr.sync_datapoint()
dps = DataPoint.objects.all()
## Test Case #4
self.assertEqual(1,len(doc_dp_ids))
def test_submission_to_datapoint(self):
'''
This simulates a new document being processed with source_object_map
records that have been sucessfullly mapped to master_ids but from a
different document.
A few things this method checks:
1. there is a record in doc_object_map even for
mappings that existed before this document was ingested.
2. WHen the submission detail is refreshed, the reiogn/campaign ids
that we mapped should exist in that row.
2. DocDataPoint records are created if the necessary mapping exists
3. There are no zero or null values allowed in doc_datapoint
4. The doc_datapoint from #3 is merged into datpaoint.
'''
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
submission_qs = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id','submission_json')[0]
ss_id, first_submission = submission_qs[0], json.loads(
submission_qs[1])
location_code = first_submission[self.location_code_input_column]
campaign_code = first_submission[self.campaign_code_input_column]
raw_indicator_list = [k for k, v in first_submission.iteritems()]
indicator_code = raw_indicator_list[-1]
map_location_id = Location.objects.all()[0].id
som_id_r = SourceObjectMap.objects.create(
content_type='location',
source_object_code=location_code,
master_object_id=map_location_id,
mapped_by_id=self.user.id)
map_campaign_id = Campaign.objects.all()[0].id
som_id_c = SourceObjectMap.objects.create(
content_type='campaign',
source_object_code=campaign_code,
master_object_id=map_campaign_id,
mapped_by_id=self.user.id)
map_indicator_id = Indicator.objects.all()[0].id
som_id_i = SourceObjectMap.objects.create(
content_type='indicator',
source_object_code=indicator_code,
master_object_id=map_indicator_id,
mapped_by_id=self.user.id)
mr.refresh_doc_meta()
doc_som_id_for_location_code = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'location',
source_object_code = location_code,
).values_list('id',flat=True)
).values_list('id',flat=True)
## Test Case 1 ##
self.assertEqual(len(doc_som_id_for_location_code), 1) # 1
mr_with_new_meta = MasterRefresh(self.user.id, self.document.id)
mr_with_new_meta.refresh_submission_details()
first_submission_detail = SourceSubmission.objects\
.get(id = ss_id)
## Test Case 2 ##
self.assertEqual(first_submission_detail.location_id, map_location_id)
self.assertEqual(first_submission_detail.campaign_id, map_campaign_id)
mr_with_new_meta.submissions_to_doc_datapoints()
doc_dp_ids = DocDataPoint.objects.filter(document_id=self.document.id)
## Test Case #3
self.assertEqual(1, len(doc_dp_ids))
mr_with_new_meta.sync_datapoint()
dps = DataPoint.objects.all()
## Test Case #4
self.assertEqual(1, len(dps))
