def test_refresh_doc_meta(self):
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values('id','submission_json')
raw_indicator_list = [k for k,v in json\
.loads(source_submissions_data[0]['submission_json']).iteritems()]
mr.refresh_doc_meta()
source_indicator_id_for_this_doc = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'indicator',
source_object_code__in = raw_indicator_list
).values_list('id',flat=True)
)
## -3 because we dont want campaign_col,indicator_col and unique_col
## showing up as indicators to map!
self.assertEqual(len(source_indicator_id_for_this_doc)\
,len(raw_indicator_list) -3 )
def do(self):
user_id = User.objects.get(username='******').id
document_id = SourceSubmission.objects.\
filter(process_status='TO_PROCESS').\
values_list('document_id',flat=True)[0]
mr = MasterRefresh(user_id, document_id)
mr.main()
def test_refresh_master_init(self):
self.set_up()
mr = MasterRefresh(self.user.id\
,self.document.id)
self.assertTrue(isinstance,(mr,MasterRefresh))
self.assertEqual(self.document.id,mr.document_id)
def refresh_master_for_document(self, list_of_object_ids):
mr = MasterRefresh(self.user_id,self.document_id)
qset = Document.objects.raw('''
DROP TABLE IF EXISTS _this_doc;
CREATE TEMP TABLE _this_doc AS
SELECT
sd.*
FROM
source_data_document sd
WHERE id = %s;
SELECT
td.*
,x.doc_datapoint_cnt
,y.source_submission_total_cnt
,z.source_submission_to_process_cnt
FROM _this_doc td
INNER JOIN (
SELECT COUNT(1) as doc_datapoint_cnt
FROM doc_datapoint dd
WHERE EXISTS (
SELECT 1 FROM _this_doc td
WHERE dd.document_id = td.id
)
)x
ON 1=1
INNER JOIN (
SELECT COUNT(1) as source_submission_total_cnt
FROM source_submission ss
WHERE EXISTS (
SELECT 1 FROM _this_doc td
WHERE ss.document_id = td.id
)
)y
ON 1=1
INNER JOIN (
SELECT COUNT(1) as source_submission_to_process_cnt
FROM source_submission ss
WHERE EXISTS (
SELECT 1 FROM _this_doc td
WHERE ss.document_id = td.id
)
AND ss.process_status = 'TO_PROCESS'
)z
ON 1=1;
''',[self.document_id])
return None, qset
def get_object_list(self, request):
document_id = request.GET['document_id']
mr = MasterRefresh(request.user.id, document_id)
mr.main()
doc_detail, created = DocumentDetail.objects.update_or_create(
document_id=document_id,
doc_detail_type_id=DocDetailType.objects.get(
name='submission_processed_count').id,
defaults={
'doc_detail_value':
SourceSubmission.objects.filter(
process_status='PROCESSED',
document_id=document_id).count()
},
)
doc_detail, created = DocumentDetail.objects.update_or_create(
document_id=document_id,
doc_detail_type_id=DocDetailType.objects.get(
name='doc_datapoint_count').id,
defaults={
'doc_detail_value':
DocDataPoint.objects.filter(document_id=document_id).count()
},
)
doc_detail, created = DocumentDetail.objects.update_or_create(
document_id=document_id,
doc_detail_type_id=DocDetailType.objects.get(
name='datapoint_count').id,
defaults={
'doc_detail_value':
DataPoint.objects.filter(
source_submission_id__in=SourceSubmission.objects.filter(
document_id=document_id).values_list(
'id', flat=True)).count()
},
)
queryset = DocumentDetail.objects \
.filter(document_id=document_id).values()
return queryset
def test_submission_detail_refresh(self, ):
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
mr.refresh_doc_meta()
## FIXME replace source_submission_data with read_csv(self.test_file)
source_submissions_data = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id',flat=True)
# = read_csv(self.test_file)
## fake the submission_data
mr.refresh_submission_details()
submission_details = SourceSubmission.objects\
.filter(document_id = self.document.id)
self.assertEqual(len(source_submissions_data)\
,len(submission_details))
def test_submission_to_datapoint(self):
'''
This simulates a new document being processed with source_object_map
records that have been sucessfullly mapped to master_ids but from a
different document.
A few things this method checks:
1. there is a record in doc_object_map even for
mappings that existed before this document was ingested.
2. WHen the submission detail is refreshed, the reiogn/campaign ids
that we mapped should exist in that row.
2. DocDataPoint records are created if the necessary mapping exists
3. There are no zero or null values allowed in doc_datapoint
4. The doc_datapoint from #3 is merged into datpaoint.
'''
self.set_up()
mr = MasterRefresh(self.user.id, self.document.id)
submission_qs = SourceSubmission.objects\
.filter(document_id = self.document.id)\
.values_list('id','submission_json')[0]
ss_id, first_submission = submission_qs[0], json.loads(
submission_qs[1])
location_code = first_submission[self.location_code_input_column]
campaign_code = first_submission[self.campaign_code_input_column]
raw_indicator_list = [k for k, v in first_submission.iteritems()]
indicator_code = raw_indicator_list[-1]
map_location_id = Location.objects.all()[0].id
som_id_r = SourceObjectMap.objects.create(
content_type='location',
source_object_code=location_code,
master_object_id=map_location_id,
mapped_by_id=self.user.id)
map_campaign_id = Campaign.objects.all()[0].id
som_id_c = SourceObjectMap.objects.create(
content_type='campaign',
source_object_code=campaign_code,
master_object_id=map_campaign_id,
mapped_by_id=self.user.id)
map_indicator_id = Indicator.objects.all()[0].id
som_id_i = SourceObjectMap.objects.create(
content_type='indicator',
source_object_code=indicator_code,
master_object_id=map_indicator_id,
mapped_by_id=self.user.id)
mr.refresh_doc_meta()
doc_som_id_for_location_code = DocumentSourceObjectMap.objects\
.filter(id__in= SourceObjectMap.objects.filter(\
content_type = 'location',
source_object_code = location_code,
).values_list('id',flat=True)
).values_list('id',flat=True)
## Test Case 1 ##
self.assertEqual(len(doc_som_id_for_location_code), 1) # 1
mr_with_new_meta = MasterRefresh(self.user.id, self.document.id)
mr_with_new_meta.refresh_submission_details()
first_submission_detail = SourceSubmission.objects\
.get(id = ss_id)
## Test Case 2 ##
self.assertEqual(first_submission_detail.location_id, map_location_id)
self.assertEqual(first_submission_detail.campaign_id, map_campaign_id)
mr_with_new_meta.submissions_to_doc_datapoints()
doc_dp_ids = DocDataPoint.objects.filter(document_id=self.document.id)
## Test Case #3
self.assertEqual(1, len(doc_dp_ids))
mr_with_new_meta.sync_datapoint()
dps = DataPoint.objects.all()
## Test Case #4
self.assertEqual(1, len(dps))
def test_source_data_points_to_doc_datapoints(self):
mr = MasterRefresh(self.user.id, self.document.id)
doc_datapoint_ids = mr.process_doc_datapoints(self.source_submissions)
self.assertTrue(doc_datapoint_ids)
def refresh_master(request, document_id):
mr = MasterRefresh(request.user.id, document_id)
mr.main()
return HttpResponseRedirect('/doc_review/overview/%s' % document_id)
