def test_refresh_no_params(self):
doc_id = self.ingest_file('eoc_post_campaign.csv')
dt = ComplexDocTransform(self.user_id, doc_id)
dt.main()
self.assertEqual(DataPoint.objects.count(), 0)
self.assertEqual(DataPointComputed.objects.count(), 0)
resp = self.ts.get(self, '/api/v1/refresh_master/')
self.assertHttpApplicationError(resp)
def test_refresh_no_params(self):
doc_id = self.ingest_file('eoc_post_campaign.csv')
dt = ComplexDocTransform(self.user_id, doc_id)
dt.main()
self.assertEqual(DataPoint.objects.count(), 0)
self.assertEqual(DataPointComputed.objects.count(), 0)
resp = self.ts.get(self, '/api/v1/refresh_master/')
self.assertHttpApplicationError(resp)
def ingest_file(self, file_name):
document = Document.objects.create(doc_title=file_name,
created_by_id=self.user.id,
guid='test')
document.docfile = file_name
document.save()
sdt = ComplexDocTransform(self.user.id, document.id)
sdt.main()
return document.id
def test_get_source_submission_by_doc(self):
doc = self.ts.create_arbitrary_document(document_docfile='eoc_post_campaign.csv')
dt = ComplexDocTransform(self.ts.user.id, doc.id)
dt.main()
get_data = {'document_id':doc.id}
resp = self.ts.get(self, '/api/v1/source_submission/', get_data)
self.assertHttpOK(resp)
resp_data = self.deserialize(resp)
self.assertEqual(len(SourceSubmission.objects.all()), len(resp_data['objects']))
def ingest_file(self, file_name):
document = Document.objects.create(
doc_title = file_name,
created_by_id = self.user.id,
guid = 'test')
document.docfile = file_name
document.save()
sdt = ComplexDocTransform(self.user.id, document.id)
sdt.main()
return document.id
def test_get_source_submission_by_id(self):
doc = self.ts.create_arbitrary_document(
document_docfile='eoc_post_campaign.csv')
dt = ComplexDocTransform(self.ts.user.id, doc.id)
dt.main()
ss_id = SourceSubmission.objects.all()[0].id
get_data = {'id': ss_id}
resp = self.ts.get(self, '/api/v1/source_submission/', get_data)
self.assertHttpOK(resp)
resp_data = self.deserialize(resp)
self.assertEqual(resp_data['objects'][0]['id'], ss_id)
def test_refresh(self):
doc_id = self.ingest_file('eoc_post_campaign.csv')
dt = ComplexDocTransform(self.user_id, doc_id)
dt.main()
self.assertEqual(DataPoint.objects.count(), 0)
self.assertEqual(DataPointComputed.objects.count(), 0)
get_data = {'document_id': doc_id}
resp = self.ts.get(self, '/api/v1/refresh_master/', get_data)
self.assertHttpOK(resp)
response_data = self.deserialize(resp)
self.assertEqual(DataPoint.objects.count(), 1)
self.assertEqual(response_data['objects'][0]['id'], doc_id)
def get_object_list(self, request):
'''
## when you upload a file, step one is getting data into source submission
## --> DocTransform <-- ##
## Step two is translating form source_submission into datapoints
## --> REfreshMaster <----
## step three is aggregation
## agg refresh ##
'''
try:
doc_id = request.GET['document_id']
except KeyError:
raise DatapointsException(
message='Document_id is a required API param')
# dt = DocTransform(request.user.id, doc_id)
ran_complex_doc_transform = False
try:
dt = ComplexDocTransform(request.user.id, doc_id)
dt.main()
ran_complex_doc_transform = True
except Exception as err:
try:
dt = DateDocTransform(request.user.id, doc_id)
ssids = dt.process_file()
except Exception as err:
raise DatapointsException(message=err.message)
mr = MasterRefresh(request.user.id, doc_id)
mr.main()
if ran_complex_doc_transform:
doc_campaign_ids = set(list(DataPoint.objects\
.filter(source_submission__document_id = doc_id)\
.values_list('campaign_id',flat=True)))
for c_id in doc_campaign_ids:
ar = AggRefresh(c_id)
# try/except block hack because tests fail otherwise
try:
with transaction.atomic():
ar.main()
except TransactionManagementError as e:
pass
return Document.objects.filter(id=doc_id).values()
def test_refresh(self):
doc_id = self.ingest_file('eoc_post_campaign.csv')
dt = ComplexDocTransform(self.user_id, doc_id)
dt.main()
self.assertEqual(DataPoint.objects.count(), 0)
self.assertEqual(DataPointComputed.objects.count(), 0)
get_data = {
'document_id':doc_id
}
resp = self.ts.get(self, '/api/v1/refresh_master/', get_data)
self.assertHttpOK(resp)
response_data = self.deserialize(resp)
self.assertEqual(DataPoint.objects.count(), 1)
self.assertEqual(response_data['objects'][0]['id'], doc_id)
def get_object_list(self, request):
'''
## when you upload a file, step one is getting data into source submission
## --> DocTransform <-- ##
## Step two is translating form source_submission into datapoints
## --> REfreshMaster <----
## step three is aggregation
## agg refresh ##
'''
try:
doc_id = request.GET['document_id']
except KeyError:
raise DatapointsException(message='Document_id is a required API param')
# dt = DocTransform(request.user.id, doc_id)
ran_complex_doc_transform = False
try:
dt = ComplexDocTransform(request.user.id, doc_id)
dt.main()
ran_complex_doc_transform = True
except Exception as err:
try:
dt = DateDocTransform(request.user.id, doc_id)
ssids = dt.process_file()
except Exception as err:
raise DatapointsException(message=err.message)
mr = MasterRefresh(request.user.id, doc_id)
mr.main()
if ran_complex_doc_transform:
doc_campaign_ids = set(list(DataPoint.objects\
.filter(source_submission__document_id = doc_id)\
.values_list('campaign_id',flat=True)))
for c_id in doc_campaign_ids:
ar = AggRefresh(c_id)
# try/except block hack because tests fail otherwise
try:
with transaction.atomic():
ar.main()
except TransactionManagementError as e:
pass
return Document.objects.filter(id=doc_id).values()
def test_duplicate_rows(self):
doc_id = self.ingest_file('dupe_datapoints.csv')
dt = ComplexDocTransform(self.user.id, doc_id)
source_submissions = dt.main()
mr = MasterRefresh(self.user.id, doc_id)
mr.main()
dps = DataPoint.objects.all()
self.assertEqual(len(dps), 1)
some_cell_value_from_the_file = 0.9029
self.assertEqual(dps[0].value, some_cell_value_from_the_file)
def set_up(self):
'''
Refresh master needs a few peices of metadata to be abel to do it's job.
Location, Campaign, User .. all of the main models that you can see
initialized in the first migrations in the datapoints application.
The set up method also runs the ComplexDocTransform method which simulates
the upload of a csv or processing of an ODK submission. Ideally this
test will run independently of this module, but for now this is how
we initialize data in the system via the .csv below.
'''
self.test_file_location = 'ebola_data.csv'
self.location_list = Location.objects.all().values_list('name',flat=True)
self.create_metadata()
self.user = User.objects.get(username = '******')
self.document = Document.objects.get(doc_title = 'test')
self.document.docfile = self.test_file_location
self.document.save()
dt = ComplexDocTransform(self.user.id, self.document.id)
dt.main()
def test_campaign_data_ingest(self):
# ./manage.py test rhizome.tests.test_refresh_master.RefreshMasterTestCase.test_campaign_data_ingest --settings=rhizome.settings.test
self.set_up()
test_file_location = 'allAccessData.csv'
test_df = read_csv('rhizome/tests/_data/' + test_file_location)
document = Document.objects.create(doc_title='allAccessData')
document.docfile = test_file_location
document.save()
## create locatino_meta ##
distinct_location_codes = test_df['geocode'].unique()
for l in distinct_location_codes:
l_id = Location.objects.create(name=l,
location_code=l,
location_type_id=1,
office_id=1).id
l_som = SourceObjectMap.objects.create(master_object_id=l_id,
content_type='location',
source_object_code=str(l))
## create campaign meta ##
distinct_campaign_codes = test_df['campaign'].unique()
for i, (c) in enumerate(distinct_campaign_codes):
c_id = Campaign.objects.create(name=c,
top_lvl_location_id=1,
top_lvl_indicator_tag_id=1,
office_id=1,
campaign_type_id=1,
start_date='2010-01-0' + str(i + 1),
end_date='2010-01-0' +
str(i + 1)).id
c_som = SourceObjectMap.objects.create(master_object_id=c_id,
content_type='campaign',
source_object_code=str(c))
## create indicator_meta ##
access_indicator_id = Indicator.objects.create(name='access',
short_name='access').id
som_obj = SourceObjectMap.objects.create(
master_object_id=access_indicator_id,
content_type='indicator',
source_object_code='# Missed children due to inaccessibility (NEPI)'
)
dt = ComplexDocTransform(self.user.id, document.id)
dt.main()
mr = MasterRefresh(self.user.id, document.id)
mr.main()
ss_id_list = SourceSubmission.objects\
.filter(document_id = document.id)\
.values_list('id', flat=True)
doc_dp_id_list = DocDataPoint.objects\
.filter(source_submission_id__in = ss_id_list)\
.values_list('id', flat=True)
dp_id_list = DataPoint.objects\
.filter(source_submission_id__in = ss_id_list)\
.values_list('id', flat=True)
self.assertEqual(len(ss_id_list), len(test_df))
self.assertEqual(len(doc_dp_id_list), len(dp_id_list))