def test_mapping_mutation(self):
file_id = self.dummy_file.id
EXTRA_INFORMATION = {
'empty_entries': {
'empty_indicator': 'Test Region',
'empty_geolocation': {
'value': '',
'type': ''
},
'empty_filter': 'Default',
'empty_value_format': {
'value format': 'Numeric'
},
'empty_date': ''
},
'multi_mapped': {
'column_heading': {},
'column_values': {},
},
'point_based_info': {
'coord': {
'lat': '',
'lon': ''
},
'subnational': '',
'country': '',
'type': '',
}
}
input_json = {
'metadata_id': file_id,
'mapping_dict': {
'indicator': [],
'geolocation': ["Region"],
'date': ['datez'],
'value_format': [],
'value': ["new infections"],
'comment': [],
},
"filter_headings": {
"filters": "filters"
},
'extra_information': EXTRA_INFORMATION
}
result = begin_mapping(input_json)
self.assertEqual(result['success'], 1)
def post(self, request):
context = {}
logger = logging.getLogger("django")
logger.info("Entering Mapping Job")
try:
context = begin_mapping(request.data)
logger.info("Successful mapping")
except Exception as e:
logger = logging.getLogger("django")
logger.exception("--Error in mapping process")
context['error'] = "Error occured when attempting to map file"
context['success'] = 0
raise #temp
return Response(context)
def mapping(instance):
file_id = instance.data.get('metadata_id')
try:
file = File.objects.get(id=file_id)
# When mapping again will be deleted all old indicator related
# to the current file
file.indicator_set.all().delete()
except File.DoesNotExist:
raise Exception(
'File {file_id} does not exists!'.format(file_id=file_id))
try:
# Begin mapping
result = begin_mapping(instance.data)
# if mapping success then continue to update relation country
# on the current indicator
if result.get('success'):
update_country_on_indicator(file=file)
except Exception as e:
raise Exception(e)
def map_iati_data():
try:
# so first we check if OIPA data has been updated
last_updated_data = requests.get(last_updated_query).json()
last_updated_time = last_updated_data['results'][0][
'last_updated_datetime']
file_source_name = 'AIDSFONDS IATI DATASET'
dataset_title = file_source_name + last_updated_time
file_source = FileSource.objects.filter(name=file_source_name)
map_it = False
if file_source.count() > 0:
file_source = file_source.get()
# we found the file source and then we find the file
file = File.objects.filter(source=file_source).get()
# so here we check if the files title is different from the
# one currently formed, if it is different, that means
# that the last_updated_time has changed meaning that
# OIPA data for this IATI/OIPA dataset of ours has changed
# and we can easily update it, otherwise nothing should be
# done
if file.title != dataset_title:
print(
'OIPA DATA HAS BEEN UPDATED, IATI DATASET REMAPPING INITIATED: ',
datetime.datetime.now())
map_it = True
# then we find the indicators
indicators = Indicator.objects.filter(file__id=file.id)
# and now we delete them all
# NOTE: no need to delete the csv file itself
# cause it will just be overwritten
indicators.delete()
file_source.delete()
file.delete()
else:
print(
'OIPA DATA HAS NOT BEEN UPDATED, NOT REMAPPING THE IATI DATASET: ',
datetime.datetime.now())
else:
print('NEW IATI DATASET BEING MAPPED', datetime.datetime.now())
map_it = True
if map_it:
print('STARTED MAPPING IATI DATA')
print('TIME:', datetime.datetime.now())
# creating the file source for the dataset
file_source = FileSource.objects.create(name=file_source_name)
full_path_to_dataset = os.path.join(
os.path.join(settings.MEDIA_ROOT, settings.DATASETS_URL),
'AIDSFONDS_IATI_DATASET.csv')
# create and saving the csv file
with open(full_path_to_dataset, 'w') as writeFile:
writer = csv.writer(writeFile)
# we write the headers
writer.writerow([
'Indicator', 'Sub-indicator', 'ISO2', 'Year',
'Value Format', 'Value'
])
# and here we retrieve and write the actual values of this dataset
for iati_ind_item in iati_indicator_metadata:
# and here we start forming the dataset
# starting from activity status
iati_data = requests.get(iati_ind_item['query']).json()
for item in iati_data['results']:
# so yeah here we form the dataset with values
writer.writerow([
iati_ind_item['ind_val'],
pydash.objects.get(item,
iati_ind_item['filter_place']),
pydash.objects.get(item,
iati_ind_item['geo_place']),
pydash.objects.get(item,
iati_ind_item['date_place']),
iati_ind_item['format_val'],
pydash.objects.get(item,
iati_ind_item['value_place'])
])
writeFile.close()
# and now out of this file we create the file object
file_man = File.objects.create(
title=dataset_title,
description=dataset_title,
contains_subnational_data=False,
organisation='Zimmerman&Zimmerman',
maintainer='Zimmerman&Zimmerman',
date_of_dataset=datetime.datetime.now(),
methodology='Iati',
define_methodology='Iati',
update_frequency='Pretty frequent',
comments='No comments',
accessibility='a',
data_quality='best',
number_of_rows=1,
file_types='csv',
source=file_source,
original_file_location=full_path_to_dataset,
file_status='1',
error_file_location='',
file=full_path_to_dataset)
# datatypes_overview_file_location
# so this is gonna be the mapping dict for that
# iati data
data = {
"metadata_id": file_man.id,
"filter_headings": {
"Sub-indicator": "Sub-indicator"
},
"extra_information": {
"empty_entries": {
"empty_indicator": '',
"empty_geolocation": {
"value": '',
"type": ''
},
"empty_filter": '',
"empty_value_format": {},
"empty_date": ''
},
"multi_mapped": {
"column_heading": {},
"column_values": {}
},
"point_based_info": {
"coord": {
"lat": '',
"lon": ''
},
"subnational": '',
"country": '',
"type": ''
}
},
"mapping_dict": {
"indicator": ['Indicator'],
"filters": ['Sub-indicator'],
"geolocation": ['ISO2'],
"date": ['Year'],
"value_format": ['Value Format'],
"value": ['Value'],
"comment": []
}
}
begin_mapping(data)
print('MAPPING IATI DATA FINISHED')
print('TIME:', datetime.datetime.now())
except Exception as e:
print('ERROR:', e)
def test_mapping_point_based_mutation(self):
file_id = self.dummy_file.id
EXTRA_INFORMATION = {
'empty_entries': {
'empty_indicator': 'Test pointbased',
'empty_geolocation': {
'value': '',
'type': ''
},
'empty_filter': 'Default',
'empty_value_format': {
'value format': 'Numeric'
},
'empty_date': '2016'
},
'multi_mapped': {
'column_heading': {},
'column_values': {},
},
'point_based_info': {
'coord': {
'lat': 'Lat location 1',
'lon': 'Long location 1'
},
'subnational': '',
'country': '',
'type': '',
}
}
# so this query call based mapping doesn't seem to work very properly
# so we comment this out for now and will adress this later
# but for the data just to be mapped out we'll just use the mapping
# function itself here ^
# input_json_str = json.dumps(input_json)
# query_input = {"input": {"data": input_json_str}}
# query = """
# mutation mapping($input: MappingMutationInput!) {
# mapping(input: $input) {
# id
# data
# }
# }"""
# result = schema.execute(query, variable_values=query_input)
input_json = {
'metadata_id': file_id,
"filter_headings": {
"filters": "filters"
},
"extra_information": EXTRA_INFORMATION,
'mapping_dict': {
'indicator': [],
'filters': [],
'geolocation': ["Lat location 1", "Long location 1"],
'date': [],
'value_format': [],
'value': ["new infections"],
'comment': [],
},
}
result = begin_mapping(input_json)
self.assertEqual(result['success'], 1)
def file_post_save(sender, instance, **kwargs):
if kwargs.get('created'):
begin_mapping(instance.data)
def setUp(self):
ci = CountryImport()
ci.update_polygon()
ci.update_alt_name()
self.dummy_file_source = factory.FileSourceFactory(
name='dummy_file_source')
self.dummy_geolocation = factory.GeolocationFactory(tag='Albania',
iso2='al',
iso3='alb',
object_id=4,
content_type_id=15,
type='country')
self.dummy_file = factory.FileFactory(
title="test",
description="test",
contains_subnational_data=True,
organisation="test",
maintainer="test",
date_of_dataset="2009-01-01",
methodology="test",
define_methodology="test",
update_frequency="test",
comments="test",
accessibility="p",
data_quality="test",
number_of_rows=1,
file_types="csv",
location=self.dummy_geolocation,
source=self.dummy_file_source,
file=os.path.abspath("samples/AIDSinfotest.csv"))
file_id = self.dummy_file.id
input_json = {
'metadata_id': file_id,
'filter_headings': {
"Subgroup": "Subgroup"
},
"extra_information": {
"empty_entries": {
"empty_indicator": '',
"empty_geolocation": {
"value": '',
"type": ''
},
"empty_filter": '',
"empty_value_format": {},
"empty_date": ''
},
"multi_mapped": {
"column_heading": {},
"column_values": {}
},
"point_based_info": {
"coord": {
"lat": '',
"lon": ''
},
"subnational": '',
"country": '',
"type": ''
}
},
'mapping_dict': {
"indicator": ["Indicator"],
"value_format": ["Unit"],
"geolocation": ["Area"],
"value": ["Data Value"],
"date": ["Time Period"],
"comment": ["Source"],
"filters": ["Subgroup"]
}
}
begin_mapping(input_json)
def setUp(self):
self.dummy_file_source = factory.FileSourceFactory(
name='dummy_file_source'
)
alb = factory.CountryFactory(name='Albania', iso2='al', iso3='alb')
self.dummy_geolocation = factory.GeolocationFactory(
tag='Albania',
iso2='al',
iso3='alb',
object_id=alb.id,
content_type_id=15,
type='country'
)
self.dummy_file = factory.FileFactory(
title="test",
description="test",
contains_subnational_data=True,
organisation="test",
maintainer="test",
date_of_dataset="2009-01-01",
methodology="test",
define_methodology="test",
update_frequency="test",
comments="test",
accessibility="p",
data_quality="test",
number_of_rows=1,
file_types="csv",
location=self.dummy_geolocation,
source=self.dummy_file_source,
file=os.path.abspath("samples/point_based.csv")
)
file_id = self.dummy_file.id
EXTRA_INFORMATION = {
'empty_entries': {
'empty_indicator': 'Test pointbased',
'empty_geolocation': {
'value': '',
'type': ''},
'empty_filter': 'Default',
'empty_value_format': {
'value format': 'Numeric'},
'empty_date': '2016'},
'multi_mapped': {
'column_heading': {},
'column_values': {},
},
'point_based_info': {
'coord': {
'lat': 'Lat location 1',
'lon': 'Long location 1'},
'subnational': '',
'country': '',
'type': '',
}}
input_json = {
'metadata_id': file_id,
"filter_headings": {"filters": "filters"},
"extra_information": EXTRA_INFORMATION,
'mapping_dict': {
'indicator': [],
'filters': [],
'geolocation': ["Lat location 1", "Long location 1"],
'date': [],
'value_format': [],
'value': ["new infections"],
'comment': [],
},
}
begin_mapping(input_json)