def fetch_locations(self):
"""
This method is responsible for delivering the list of locations for which we have expert created data.
This currently grabs a csv file containing information from the Digital Library. It parses that file
and then looks through the location data in the Data Warehouse. If it can't find a match it hides that
result from the user.
NOTE: During the last Digital Library outage we moved the file into the adapter itself.
:returns: Dictionary containing location information where the keys are the location indexes
"""
# TODO: Raise exceptions instead of returning -1
try:
if self.model_id == 1:
csv = self.EMOD_buffer
elif self.model_id == 2:
csv = self.OM_buffer
# csv = urllib2.urlopen(self.csv_url, timeout=30) # Creates a "file-like" object containing the csv information
except urllib2.URLError:
print "There has been an error retrieving the csv from the meta data catalog"
return -1
# Skip the first line, as it contains header information
contents = csv.readline()
contents = csv.read().split('\n')
# If self.location_dict exists, wipe it out
self.locations_dict.clear()
for line in contents:
elements = line.split(',')
if line == '': continue
# Find locations and get location id
loc_filter = DimLocation.objects.filter(
Q(admin3=elements[1]) |
Q(admin2=elements[1]) |
Q(admin1=elements[1]) |
Q(admin0=elements[1])
)
if loc_filter.exists():
# Takes the first one it finds
location = loc_filter[0]
else:
loc_filter = GisBaseTable.objects.filter(s_name=elements[1])
if not loc_filter.exists():
#Not in our database, so it shouldn't be presented to the tools
continue
contains = GisBaseTable.objects.filter(geom__contains=loc_filter[0].geom)
geo_dict = {
-1: None,
0: None,
1: None,
2: None,
3: None
}
for geo in contains:
geodict[geo.admin_level] = geo.s_name
location = DimLocation(
admin007=geo_dict[-1],
admin0=geo_dict[0],
admin1=geo_dict[1],
admin2=geo_dict[2],
admin3=geo_dict[3],
geom_key_id=loc_filter[0].id
)
location.save()
if elements[3] == 'NA':
start_date = ''
end_date = ''
else:
start = elements[3].split('-')[0]
end = elements[3].split('-')[1]
start_date = '{0}-{1}-{2}'.format(start[0:4], start[4:6], start[6:8])
end_date = '{0}-{1}-{2}'.format(end[0:4], end[4:6], end[6:8])
# Fill the location dictionary
self.locations_dict[location.id] = {
"country": elements[0],
"place": elements[1],
"resolution": elements[2],
"start_date": start_date,
"end_date": end_date,
"link": self.library_url + elements[4]
}
self.EMOD_buffer.seek(0)
self.OM_buffer.seek(0)
return self.locations_dict
def locationMap(self, obj, row, cols):
"""This method is used to map the location key.
Due to the complicated nature of gis data, a custom method was necessary
to handle location fields. This method is responsible for ingesting location data
into the database. It relies on Django's raw query capabilities.
:return: A DimLocation object.
"""
tmpdict = {}
# if lattitude and longitude were given, use a stored procedure to find the location
# containing the given lat/lon. Return the corresponding location, creating it if necessary.
if 'lattitude' in obj and 'longitude' in obj:
# Note this hasn't been tested yet (no test cases are available for ingestor)
return DimLocation.vecnet_fill_location_from_point(row[cols[obj['lattitude']]], row[cols[obj['longitude']]])
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Query below (SELECT vecnet_fill_location_from_point) will no longer works after removing
# partition on gis_base_table. Stored procedure vecnet_fill_location_from_point assumes that
# gis_adm_007, gis_adm_0lev and other tables exist.
# This query can be replaced with
# SELECT * FROM gis_base_table WHERE ST_CONTAINS(geom, ST_GeomFromText('point(%s %s)', 4326));
# There are no test cases for ingester yet to update the code
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# conn = connections['default']
# cursor = conn.cursor()
# cursor.execute("SELECT vecnet_fill_location_from_point(%s, %s);", [row[cols[obj['lattitude']]], row[cols[obj['longitude']]]])
# tmp = cursor.fetchone()
# tmpstr = tmp[0][2:][:-1]
# tmplist = tmpstr.split(',')
# tmpdict['geom_key'] = tmplist[0]
# tmpdict['admin007'] = tmplist[1]
# tmpdict['admin0'] = tmplist[2]
# tmpdict['admin1'] = tmplist[3]
# tmpdict['admin2'] = tmplist[4]
# model, created = DimLocation.objects.get_or_create(**tmpdict)
# conn.close()
# return model
# if lat/lon wasn't given, use the admin levels to determine the location. Create a
# location object if necessary, and return the location object.
else:
if 'admin2' in obj:
# Could be replaced with
# GisBaseTable.object.get(s_name=row[obj['admin2']], admin_level = 2)
# Double-check admin_level though
gisobj = GisBaseTable.objects.get(s_name=row[cols[obj['admin2']]], admin_level=2)
tmpdict['geom_key'] = gisobj.id
tmpdict['admin2'] = gisobj.s_name
elif 'admin1' in obj:
gisobj = GisBaseTable.objects.get(s_name=row[cols[obj['admin1']]], admin_level=1)
tmpdict['geom_key'] = gisobj.id
tmpdict['admin1'] = gisobj.s_name
elif 'admin0' in obj:
gisobj = GisBaseTable.objects.get(s_name=row[cols[obj['admin0']]], admin_level=0)
tmpdict['geom_key'] = gisobj.id
tmpdict['admin0'] = gisobj.s_name
elif 'admin007' in obj:
gisobj = GisBaseTable.objects.get(s_name=row[cols[obj['admin007']]], admin_level=-1)
tmpdict['geom_key'] = gisobj.id
tmpdict['admin007'] = gisobj.s_name
model, created = DimLocation.objects.get_or_create(**tmpdict)
return model