def add_worldmap_layerinfo_if_exists(tabular_info):
"""
Does the WorldMap already have a layer for this Dataverse DataFile?
Check the WorldMap API. If a layer exists, for this "tabular_info",
create one of the following:
- WorldMapLatLngInfo
- WorldMapJoinLayerInfo
Expects tabular_info to have these fields:
- dataverse_installation_name
- datafile_id
"""
if tabular_info is None:
return False
success, dict_or_err_msg = get_layer_info_using_dv_info(
tabular_info.__dict__)
if not success:
return False
worldmap_tabular_info = WorldMapTabularLayerInfo.build_from_worldmap_json(\
tabular_info,\
dict_or_err_msg)
if worldmap_tabular_info is None:
LOGGER.error("Failed to create WorldMapTabularLayerInfo using %s",\
dict_or_err_msg)
return False
return True
def add_worldmap_layerinfo_if_exists(tabular_info):
"""
Does the WorldMap already have a layer for this Dataverse DataFile?
Check the WorldMap API. If a layer exists, for this "tabular_info",
create one of the following:
- WorldMapLatLngInfo
- WorldMapJoinLayerInfo
Expects tabular_info to have these fields:
- dataverse_installation_name
- datafile_id
"""
if tabular_info is None:
return False
success, dict_or_err_msg = get_layer_info_using_dv_info(tabular_info.__dict__)
if not success:
return False
worldmap_tabular_info = WorldMapTabularLayerInfo.build_from_worldmap_json(\
tabular_info,\
dict_or_err_msg)
if worldmap_tabular_info is None:
LOGGER.error("Failed to create WorldMapTabularLayerInfo using %s",\
dict_or_err_msg)
return False
return True
def test_01_initial_join(self):
"""Using WorldMap successful JSON response to test "build_from_worldmap_json"""
msgt(self.test_01_initial_join.__doc__)
tab_file_info = TabularFileInfo.objects.get(
pk=15) # Election precinct test
# --------------------------------------------
# Attach actual file -- path from fixture is not correct
# --------------------------------------------
elect_filepath = join(dirname(__file__), 'input',
'election_precincts2.csv')
tab_file_info.dv_file.save(\
'election_precincts2.csv',
File(open(elect_filepath, 'r')),
save=False)
self.assertEqual(tab_file_info.id, 15)
# ------------------------------------------
# Fail by passing a string instead of JSON
# ------------------------------------------
tab_map_info = WorldMapTabularLayerInfo.build_from_worldmap_json(
tab_file_info, self.json_join_data_string)
self.assertEqual(tab_map_info, None)
# ------------------------------------------
# Load successful info
# ------------------------------------------
tab_map_info = WorldMapTabularLayerInfo.build_from_worldmap_json(\
tab_file_info,\
json.loads(self.json_join_data_string))
self.assertTrue(tab_map_info.id is not None)
# ------------------------------------------
# Make sure data loading as expected
# ------------------------------------------
self.assertEqual(type(tab_map_info.core_data), dict)
self.assertEqual(type(tab_map_info.attribute_data), list)
self.assertEqual(type(tab_map_info.download_links), dict)
def view_map_tabular_file_form(request):
"""
AJAX call: Join your tabular file to an existing WorldMap layer
using the column selected in this form
"""
#for k, v in request.POST.items():
# print k, v
# -----------------------------------------
# Retrieve the id of the Tabular info object
# -----------------------------------------
tabular_file_info_id = request.POST.get('tabular_file_info_id', -1)
try:
tabular_info = TabularFileInfo.objects.get(pk=tabular_file_info_id)
except TabularFileInfo.DoesNotExist:
err_msg = 'Sorry! The Tabular File was not found. (tabular_file_info_id)'
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
#raise Http404('No TabularFileInfo for id: %s' % tabular_file_info_id)
# -----------------------------------------
# Retrieve available Geocode types and join Layers
# note: geocode_types_from_worldmap not needed here
# -----------------------------------------
(geocode_types_from_worldmap,
available_layers_list) = get_geocode_types_and_join_layers()
# -----------------------------------------
# Create form with initial + POST data
# -----------------------------------------
form_single_column = ChooseSingleColumnForm(tabular_info.id,\
available_layers_list,\
tabular_info.column_names,\
request.POST)
# -----------------------------------------
# Check the form's validity
# -----------------------------------------
if not form_single_column.is_valid():
json_msg = MessageHelperJSON.get_json_fail_msg(\
format_errors_as_text(form_single_column, for_web=True))
return HttpResponse(json_msg,
content_type="application/json",
status=200)
print 'cleaned_data', form_single_column.cleaned_data
# -----------------------------------------
# Get Dataverse info dict
# -----------------------------------------
dataverse_metadata_dict = get_dataverse_info_dict(tabular_info)
# -----------------------------------------
# Use the WorldMap API and
# try to create a layer
# -----------------------------------------
tj_map_maker = TableJoinMapMaker(
tabular_info,
dataverse_metadata_dict,
form_single_column.cleaned_data.get('chosen_column'),
form_single_column.cleaned_data.get('chosen_layer'),
)
success = tj_map_maker.run_map_create()
msg('success: %s' % success)
if not success:
json_msg = MessageHelperJSON.get_json_fail_msg(\
'Sorry! ' + tj_map_maker.get_error_msg())
msg('error msg: %s' % json_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Succeeded! Create a WorldMapTabularLayerInfo object
# -----------------------------------------
worldmap_tabular_info = WorldMapTabularLayerInfo.build_from_worldmap_json(tabular_info,\
tj_map_maker.get_map_info())
if worldmap_tabular_info is None:
LOGGER.error("Failed to create WorldMapTabularLayerInfo using %s",\
tj_map_maker.get_map_info())
user_msg = 'Sorry! Failed to create map. Please try again. (code: s1)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Notify Dataverse of the new map
# -----------------------------------------
MetadataUpdater.update_dataverse_with_metadata(worldmap_tabular_info)
# -----------------------------------------
# Build the Map HTML chunk to replace the form
# -----------------------------------------
map_html, user_message_html = build_map_html(request,
worldmap_tabular_info)
if map_html is None: # Failed! Send an error
LOGGER.error("Failed to create map HTML using WorldMapTabularLayerInfo: %s (%d)",\
worldmap_tabular_info, worldmap_tabular_info.id)
user_msg = 'Sorry! Failed to create map. Please try again. (code: s2)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Looks good. In the JSON response, send
# back the map HTML
# -----------------------------------------
data_dict = dict(map_html=map_html,
user_message_html=user_message_html,
id_main_panel_title=PANEL_TITLE_STYLE_MAP)
json_msg = MessageHelperJSON.get_json_success_msg("great job",
data_dict=data_dict)
return HttpResponse(json_msg, content_type="application/json", status=200)
def view_process_lat_lng_form(request):
"""
Create a WorldMap layer from your tabular file
using the latitude and longitude columns selected in this form
"""
tabular_file_info_id = request.POST.get('tabular_file_info_id', -1)
try:
tabular_info = TabularFileInfo.objects.get(pk=tabular_file_info_id)
except TabularFileInfo.DoesNotExist:
err_msg = 'Sorry! The Tabular File was not found. (tabular_file_info_id)'
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
#raise Http404('No TabularFileInfo for id: %s' % tabular_file_info_id)
form_lat_lng = LatLngColumnsForm(tabular_info.id,\
tabular_info.column_names,\
request.POST)
if not form_lat_lng.is_valid():
json_msg = MessageHelperJSON.get_json_fail_msg(\
format_errors_as_text(form_lat_lng,\
for_web=True)\
)
#json_msg = MessageHelperJSON.get_json_fail_msg(f.err_msg_for_web)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
(success, worldmap_data_or_err_msg) = create_map_from_datatable_lat_lng(\
tabular_info,\
form_lat_lng.get_latitude_colname(),\
form_lat_lng.get_longitude_colname(),\
)
# -----------------------------------------
# Failed! Send error message
# -----------------------------------------
if not success:
json_msg = MessageHelperJSON.get_json_fail_msg(\
'Sorry! ' + worldmap_data_or_err_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Succeeded! Create a WorldMapTabularLayerInfo object
# -----------------------------------------
user_msg, response_data = worldmap_data_or_err_msg
#json_msg = MessageHelperJSON.get_json_success_msg(user_msg, data_dict=response_data)
#return HttpResponse(json_msg, content_type="application/json", status=200)
worldmap_latlng_info = WorldMapTabularLayerInfo.build_from_worldmap_json(tabular_info,\
response_data)
if worldmap_latlng_info is None:
LOGGER.error("Failed to create WorldMapLatLngInfo using data: %s",\
response_data)
user_msg = 'Sorry! Failed to create map. Please try again. (code: s4)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Notify Dataverse of the new map
# -----------------------------------------
MetadataUpdater.update_dataverse_with_metadata(worldmap_latlng_info)
# -----------------------------------------
# Possible that this failed?
# Make sure at least 1 row mapped
# -----------------------------------------
# Skip for now! Error in row counts for Lat/Lng!
"""
if worldmap_latlng_info.did_any_rows_map() is False:
# Delete the worldmap_latlng_info object
worldmap_latlng_info.delete()
# Send back a user error message
user_msg = "Sorry! We couldn't map any of those latitude and longitude values."
return HttpResponse(MessageHelperJSON.get_json_fail_msg(user_msg),\
content_type="application/json",\
status=200)
"""
# -----------------------------------------
# Build the Map HTML chunk to replace the form
# -----------------------------------------
map_html, user_message_html = build_map_html(request, worldmap_latlng_info)
if map_html is None: # Failed! Send an error
LOGGER.error("Failed to create map HTML using WorldMapLatLngInfo: %s (%d)",\
worldmap_latlng_info, worldmap_latlng_info.id)
user_msg = 'Sorry! Failed to create map. Please try again. (code: s5)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Looks good. In the JSON response, send
# back the map HTML
# -----------------------------------------
data_dict = dict(map_html=map_html,
user_message_html=user_message_html,
id_main_panel_title=PANEL_TITLE_STYLE_MAP)
json_msg = MessageHelperJSON.get_json_success_msg("great job",
data_dict=data_dict)
return HttpResponse(json_msg, content_type="application/json", status=200)
def test_01_update_dataverse_metadata(self):
"""Test Dataverse "update metadata" url endpoint. Only testing
fail conditions, e.g. can't contact server, etc."""
msgt(self.test_01_update_dataverse_metadata.__doc__)
tab_file_info = TabularFileInfo.objects.get(
pk=15) # Election precinct test
# --------------------------------------------
# Attach actual file -- path from fixture is not correct
# --------------------------------------------
elect_filepath = join(dirname(__file__), 'input',
'election_precincts2.csv')
tab_file_info.dv_file.save(\
'election_precincts2.csv',
File(open(elect_filepath, 'r')),
save=False)
self.assertEqual(tab_file_info.id, 15)
# ------------------------------------------
# Load successful info
# ------------------------------------------
tab_map_info = WorldMapTabularLayerInfo.build_from_worldmap_json(\
tab_file_info,\
json.loads(self.json_join_data_string))
self.assertTrue(tab_map_info.id is not None)
# ------------------------------------------
# Make sure data loading as expected
# ------------------------------------------
self.assertEqual(type(tab_map_info.core_data), dict)
self.assertEqual(type(tab_map_info.attribute_data), list)
self.assertEqual(type(tab_map_info.download_links), dict)
# ------------------------------------------
# Send message to non-existent server
# ------------------------------------------
msgt('Send message to non-existent server')
url_non_existent = 'https://nope.dataverse.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_non_existent)
self.assertEqual(success, False)
self.assertTrue(resp_dict['message'].startswith(\
ERROR_DV_NO_SERVER_RESPONSE))
# ------------------------------------------
# Send message to server without an endpoint
# ------------------------------------------
msgt('Send message to server without an endpoint')
url_no_endpoint = 'http://www.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_no_endpoint)
self.assertEqual(success, False)
self.assertTrue(resp_dict['message'].startswith(\
ERROR_DV_PAGE_NOT_FOUND))
# ------------------------------------------
# No token in request to Dataverse
# ------------------------------------------
msgt(('No token in request to Dataverse'
' (requires working endpoint at https://dataverse.harvard.edu)'))
url_no_endpoint = 'https://dataverse.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_no_endpoint)
self.assertEqual(success, False)
self.assertEqual(resp_dict['message'],
'Token not found in JSON request.')
def test_01_update_dataverse_metadata(self):
"""Test Dataverse "update metadata" url endpoint. Only testing
fail conditions, e.g. can't contact server, etc."""
msgt(self.test_01_update_dataverse_metadata.__doc__)
tab_file_info = TabularFileInfo.objects.get(pk=15) # Election precinct test
# --------------------------------------------
# Attach actual file -- path from fixture is not correct
# --------------------------------------------
elect_filepath = join(dirname(__file__),
'input',
'election_precincts2.csv')
tab_file_info.dv_file.save(\
'election_precincts2.csv',
File(open(elect_filepath, 'r')),
save=False)
self.assertEqual(tab_file_info.id, 15)
# ------------------------------------------
# Load successful info
# ------------------------------------------
tab_map_info = WorldMapTabularLayerInfo.build_from_worldmap_json(\
tab_file_info,\
json.loads(self.json_join_data_string))
self.assertTrue(tab_map_info.id is not None)
# ------------------------------------------
# Make sure data loading as expected
# ------------------------------------------
self.assertEqual(type(tab_map_info.core_data), dict)
self.assertEqual(type(tab_map_info.attribute_data), list)
self.assertEqual(type(tab_map_info.download_links), dict)
# ------------------------------------------
# Send message to non-existent server
# ------------------------------------------
msgt('Send message to non-existent server')
url_non_existent = 'https://nope.dataverse.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_non_existent)
self.assertEqual(success, False)
self.assertTrue(resp_dict['message'].startswith(\
ERROR_DV_NO_SERVER_RESPONSE))
# ------------------------------------------
# Send message to server without an endpoint
# ------------------------------------------
msgt('Send message to server without an endpoint')
url_no_endpoint = 'http://www.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_no_endpoint)
self.assertEqual(success, False)
self.assertTrue(resp_dict['message'].startswith(\
ERROR_DV_PAGE_NOT_FOUND))
# ------------------------------------------
# No token in request to Dataverse
# ------------------------------------------
msgt(('No token in request to Dataverse'
' (requires working endpoint at https://dataverse.harvard.edu)'))
url_no_endpoint = 'https://dataverse.harvard.edu'
success, resp_dict = MetadataUpdater.update_dataverse_with_metadata(\
tab_map_info,
url_no_endpoint)
self.assertEqual(success, False)
self.assertEqual(resp_dict['message'], 'Token not found in JSON request.')