def add_message_line(self, mline, prepend=False):
"""Add message line"""
if not mline:
return
msg(mline)
if prepend:
self.message_lines.insert(0, mline)
else:
self.message_lines.append(mline)
def handle(self, *args, **options):
really_delete = options.get('really_delete', False)
email_notice = options.get('email_notice', False)
dashes()
if really_delete:
msg('Check for old objects and DELETE them')
else:
msg('Check for old objects but DO NOT delete them')
stale_data_remover = StaleDataRemover(really_delete=really_delete)
stale_data_remover.remove_stale_data(\
settings.STALE_DATA_SECONDS_TO_EXPIRATION)
if email_notice is True:
stale_data_remover.send_email_notice()
def get_tablejoin_info(tablejoin_id):
"""
Use the WorldMap API to retrieve TableJoin data
from gc_apps.worldmap_connect.join_layer_service import get_tablejoin_info
get_tablejoin_info(215)
"""
if not tablejoin_id:
LOGGER.error('A tablejoin_id was not specified')
return (False, 'You must specifiy a tablejoin_id')
api_url = '%s%s/' % (GET_TABLEJOIN_INFO, tablejoin_id)
msg('api_url: %s' % api_url)
try:
r = requests.get(api_url,
auth=settings.WORLDMAP_ACCOUNT_AUTH,
timeout=settings.WORLDMAP_SHORT_TIMEOUT
)
except RequestsConnectionError as e:
print 'err', e
err_msg = 'Error connecting to WorldMap server: %s' % e.message
LOGGER.error('Error trying to retrive TableJoin with id: %s', tablejoin_id)
LOGGER.error(err_msg)
return (False, err_msg)
except:
err_msg = "Unexpected error: %s" % sys.exc_info()[0]
LOGGER.error(err_msg)
return (False, err_msg)
msg(r.text)
try:
rjson = r.json()
except TypeError:
return (False, "Sorry! Could not retrieve the TableJoin information. (%s)" % r.text)
if rjson.get('success', False) is True:
return (True, rjson)
else:
return (False, rjson.get('message', '(no message sent)'))
def send_email_notice(self):
"""Send email notice to settings.ADMINS"""
msgt('Send email notice!')
subject = 'GeoConnect: Clear stale data (%s)' % timezone.now()
self.add_message_line(\
'This is an email notice from Geoconnect',
prepend=True)
self.add_message_title_line('(end of message)')
if len(settings.ADMINS) == 0:
msg('No one to email! (no one in settings.ADMINS)')
return
to_addresses = [x[1] for x in settings.ADMINS]
if len(settings.ADMINS) == 0:
msg('No one to email! (no one in settings.ADMINS)')
return
#email_msg = render_to_string('task_scripts/prune_scratch_directories_email.txt', d)
#msg(subject)
#msg(email_msg)
from_email = to_addresses[0]
email_msg = '\n'.join(self.message_lines)
send_mail(subject, email_msg,
from_email, to_addresses,
fail_silently=False)
msg('email sent to: %s' % to_addresses)
def get_tablejoin_info(tablejoin_id):
"""
Use the WorldMap API to retrieve TableJoin data
from gc_apps.worldmap_connect.join_layer_service import get_tablejoin_info
get_tablejoin_info(215)
"""
if not tablejoin_id:
LOGGER.error('A tablejoin_id was not specified')
return (False, 'You must specifiy a tablejoin_id')
api_url = '%s%s/' % (GET_TABLEJOIN_INFO, tablejoin_id)
msg('api_url: %s' % api_url)
try:
r = requests.get(api_url,
auth=settings.WORLDMAP_ACCOUNT_AUTH,
timeout=settings.WORLDMAP_SHORT_TIMEOUT
)
except RequestsConnectionError as e:
err_msg = 'Error connecting to WorldMap server: %s' % e.message
LOGGER.error('Error trying to retrieve TableJoin with id: %s', tablejoin_id)
LOGGER.error(err_msg)
return (False, err_msg)
except:
err_msg = "Unexpected error: %s" % sys.exc_info()[0]
LOGGER.error(err_msg)
return (False, err_msg)
msg(r.text)
try:
rjson = r.json()
except TypeError:
return (False, "Sorry! Could not retrieve the TableJoin information. (%s)" % r.text)
if rjson.get('success', False) is True:
return (True, rjson)
else:
return (False, rjson.get('message', '(no message sent)'))
def test_target_read_functions(self):
"""Check expected format type functions including:
- is_target_column_string()
- requires_zero_padding()
- get_zero_pad_length()
- does_join_column_potentially_need_formatting()
"""
msgt(self.test_target_read_functions.__doc__)
j = JoinTargetInformation(name='test',
target_info=self.join_targets_json)
j.save()
self.assertEqual(type(j.target_info), dict)
cnt = 0
for info in j.target_info['data']:
target_info = SingleJoinTargetInfo(info)
cnt += 1
#target_info.show()
if target_info.target_layer_name == 'geonode:census_tracts_2010_boston_6f6':
msg('a) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), True)
self.assertEqual(
target_info.does_join_column_potentially_need_formatting(),
True)
elif target_info.target_layer_name == 'geonode:us_zip_code_2015_boston_v3q':
msg('b) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), True)
self.assertEqual(target_info.requires_zero_padding(), True)
self.assertEqual(target_info.get_zero_pad_length(), 5)
self.assertEqual(
target_info.does_join_column_potentially_need_formatting(),
True)
elif target_info.name == 'Roads, Boston':
msg('c) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), False)
self.assertEqual(target_info.requires_zero_padding(), False)
self.assertEqual(target_info.get_zero_pad_length(), None)
self.assertEqual(
target_info.does_join_column_potentially_need_formatting(),
False)
def test_target_read_functions(self):
"""Check expected format type functions including:
- is_target_column_string()
- requires_zero_padding()
- get_zero_pad_length()
- does_join_column_potentially_need_formatting()
"""
msgt(self.test_target_read_functions.__doc__)
j = JoinTargetInformation(name='test', target_info=self.join_targets_json)
j.save()
self.assertEqual(type(j.target_info), dict)
cnt = 0
for info in j.target_info['data']:
target_info = SingleJoinTargetInfo(info)
cnt += 1
#target_info.show()
if target_info.target_layer_name == 'geonode:census_tracts_2010_boston_6f6':
msg('a) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), True)
self.assertEqual(target_info.does_join_column_potentially_need_formatting(), True)
elif target_info.target_layer_name == 'geonode:us_zip_code_2015_boston_v3q':
msg('b) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), True)
self.assertEqual(target_info.requires_zero_padding(), True)
self.assertEqual(target_info.get_zero_pad_length(), 5)
self.assertEqual(target_info.does_join_column_potentially_need_formatting(), True)
elif target_info.name == 'Roads, Boston':
msg('c) checking target: %s' % (target_info.target_layer_name))
self.assertEqual(target_info.is_target_column_string(), False)
self.assertEqual(target_info.requires_zero_padding(), False)
self.assertEqual(target_info.get_zero_pad_length(), None)
self.assertEqual(target_info.does_join_column_potentially_need_formatting(), False)
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict):
"""Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object.
This function should only return successful responses.
return True/False, shp_md5 or ErrResultMsg
Examples: True, md5 from TabularFileInfo
False, ErrResultMsg
"""
assert dv_session_token is not None, "dv_session_token cannot be None"
assert type(dataverse_info_dict) is dict,\
"dataverse_info_dict must be type 'dict'"
msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict))
#------------------------------
# (1) Validate the data (DataverseInfoValidationForm)
#------------------------------
#dataverse_info_dict.update({'datafile_id':None}) # for testing
validation_form = DataverseInfoValidationForm(dataverse_info_dict)
if not validation_form.is_valid():
errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()]
LOGGER.debug('errors: %s', errs)
form_errs = '\n'.join(errs)
return False, ErrResultMsg(None, form_errs)
#-------------------------------------------------
# (2) Check if this is a Registered Dataverse
#-------------------------------------------------
registered_dataverse = find_registered_dataverse(
dataverse_info_dict['return_to_dataverse_url'])
if registered_dataverse is None:
return False, ErrResultMsg(\
FAILED_NOT_A_REGISTERED_DATAVERSE,
"This dataverse url was not recognized: %s" %\
dataverse_info_dict['return_to_dataverse_url'])
#-------------------------------------------------
# (3b) Look for existing Dataverse files in the database
# ShapefileInfo and TabularFileInfo objects are routinely
# deleted, but if file is already here, use it
#-------------------------------------------------
params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\
, dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\
)
existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\
).values_list('id', flat=True\
).order_by('created')
existing_tabular_info_ids = list(existing_sets)
msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids)
#-------------------------------------------------
# add dv_session_token and registered_dataverse to dataverse_info_dict
#-------------------------------------------------
dataverse_info_dict['dv_session_token'] = dv_session_token
dataverse_info_dict['registered_dataverse'] = registered_dataverse
#------------------------------
# (4) Existing TabularFileInfo(s) found:
# (a) Update the TabularFileInfo object
# (b) Delete other groups TabularFileInfo object for this datafile and user
# (c) Return the md5
#------------------------------
if len(existing_tabular_info_ids) > 1:
# pop the last TabularFileInfo id off the list of existing_tabular_info_ids
shp_id = existing_tabular_info_ids.pop()
# delete the rest
if len(existing_sets) > 0:
# delete older TabularFileInfo objects
TabularFileInfo.objects.filter(
id__in=existing_tabular_info_ids).delete()
#------------------------------
# (5) Get or create a new TabularFileInfo object
#------------------------------
msgt('(5) Get or create a new TabularFileInfo object')
try:
# Existing TabularFileInfo:
# (1) Assume file is already saved
# (2) update the data
#
tabular_info = TabularFileInfo.objects.get(**params_for_existing_check)
for key, value in dataverse_info_dict.iteritems():
if key == 'column_names':
tabular_info.add_column_names(value)
else:
setattr(tabular_info, key, value)
# Save
tabular_info.save()
msg('tabular_info info saved')
# If the file is still available, return it
if tabular_info.is_dv_file_available():
add_worldmap_layerinfo_if_exists(tabular_info)
return True, tabular_info.md5
else:
# But the file isn't there!! Delete TabularFileInfo and make a new one
tabular_info.delete()
except TabularFileInfo.DoesNotExist:
pass
#except:
# msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one')
# #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object')
msg('new file')
#------------------------------
# New tabular_info, create object and attach file
#------------------------------
# Add name parameter
dataverse_info_dict['name'] = dataverse_info_dict.get(
'datafile_label', '(no datafile_label found)')
tabular_info = TabularFileInfo(**dataverse_info_dict)
tabular_info.save()
#------------------------------
# Download and attach file
#------------------------------
datafile_download_url = dataverse_info_dict.get('datafile_download_url',
'')
# Add session token. Gives permission to download/retrieve the file
# - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY
#
datafile_download_url = '%s?key=%s' % (datafile_download_url,
dv_session_token)
msg('datafile_download_url: %s' % datafile_download_url)
datafile_filename = dataverse_info_dict.get('datafile_label', '')
img_temp = NamedTemporaryFile(delete=True)
try:
img_temp.write(urllib2.urlopen(datafile_download_url).read())
except urllib2.HTTPError as e:
tabular_info.delete() # clear tabular info
err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % (
str(e), datafile_download_url)
return False, ErrResultMsg(None, err_msg)
img_temp.flush()
tabular_info.dv_file.save(datafile_filename, File(img_temp))
tabular_info.save()
add_worldmap_layerinfo_if_exists(tabular_info)
return True, tabular_info.md5
def view_map_tabular_file_form(request):
"""
AJAX call: Join your tabular file to an existing WorldMap layer
using the column selected in this form
"""
#for k, v in request.POST.items():
# print k, v
# -----------------------------------------
# Retrieve the id of the Tabular info object
# -----------------------------------------
tabular_file_info_id = request.POST.get('tabular_file_info_id', -1)
try:
tabular_info = TabularFileInfo.objects.get(pk=tabular_file_info_id)
except TabularFileInfo.DoesNotExist:
err_msg = 'Sorry! The Tabular File was not found. (tabular_file_info_id)'
json_msg = MessageHelperJSON.get_json_fail_msg(err_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
#raise Http404('No TabularFileInfo for id: %s' % tabular_file_info_id)
# -----------------------------------------
# Retrieve available Geocode types and join Layers
# note: geocode_types_from_worldmap not needed here
# -----------------------------------------
(geocode_types_from_worldmap,
available_layers_list) = get_geocode_types_and_join_layers()
# -----------------------------------------
# Create form with initial + POST data
# -----------------------------------------
form_single_column = ChooseSingleColumnForm(tabular_info.id,\
available_layers_list,\
tabular_info.column_names,\
request.POST)
# -----------------------------------------
# Check the form's validity
# -----------------------------------------
if not form_single_column.is_valid():
json_msg = MessageHelperJSON.get_json_fail_msg(\
format_errors_as_text(form_single_column, for_web=True))
return HttpResponse(json_msg,
content_type="application/json",
status=200)
print 'cleaned_data', form_single_column.cleaned_data
# -----------------------------------------
# Get Dataverse info dict
# -----------------------------------------
dataverse_metadata_dict = get_dataverse_info_dict(tabular_info)
# -----------------------------------------
# Use the WorldMap API and
# try to create a layer
# -----------------------------------------
tj_map_maker = TableJoinMapMaker(
tabular_info,
dataverse_metadata_dict,
form_single_column.cleaned_data.get('chosen_column'),
form_single_column.cleaned_data.get('chosen_layer'),
)
success = tj_map_maker.run_map_create()
msg('success: %s' % success)
if not success:
json_msg = MessageHelperJSON.get_json_fail_msg(\
'Sorry! ' + tj_map_maker.get_error_msg())
msg('error msg: %s' % json_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Succeeded! Create a WorldMapTabularLayerInfo object
# -----------------------------------------
worldmap_tabular_info = WorldMapTabularLayerInfo.build_from_worldmap_json(tabular_info,\
tj_map_maker.get_map_info())
if worldmap_tabular_info is None:
LOGGER.error("Failed to create WorldMapTabularLayerInfo using %s",\
tj_map_maker.get_map_info())
user_msg = 'Sorry! Failed to create map. Please try again. (code: s1)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Notify Dataverse of the new map
# -----------------------------------------
MetadataUpdater.update_dataverse_with_metadata(worldmap_tabular_info)
# -----------------------------------------
# Build the Map HTML chunk to replace the form
# -----------------------------------------
map_html, user_message_html = build_map_html(request,
worldmap_tabular_info)
if map_html is None: # Failed! Send an error
LOGGER.error("Failed to create map HTML using WorldMapTabularLayerInfo: %s (%d)",\
worldmap_tabular_info, worldmap_tabular_info.id)
user_msg = 'Sorry! Failed to create map. Please try again. (code: s2)'
json_msg = MessageHelperJSON.get_json_fail_msg(user_msg)
return HttpResponse(json_msg,
content_type="application/json",
status=200)
# -----------------------------------------
# Looks good. In the JSON response, send
# back the map HTML
# -----------------------------------------
data_dict = dict(map_html=map_html,
user_message_html=user_message_html,
id_main_panel_title=PANEL_TITLE_STYLE_MAP)
json_msg = MessageHelperJSON.get_json_success_msg("great job",
data_dict=data_dict)
return HttpResponse(json_msg, content_type="application/json", status=200)
def run_map_create(self):
"""
Format parameters and make a WorldMap API call
"""
if self.err_found:
return False
# -------------------------------------------
# Based on the target layer ID,
# Retrieve the layer name, column name, and zero_pad_length
# -------------------------------------------
all_join_target_info = get_latest_jointarget_information()
single_join_target_info =\
all_join_target_info.get_single_join_target_info(self.target_layer_id)
if single_join_target_info is None:
self.add_error('Failed to retrieve target layer information.')
return False
#single_join_target_info.show()
# Update instance info
#
self.zero_pad_length = single_join_target_info.zero_pad_length
# --------------------------------------------------
# Do we need to format the data in the join column?
# --------------------------------------------------
format_success = self.format_data_table_for_join(
single_join_target_info)
if not format_success:
return False
# --------------------------------
# Prepare parameters
# --------------------------------
map_params = dict(\
title=self.datatable_obj.name,
abstract=self.datatable_obj.get_abstract_for_worldmap(),
delimiter=self.datatable_obj.delimiter,
table_attribute=self.table_attribute_for_join,
layer_name=single_join_target_info.target_layer_name,
layer_attribute=single_join_target_info.target_column_name)
# Add dataverse dict info
#
map_params.update(self.dataverse_metadata_dict)
pretty_printer = pprint.PrettyPrinter(indent=4)
msg(pretty_printer.pprint(map_params))
# --------------------------------
# Prepare file
# --------------------------------
file_params = self.get_file_params()
if file_params is None:
return False
LOGGER.info('make request to: %s', UPLOAD_JOIN_DATATABLE_API_PATH)
try:
resp = requests.post(\
UPLOAD_JOIN_DATATABLE_API_PATH,
data=map_params,
files=file_params,
auth=settings.WORLDMAP_ACCOUNT_AUTH,
timeout=settings.WORLDMAP_DEFAULT_TIMEOUT)
except RequestsConnectionError as ex_obj:
err_msg = 'Error connecting to WorldMap server: %s' % ex_obj.message
LOGGER.error('Error trying to join to datatable with id: %s',\
self.datatable_obj.id)
LOGGER.error(err_msg)
self.add_error(err_msg)
return False
except:
err_msg = "Unexpected error: %s" % sys.exc_info()[0]
LOGGER.error(err_msg)
self.add_error(err_msg)
return False
try:
rjson = resp.json()
except:
self.add_error("Sorry! The mapping failed. (%s)" % resp.text)
return False
msg('rjson: %s' % json.dumps(rjson, indent=4))
if rjson.get('success', False) is True:
self.rjson_output = rjson
# (True, (message, data))
return True
else:
self.add_error(rjson.get('message', '(no message sent)'))
return False
def run_map_create(self):
"""
Format parameters and make a WorldMap API call
"""
if self.err_found:
return False
# -------------------------------------------
# Based on the target layer ID,
# Retrieve the layer name, column name, and zero_pad_length
# -------------------------------------------
all_join_target_info = get_latest_jointarget_information()
single_join_target_info =\
all_join_target_info.get_single_join_target_info(self.target_layer_id)
if single_join_target_info is None:
self.add_error('Failed to retrieve target layer information.')
return False
#single_join_target_info.show()
# Update instance info
#
self.zero_pad_length = single_join_target_info.zero_pad_length
# --------------------------------------------------
# Do we need to format the data in the join column?
# --------------------------------------------------
format_success = self.format_data_table_for_join(single_join_target_info)
if not format_success:
return False
# --------------------------------
# Prepare parameters
# --------------------------------
map_params = dict(\
title=self.datatable_obj.name,
abstract=self.datatable_obj.get_abstract_for_worldmap(),
delimiter=self.datatable_obj.delimiter,
table_attribute=self.table_attribute_for_join,
layer_name=single_join_target_info.target_layer_name,
layer_attribute=single_join_target_info.target_column_name)
# Add dataverse dict info
#
map_params.update(self.dataverse_metadata_dict)
pretty_printer = pprint.PrettyPrinter(indent=4)
msg(pretty_printer.pprint(map_params))
# --------------------------------
# Prepare file
# --------------------------------
file_params = self.get_file_params()
if file_params is None:
return False
LOGGER.info('make request to: %s', UPLOAD_JOIN_DATATABLE_API_PATH)
try:
resp = requests.post(\
UPLOAD_JOIN_DATATABLE_API_PATH,
data=map_params,
files=file_params,
auth=settings.WORLDMAP_ACCOUNT_AUTH,
timeout=settings.WORLDMAP_DEFAULT_TIMEOUT)
except RequestsConnectionError as ex_obj:
err_msg = 'Error connecting to WorldMap server: %s' % ex_obj.message
LOGGER.error('Error trying to join to datatable with id: %s',\
self.datatable_obj.id)
LOGGER.error(err_msg)
self.add_error(err_msg)
return False
except:
err_msg = "Unexpected error: %s" % sys.exc_info()[0]
LOGGER.error(err_msg)
self.add_error(err_msg)
return False
try:
rjson = resp.json()
except:
self.add_error("Sorry! The mapping failed. (%s)" % resp.text)
return False
msg('rjson: %s' % json.dumps(rjson, indent=4))
if rjson.get('success', False) is True:
self.rjson_output = rjson
# (True, (message, data))
return True
else:
self.add_error(rjson.get('message', '(no message sent)'))
return False
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict):
"""Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object.
This function should only return successful responses.
return True/False, shp_md5 or ErrResultMsg
Examples: True, md5 from TabularFileInfo
False, ErrResultMsg
"""
assert dv_session_token is not None, "dv_session_token cannot be None"
assert type(dataverse_info_dict) is dict,\
"dataverse_info_dict must be type 'dict'"
msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict))
#------------------------------
# (1) Validate the data (DataverseInfoValidationForm)
#------------------------------
#dataverse_info_dict.update({'datafile_id':None}) # for testing
validation_form = DataverseInfoValidationForm(dataverse_info_dict)
if not validation_form.is_valid():
errs = ['%s: %s' % (k, v) for k,v in validation_form.errors.items()]
LOGGER.debug('errors: %s', errs)
form_errs = '\n'.join(errs)
return False, ErrResultMsg(None, form_errs)
#-------------------------------------------------
# (2) Check if this is a Registered Dataverse
#-------------------------------------------------
registered_dataverse = find_registered_dataverse(dataverse_info_dict['return_to_dataverse_url'])
if registered_dataverse is None:
return False, ErrResultMsg(\
FAILED_NOT_A_REGISTERED_DATAVERSE,
"This dataverse url was not recognized: %s" %\
dataverse_info_dict['return_to_dataverse_url'])
#-------------------------------------------------
# (3b) Look for existing Dataverse files in the database
# ShapefileInfo and TabularFileInfo objects are routinely
# deleted, but if file is already here, use it
#-------------------------------------------------
params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\
, dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\
)
existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\
).values_list('id', flat=True\
).order_by('created')
existing_tabular_info_ids = list(existing_sets)
msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids)
#-------------------------------------------------
# add dv_session_token and registered_dataverse to dataverse_info_dict
#-------------------------------------------------
dataverse_info_dict['dv_session_token'] = dv_session_token
dataverse_info_dict['registered_dataverse'] = registered_dataverse
#------------------------------
# (4) Existing TabularFileInfo(s) found:
# (a) Update the TabularFileInfo object
# (b) Delete other groups TabularFileInfo object for this datafile and user
# (c) Return the md5
#------------------------------
if len(existing_tabular_info_ids) > 1:
# pop the last TabularFileInfo id off the list of existing_tabular_info_ids
shp_id = existing_tabular_info_ids.pop()
# delete the rest
if len(existing_sets) > 0:
# delete older TabularFileInfo objects
TabularFileInfo.objects.filter(id__in=existing_tabular_info_ids).delete()
#------------------------------
# (5) Get or create a new TabularFileInfo object
#------------------------------
msgt('(5) Get or create a new TabularFileInfo object')
try:
# Existing TabularFileInfo:
# (1) Assume file is already saved
# (2) update the data
#
tabular_info = TabularFileInfo.objects.get(**params_for_existing_check)
for key, value in dataverse_info_dict.iteritems():
if key == 'column_names':
tabular_info.add_column_names(value)
else:
setattr(tabular_info, key, value)
# Save
tabular_info.save()
msg('tabular_info info saved')
# If the file is still available, return it
if tabular_info.is_dv_file_available():
add_worldmap_layerinfo_if_exists(tabular_info)
return True, tabular_info.md5
else:
# But the file isn't there!! Delete TabularFileInfo and make a new one
tabular_info.delete()
except TabularFileInfo.DoesNotExist:
pass
#except:
# msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one')
# #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object')
msg('new file')
#------------------------------
# New tabular_info, create object and attach file
#------------------------------
# Add name parameter
dataverse_info_dict['name'] = dataverse_info_dict.get('datafile_label', '(no datafile_label found)')
tabular_info = TabularFileInfo(**dataverse_info_dict)
tabular_info.save()
#------------------------------
# Download and attach file
#------------------------------
datafile_download_url = dataverse_info_dict.get('datafile_download_url', '')
# Add session token. Gives permission to download/retrieve the file
# - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY
#
datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token)
msg('datafile_download_url: %s' % datafile_download_url)
datafile_filename = dataverse_info_dict.get('datafile_label', '')
img_temp = NamedTemporaryFile(delete=True)
try:
img_temp.write(urllib2.urlopen(datafile_download_url).read())
except urllib2.HTTPError as e:
tabular_info.delete() # clear tabular info
err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % (str(e), datafile_download_url)
return False, ErrResultMsg(None, err_msg)
img_temp.flush()
tabular_info.dv_file.save(datafile_filename, File(img_temp))
tabular_info.save()
add_worldmap_layerinfo_if_exists(tabular_info)
return True, tabular_info.md5
def get_shapefile_from_dv_api_info(dv_session_token, dv_info_dict):
"""Using Dataverse API information, create a "ShapefileInfo" object.
This function should only result in successful responses.
return True/False, shp_md5 or ErrResultMsg
Examples: True, md5 from ShapefileInfo
False, ErrResultMsg
To do: Make this into a separate class
"""
assert dv_session_token is not None, "dv_session_token cannot be None"
assert type(dv_info_dict) is dict, "dv_info_dict must be type 'dict'"
#------------------------------
# (1) Validate the data (DataverseInfoValidationForm)
#------------------------------
validation_form = DataverseInfoValidationForm(dv_info_dict)
if not validation_form.is_valid():
errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()]
print(errs)
form_errs = '\n'.join(errs)
return False, ErrResultMsg(None, form_errs)
#-------------------------------------------------
# (2) Check if this is a Registered Dataverse
#-------------------------------------------------
registered_dataverse = find_registered_dataverse(
dv_info_dict['return_to_dataverse_url'])
if registered_dataverse is None:
return False, ErrResultMsg(FAILED_NOT_A_REGISTERED_DATAVERSE\
, "This dataverse url was not recognized: %s" % dv_info_dict['return_to_dataverse_url']\
)
#-------------------------------------------------
# (3) Look for existing ShapefileInfo objects in the database
# ShapefileInfo objects are routinely deleted, but if file is already here, use it
# * todo: check for staleness, if the data is old delete it
#-------------------------------------------------
params_for_existing_check = dict(datafile_id=dv_info_dict.get('datafile_id', -1),\
dataverse_installation_name=dv_info_dict.get('dataverse_installation_name', -1),\
)
existing_sets = ShapefileInfo.objects.filter(**params_for_existing_check\
).values_list('id', flat=True\
).order_by('created')
existing_shapefile_info_ids = list(existing_sets)
msgt('existing_shapefile_info_ids: %s' % existing_shapefile_info_ids)
#-------------------------------------------------
# add dv_session_token and registered_dataverse to dv_info_dict
#-------------------------------------------------
dv_info_dict['dv_session_token'] = dv_session_token
dv_info_dict['registered_dataverse'] = registered_dataverse
#------------------------------
# (4) Existing ShapefileInfo(s) found:
# (a) Update the ShapefileInfo object
# (b) Delete other ShapefileInfo objects for this datafile and user
# (c) Return the md5
#------------------------------
if len(existing_shapefile_info_ids) > 1:
# pop the last ShapefileInfo id off the list of existing_shapefile_info_ids
shp_id = existing_shapefile_info_ids.pop()
# delete the rest
if len(existing_sets) > 0:
ShapefileInfo.objects.filter(
id__in=existing_shapefile_info_ids).delete(
) # delete older ShapefileInfo(s)
#------------------------------
# (5) Get or create a new ShapefileInfo object
#------------------------------
msgt('(5) Get or create a new ShapefileInfo object')
try:
# Existing ShapefileInfo:
# (1) Assume file is already saved
# (2) update the data
#
shapefile_info = ShapefileInfo.objects.get(**params_for_existing_check)
for key, value in dv_info_dict.iteritems():
setattr(shapefile_info, key, value)
# Save
shapefile_info.save()
msg('shapefile info saved')
# If the file is still available, return it
if shapefile_info.is_dv_file_available():
return True, shapefile_info.md5
else:
# But the file isn't there!! Delete ShapefileInfo and make a new one
shapefile_info.delete()
except ShapefileInfo.DoesNotExist:
pass
#except:
# msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one')
# #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object')
msg('new file')
#------------------------------
# New shapefile info, create object and attach file
#------------------------------
shapefile_info = ShapefileInfo(**dv_info_dict)
shapefile_info.save()
#------------------------------
# Download and attach file
#------------------------------
datafile_download_url = dv_info_dict.get('datafile_download_url', '')
# Add session token. Gives permission to download/retrieve the file
# - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY
#
datafile_download_url = '%s?key=%s' % (datafile_download_url,
dv_session_token)
msg('datafile_download_url: %s' % datafile_download_url)
datafile_filename = dv_info_dict.get('datafile_label', '')
tmp_shapefile = NamedTemporaryFile(delete=True)
try:
tmp_shapefile.write(urllib2.urlopen(datafile_download_url).read())
except urllib2.HTTPError as e:
shapefile_info.delete() # clear shapefile
err_msg = 'Failed to download shapefile. HTTPError: %s \n\nurl: %s' % (
str(e), datafile_download_url)
return False, ErrResultMsg(None, err_msg)
tmp_shapefile.flush()
shapefile_info.dv_file.save(datafile_filename, File(tmp_shapefile))
shapefile_info.save()
return True, shapefile_info.md5
def handle(self, *args, **options):
print (options)
# For Retrieving the WorldMap Layer Info stored in Geoconnect
#
worldmap_info_md5 = options.get('worldmap_info_md5', None)
if worldmap_info_md5 is None:
error_note = "Please specify the md5 of the WorldMapLayerInfo object"
msg(error_note)
return
# File type, to differntiate which WorldMap Layer Info is stored in Geoconnect
#
file_type = options.get('file_type', None)
if file_type is None:
error_note = "Please specify the file type. e.g. --type=tabular"
msg(error_note)
return
delay_seconds = options.get('delay_seconds')
try:
delay_seconds = int(delay_seconds)
except ValueError:
error_note = "Please use an integer for 'delay_seconds'"
msg(error_note)
return
except TypeError:
error_note = "Please use an integer for 'delay_seconds'"
msg(error_note)
return
num_attempts = options.get('num_attempts')
try:
num_attempts = int(num_attempts)
except ValueError:
error_note = "Please use an integer for 'num_attempts'"
msg(error_note)
return
except TypeError:
error_note = "Please use an integer for 'num_attempts'"
msg(error_note)
return
for attempt_num in range(1, num_attempts+1):
msg('(Attempt %s) Pausing for %s second(s)' % (attempt_num, delay_seconds))
time.sleep(delay_seconds)
success, err_info = MetadataUpdater.run_metadata_update_with_thumbnail_check(\
worldmap_info_md5, file_type)
msg('success: %s' % success)
msg('err_info: %s' % err_info)
if success is True:
break
def handle(self, *args, **options):
print(options)
# For Retrieving the WorldMap Layer Info stored in Geoconnect
#
worldmap_info_md5 = options.get('worldmap_info_md5', None)
if worldmap_info_md5 is None:
error_note = "Please specify the md5 of the WorldMapLayerInfo object"
msg(error_note)
return
# File type, to differntiate which WorldMap Layer Info is stored in Geoconnect
#
file_type = options.get('file_type', None)
if file_type is None:
error_note = "Please specify the file type. e.g. --type=tabular"
msg(error_note)
return
delay_seconds = options.get('delay_seconds')
try:
delay_seconds = int(delay_seconds)
except ValueError:
error_note = "Please use an integer for 'delay_seconds'"
msg(error_note)
return
except TypeError:
error_note = "Please use an integer for 'delay_seconds'"
msg(error_note)
return
num_attempts = options.get('num_attempts')
try:
num_attempts = int(num_attempts)
except ValueError:
error_note = "Please use an integer for 'num_attempts'"
msg(error_note)
return
except TypeError:
error_note = "Please use an integer for 'num_attempts'"
msg(error_note)
return
for attempt_num in range(1, num_attempts + 1):
msg('(Attempt %s) Pausing for %s second(s)' %
(attempt_num, delay_seconds))
time.sleep(delay_seconds)
success, err_info = MetadataUpdater.run_metadata_update_with_thumbnail_check(\
worldmap_info_md5, file_type)
msg('success: %s' % success)
msg('err_info: %s' % err_info)
if success is True:
break
