def add_message_line(self, mline, prepend=False): """Add message line""" if not mline: return msg(mline) if prepend: self.message_lines.insert(0, mline) else: self.message_lines.append(mline)
def handle(self, *args, **options): really_delete = options.get('really_delete', False) email_notice = options.get('email_notice', False) dashes() if really_delete: msg('Check for old objects and DELETE them') else: msg('Check for old objects but DO NOT delete them') stale_data_remover = StaleDataRemover(really_delete=really_delete) stale_data_remover.remove_stale_data(\ settings.STALE_DATA_SECONDS_TO_EXPIRATION) if email_notice is True: stale_data_remover.send_email_notice()
def get_tablejoin_info(tablejoin_id): """ Use the WorldMap API to retrieve TableJoin data from gc_apps.worldmap_connect.join_layer_service import get_tablejoin_info get_tablejoin_info(215) """ if not tablejoin_id: LOGGER.error('A tablejoin_id was not specified') return (False, 'You must specifiy a tablejoin_id') api_url = '%s%s/' % (GET_TABLEJOIN_INFO, tablejoin_id) msg('api_url: %s' % api_url) try: r = requests.get(api_url, auth=settings.WORLDMAP_ACCOUNT_AUTH, timeout=settings.WORLDMAP_SHORT_TIMEOUT ) except RequestsConnectionError as e: print 'err', e err_msg = 'Error connecting to WorldMap server: %s' % e.message LOGGER.error('Error trying to retrive TableJoin with id: %s', tablejoin_id) LOGGER.error(err_msg) return (False, err_msg) except: err_msg = "Unexpected error: %s" % sys.exc_info()[0] LOGGER.error(err_msg) return (False, err_msg) msg(r.text) try: rjson = r.json() except TypeError: return (False, "Sorry! Could not retrieve the TableJoin information. (%s)" % r.text) if rjson.get('success', False) is True: return (True, rjson) else: return (False, rjson.get('message', '(no message sent)'))
def send_email_notice(self): """Send email notice to settings.ADMINS""" msgt('Send email notice!') subject = 'GeoConnect: Clear stale data (%s)' % timezone.now() self.add_message_line(\ 'This is an email notice from Geoconnect', prepend=True) self.add_message_title_line('(end of message)') if len(settings.ADMINS) == 0: msg('No one to email! (no one in settings.ADMINS)') return to_addresses = [x[1] for x in settings.ADMINS] if len(settings.ADMINS) == 0: msg('No one to email! (no one in settings.ADMINS)') return #email_msg = render_to_string('task_scripts/prune_scratch_directories_email.txt', d) #msg(subject) #msg(email_msg) from_email = to_addresses[0] email_msg = '\n'.join(self.message_lines) send_mail(subject, email_msg, from_email, to_addresses, fail_silently=False) msg('email sent to: %s' % to_addresses)
def get_tablejoin_info(tablejoin_id): """ Use the WorldMap API to retrieve TableJoin data from gc_apps.worldmap_connect.join_layer_service import get_tablejoin_info get_tablejoin_info(215) """ if not tablejoin_id: LOGGER.error('A tablejoin_id was not specified') return (False, 'You must specifiy a tablejoin_id') api_url = '%s%s/' % (GET_TABLEJOIN_INFO, tablejoin_id) msg('api_url: %s' % api_url) try: r = requests.get(api_url, auth=settings.WORLDMAP_ACCOUNT_AUTH, timeout=settings.WORLDMAP_SHORT_TIMEOUT ) except RequestsConnectionError as e: err_msg = 'Error connecting to WorldMap server: %s' % e.message LOGGER.error('Error trying to retrieve TableJoin with id: %s', tablejoin_id) LOGGER.error(err_msg) return (False, err_msg) except: err_msg = "Unexpected error: %s" % sys.exc_info()[0] LOGGER.error(err_msg) return (False, err_msg) msg(r.text) try: rjson = r.json() except TypeError: return (False, "Sorry! Could not retrieve the TableJoin information. (%s)" % r.text) if rjson.get('success', False) is True: return (True, rjson) else: return (False, rjson.get('message', '(no message sent)'))
def test_target_read_functions(self): """Check expected format type functions including: - is_target_column_string() - requires_zero_padding() - get_zero_pad_length() - does_join_column_potentially_need_formatting() """ msgt(self.test_target_read_functions.__doc__) j = JoinTargetInformation(name='test', target_info=self.join_targets_json) j.save() self.assertEqual(type(j.target_info), dict) cnt = 0 for info in j.target_info['data']: target_info = SingleJoinTargetInfo(info) cnt += 1 #target_info.show() if target_info.target_layer_name == 'geonode:census_tracts_2010_boston_6f6': msg('a) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), True) self.assertEqual( target_info.does_join_column_potentially_need_formatting(), True) elif target_info.target_layer_name == 'geonode:us_zip_code_2015_boston_v3q': msg('b) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), True) self.assertEqual(target_info.requires_zero_padding(), True) self.assertEqual(target_info.get_zero_pad_length(), 5) self.assertEqual( target_info.does_join_column_potentially_need_formatting(), True) elif target_info.name == 'Roads, Boston': msg('c) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), False) self.assertEqual(target_info.requires_zero_padding(), False) self.assertEqual(target_info.get_zero_pad_length(), None) self.assertEqual( target_info.does_join_column_potentially_need_formatting(), False)
def test_target_read_functions(self): """Check expected format type functions including: - is_target_column_string() - requires_zero_padding() - get_zero_pad_length() - does_join_column_potentially_need_formatting() """ msgt(self.test_target_read_functions.__doc__) j = JoinTargetInformation(name='test', target_info=self.join_targets_json) j.save() self.assertEqual(type(j.target_info), dict) cnt = 0 for info in j.target_info['data']: target_info = SingleJoinTargetInfo(info) cnt += 1 #target_info.show() if target_info.target_layer_name == 'geonode:census_tracts_2010_boston_6f6': msg('a) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), True) self.assertEqual(target_info.does_join_column_potentially_need_formatting(), True) elif target_info.target_layer_name == 'geonode:us_zip_code_2015_boston_v3q': msg('b) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), True) self.assertEqual(target_info.requires_zero_padding(), True) self.assertEqual(target_info.get_zero_pad_length(), 5) self.assertEqual(target_info.does_join_column_potentially_need_formatting(), True) elif target_info.name == 'Roads, Boston': msg('c) checking target: %s' % (target_info.target_layer_name)) self.assertEqual(target_info.is_target_column_string(), False) self.assertEqual(target_info.requires_zero_padding(), False) self.assertEqual(target_info.get_zero_pad_length(), None) self.assertEqual(target_info.does_join_column_potentially_need_formatting(), False)
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict): """Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object. This function should only return successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from TabularFileInfo False, ErrResultMsg """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dataverse_info_dict) is dict,\ "dataverse_info_dict must be type 'dict'" msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict)) #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ #dataverse_info_dict.update({'datafile_id':None}) # for testing validation_form = DataverseInfoValidationForm(dataverse_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()] LOGGER.debug('errors: %s', errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse( dataverse_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(\ FAILED_NOT_A_REGISTERED_DATAVERSE, "This dataverse url was not recognized: %s" %\ dataverse_info_dict['return_to_dataverse_url']) #------------------------------------------------- # (3b) Look for existing Dataverse files in the database # ShapefileInfo and TabularFileInfo objects are routinely # deleted, but if file is already here, use it #------------------------------------------------- params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\ , dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\ ) existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_tabular_info_ids = list(existing_sets) msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids) #------------------------------------------------- # add dv_session_token and registered_dataverse to dataverse_info_dict #------------------------------------------------- dataverse_info_dict['dv_session_token'] = dv_session_token dataverse_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing TabularFileInfo(s) found: # (a) Update the TabularFileInfo object # (b) Delete other groups TabularFileInfo object for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_tabular_info_ids) > 1: # pop the last TabularFileInfo id off the list of existing_tabular_info_ids shp_id = existing_tabular_info_ids.pop() # delete the rest if len(existing_sets) > 0: # delete older TabularFileInfo objects TabularFileInfo.objects.filter( id__in=existing_tabular_info_ids).delete() #------------------------------ # (5) Get or create a new TabularFileInfo object #------------------------------ msgt('(5) Get or create a new TabularFileInfo object') try: # Existing TabularFileInfo: # (1) Assume file is already saved # (2) update the data # tabular_info = TabularFileInfo.objects.get(**params_for_existing_check) for key, value in dataverse_info_dict.iteritems(): if key == 'column_names': tabular_info.add_column_names(value) else: setattr(tabular_info, key, value) # Save tabular_info.save() msg('tabular_info info saved') # If the file is still available, return it if tabular_info.is_dv_file_available(): add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5 else: # But the file isn't there!! Delete TabularFileInfo and make a new one tabular_info.delete() except TabularFileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') msg('new file') #------------------------------ # New tabular_info, create object and attach file #------------------------------ # Add name parameter dataverse_info_dict['name'] = dataverse_info_dict.get( 'datafile_label', '(no datafile_label found)') tabular_info = TabularFileInfo(**dataverse_info_dict) tabular_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dataverse_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) msg('datafile_download_url: %s' % datafile_download_url) datafile_filename = dataverse_info_dict.get('datafile_label', '') img_temp = NamedTemporaryFile(delete=True) try: img_temp.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: tabular_info.delete() # clear tabular info err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % ( str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) img_temp.flush() tabular_info.dv_file.save(datafile_filename, File(img_temp)) tabular_info.save() add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5
def view_map_tabular_file_form(request): """ AJAX call: Join your tabular file to an existing WorldMap layer using the column selected in this form """ #for k, v in request.POST.items(): # print k, v # ----------------------------------------- # Retrieve the id of the Tabular info object # ----------------------------------------- tabular_file_info_id = request.POST.get('tabular_file_info_id', -1) try: tabular_info = TabularFileInfo.objects.get(pk=tabular_file_info_id) except TabularFileInfo.DoesNotExist: err_msg = 'Sorry! The Tabular File was not found. (tabular_file_info_id)' json_msg = MessageHelperJSON.get_json_fail_msg(err_msg) return HttpResponse(json_msg, content_type="application/json", status=200) #raise Http404('No TabularFileInfo for id: %s' % tabular_file_info_id) # ----------------------------------------- # Retrieve available Geocode types and join Layers # note: geocode_types_from_worldmap not needed here # ----------------------------------------- (geocode_types_from_worldmap, available_layers_list) = get_geocode_types_and_join_layers() # ----------------------------------------- # Create form with initial + POST data # ----------------------------------------- form_single_column = ChooseSingleColumnForm(tabular_info.id,\ available_layers_list,\ tabular_info.column_names,\ request.POST) # ----------------------------------------- # Check the form's validity # ----------------------------------------- if not form_single_column.is_valid(): json_msg = MessageHelperJSON.get_json_fail_msg(\ format_errors_as_text(form_single_column, for_web=True)) return HttpResponse(json_msg, content_type="application/json", status=200) print 'cleaned_data', form_single_column.cleaned_data # ----------------------------------------- # Get Dataverse info dict # ----------------------------------------- dataverse_metadata_dict = get_dataverse_info_dict(tabular_info) # ----------------------------------------- # Use the WorldMap API and # try to create a layer # ----------------------------------------- tj_map_maker = TableJoinMapMaker( tabular_info, dataverse_metadata_dict, form_single_column.cleaned_data.get('chosen_column'), form_single_column.cleaned_data.get('chosen_layer'), ) success = tj_map_maker.run_map_create() msg('success: %s' % success) if not success: json_msg = MessageHelperJSON.get_json_fail_msg(\ 'Sorry! ' + tj_map_maker.get_error_msg()) msg('error msg: %s' % json_msg) return HttpResponse(json_msg, content_type="application/json", status=200) # ----------------------------------------- # Succeeded! Create a WorldMapTabularLayerInfo object # ----------------------------------------- worldmap_tabular_info = WorldMapTabularLayerInfo.build_from_worldmap_json(tabular_info,\ tj_map_maker.get_map_info()) if worldmap_tabular_info is None: LOGGER.error("Failed to create WorldMapTabularLayerInfo using %s",\ tj_map_maker.get_map_info()) user_msg = 'Sorry! Failed to create map. Please try again. (code: s1)' json_msg = MessageHelperJSON.get_json_fail_msg(user_msg) return HttpResponse(json_msg, content_type="application/json", status=200) # ----------------------------------------- # Notify Dataverse of the new map # ----------------------------------------- MetadataUpdater.update_dataverse_with_metadata(worldmap_tabular_info) # ----------------------------------------- # Build the Map HTML chunk to replace the form # ----------------------------------------- map_html, user_message_html = build_map_html(request, worldmap_tabular_info) if map_html is None: # Failed! Send an error LOGGER.error("Failed to create map HTML using WorldMapTabularLayerInfo: %s (%d)",\ worldmap_tabular_info, worldmap_tabular_info.id) user_msg = 'Sorry! Failed to create map. Please try again. (code: s2)' json_msg = MessageHelperJSON.get_json_fail_msg(user_msg) return HttpResponse(json_msg, content_type="application/json", status=200) # ----------------------------------------- # Looks good. In the JSON response, send # back the map HTML # ----------------------------------------- data_dict = dict(map_html=map_html, user_message_html=user_message_html, id_main_panel_title=PANEL_TITLE_STYLE_MAP) json_msg = MessageHelperJSON.get_json_success_msg("great job", data_dict=data_dict) return HttpResponse(json_msg, content_type="application/json", status=200)
def run_map_create(self): """ Format parameters and make a WorldMap API call """ if self.err_found: return False # ------------------------------------------- # Based on the target layer ID, # Retrieve the layer name, column name, and zero_pad_length # ------------------------------------------- all_join_target_info = get_latest_jointarget_information() single_join_target_info =\ all_join_target_info.get_single_join_target_info(self.target_layer_id) if single_join_target_info is None: self.add_error('Failed to retrieve target layer information.') return False #single_join_target_info.show() # Update instance info # self.zero_pad_length = single_join_target_info.zero_pad_length # -------------------------------------------------- # Do we need to format the data in the join column? # -------------------------------------------------- format_success = self.format_data_table_for_join( single_join_target_info) if not format_success: return False # -------------------------------- # Prepare parameters # -------------------------------- map_params = dict(\ title=self.datatable_obj.name, abstract=self.datatable_obj.get_abstract_for_worldmap(), delimiter=self.datatable_obj.delimiter, table_attribute=self.table_attribute_for_join, layer_name=single_join_target_info.target_layer_name, layer_attribute=single_join_target_info.target_column_name) # Add dataverse dict info # map_params.update(self.dataverse_metadata_dict) pretty_printer = pprint.PrettyPrinter(indent=4) msg(pretty_printer.pprint(map_params)) # -------------------------------- # Prepare file # -------------------------------- file_params = self.get_file_params() if file_params is None: return False LOGGER.info('make request to: %s', UPLOAD_JOIN_DATATABLE_API_PATH) try: resp = requests.post(\ UPLOAD_JOIN_DATATABLE_API_PATH, data=map_params, files=file_params, auth=settings.WORLDMAP_ACCOUNT_AUTH, timeout=settings.WORLDMAP_DEFAULT_TIMEOUT) except RequestsConnectionError as ex_obj: err_msg = 'Error connecting to WorldMap server: %s' % ex_obj.message LOGGER.error('Error trying to join to datatable with id: %s',\ self.datatable_obj.id) LOGGER.error(err_msg) self.add_error(err_msg) return False except: err_msg = "Unexpected error: %s" % sys.exc_info()[0] LOGGER.error(err_msg) self.add_error(err_msg) return False try: rjson = resp.json() except: self.add_error("Sorry! The mapping failed. (%s)" % resp.text) return False msg('rjson: %s' % json.dumps(rjson, indent=4)) if rjson.get('success', False) is True: self.rjson_output = rjson # (True, (message, data)) return True else: self.add_error(rjson.get('message', '(no message sent)')) return False
def run_map_create(self): """ Format parameters and make a WorldMap API call """ if self.err_found: return False # ------------------------------------------- # Based on the target layer ID, # Retrieve the layer name, column name, and zero_pad_length # ------------------------------------------- all_join_target_info = get_latest_jointarget_information() single_join_target_info =\ all_join_target_info.get_single_join_target_info(self.target_layer_id) if single_join_target_info is None: self.add_error('Failed to retrieve target layer information.') return False #single_join_target_info.show() # Update instance info # self.zero_pad_length = single_join_target_info.zero_pad_length # -------------------------------------------------- # Do we need to format the data in the join column? # -------------------------------------------------- format_success = self.format_data_table_for_join(single_join_target_info) if not format_success: return False # -------------------------------- # Prepare parameters # -------------------------------- map_params = dict(\ title=self.datatable_obj.name, abstract=self.datatable_obj.get_abstract_for_worldmap(), delimiter=self.datatable_obj.delimiter, table_attribute=self.table_attribute_for_join, layer_name=single_join_target_info.target_layer_name, layer_attribute=single_join_target_info.target_column_name) # Add dataverse dict info # map_params.update(self.dataverse_metadata_dict) pretty_printer = pprint.PrettyPrinter(indent=4) msg(pretty_printer.pprint(map_params)) # -------------------------------- # Prepare file # -------------------------------- file_params = self.get_file_params() if file_params is None: return False LOGGER.info('make request to: %s', UPLOAD_JOIN_DATATABLE_API_PATH) try: resp = requests.post(\ UPLOAD_JOIN_DATATABLE_API_PATH, data=map_params, files=file_params, auth=settings.WORLDMAP_ACCOUNT_AUTH, timeout=settings.WORLDMAP_DEFAULT_TIMEOUT) except RequestsConnectionError as ex_obj: err_msg = 'Error connecting to WorldMap server: %s' % ex_obj.message LOGGER.error('Error trying to join to datatable with id: %s',\ self.datatable_obj.id) LOGGER.error(err_msg) self.add_error(err_msg) return False except: err_msg = "Unexpected error: %s" % sys.exc_info()[0] LOGGER.error(err_msg) self.add_error(err_msg) return False try: rjson = resp.json() except: self.add_error("Sorry! The mapping failed. (%s)" % resp.text) return False msg('rjson: %s' % json.dumps(rjson, indent=4)) if rjson.get('success', False) is True: self.rjson_output = rjson # (True, (message, data)) return True else: self.add_error(rjson.get('message', '(no message sent)')) return False
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict): """Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object. This function should only return successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from TabularFileInfo False, ErrResultMsg """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dataverse_info_dict) is dict,\ "dataverse_info_dict must be type 'dict'" msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict)) #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ #dataverse_info_dict.update({'datafile_id':None}) # for testing validation_form = DataverseInfoValidationForm(dataverse_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k,v in validation_form.errors.items()] LOGGER.debug('errors: %s', errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse(dataverse_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(\ FAILED_NOT_A_REGISTERED_DATAVERSE, "This dataverse url was not recognized: %s" %\ dataverse_info_dict['return_to_dataverse_url']) #------------------------------------------------- # (3b) Look for existing Dataverse files in the database # ShapefileInfo and TabularFileInfo objects are routinely # deleted, but if file is already here, use it #------------------------------------------------- params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\ , dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\ ) existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_tabular_info_ids = list(existing_sets) msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids) #------------------------------------------------- # add dv_session_token and registered_dataverse to dataverse_info_dict #------------------------------------------------- dataverse_info_dict['dv_session_token'] = dv_session_token dataverse_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing TabularFileInfo(s) found: # (a) Update the TabularFileInfo object # (b) Delete other groups TabularFileInfo object for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_tabular_info_ids) > 1: # pop the last TabularFileInfo id off the list of existing_tabular_info_ids shp_id = existing_tabular_info_ids.pop() # delete the rest if len(existing_sets) > 0: # delete older TabularFileInfo objects TabularFileInfo.objects.filter(id__in=existing_tabular_info_ids).delete() #------------------------------ # (5) Get or create a new TabularFileInfo object #------------------------------ msgt('(5) Get or create a new TabularFileInfo object') try: # Existing TabularFileInfo: # (1) Assume file is already saved # (2) update the data # tabular_info = TabularFileInfo.objects.get(**params_for_existing_check) for key, value in dataverse_info_dict.iteritems(): if key == 'column_names': tabular_info.add_column_names(value) else: setattr(tabular_info, key, value) # Save tabular_info.save() msg('tabular_info info saved') # If the file is still available, return it if tabular_info.is_dv_file_available(): add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5 else: # But the file isn't there!! Delete TabularFileInfo and make a new one tabular_info.delete() except TabularFileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') msg('new file') #------------------------------ # New tabular_info, create object and attach file #------------------------------ # Add name parameter dataverse_info_dict['name'] = dataverse_info_dict.get('datafile_label', '(no datafile_label found)') tabular_info = TabularFileInfo(**dataverse_info_dict) tabular_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dataverse_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) msg('datafile_download_url: %s' % datafile_download_url) datafile_filename = dataverse_info_dict.get('datafile_label', '') img_temp = NamedTemporaryFile(delete=True) try: img_temp.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: tabular_info.delete() # clear tabular info err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % (str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) img_temp.flush() tabular_info.dv_file.save(datafile_filename, File(img_temp)) tabular_info.save() add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5
def get_shapefile_from_dv_api_info(dv_session_token, dv_info_dict): """Using Dataverse API information, create a "ShapefileInfo" object. This function should only result in successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from ShapefileInfo False, ErrResultMsg To do: Make this into a separate class """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dv_info_dict) is dict, "dv_info_dict must be type 'dict'" #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ validation_form = DataverseInfoValidationForm(dv_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()] print(errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse( dv_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(FAILED_NOT_A_REGISTERED_DATAVERSE\ , "This dataverse url was not recognized: %s" % dv_info_dict['return_to_dataverse_url']\ ) #------------------------------------------------- # (3) Look for existing ShapefileInfo objects in the database # ShapefileInfo objects are routinely deleted, but if file is already here, use it # * todo: check for staleness, if the data is old delete it #------------------------------------------------- params_for_existing_check = dict(datafile_id=dv_info_dict.get('datafile_id', -1),\ dataverse_installation_name=dv_info_dict.get('dataverse_installation_name', -1),\ ) existing_sets = ShapefileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_shapefile_info_ids = list(existing_sets) msgt('existing_shapefile_info_ids: %s' % existing_shapefile_info_ids) #------------------------------------------------- # add dv_session_token and registered_dataverse to dv_info_dict #------------------------------------------------- dv_info_dict['dv_session_token'] = dv_session_token dv_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing ShapefileInfo(s) found: # (a) Update the ShapefileInfo object # (b) Delete other ShapefileInfo objects for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_shapefile_info_ids) > 1: # pop the last ShapefileInfo id off the list of existing_shapefile_info_ids shp_id = existing_shapefile_info_ids.pop() # delete the rest if len(existing_sets) > 0: ShapefileInfo.objects.filter( id__in=existing_shapefile_info_ids).delete( ) # delete older ShapefileInfo(s) #------------------------------ # (5) Get or create a new ShapefileInfo object #------------------------------ msgt('(5) Get or create a new ShapefileInfo object') try: # Existing ShapefileInfo: # (1) Assume file is already saved # (2) update the data # shapefile_info = ShapefileInfo.objects.get(**params_for_existing_check) for key, value in dv_info_dict.iteritems(): setattr(shapefile_info, key, value) # Save shapefile_info.save() msg('shapefile info saved') # If the file is still available, return it if shapefile_info.is_dv_file_available(): return True, shapefile_info.md5 else: # But the file isn't there!! Delete ShapefileInfo and make a new one shapefile_info.delete() except ShapefileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') msg('new file') #------------------------------ # New shapefile info, create object and attach file #------------------------------ shapefile_info = ShapefileInfo(**dv_info_dict) shapefile_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dv_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) msg('datafile_download_url: %s' % datafile_download_url) datafile_filename = dv_info_dict.get('datafile_label', '') tmp_shapefile = NamedTemporaryFile(delete=True) try: tmp_shapefile.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: shapefile_info.delete() # clear shapefile err_msg = 'Failed to download shapefile. HTTPError: %s \n\nurl: %s' % ( str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) tmp_shapefile.flush() shapefile_info.dv_file.save(datafile_filename, File(tmp_shapefile)) shapefile_info.save() return True, shapefile_info.md5
def handle(self, *args, **options): print (options) # For Retrieving the WorldMap Layer Info stored in Geoconnect # worldmap_info_md5 = options.get('worldmap_info_md5', None) if worldmap_info_md5 is None: error_note = "Please specify the md5 of the WorldMapLayerInfo object" msg(error_note) return # File type, to differntiate which WorldMap Layer Info is stored in Geoconnect # file_type = options.get('file_type', None) if file_type is None: error_note = "Please specify the file type. e.g. --type=tabular" msg(error_note) return delay_seconds = options.get('delay_seconds') try: delay_seconds = int(delay_seconds) except ValueError: error_note = "Please use an integer for 'delay_seconds'" msg(error_note) return except TypeError: error_note = "Please use an integer for 'delay_seconds'" msg(error_note) return num_attempts = options.get('num_attempts') try: num_attempts = int(num_attempts) except ValueError: error_note = "Please use an integer for 'num_attempts'" msg(error_note) return except TypeError: error_note = "Please use an integer for 'num_attempts'" msg(error_note) return for attempt_num in range(1, num_attempts+1): msg('(Attempt %s) Pausing for %s second(s)' % (attempt_num, delay_seconds)) time.sleep(delay_seconds) success, err_info = MetadataUpdater.run_metadata_update_with_thumbnail_check(\ worldmap_info_md5, file_type) msg('success: %s' % success) msg('err_info: %s' % err_info) if success is True: break
def handle(self, *args, **options): print(options) # For Retrieving the WorldMap Layer Info stored in Geoconnect # worldmap_info_md5 = options.get('worldmap_info_md5', None) if worldmap_info_md5 is None: error_note = "Please specify the md5 of the WorldMapLayerInfo object" msg(error_note) return # File type, to differntiate which WorldMap Layer Info is stored in Geoconnect # file_type = options.get('file_type', None) if file_type is None: error_note = "Please specify the file type. e.g. --type=tabular" msg(error_note) return delay_seconds = options.get('delay_seconds') try: delay_seconds = int(delay_seconds) except ValueError: error_note = "Please use an integer for 'delay_seconds'" msg(error_note) return except TypeError: error_note = "Please use an integer for 'delay_seconds'" msg(error_note) return num_attempts = options.get('num_attempts') try: num_attempts = int(num_attempts) except ValueError: error_note = "Please use an integer for 'num_attempts'" msg(error_note) return except TypeError: error_note = "Please use an integer for 'num_attempts'" msg(error_note) return for attempt_num in range(1, num_attempts + 1): msg('(Attempt %s) Pausing for %s second(s)' % (attempt_num, delay_seconds)) time.sleep(delay_seconds) success, err_info = MetadataUpdater.run_metadata_update_with_thumbnail_check(\ worldmap_info_md5, file_type) msg('success: %s' % success) msg('err_info: %s' % err_info) if success is True: break