def test_form_validation3(self): msgt( '(3) Test valid data with extra fields. Are the extra fields removed?' ) # retrieve data and add extra fields tdata = self.test_data.copy() tdata.update({ 'map_layer' : 'extra field 1'\ , 'comment' : 'extra field 2'\ , 'token_check' : 'extra field 3'\ }) validation_form = DataverseInfoValidationForm(tdata) msg('is valid: %s' % validation_form.is_valid()) err_msgs = validation_form.errors.values() msg('errs: %s' % err_msgs) self.assertEqual(validation_form.is_valid(), True) cleaned_data = validation_form.cleaned_data self.assertEqual(cleaned_data.has_key('map_layer'), False) self.assertEqual(cleaned_data.has_key('comment'), False) self.assertEqual(cleaned_data.has_key('token_check'), False) msg('Yes, strips out the extra fields?')
def test_form_validation1(self): msgt('(1) Test valid data') validation_form = DataverseInfoValidationForm(self.test_data) #print 'valid',validation_form.is_valid() self.assertEqual(validation_form.is_valid(), True)
def test_form_validation3(self): msgt('(3) Test valid data with extra fields. Are the extra fields removed?') # retrieve data and add extra fields tdata = self.test_data.copy() tdata.update({ 'map_layer' : 'extra field 1'\ , 'comment' : 'extra field 2'\ , 'token_check' : 'extra field 3'\ }) validation_form = DataverseInfoValidationForm(tdata) msg ('is valid: %s' % validation_form.is_valid()) err_msgs = validation_form.errors.values() msg('errs: %s' % err_msgs) self.assertEqual(validation_form.is_valid(), True) cleaned_data = validation_form.cleaned_data self.assertEqual(cleaned_data.has_key('map_layer'), False) self.assertEqual(cleaned_data.has_key('comment'), False) self.assertEqual(cleaned_data.has_key('token_check'), False) msg('Yes, strips out the extra fields?') #msg(cleaned_data) #dvinfo_obj = validation_form.save(commit=False) #self.assertEqual(type(dvinfo_obj), DataverseInfo) #msg('yes, converts into a DataverseInfo object (minus the map_layer)')
def test_form_validation1(self): msgt('(1) Test valid data') validation_form = DataverseInfoValidationForm(self.test_data) #print 'valid',validation_form.is_valid() self.assertEqual(validation_form.is_valid(), True)
def get_dataverse_info_dict(gis_data_file): """ Convert a GISDataFile or ShapefileInfo object into a dict containing only DataverseInfo attributes GISDataFile and ShapefileInfo should always pass the DataverseInfoValidationForm """ assert isinstance(gis_data_file, GISDataFile), "gis_data_file must be a GISDataFile object. (Note: A ShapefileInfo is also a GISDataFile object)" f = DataverseInfoValidationForm(gis_data_file.__dict__) if f.is_valid(): return f.cleaned_data raise Exception('Dataverse Info is not valid')
def get_dataverse_info_dict(gis_data_file): """ Convert a GISDataFile or ShapefileInfo object into a dict containing only DataverseInfo attributes GISDataFile and ShapefileInfo should always pass the DataverseInfoValidationForm """ assert isinstance( gis_data_file, GISDataFile ), "gis_data_file must be a GISDataFile object. (Note: A ShapefileInfo is also a GISDataFile object)" f = DataverseInfoValidationForm(gis_data_file.__dict__) if f.is_valid(): return f.cleaned_data raise Exception('Dataverse Info is not valid')
def test_form_validation2(self): msgt('(2) Test invalid data') tdata = self.test_data.copy() tdata['dataset_id'] = '11z' tdata['return_to_dataverse_url'] = 'ha' #msg(tdata) validation_form = DataverseInfoValidationForm(tdata) #msg('valid: %s' % validation_form.is_valid()) self.assertEqual(validation_form.is_valid(), False) msg('check for attributes in error') err_keys = validation_form.errors.keys() msg(err_keys) self.assertEqual('return_to_dataverse_url' in err_keys, True) self.assertEqual('dataset_id' in err_keys, True) msg('check for err messages') err_msgs = validation_form.errors.values() msg(err_msgs) self.assertEqual([u'Enter a valid URL.'] in err_msgs, True) self.assertEqual([u'Enter a whole number.'] in err_msgs, True)
def test_form_validation2(self): msgt('(2) Test invalid data') tdata = self.test_data.copy() tdata['dataset_id'] = '11z' tdata['return_to_dataverse_url'] = 'ha' #msg(tdata) validation_form = DataverseInfoValidationForm(tdata) #msg('valid: %s' % validation_form.is_valid()) self.assertEqual(validation_form.is_valid(), False) msg('check for attributes in error') err_keys = validation_form.errors.keys() msg(err_keys) self.assertEqual('return_to_dataverse_url' in err_keys, True) self.assertEqual('dataset_id' in err_keys, True) msg('check for err messages') err_msgs = validation_form.errors.values() msg(err_msgs) self.assertEqual([u'Enter a valid URL.'] in err_msgs, True) self.assertEqual([u'Enter a whole number.'] in err_msgs, True)
def run_test01_datafile_metadata(self): #----------------------------------------------------------- msgt("--- Retrieve metadata ---") #----------------------------------------------------------- api_url = '%s/api/worldmap/datafile/' % (self.dataverse_server) #----------------------------------------------------------- msgn("(1a) Try with no json params") #----------------------------------------------------------- msg('api_url: %s' % api_url) try: r = requests.post(api_url) except requests.exceptions.ConnectionError as e: msg('error: %s' % e.message) return msgx('Connection error: %s' % e.message) except: msg('error: %s' % sys.exc_info()[0]) #msgx("Unexpected error: %s" % sys.exc_info()[0]) return msg(r.status_code) self.assertEqual(r.status_code, 400, "Try with no json params") #----------------------------------------------------------- msgn("(1b) Try with empty string token") #----------------------------------------------------------- msg('api_url: %s' % api_url) try: r = requests.post(api_url, data=json.dumps({ self.wm_token_name: ''} )) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) self.assertEqual(r.status_code, 400, "Try without a token") #----------------------------------------------------------- msgn("(1c) Try a random token") #----------------------------------------------------------- msg('api_url: %s' % api_url) try: r = requests.post(api_url, data=json.dumps({ self.wm_token_name: self.get_random_token() } )) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) self.assertEqual(r.status_code, 401, "Try without a random token") #----------------------------------------------------------- msgn("(1d) Retrieve metadata") #----------------------------------------------------------- params = self.get_worldmap_token_dict() msg('api_url: %s' % api_url) msg('params: %s' % params) try: r = requests.post(api_url, data=json.dumps(params)) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) #----------------------------------------------------------- msgn("(1e) Check metadata") #----------------------------------------------------------- msg(r.text) self.assertEqual(r.status_code, 200, "API call successful, with a 200 response?") json_resp = r.json() self.assertEqual(json_resp.get('status'), 'OK', "status is 'OK'") metadata_json = json_resp.get('data', None) self.assertTrue(type(metadata_json) is not None, "Check that metadata_json is a dict") #----------------------------------------------------------- msgn("(1f) Check metadata with DataverseInfoValidationForm") #----------------------------------------------------------- # Metadata validation form (used directly by GeoConnect and WorldMap) # f = DataverseInfoValidationForm(metadata_json) msg('metadata valid? %s' % f.is_valid()) if not f.is_valid(): msg(f.errors) self.assertTrue(f.is_valid(), "Check Metadata in validation form. Errors:\n%s" % f.errors) self.assertTrue(metadata_json.has_key('datafile_download_url') is True, "Check that metadata_json has 'datafile_download_url'") self.assertTrue(metadata_json.has_key('datafile_filesize') is True, "Check that metadata_json has 'datafile_filesize'") #----------------------------------------------------------- msgt("(2) Retrieve file") #----------------------------------------------------------- #msgn("(2a) Try without token--should be unauthorized") msgn("(2a) Try without token--should be ok b/c dataset is published") #----------------------------------------------------------- download_api_url = metadata_json['datafile_download_url'] msg('download_api_url: %s' % download_api_url) try: r = requests.get(download_api_url) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) #self.assertEqual(r.status_code, 401, "API call should be forbidden--no token") self.assertEqual(r.status_code, 200, "API call should be ok b/c dataset is published") """ #----------------------------------------------------------- msgn("(2b) Try with bad token, not WorldMap token length--should be forbidden.") #----------------------------------------------------------- random_non_worldmap_token = self.get_random_token(36) download_api_url = '%s?key=%s' % (metadata_json['datafile_download_url'], random_non_worldmap_token) msg('download_api_url: %s' % download_api_url) try: r = requests.get(download_api_url) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) self.assertEqual(r.status_code, 403, "API call should be forbidden--bad token") #----------------------------------------------------------- msgn("(2c) Try with bad token, WorldMap token length, but random") #----------------------------------------------------------- random_worldmap_token = self.get_random_token() download_api_url = '%s?key=%s' % (metadata_json['datafile_download_url'], random_worldmap_token) msg('download_api_url: %s' % download_api_url) try: r = requests.get(download_api_url) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) self.assertEqual(r.status_code, 403, "API call should be forbidden--no token") """ #----------------------------------------------------------- msgn("(2d) Legit request with real token (takes a couple of seconds to get file)") #----------------------------------------------------------- download_api_url = '%s?key=%s' % (metadata_json['datafile_download_url'], GEOCONNECT_TOKEN_VALUE) msg('download_api_url: %s' % download_api_url) try: r = requests.get(download_api_url) except requests.exceptions.ConnectionError as e: msgx('Connection error: %s' % e.message) except: msgx("Unexpected error: %s" % sys.exc_info()[0]) msg(r.status_code) msg('downloaded file size: %s' % len(r.text)) msg('expected file size: %s' % metadata_json['datafile_filesize']) self.assertEqual(r.status_code, 200, "API call successful to file download") self.assertEqual(metadata_json['datafile_filesize'], len(r.text), "Actual file size matches size in metadata")
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict): """Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object. This function should only return successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from TabularFileInfo False, ErrResultMsg """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dataverse_info_dict) is dict,\ "dataverse_info_dict must be type 'dict'" msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict)) #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ #dataverse_info_dict.update({'datafile_id':None}) # for testing validation_form = DataverseInfoValidationForm(dataverse_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()] LOGGER.debug('errors: %s', errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse( dataverse_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(\ FAILED_NOT_A_REGISTERED_DATAVERSE, "This dataverse url was not recognized: %s" %\ dataverse_info_dict['return_to_dataverse_url']) #------------------------------------------------- # (3b) Look for existing Dataverse files in the database # ShapefileInfo and TabularFileInfo objects are routinely # deleted, but if file is already here, use it #------------------------------------------------- params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\ , dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\ ) existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_tabular_info_ids = list(existing_sets) msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids) #------------------------------------------------- # add dv_session_token and registered_dataverse to dataverse_info_dict #------------------------------------------------- dataverse_info_dict['dv_session_token'] = dv_session_token dataverse_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing TabularFileInfo(s) found: # (a) Update the TabularFileInfo object # (b) Delete other groups TabularFileInfo object for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_tabular_info_ids) > 1: # pop the last TabularFileInfo id off the list of existing_tabular_info_ids shp_id = existing_tabular_info_ids.pop() # delete the rest if len(existing_sets) > 0: # delete older TabularFileInfo objects TabularFileInfo.objects.filter( id__in=existing_tabular_info_ids).delete() #------------------------------ # (5) Get or create a new TabularFileInfo object #------------------------------ msgt('(5) Get or create a new TabularFileInfo object') try: # Existing TabularFileInfo: # (1) Assume file is already saved # (2) update the data # tabular_info = TabularFileInfo.objects.get(**params_for_existing_check) for key, value in dataverse_info_dict.iteritems(): if key == 'column_names': tabular_info.add_column_names(value) else: setattr(tabular_info, key, value) # Save tabular_info.save() msg('tabular_info info saved') # If the file is still available, return it if tabular_info.is_dv_file_available(): add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5 else: # But the file isn't there!! Delete TabularFileInfo and make a new one tabular_info.delete() except TabularFileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') msg('new file') #------------------------------ # New tabular_info, create object and attach file #------------------------------ # Add name parameter dataverse_info_dict['name'] = dataverse_info_dict.get( 'datafile_label', '(no datafile_label found)') tabular_info = TabularFileInfo(**dataverse_info_dict) tabular_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dataverse_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) msg('datafile_download_url: %s' % datafile_download_url) datafile_filename = dataverse_info_dict.get('datafile_label', '') img_temp = NamedTemporaryFile(delete=True) try: img_temp.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: tabular_info.delete() # clear tabular info err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % ( str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) img_temp.flush() tabular_info.dv_file.save(datafile_filename, File(img_temp)) tabular_info.save() add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5
def get_shapefile_from_dv_api_info(dv_session_token, dv_info_dict): """Using Dataverse API information, create a "ShapefileInfo" object. This function should only result in successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from ShapefileInfo False, ErrResultMsg To do: Make this into a separate class """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dv_info_dict) is dict, "dv_info_dict must be type 'dict'" #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ validation_form = DataverseInfoValidationForm(dv_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k, v in validation_form.errors.items()] LOGGER.debug(errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse( dv_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(FAILED_NOT_A_REGISTERED_DATAVERSE\ , "This dataverse url was not recognized: %s" % dv_info_dict['return_to_dataverse_url']\ ) #------------------------------------------------- # (3) Look for existing ShapefileInfo objects in the database # ShapefileInfo objects are routinely deleted, but if file is already here, use it # * todo: check for staleness, if the data is old delete it #------------------------------------------------- LOGGER.debug('(3) Look for existing ShapefileInfo objects in the database') params_for_existing_check = dict(datafile_id=dv_info_dict.get('datafile_id', -1),\ dataverse_installation_name=dv_info_dict.get('dataverse_installation_name', -1),\ ) existing_sets = ShapefileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_shapefile_info_ids = list(existing_sets) #------------------------------------------------- # add dv_session_token and registered_dataverse to dv_info_dict #------------------------------------------------- dv_info_dict['dv_session_token'] = dv_session_token dv_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing ShapefileInfo(s) found: # (a) Update the ShapefileInfo object # (b) Delete other ShapefileInfo objects for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_shapefile_info_ids) > 1: # pop the last ShapefileInfo id off the list of existing_shapefile_info_ids shp_id = existing_shapefile_info_ids.pop() # delete the rest if len(existing_sets) > 0: ShapefileInfo.objects.filter( id__in=existing_shapefile_info_ids).delete( ) # delete older ShapefileInfo(s) #------------------------------ # (5) Get or create a new ShapefileInfo object #------------------------------ LOGGER.debug('(5) Get or create a new ShapefileInfo object') try: # Existing ShapefileInfo: # (1) Assume file is already saved # (2) update the data # shapefile_info = ShapefileInfo.objects.get(**params_for_existing_check) for key, value in dv_info_dict.iteritems(): setattr(shapefile_info, key, value) # Save shapefile_info.save() LOGGER.debug('shapefile info updated') LOGGER.debug('shapefile_info: %s' % shapefile_info.id) # If the file is still available, return it if shapefile_info.is_dv_file_available(): return True, shapefile_info.md5 else: # But the file isn't there!! Delete ShapefileInfo and make a new one shapefile_info.delete() except ShapefileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') LOGGER.debug('Create a new file') #------------------------------ # New shapefile info, create object and attach file #------------------------------ shapefile_info = ShapefileInfo(**dv_info_dict) shapefile_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dv_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) LOGGER.debug('datafile_download_url: %s' % datafile_download_url) datafile_filename = dv_info_dict.get('datafile_label', '') tmp_shapefile = NamedTemporaryFile(delete=True) try: tmp_shapefile.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: shapefile_info.delete() # clear shapefile err_msg = 'Failed to download shapefile. HTTPError: %s \n\nurl: %s' % ( str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) tmp_shapefile.flush() shapefile_info.dv_file.save(datafile_filename, File(tmp_shapefile)) shapefile_info.save() return True, shapefile_info.md5
def get_tabular_file_from_dv_api_info(dv_session_token, dataverse_info_dict): """Using Dataverse API information, create a :model:`gis_tabular.TabularFileInfo' object. This function should only return successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from TabularFileInfo False, ErrResultMsg """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dataverse_info_dict) is dict,\ "dataverse_info_dict must be type 'dict'" msgt('dataverse_info_dict: {0}'.format(dataverse_info_dict)) #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ #dataverse_info_dict.update({'datafile_id':None}) # for testing validation_form = DataverseInfoValidationForm(dataverse_info_dict) if not validation_form.is_valid(): errs = ['%s: %s' % (k, v) for k,v in validation_form.errors.items()] LOGGER.debug('errors: %s', errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse(dataverse_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(\ FAILED_NOT_A_REGISTERED_DATAVERSE, "This dataverse url was not recognized: %s" %\ dataverse_info_dict['return_to_dataverse_url']) #------------------------------------------------- # (3b) Look for existing Dataverse files in the database # ShapefileInfo and TabularFileInfo objects are routinely # deleted, but if file is already here, use it #------------------------------------------------- params_for_existing_check = dict(datafile_id=dataverse_info_dict.get('datafile_id', -1)\ , dataverse_installation_name=dataverse_info_dict.get('dataverse_installation_name', -1)\ ) existing_sets = TabularFileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_tabular_info_ids = list(existing_sets) msgt('existing_tabular_info_ids: %s' % existing_tabular_info_ids) #------------------------------------------------- # add dv_session_token and registered_dataverse to dataverse_info_dict #------------------------------------------------- dataverse_info_dict['dv_session_token'] = dv_session_token dataverse_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing TabularFileInfo(s) found: # (a) Update the TabularFileInfo object # (b) Delete other groups TabularFileInfo object for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_tabular_info_ids) > 1: # pop the last TabularFileInfo id off the list of existing_tabular_info_ids shp_id = existing_tabular_info_ids.pop() # delete the rest if len(existing_sets) > 0: # delete older TabularFileInfo objects TabularFileInfo.objects.filter(id__in=existing_tabular_info_ids).delete() #------------------------------ # (5) Get or create a new TabularFileInfo object #------------------------------ msgt('(5) Get or create a new TabularFileInfo object') try: # Existing TabularFileInfo: # (1) Assume file is already saved # (2) update the data # tabular_info = TabularFileInfo.objects.get(**params_for_existing_check) for key, value in dataverse_info_dict.iteritems(): if key == 'column_names': tabular_info.add_column_names(value) else: setattr(tabular_info, key, value) # Save tabular_info.save() msg('tabular_info info saved') # If the file is still available, return it if tabular_info.is_dv_file_available(): add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5 else: # But the file isn't there!! Delete TabularFileInfo and make a new one tabular_info.delete() except TabularFileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') msg('new file') #------------------------------ # New tabular_info, create object and attach file #------------------------------ # Add name parameter dataverse_info_dict['name'] = dataverse_info_dict.get('datafile_label', '(no datafile_label found)') tabular_info = TabularFileInfo(**dataverse_info_dict) tabular_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dataverse_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) msg('datafile_download_url: %s' % datafile_download_url) datafile_filename = dataverse_info_dict.get('datafile_label', '') img_temp = NamedTemporaryFile(delete=True) try: img_temp.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: tabular_info.delete() # clear tabular info err_msg = 'Failed to download tabular file. HTTPError: %s \n\nurl: %s' % (str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) img_temp.flush() tabular_info.dv_file.save(datafile_filename, File(img_temp)) tabular_info.save() add_worldmap_layerinfo_if_exists(tabular_info) return True, tabular_info.md5
def get_shapefile_from_dv_api_info(dv_session_token, dv_info_dict): """Using Dataverse API information, create a "ShapefileInfo" object. This function should only result in successful responses. return True/False, shp_md5 or ErrResultMsg Examples: True, md5 from ShapefileInfo False, ErrResultMsg To do: Make this into a separate class """ assert dv_session_token is not None, "dv_session_token cannot be None" assert type(dv_info_dict) is dict, "dv_info_dict must be type 'dict'" #------------------------------ # (1) Validate the data (DataverseInfoValidationForm) #------------------------------ validation_form = DataverseInfoValidationForm(dv_info_dict) if not validation_form.is_valid(): errs = [ '%s: %s' % (k, v) for k,v in validation_form.errors.items()] LOGGER.debug(errs) form_errs = '\n'.join(errs) return False, ErrResultMsg(None, form_errs) #------------------------------------------------- # (2) Check if this is a Registered Dataverse #------------------------------------------------- registered_dataverse = find_registered_dataverse(dv_info_dict['return_to_dataverse_url']) if registered_dataverse is None: return False, ErrResultMsg(FAILED_NOT_A_REGISTERED_DATAVERSE\ , "This dataverse url was not recognized: %s" % dv_info_dict['return_to_dataverse_url']\ ) #------------------------------------------------- # (3) Look for existing ShapefileInfo objects in the database # ShapefileInfo objects are routinely deleted, but if file is already here, use it # * todo: check for staleness, if the data is old delete it #------------------------------------------------- LOGGER.debug('(3) Look for existing ShapefileInfo objects in the database') params_for_existing_check = dict(datafile_id=dv_info_dict.get('datafile_id', -1),\ dataverse_installation_name=dv_info_dict.get('dataverse_installation_name', -1),\ ) existing_sets = ShapefileInfo.objects.filter(**params_for_existing_check\ ).values_list('id', flat=True\ ).order_by('created') existing_shapefile_info_ids = list(existing_sets) #------------------------------------------------- # add dv_session_token and registered_dataverse to dv_info_dict #------------------------------------------------- dv_info_dict['dv_session_token'] = dv_session_token dv_info_dict['registered_dataverse'] = registered_dataverse #------------------------------ # (4) Existing ShapefileInfo(s) found: # (a) Update the ShapefileInfo object # (b) Delete other ShapefileInfo objects for this datafile and user # (c) Return the md5 #------------------------------ if len(existing_shapefile_info_ids) > 1: # pop the last ShapefileInfo id off the list of existing_shapefile_info_ids shp_id = existing_shapefile_info_ids.pop() # delete the rest if len(existing_sets) > 0: ShapefileInfo.objects.filter(id__in=existing_shapefile_info_ids).delete() # delete older ShapefileInfo(s) #------------------------------ # (5) Get or create a new ShapefileInfo object #------------------------------ LOGGER.debug('(5) Get or create a new ShapefileInfo object') try: # Existing ShapefileInfo: # (1) Assume file is already saved # (2) update the data # shapefile_info = ShapefileInfo.objects.get(**params_for_existing_check) for key, value in dv_info_dict.iteritems(): setattr(shapefile_info, key, value) # Save shapefile_info.save() LOGGER.debug('shapefile info updated') LOGGER.debug('shapefile_info: %s' % shapefile_info.id) # If the file is still available, return it if shapefile_info.is_dv_file_available(): return True, shapefile_info.md5 else: # But the file isn't there!! Delete ShapefileInfo and make a new one shapefile_info.delete() except ShapefileInfo.DoesNotExist: pass #except: # msg('Failed to retrieve an existing ShapefileInfo object -- so create a new one') # #return False, ErrResultMsg(None, 'Failed to retrieve an existing ShapefileInfo object') LOGGER.debug('Create a new file') #------------------------------ # New shapefile info, create object and attach file #------------------------------ shapefile_info = ShapefileInfo(**dv_info_dict) shapefile_info.save() #------------------------------ # Download and attach file #------------------------------ datafile_download_url = dv_info_dict.get('datafile_download_url', '') # Add session token. Gives permission to download/retrieve the file # - http://localhost:8080/api/access/datafile/FILEID?key=YOURAPIKEY # datafile_download_url = '%s?key=%s' % (datafile_download_url, dv_session_token) LOGGER.debug('datafile_download_url: %s' % datafile_download_url) datafile_filename = dv_info_dict.get('datafile_label', '') tmp_shapefile = NamedTemporaryFile(delete=True) try: tmp_shapefile.write(urllib2.urlopen(datafile_download_url).read()) except urllib2.HTTPError as e: shapefile_info.delete() # clear shapefile err_msg = 'Failed to download shapefile. HTTPError: %s \n\nurl: %s' % (str(e), datafile_download_url) return False, ErrResultMsg(None, err_msg) tmp_shapefile.flush() shapefile_info.dv_file.save(datafile_filename, File(tmp_shapefile)) shapefile_info.save() return True, shapefile_info.md5