def test_west_turkana_occurrence_save_simple(self): """ Test west_turkana_occurrence instance save method with the simplest possible attributes. """ starting_record_count = Occurrence.objects.count() # get current number of occurrence records # The simplest occurrence instance we can create needs six bits of data. # Using the instance creation and then save methods new_occurrence = Occurrence(id=1, item_type="Faunal", basis_of_record="HumanObservation", collecting_method="Surface Standard", field_number=datetime.now(), geom="POINT (40.8352906016 11.5303732536)") new_occurrence.save() now = datetime.now() self.assertEqual(Occurrence.objects.count(), starting_record_count+1) # test that one record has been added self.assertEqual(new_occurrence.date_last_modified.day, now.day) # test date last modified is correct self.assertEqual(new_occurrence.point_x(), 40.8352906016) self.assertEqual(new_occurrence.point_y(), 11.5303732536)
def test_west_turkana_save_method_invalid_item_type(self): """ """ starting_record_count = Occurrence.objects.count() new_occurrence = Occurrence() new_occurrence.item_type = "Fake" new_occurrence.basis_of_record = "HumanObservation" new_occurrence.collecting_method = "Surface Standard" new_occurrence.field_number = datetime.now() new_occurrence.geom = "POINT (40.8352906016 11.5303732536)" new_occurrence.save() now = datetime.now() self.assertEqual(Occurrence.objects.count(), starting_record_count+1) # test that one record has been added self.assertEqual(new_occurrence.date_last_modified.day, now.day) # test date last modified is correct self.assertEqual(new_occurrence.point_x(), 40.8352906016) self.assertEqual(new_occurrence.item_type, "Fake")
def import_placemarks(placemark_list): feature_count = 0 for o in placemark_list: # Check to make sure that the object is a Placemark, filter out folder objects if type(o) is Placemark: table = etree.fromstring(o.description) attributes = table.xpath("//text()") # TODO test attributes is even length attributes_dict = dict(zip(attributes[0::2], attributes[1::2])) mlp_occ = Occurrence() ################### # REQUIRED FIELDS # ################### # Validate Basis of Record if attributes_dict.get("Basis Of Record") in ("Fossil", "FossilSpecimen", "Collection"): mlp_occ.basis_of_record = "FossilSpecimen" elif attributes_dict.get("Basis Of Record") in ("Observation", "HumanObservation"): mlp_occ.basis_of_record = "HumanObservation" # Validate Item Type item_type = attributes_dict.get("Item Type") if item_type in ("Artifact", "Artifactual", "Archeology", "Archaeological"): mlp_occ.item_type = "Artifactual" elif item_type in ("Faunal", "Fauna"): mlp_occ.item_type = "Faunal" elif item_type in ("Floral", "Flora"): mlp_occ.item_type = "Floral" elif item_type in ("Geological", "Geology"): mlp_occ.item_type = "Geological" # Field Number try: mlp_occ.field_number = datetime.strptime(attributes_dict.get("Time"), "%b %d, %Y, %I:%M %p") # parse field nubmer mlp_occ.year_collected = mlp_occ.field_number.year # set the year collected form field number except ValueError: mlp_occ.field_number = datetime.now() mlp_occ.problem = True try: error_string = "Upload error, missing field number, using current date and time instead." mlp_occ.problem_comment = mlp_occ.problem_comment + " " +error_string except TypeError: mlp_occ.problem_comment = error_string #utmPoint = utm.from_latlon(o.geometry.y, o.geometry.x) # Process point, comes in as well known text string # Assuming point is in GCS WGS84 datum = SRID 4326 pnt = GEOSGeometry("POINT (" + str(o.geometry.x) + " " + str(o.geometry.y) + ")", 4326) # WKT mlp_occ.geom = pnt ####################### # NON-REQUIRED FIELDS # ####################### mlp_occ.barcode = attributes_dict.get("Barcode") mlp_occ.item_number = mlp_occ.barcode mlp_occ.catalog_number = "MLP-" + str(mlp_occ.item_number) mlp_occ.remarks = attributes_dict.get("Remarks") mlp_occ.item_scientific_name = attributes_dict.get("Scientific Name") mlp_occ.item_description = attributes_dict.get("Description") # Validate Collecting Method collection_method = attributes_dict.get("Collection Method") if collection_method in ("Surface Standard", "Standard"): mlp_occ.collecting_method = "Surface Standard" elif collection_method in ("Surface Intensive", "Intensive"): mlp_occ.collecting_method = "Surface Intensive" elif collection_method in ("Surface Complete", "Complete"): mlp_occ.collecting_method = "Surface Complete" elif collection_method in ("Exploratory Survey", "Exploratory"): mlp_occ.collecting_method = "Exploratory Survey" elif collection_method in ("Dry Screen 5mm", "Dry Screen 5 Mm", "Dry Screen 5 mm"): mlp_occ.collecting_method = "Dry Screen 5mm" elif collection_method in ("Dry Screen 2mm", "Dry Screen 2 Mm", "Dry Screen 2 mm"): mlp_occ.collecting_method = "Dry Screen 2mm" elif collection_method in ("Dry Screen 1mm", "Dry Screen 1 Mm", "Dry Screen 1 mm"): mlp_occ.collecting_method = "Dry Screen 1mm" # else: # mlp_occ.collecting_method = None # mlp_occ.problem = True # mlp_occ.problem_comment = mlp_occ.problem_comment + " problem importing collecting method" mlp_occ.collecting_method = attributes_dict.get("Collection Method") mlp_occ.collector = attributes_dict.get("Collector") mlp_occ.individual_count = attributes_dict.get("Count") #if mlp_occ: # mlp_occ.year_collected = mlp_occ.field_number.year if attributes_dict.get("In Situ") in ('No', "NO", 'no'): mlp_occ.in_situ = False elif attributes_dict.get("In Situ") in ('Yes', "YES", 'yes'): mlp_occ.in_situ = True ############## # Save Image # ############## image_file = "" image_added = False # Now look for images if this is a KMZ if KML_file_extension.lower() == "kmz": # grab image names from XML image_tags = table.xpath("//img/@src") # grab the name of the first image try: image_tag = image_tags[0] # grab the file info from the zip list for file_info in KMZ_file.filelist: if image_tag == file_info.orig_filename: # grab the image file itself image_file = KMZ_file.extract(file_info, "media/uploads/images/mlp") image_added = True break except IndexError: pass mlp_occ.save() # need to save record before adding image in order to obtain the DB ID if image_added: # strip off the file name from the path image_path = image_file[:image_file.rfind(os.sep)] # construct new file name new_file_name = image_path + os.sep + str(mlp_occ.id) + "_" + image_tag # rename extracted file with DB record ID os.rename(image_file, new_file_name) # need to strip off "media" folder from relative path saved to DB mlp_occ.image = new_file_name[new_file_name.find(os.sep)+1:] mlp_occ.save() feature_count += 1 elif type(o) is not Placemark: raise IOError("KML File is badly formatted")
def create_data_csv(self, request, queryset): response = HttpResponse( content_type='text/csv') # declare the response type response[ 'Content-Disposition'] = 'attachment; filename="SF_data.csv"' # declare the file name writer = unicodecsv.writer(response) # open a .csv writer o = Occurrence() # create an empty instance of an occurrence object b = Biology() # create an empty instance of a biology object occurrence_field_list = o.__dict__.keys( ) # fetch the fields names from the instance dictionary try: # try removing the state field from the list occurrence_field_list.remove('_state') # remove the _state field except ValueError: # raised if _state field is not in the dictionary list pass try: # try removing the geom field from the list occurrence_field_list.remove('geom') except ValueError: # raised if geom field is not in the dictionary list pass # Replace the geom field with two new fields occurrence_field_list.append( "point_x") # add new fields for coordinates of the geom object occurrence_field_list.append("point_y") biology_field_list = b.__dict__.keys() # get biology fields try: # try removing the state field biology_field_list.remove('_state') except ValueError: # raised if _state field is not in the dictionary list pass ################################################################# # For now this method handles all occurrences and corresponding # # data from the biology table for faunal occurrences. # ################################################################# writer.writerow(occurrence_field_list + biology_field_list) # write column headers for occurrence in queryset: # iterate through the occurrence instances selected in the admin # The next line uses string comprehension to build a list of values for each field occurrence_dict = occurrence.__dict__ # Check that instance has geom try: occurrence_dict['point_x'] = occurrence.geom.get_x( ) # translate the occurrence geom object occurrence_dict['point_y'] = occurrence.geom.get_y() except AttributeError: # If no geom data exists write None to the dictionary occurrence_dict['point_x'] = None occurrence_dict['point_y'] = None # Next we use the field list to fetch the values from the dictionary. # Dictionaries do not have a reliable ordering. This code insures we get the values # in the same order as the field list. try: # Try writing values for all keys listed in both the occurrence and biology tables writer.writerow([ occurrence.__dict__.get(k) for k in occurrence_field_list ] + [ occurrence.Biology.__dict__.get(k) for k in biology_field_list ]) except ObjectDoesNotExist: # Django specific exception writer.writerow([ occurrence.__dict__.get(k) for k in occurrence_field_list ]) except AttributeError: # Django specific exception writer.writerow([ occurrence.__dict__.get(k) for k in occurrence_field_list ]) return response
def import_placemarks(placemark_list): feature_count = 0 for o in placemark_list: # iterate through all the placemarks in the KML/KMZ file and process each one # Check to make sure that the object is a Placemark, filter out folder objects if type(o) is Placemark: # save the placemark attribute data located in the description element as an element tree table = etree.fromstring(o.description) attributes = table.xpath("//text()") # TODO test attributes is even length attributes_dict = dict(zip(attributes[0::2], attributes[1::2])) # Create a new, empty occurrence instance omo_mursi_occ = Occurrence() ################### # REQUIRED FIELDS # ################### # Validate Basis of Record if attributes_dict.get("Basis Of Record") in ("Fossil", "FossilSpecimen", "Collection"): omo_mursi_occ.basis_of_record = "FossilSpecimen" elif attributes_dict.get("Basis Of Record") in ("Observation", "HumanObservation"): omo_mursi_occ.basis_of_record = "HumanObservation" # Validate Item Type item_type = attributes_dict.get("Item Type") if item_type in ("Artifact", "Artifactual", "Archeology", "Archaeological"): omo_mursi_occ.item_type = "Artifactual" elif item_type in ("Faunal", "Fauna"): omo_mursi_occ.item_type = "Faunal" elif item_type in ("Floral", "Flora"): omo_mursi_occ.item_type = "Floral" elif item_type in ("Geological", "Geology"): omo_mursi_occ.item_type = "Geological" # Field Number and Year Collected try: # parse field number omo_mursi_occ.field_number = datetime.strptime(attributes_dict.get("Time"), "%b %d, %Y, %I:%M %p") # set the year collected from field number omo_mursi_occ.year_collected = omo_mursi_occ.field_number.year except ValueError: omo_mursi_occ.field_number = datetime.now() omo_mursi_occ.problem = True try: error_string = "Upload error, missing field number, using current date and time instead." omo_mursi_occ.problem_comment = omo_mursi_occ.problem_comment + " " +error_string except TypeError: omo_mursi_occ.problem_comment = error_string # Process point, comes in as well known text string # Assuming point is in GCS WGS84 datum = SRID 4326 pnt = GEOSGeometry("POINT (" + str(o.geometry.x) + " " + str(o.geometry.y) + ")", 4326) # WKT omo_mursi_occ.geom = pnt ####################### # NON-REQUIRED FIELDS # ####################### omo_mursi_occ.barcode = attributes_dict.get("Barcode") omo_mursi_occ.item_number = omo_mursi_occ.barcode omo_mursi_occ.catalog_number = "MUR-" + str(omo_mursi_occ.item_number) omo_mursi_occ.remarks = attributes_dict.get("Remarks") omo_mursi_occ.item_scientific_name = attributes_dict.get("Scientific Name") omo_mursi_occ.item_description = attributes_dict.get("Description") # Validate Collecting Method collection_method = attributes_dict.get("Collection Method") if collection_method in ("Surface Standard", "Standard"): omo_mursi_occ.collecting_method = "Surface Standard" elif collection_method in ("Surface Intensive", "Intensive"): omo_mursi_occ.collecting_method = "Surface Intensive" elif collection_method in ("Surface Complete", "Complete"): omo_mursi_occ.collecting_method = "Surface Complete" elif collection_method in ("Exploratory Survey", "Exploratory"): omo_mursi_occ.collecting_method = "Exploratory Survey" elif collection_method in ("Dry Screen 5mm", "Dry Screen 5 Mm", "Dry Screen 5 mm"): omo_mursi_occ.collecting_method = "Dry Screen 5mm" elif collection_method in ("Dry Screen 2mm", "Dry Screen 2 Mm", "Dry Screen 2 mm"): omo_mursi_occ.collecting_method = "Dry Screen 2mm" elif collection_method in ("Dry Screen 1mm", "Dry Screen 1 Mm", "Dry Screen 1 mm"): omo_mursi_occ.collecting_method = "Dry Screen 1mm" omo_mursi_occ.collecting_method = attributes_dict.get("Collection Method") omo_mursi_occ.collector = attributes_dict.get("Collector") omo_mursi_occ.individual_count = attributes_dict.get("Count") # In Situ if attributes_dict.get("In Situ") in ('No', "NO", 'no'): omo_mursi_occ.in_situ = False elif attributes_dict.get("In Situ") in ('Yes', "YES", 'yes'): omo_mursi_occ.in_situ = True ############## # Save Image # ############## omo_mursi_occ.save() # need to save record before adding image in order to obtain the DB ID # Now look for images if this is a KMZ if kml_file_extension.lower() == "kmz": # grab image names from XML element tree image_file_name_list = table.xpath("//img/@src") # grab the name of the first image try: image_file_name = image_file_name_list[0] # grab the file info from the zip list for file_info in kmz_file.filelist: if image_file_name == file_info.orig_filename: # grab the image file itself image = kmz_file.open(image_file_name) # get a handle on the image in the kmz file image_stream = BytesIO(image.read()) # read the image without saving to disk save_file_name = str(omo_mursi_occ.id) + "_" + image_file_name # rename omo_mursi_occ.image.save(save_file_name, File(image_stream)) # save the image break except IndexError: pass feature_count += 1 elif type(o) is not Placemark: raise IOError("KML File is badly formatted")