def test_west_turkana_occurrence_save_simple(self):
"""
Test west_turkana_occurrence instance save method with the simplest possible attributes.
"""
starting_record_count = Occurrence.objects.count() # get current number of occurrence records
# The simplest occurrence instance we can create needs six bits of data.
# Using the instance creation and then save methods
new_occurrence = Occurrence(id=1, item_type="Faunal", basis_of_record="HumanObservation",
collecting_method="Surface Standard", field_number=datetime.now(),
geom="POINT (40.8352906016 11.5303732536)")
new_occurrence.save()
now = datetime.now()
self.assertEqual(Occurrence.objects.count(), starting_record_count+1) # test that one record has been added
self.assertEqual(new_occurrence.date_last_modified.day, now.day) # test date last modified is correct
self.assertEqual(new_occurrence.point_x(), 40.8352906016)
self.assertEqual(new_occurrence.point_y(), 11.5303732536)
def test_west_turkana_save_method_invalid_item_type(self):
"""
"""
starting_record_count = Occurrence.objects.count()
new_occurrence = Occurrence()
new_occurrence.item_type = "Fake"
new_occurrence.basis_of_record = "HumanObservation"
new_occurrence.collecting_method = "Surface Standard"
new_occurrence.field_number = datetime.now()
new_occurrence.geom = "POINT (40.8352906016 11.5303732536)"
new_occurrence.save()
now = datetime.now()
self.assertEqual(Occurrence.objects.count(), starting_record_count+1) # test that one record has been added
self.assertEqual(new_occurrence.date_last_modified.day, now.day) # test date last modified is correct
self.assertEqual(new_occurrence.point_x(), 40.8352906016)
self.assertEqual(new_occurrence.item_type, "Fake")
def import_placemarks(placemark_list):
feature_count = 0
for o in placemark_list:
# Check to make sure that the object is a Placemark, filter out folder objects
if type(o) is Placemark:
table = etree.fromstring(o.description)
attributes = table.xpath("//text()")
# TODO test attributes is even length
attributes_dict = dict(zip(attributes[0::2], attributes[1::2]))
mlp_occ = Occurrence()
###################
# REQUIRED FIELDS #
###################
# Validate Basis of Record
if attributes_dict.get("Basis Of Record") in ("Fossil", "FossilSpecimen", "Collection"):
mlp_occ.basis_of_record = "FossilSpecimen"
elif attributes_dict.get("Basis Of Record") in ("Observation", "HumanObservation"):
mlp_occ.basis_of_record = "HumanObservation"
# Validate Item Type
item_type = attributes_dict.get("Item Type")
if item_type in ("Artifact", "Artifactual", "Archeology", "Archaeological"):
mlp_occ.item_type = "Artifactual"
elif item_type in ("Faunal", "Fauna"):
mlp_occ.item_type = "Faunal"
elif item_type in ("Floral", "Flora"):
mlp_occ.item_type = "Floral"
elif item_type in ("Geological", "Geology"):
mlp_occ.item_type = "Geological"
# Field Number
try:
mlp_occ.field_number = datetime.strptime(attributes_dict.get("Time"), "%b %d, %Y, %I:%M %p") # parse field nubmer
mlp_occ.year_collected = mlp_occ.field_number.year # set the year collected form field number
except ValueError:
mlp_occ.field_number = datetime.now()
mlp_occ.problem = True
try:
error_string = "Upload error, missing field number, using current date and time instead."
mlp_occ.problem_comment = mlp_occ.problem_comment + " " +error_string
except TypeError:
mlp_occ.problem_comment = error_string
#utmPoint = utm.from_latlon(o.geometry.y, o.geometry.x)
# Process point, comes in as well known text string
# Assuming point is in GCS WGS84 datum = SRID 4326
pnt = GEOSGeometry("POINT (" + str(o.geometry.x) + " " + str(o.geometry.y) + ")", 4326) # WKT
mlp_occ.geom = pnt
#######################
# NON-REQUIRED FIELDS #
#######################
mlp_occ.barcode = attributes_dict.get("Barcode")
mlp_occ.item_number = mlp_occ.barcode
mlp_occ.catalog_number = "MLP-" + str(mlp_occ.item_number)
mlp_occ.remarks = attributes_dict.get("Remarks")
mlp_occ.item_scientific_name = attributes_dict.get("Scientific Name")
mlp_occ.item_description = attributes_dict.get("Description")
# Validate Collecting Method
collection_method = attributes_dict.get("Collection Method")
if collection_method in ("Surface Standard", "Standard"):
mlp_occ.collecting_method = "Surface Standard"
elif collection_method in ("Surface Intensive", "Intensive"):
mlp_occ.collecting_method = "Surface Intensive"
elif collection_method in ("Surface Complete", "Complete"):
mlp_occ.collecting_method = "Surface Complete"
elif collection_method in ("Exploratory Survey", "Exploratory"):
mlp_occ.collecting_method = "Exploratory Survey"
elif collection_method in ("Dry Screen 5mm", "Dry Screen 5 Mm", "Dry Screen 5 mm"):
mlp_occ.collecting_method = "Dry Screen 5mm"
elif collection_method in ("Dry Screen 2mm", "Dry Screen 2 Mm", "Dry Screen 2 mm"):
mlp_occ.collecting_method = "Dry Screen 2mm"
elif collection_method in ("Dry Screen 1mm", "Dry Screen 1 Mm", "Dry Screen 1 mm"):
mlp_occ.collecting_method = "Dry Screen 1mm"
# else:
# mlp_occ.collecting_method = None
# mlp_occ.problem = True
# mlp_occ.problem_comment = mlp_occ.problem_comment + " problem importing collecting method"
mlp_occ.collecting_method = attributes_dict.get("Collection Method")
mlp_occ.collector = attributes_dict.get("Collector")
mlp_occ.individual_count = attributes_dict.get("Count")
#if mlp_occ:
# mlp_occ.year_collected = mlp_occ.field_number.year
if attributes_dict.get("In Situ") in ('No', "NO", 'no'):
mlp_occ.in_situ = False
elif attributes_dict.get("In Situ") in ('Yes', "YES", 'yes'):
mlp_occ.in_situ = True
##############
# Save Image #
##############
image_file = ""
image_added = False
# Now look for images if this is a KMZ
if KML_file_extension.lower() == "kmz":
# grab image names from XML
image_tags = table.xpath("//img/@src")
# grab the name of the first image
try:
image_tag = image_tags[0]
# grab the file info from the zip list
for file_info in KMZ_file.filelist:
if image_tag == file_info.orig_filename:
# grab the image file itself
image_file = KMZ_file.extract(file_info, "media/uploads/images/mlp")
image_added = True
break
except IndexError:
pass
mlp_occ.save()
# need to save record before adding image in order to obtain the DB ID
if image_added:
# strip off the file name from the path
image_path = image_file[:image_file.rfind(os.sep)]
# construct new file name
new_file_name = image_path + os.sep + str(mlp_occ.id) + "_" + image_tag
# rename extracted file with DB record ID
os.rename(image_file, new_file_name)
# need to strip off "media" folder from relative path saved to DB
mlp_occ.image = new_file_name[new_file_name.find(os.sep)+1:]
mlp_occ.save()
feature_count += 1
elif type(o) is not Placemark:
raise IOError("KML File is badly formatted")
def import_placemarks(placemark_list):
feature_count = 0
for o in placemark_list:
# Check to make sure that the object is a Placemark, filter out folder objects
if type(o) is Placemark:
table = etree.fromstring(o.description)
attributes = table.xpath("//text()")
# TODO test attributes is even length
attributes_dict = dict(zip(attributes[0::2], attributes[1::2]))
mlp_occ = Occurrence()
###################
# REQUIRED FIELDS #
###################
# Validate Basis of Record
if attributes_dict.get("Basis Of Record") in ("Fossil", "FossilSpecimen", "Collection"):
mlp_occ.basis_of_record = "FossilSpecimen"
elif attributes_dict.get("Basis Of Record") in ("Observation", "HumanObservation"):
mlp_occ.basis_of_record = "HumanObservation"
# Validate Item Type
item_type = attributes_dict.get("Item Type")
if item_type in ("Artifact", "Artifactual", "Archeology", "Archaeological"):
mlp_occ.item_type = "Artifactual"
elif item_type in ("Faunal", "Fauna"):
mlp_occ.item_type = "Faunal"
elif item_type in ("Floral", "Flora"):
mlp_occ.item_type = "Floral"
elif item_type in ("Geological", "Geology"):
mlp_occ.item_type = "Geological"
# Field Number
try:
mlp_occ.field_number = datetime.strptime(attributes_dict.get("Time"), "%b %d, %Y, %I:%M %p") # parse field nubmer
mlp_occ.year_collected = mlp_occ.field_number.year # set the year collected form field number
except ValueError:
mlp_occ.field_number = datetime.now()
mlp_occ.problem = True
try:
error_string = "Upload error, missing field number, using current date and time instead."
mlp_occ.problem_comment = mlp_occ.problem_comment + " " +error_string
except TypeError:
mlp_occ.problem_comment = error_string
#utmPoint = utm.from_latlon(o.geometry.y, o.geometry.x)
# Process point, comes in as well known text string
# Assuming point is in GCS WGS84 datum = SRID 4326
pnt = GEOSGeometry("POINT (" + str(o.geometry.x) + " " + str(o.geometry.y) + ")", 4326) # WKT
mlp_occ.geom = pnt
#######################
# NON-REQUIRED FIELDS #
#######################
mlp_occ.barcode = attributes_dict.get("Barcode")
mlp_occ.item_number = mlp_occ.barcode
mlp_occ.catalog_number = "MLP-" + str(mlp_occ.item_number)
mlp_occ.remarks = attributes_dict.get("Remarks")
mlp_occ.item_scientific_name = attributes_dict.get("Scientific Name")
mlp_occ.item_description = attributes_dict.get("Description")
# Validate Collecting Method
collection_method = attributes_dict.get("Collection Method")
if collection_method in ("Surface Standard", "Standard"):
mlp_occ.collecting_method = "Surface Standard"
elif collection_method in ("Surface Intensive", "Intensive"):
mlp_occ.collecting_method = "Surface Intensive"
elif collection_method in ("Surface Complete", "Complete"):
mlp_occ.collecting_method = "Surface Complete"
elif collection_method in ("Exploratory Survey", "Exploratory"):
mlp_occ.collecting_method = "Exploratory Survey"
elif collection_method in ("Dry Screen 5mm", "Dry Screen 5 Mm", "Dry Screen 5 mm"):
mlp_occ.collecting_method = "Dry Screen 5mm"
elif collection_method in ("Dry Screen 2mm", "Dry Screen 2 Mm", "Dry Screen 2 mm"):
mlp_occ.collecting_method = "Dry Screen 2mm"
elif collection_method in ("Dry Screen 1mm", "Dry Screen 1 Mm", "Dry Screen 1 mm"):
mlp_occ.collecting_method = "Dry Screen 1mm"
# else:
# mlp_occ.collecting_method = None
# mlp_occ.problem = True
# mlp_occ.problem_comment = mlp_occ.problem_comment + " problem importing collecting method"
mlp_occ.collecting_method = attributes_dict.get("Collection Method")
mlp_occ.collector = attributes_dict.get("Collector")
mlp_occ.individual_count = attributes_dict.get("Count")
#if mlp_occ:
# mlp_occ.year_collected = mlp_occ.field_number.year
if attributes_dict.get("In Situ") in ('No', "NO", 'no'):
mlp_occ.in_situ = False
elif attributes_dict.get("In Situ") in ('Yes', "YES", 'yes'):
mlp_occ.in_situ = True
##############
# Save Image #
##############
image_file = ""
image_added = False
# Now look for images if this is a KMZ
if KML_file_extension.lower() == "kmz":
# grab image names from XML
image_tags = table.xpath("//img/@src")
# grab the name of the first image
try:
image_tag = image_tags[0]
# grab the file info from the zip list
for file_info in KMZ_file.filelist:
if image_tag == file_info.orig_filename:
# grab the image file itself
image_file = KMZ_file.extract(file_info, "media/uploads/images/mlp")
image_added = True
break
except IndexError:
pass
mlp_occ.save()
# need to save record before adding image in order to obtain the DB ID
if image_added:
# strip off the file name from the path
image_path = image_file[:image_file.rfind(os.sep)]
# construct new file name
new_file_name = image_path + os.sep + str(mlp_occ.id) + "_" + image_tag
# rename extracted file with DB record ID
os.rename(image_file, new_file_name)
# need to strip off "media" folder from relative path saved to DB
mlp_occ.image = new_file_name[new_file_name.find(os.sep)+1:]
mlp_occ.save()
feature_count += 1
elif type(o) is not Placemark:
raise IOError("KML File is badly formatted")
def create_data_csv(self, request, queryset):
response = HttpResponse(
content_type='text/csv') # declare the response type
response[
'Content-Disposition'] = 'attachment; filename="SF_data.csv"' # declare the file name
writer = unicodecsv.writer(response) # open a .csv writer
o = Occurrence() # create an empty instance of an occurrence object
b = Biology() # create an empty instance of a biology object
occurrence_field_list = o.__dict__.keys(
) # fetch the fields names from the instance dictionary
try: # try removing the state field from the list
occurrence_field_list.remove('_state') # remove the _state field
except ValueError: # raised if _state field is not in the dictionary list
pass
try: # try removing the geom field from the list
occurrence_field_list.remove('geom')
except ValueError: # raised if geom field is not in the dictionary list
pass
# Replace the geom field with two new fields
occurrence_field_list.append(
"point_x") # add new fields for coordinates of the geom object
occurrence_field_list.append("point_y")
biology_field_list = b.__dict__.keys() # get biology fields
try: # try removing the state field
biology_field_list.remove('_state')
except ValueError: # raised if _state field is not in the dictionary list
pass
#################################################################
# For now this method handles all occurrences and corresponding #
# data from the biology table for faunal occurrences. #
#################################################################
writer.writerow(occurrence_field_list +
biology_field_list) # write column headers
for occurrence in queryset: # iterate through the occurrence instances selected in the admin
# The next line uses string comprehension to build a list of values for each field
occurrence_dict = occurrence.__dict__
# Check that instance has geom
try:
occurrence_dict['point_x'] = occurrence.geom.get_x(
) # translate the occurrence geom object
occurrence_dict['point_y'] = occurrence.geom.get_y()
except AttributeError: # If no geom data exists write None to the dictionary
occurrence_dict['point_x'] = None
occurrence_dict['point_y'] = None
# Next we use the field list to fetch the values from the dictionary.
# Dictionaries do not have a reliable ordering. This code insures we get the values
# in the same order as the field list.
try: # Try writing values for all keys listed in both the occurrence and biology tables
writer.writerow([
occurrence.__dict__.get(k) for k in occurrence_field_list
] + [
occurrence.Biology.__dict__.get(k)
for k in biology_field_list
])
except ObjectDoesNotExist: # Django specific exception
writer.writerow([
occurrence.__dict__.get(k) for k in occurrence_field_list
])
except AttributeError: # Django specific exception
writer.writerow([
occurrence.__dict__.get(k) for k in occurrence_field_list
])
return response
def import_placemarks(placemark_list):
feature_count = 0
for o in placemark_list: # iterate through all the placemarks in the KML/KMZ file and process each one
# Check to make sure that the object is a Placemark, filter out folder objects
if type(o) is Placemark:
# save the placemark attribute data located in the description element as an element tree
table = etree.fromstring(o.description)
attributes = table.xpath("//text()")
# TODO test attributes is even length
attributes_dict = dict(zip(attributes[0::2], attributes[1::2]))
# Create a new, empty occurrence instance
omo_mursi_occ = Occurrence()
###################
# REQUIRED FIELDS #
###################
# Validate Basis of Record
if attributes_dict.get("Basis Of Record") in ("Fossil", "FossilSpecimen", "Collection"):
omo_mursi_occ.basis_of_record = "FossilSpecimen"
elif attributes_dict.get("Basis Of Record") in ("Observation", "HumanObservation"):
omo_mursi_occ.basis_of_record = "HumanObservation"
# Validate Item Type
item_type = attributes_dict.get("Item Type")
if item_type in ("Artifact", "Artifactual", "Archeology", "Archaeological"):
omo_mursi_occ.item_type = "Artifactual"
elif item_type in ("Faunal", "Fauna"):
omo_mursi_occ.item_type = "Faunal"
elif item_type in ("Floral", "Flora"):
omo_mursi_occ.item_type = "Floral"
elif item_type in ("Geological", "Geology"):
omo_mursi_occ.item_type = "Geological"
# Field Number and Year Collected
try:
# parse field number
omo_mursi_occ.field_number = datetime.strptime(attributes_dict.get("Time"),
"%b %d, %Y, %I:%M %p")
# set the year collected from field number
omo_mursi_occ.year_collected = omo_mursi_occ.field_number.year
except ValueError:
omo_mursi_occ.field_number = datetime.now()
omo_mursi_occ.problem = True
try:
error_string = "Upload error, missing field number, using current date and time instead."
omo_mursi_occ.problem_comment = omo_mursi_occ.problem_comment + " " +error_string
except TypeError:
omo_mursi_occ.problem_comment = error_string
# Process point, comes in as well known text string
# Assuming point is in GCS WGS84 datum = SRID 4326
pnt = GEOSGeometry("POINT (" + str(o.geometry.x) + " " + str(o.geometry.y) + ")", 4326) # WKT
omo_mursi_occ.geom = pnt
#######################
# NON-REQUIRED FIELDS #
#######################
omo_mursi_occ.barcode = attributes_dict.get("Barcode")
omo_mursi_occ.item_number = omo_mursi_occ.barcode
omo_mursi_occ.catalog_number = "MUR-" + str(omo_mursi_occ.item_number)
omo_mursi_occ.remarks = attributes_dict.get("Remarks")
omo_mursi_occ.item_scientific_name = attributes_dict.get("Scientific Name")
omo_mursi_occ.item_description = attributes_dict.get("Description")
# Validate Collecting Method
collection_method = attributes_dict.get("Collection Method")
if collection_method in ("Surface Standard", "Standard"):
omo_mursi_occ.collecting_method = "Surface Standard"
elif collection_method in ("Surface Intensive", "Intensive"):
omo_mursi_occ.collecting_method = "Surface Intensive"
elif collection_method in ("Surface Complete", "Complete"):
omo_mursi_occ.collecting_method = "Surface Complete"
elif collection_method in ("Exploratory Survey", "Exploratory"):
omo_mursi_occ.collecting_method = "Exploratory Survey"
elif collection_method in ("Dry Screen 5mm", "Dry Screen 5 Mm", "Dry Screen 5 mm"):
omo_mursi_occ.collecting_method = "Dry Screen 5mm"
elif collection_method in ("Dry Screen 2mm", "Dry Screen 2 Mm", "Dry Screen 2 mm"):
omo_mursi_occ.collecting_method = "Dry Screen 2mm"
elif collection_method in ("Dry Screen 1mm", "Dry Screen 1 Mm", "Dry Screen 1 mm"):
omo_mursi_occ.collecting_method = "Dry Screen 1mm"
omo_mursi_occ.collecting_method = attributes_dict.get("Collection Method")
omo_mursi_occ.collector = attributes_dict.get("Collector")
omo_mursi_occ.individual_count = attributes_dict.get("Count")
# In Situ
if attributes_dict.get("In Situ") in ('No', "NO", 'no'):
omo_mursi_occ.in_situ = False
elif attributes_dict.get("In Situ") in ('Yes', "YES", 'yes'):
omo_mursi_occ.in_situ = True
##############
# Save Image #
##############
omo_mursi_occ.save() # need to save record before adding image in order to obtain the DB ID
# Now look for images if this is a KMZ
if kml_file_extension.lower() == "kmz":
# grab image names from XML element tree
image_file_name_list = table.xpath("//img/@src")
# grab the name of the first image
try:
image_file_name = image_file_name_list[0]
# grab the file info from the zip list
for file_info in kmz_file.filelist:
if image_file_name == file_info.orig_filename:
# grab the image file itself
image = kmz_file.open(image_file_name) # get a handle on the image in the kmz file
image_stream = BytesIO(image.read()) # read the image without saving to disk
save_file_name = str(omo_mursi_occ.id) + "_" + image_file_name # rename
omo_mursi_occ.image.save(save_file_name, File(image_stream)) # save the image
break
except IndexError:
pass
feature_count += 1
elif type(o) is not Placemark:
raise IOError("KML File is badly formatted")
