class TestHerbieFacade(TestCase):
def setUp(self):
self.facade = HerbieFacade()
def test_all_species(self):
try:
species = self.facade.get_all_species()
# at least 10000 species
self.assertTrue(len(species) > 1000)
# check the properties
sp = species[0]
props = sp.keys()
# at least 10 of them
self.assertTrue(len(props) > 10)
# some important ones
self.assertTrue('species_name' in sp)
self.assertTrue('name_id' in sp)
except Exception as e:
self.fail("Should not raise an exception!: {}: '{}'".format(e.__class__, e))
def test_property_filter(self):
try:
species = self.facade.get_all_species([self.facade.PROPERTY_SPECIES_NAME, self.facade.PROPERTY_NAME_ID])
# at least 10000 species
self.assertTrue(len(species) > 1000)
# check the properties
sp = species[0]
props = sp.keys()
self.assertTrue(len(props) == 2)
# some important ones
self.assertTrue(self.facade.PROPERTY_SPECIES_NAME.herbie_name in sp)
self.assertTrue(self.facade.PROPERTY_NAME_ID.herbie_name in sp)
except Exception as e:
self.fail("Should not raise an exception!: {}: '{}'".format(e.__class__, e))
def setUp(self):
self.facade = HerbieFacade()
def form_valid(self, form):
pk = self.kwargs.get('pk')
dataset = get_object_or_404(Dataset, pk=pk)
error_url = reverse_lazy('admin:main_dataset_change', args=[pk])
src_file = DatasetFile(file=self.request.FILES['file'], dataset=dataset, uploaded_by=self.request.user)
src_file.save()
is_append = form.cleaned_data['append_mode']
create_site = form.cleaned_data['create_site']
schema = dataset.schema_class(dataset.schema_data)
record_model = dataset.record_model
# if species. First load species list from herbie. Should raise an exception if problem.
species_id_by_name = None
if dataset.type == Dataset.TYPE_SPECIES_OBSERVATION:
species_id_by_name = HerbieFacade().name_id_by_species_name()
with open(src_file.path) as csvfile:
reader = csv.DictReader(csvfile)
records = []
row_number = 1
warnings = []
errors = []
site_fk = schema.get_fk_for_model('Site')
if site_fk is None or site_fk.model_field is None:
warnings.append(
"The schema doesn't include a a related link to a Site model. The data won't be linked to a Site")
for row in reader:
row_number += 1
field_errors = schema.get_error_fields(row)
if len(field_errors) > 0:
for field_name, data in field_errors:
msg = "Row #{}, '{}': {}".format(row_number, field_name, data.get('error'))
errors.append(msg)
else:
# data valid
# find site
site = None
if site_fk:
model_field = site_fk.model_field
data = row.get(site_fk.data_field)
kwargs = {
"project": dataset.project,
model_field: data
}
site = Site.objects.filter(**kwargs).first()
if site is None:
if create_site:
try:
site = Site.objects.create(**kwargs)
except Exception as e:
errors.append("Error while creating the site '{}': {}".format(
data,
e
))
else:
msg = "Row #{}: could not find the site '{}':".format(row_number, data)
errors.append(msg)
record = record_model(
site=site,
dataset=dataset,
data=row,
)
# specific fields
try:
if dataset.type == Dataset.TYPE_OBSERVATION or dataset.type == Dataset.TYPE_SPECIES_OBSERVATION:
observation_date = schema.cast_record_observation_date(row)
# convert to datetime with timezone awareness
if isinstance(observation_date, datetime.date):
observation_date = datetime.datetime.combine(observation_date, datetime.time.min)
tz = dataset.project.timezone or timezone.get_current_timezone()
record.datetime = timezone.make_aware(observation_date, tz)
# geometry
geometry = schema.cast_geometry(row, default_srid=MODEL_SRID)
record.geometry = geometry
if dataset.type == Dataset.TYPE_SPECIES_OBSERVATION:
# species stuff. Lookup for species match in herbie
species_name = schema.cast_species_name(row)
name_id = int(species_id_by_name.get(species_name, -1))
record.species_name = species_name
record.name_id = name_id
except Exception as e:
msg = "> Row #{}: problem while extracting the Observation data: {}. [{}]".format(row_number, e,
row)
errors.append(msg)
records.append(record)
if not errors:
if not is_append:
record_model.objects.filter(dataset=dataset).delete()
record_model.objects.bulk_create(records)
if warnings:
msg = "{} records imported but with the following warnings: \n {}".format(
len(records),
'<br>'.join(warnings)
)
messages.warning(self.request, msg)
return HttpResponseRedirect(error_url)
else:
msg = "{} records successfully imported in dataset '{}'".format(
len(records), dataset
)
messages.success(self.request, msg)
else:
src_file.delete()
msg = '<br>'.join(errors)
messages.error(self.request, msg)
return HttpResponseRedirect(error_url)
return super(UploadDataSetView, self).form_valid(form)