def test_reference_document_not_exist(self):
message, source_reference = process_source_reference(
document_author='dimas',
reference='name',
document_link=self.document_link,
reference_category='published report'
)
self.assertTrue('does not exist' in message)
def test_reference_missing_author(self):
# Missing author
message, source_reference = process_source_reference(
reference='name',
document_link=self.document_link,
reference_category='published report'
)
self.assertTrue('missing author' in message.lower())
def test_reference_bibliography_not_created(self):
message, source_reference = process_source_reference(
document_author=self.owner.first_name,
reference=self.reference_title,
doi='1123',
reference_category='peer-reviewed'
)
self.assertIsNone(
source_reference
)
def test_reference_database_created(self):
message, source_reference = process_source_reference(
document_author=self.owner.first_name,
reference=self.reference_title,
source_year='2012',
reference_category='database'
)
self.assertTrue(
'2012' in source_reference.title
)
self.assertIsNotNone(
source_reference
)
def test_reference_bibliography_created(self):
doc = DocumentF.create(
id=11,
owner=self.owner,
title=self.reference_title
)
message, source_reference = process_source_reference(
document_link=self.document_link,
document_author=self.owner.first_name,
reference=self.reference_title,
doi='10.1016/j.aaen.2007.05.002',
reference_category='peer-reviewed'
)
self.assertIsNotNone(
source_reference
)
self.assertEqual(
source_reference.document,
doc
)
def test_reference_published_report_created(self):
# Published report source reference created
DocumentF.create(
id=11,
owner=self.owner,
title=self.reference_title
)
message, source_reference = process_source_reference(
document_author='dimas',
reference=self.reference_title,
document_link=self.document_link,
reference_category='published report'
)
self.assertIsNotNone(source_reference)
self.assertEqual(
source_reference.source_name,
self.reference_title
)
self.assertEqual(
source_reference.title,
self.reference_title
)
def process_row(self, row):
optional_data = {}
# -- Location site
location_site = self.location_site(row)
if not location_site:
return
# -- UUID
# If no uuid provided then it will be generated after collection record
# saved
uuid_value = ''
if self.row_value(row, UUID):
try:
uuid_value = uuid.UUID(self.row_value(row, UUID)[0:36]).hex
except ValueError:
self.error_file(error_row=row, error_message='Bad UUID format')
return
# -- Source reference
message, source_reference = process_source_reference(
reference=self.row_value(row, SOURCE),
reference_category=self.row_value(row, REFERENCE_CATEGORY),
doi=self.row_value(row, DOI),
document_title=self.row_value(row, DOCUMENT_TITLE),
document_link=self.row_value(row, DOCUMENT_UPLOAD_LINK),
document_url=self.row_value(row, DOCUMENT_URL),
document_author=self.row_value(row, DOCUMENT_AUTHOR),
source_year=self.row_value(row, SOURCE_YEAR))
if message and not source_reference:
# Source reference data from csv exists but not created
self.error_file(error_row=row, error_message=message)
return
else:
optional_data['source_reference'] = source_reference
# -- Sampling date
sampling_date = self.parse_date(row)
if not sampling_date:
return
# -- Processing Taxonomy
taxonomy = self.taxonomy(row)
if not taxonomy:
return
# -- Processing collectors
custodian = self.row_value(row, CUSTODIAN)
collectors = create_users_from_string(
self.row_value(row, COLLECTOR_OR_OWNER))
if not collectors:
self.error_file(error_row=row,
error_message='Missing collector/owner')
return
collector = collectors
optional_data['collector'] = self.row_value(row, COLLECTOR_OR_OWNER)
if len(collectors) > 0:
collector = collectors[0]
optional_data['collector_user'] = collectors[0]
optional_data['owner'] = collectors[0]
# Add owner and creator to location site
# if it doesnt exist yet
if not location_site.owner:
location_site.owner = collectors[0]
if not location_site.creator:
location_site.creator = collectors[0]
location_site.save()
if custodian:
for _collector in collectors:
_collector.organization = self.row_value(row, CUSTODIAN)
_collector.save()
# -- Get or create a survey
self.process_survey(
row,
location_site,
sampling_date,
collector=collectors[0],
)
# -- Optional data - Present
if PRESENT in row:
optional_data['present'] = bool(self.row_value(row, PRESENT))
# -- Process origin
category = None
if CATEGORY in row:
category = self.row_value(row, CATEGORY).lower()
if ORIGIN in row and self.row_value(row, ORIGIN):
origin = self.row_value(row, ORIGIN)
if ('translocated' in origin.lower()
or 'non-native' in origin.lower()):
category = 'alien'
elif 'native' == origin.lower():
category = 'native'
else:
category = None
if not category:
category = taxonomy.origin
optional_data['category'] = category
# -- Optional data - Habitat
if HABITAT in row and self.row_value(row, HABITAT):
habitat_choices = {
v: k
for k, v in BiologicalCollectionRecord.HABITAT_CHOICES
}
optional_data['collection_habitat'] = (
habitat_choices[self.row_value(row, HABITAT)])
# -- Optional data - Sampling method
sampling_method = None
if SAMPLING_METHOD in row and self.row_value(row, SAMPLING_METHOD):
if self.row_value(row, SAMPLING_METHOD).lower() != 'unspecified':
try:
sampling_method, sm_created = (
SamplingMethod.objects.get_or_create(
sampling_method=self.row_value(
row, SAMPLING_METHOD)))
except SamplingMethod.MultipleObjectsReturned:
sampling_method = (SamplingMethod.objects.filter(
sampling_method=self.row_value(row, SAMPLING_METHOD))
)[0]
if sampling_method:
optional_data['sampling_method'] = sampling_method
# -- Optional data - Sampling effort
sampling_effort = ''
if SAMPLING_EFFORT_VALUE in row and self.row_value(
row, SAMPLING_EFFORT_VALUE):
sampling_effort += self.row_value(row, SAMPLING_EFFORT_VALUE) + ' '
if self.row_value(row, SAMPLING_EFFORT):
sampling_effort += self.row_value(row, SAMPLING_EFFORT)
optional_data['sampling_effort'] = sampling_effort
# -- Optional data - Processing biotope
# Broad biotope
optional_data['biotope'] = self.biotope(row, BROAD_BIOTOPE,
BIOTOPE_TYPE_BROAD)
# -- Optional data - Specific biotope
optional_data['specific_biotope'] = self.biotope(
row, SPECIFIC_BIOTOPE, BIOTOPE_TYPE_SPECIFIC)
# -- Optional data - Substratum
optional_data['substratum'] = self.biotope(row, SUBSTRATUM,
BIOTOPE_TYPE_SUBSTRATUM)
# -- Optional data - Abundance
abundance_type = ''
abundance_number = None
if self.row_value(row, ABUNDANCE_MEASURE):
abundance_type = self.row_value(row, ABUNDANCE_MEASURE).lower()
if 'count' in abundance_type:
abundance_type = 'number'
elif 'density' in abundance_type:
abundance_type = 'density'
elif 'percentage' in abundance_type:
abundance_type = 'percentage'
if self.row_value(row, ABUNDANCE_VALUE):
try:
abundance_number = float(self.row_value(row, ABUNDANCE_VALUE))
except ValueError:
pass
if abundance_number:
optional_data['abundance_number'] = abundance_number
if abundance_type:
optional_data['abundance_type'] = abundance_type
# -- Processing chemical records
self.chemical_records(row, location_site, sampling_date)
record = None
fields = {
'site': location_site,
'original_species_name': self.row_value(row, SPECIES_NAME),
'collection_date': sampling_date,
'taxonomy': taxonomy,
'category': category,
'collector_user': collector
}
if uuid_value:
records = BiologicalCollectionRecord.objects.filter(
uuid=uuid_value)
if records.exists():
records.update(**fields)
record = records[0]
else:
fields['uuid'] = uuid_value
if not record:
try:
record, _ = (BiologicalCollectionRecord.objects.get_or_create(
**fields))
except Exception as e: # noqa
self.error_file(error_row=row, error_message=str(e))
return
if not uuid_value:
row[UUID] = record.uuid
# Update existing data
if self.survey:
record.survey = self.survey
if self.upload_session.module_group:
record.module_group = self.upload_session.module_group
for field in optional_data:
setattr(record, field, optional_data[field])
# -- Additional data
record.additional_data = json.dumps(row)
record.validated = True
record.save()
if not str(record.site.id) in self.site_ids:
self.site_ids.append(str(record.site.id))
self.success_file(success_row=row, data_id=record.id)