def handle(self, *args, **options):
display_order = 1
excel_file_name = 'SASS5_Data_sheet.xls'
sheet_title = 'ModifiedFieldSheet'
sass_sheet_file = os.path.join(settings.MEDIA_ROOT, excel_file_name)
if not os.path.isfile(sass_sheet_file):
print('%s not found in media directory' % sass_sheet_file)
return False
xl = pd.ExcelFile(sass_sheet_file)
df1 = xl.parse(sheet_title)
taxon_data = False
taxon_dict = {}
for index, row in df1.iterrows():
if taxon_data:
if row.isnull().all():
taxon_data = False
continue
row_index = 0
for column_data in row:
if isinstance(column_data, basestring):
if (not column_data.isspace()
and column_data not in self.not_taxon_string):
if row_index not in taxon_dict:
taxon_dict[row_index] = []
taxon_dict[row_index].append(
{column_data: row[row_index + 1]})
row_index += 1
if row[0] == 'Taxon':
taxon_data = True
for taxon_list in taxon_dict:
for taxon_data in taxon_dict[taxon_list]:
for taxon_complete_name, taxon_score in taxon_data.iteritems():
taxon_name = taxon_complete_name.split('*')[0]
taxon_name = taxon_name.split(' ')[0]
taxonomy = search_taxon_identifier(taxon_name)
sass_taxon, created = SassTaxon.objects.get_or_create(
taxon=taxonomy,
taxon_sass_5=taxon_complete_name,
display_order_sass_5=display_order)
if not math.isnan(taxon_score):
sass_taxon.sass_5_score = int(taxon_score)
sass_taxon.save()
display_order += 1
print('Added new taxon : %s' % taxonomy.scientific_name)
def test_search_taxon_identifier(self):
species_name = 'Enteromius anoplus (Weber, 1897)'
species = search_taxon_identifier(species_name, False) # noqa
def process_row(self, row, index):
taxon = self.get_object_from_uuid(column='TaxonID', model=SassTaxon)
if self.only_missing and taxon:
print('{} already exist'.format(taxon))
return
taxon_name = self.get_row_value('TaxonName')
taxon_name = taxon_name.split(' ')[0]
taxon_name = taxon_name.split('(')[0]
taxon_name = taxon_name.split('/')[0]
group = self.get_object_from_uuid('GroupID', TaxonGroup)
# Check existing taxonomy
taxa = Taxonomy.objects.filter(
Q(canonical_name__icontains=taxon_name)
| Q(legacy_canonical_name__icontains=taxon_name))
if taxa.exists():
taxonomy = taxa[0]
else:
taxonomy = search_taxon_identifier(taxon_name)
if not taxonomy:
# Create one
taxonomy = Taxonomy.objects.create(
scientific_name=taxon_name,
legacy_canonical_name=taxon_name,
canonical_name=taxon_name,
rank='FAMILY')
air_breather_value = self.get_row_value('AirBreather',
return_none_if_empty=True)
if not air_breather_value:
air_breather_value = 0
taxon_sass, created = SassTaxon.objects.get_or_create(
taxon=taxonomy,
taxon_sass_4=self.get_row_value('TaxonSASS4'),
score=self.get_row_value('Score', return_none_if_empty=True),
sass_5_score=self.get_row_value('SASS5Score',
return_none_if_empty=True),
air_breather=air_breather_value,
display_order_sass_4=self.get_row_value('DisplayOrderSASS4',
return_none_if_empty=True),
display_order_sass_5=self.get_row_value('DisplayOrderSASS5',
return_none_if_empty=True))
if created:
self.new_data.append(taxon_sass.id)
biobase_id = self.get_row_value('BioBaseID')
if biobase_id:
biobase_id = str(biobase_id)
lifestage = self.get_row_value('Lifestage')
if lifestage:
lifestage = str(lifestage)
taxon_sass.additional_data = {
'TaxonK': self.get_row_value('TaxonK'),
'TaxonOrder': self.get_row_value('TaxonOrder'),
'Lifestage': lifestage,
'BioBaseID': biobase_id,
}
taxon_sass.save()
if group:
group.taxonomies.add(taxonomy)
group.save()
self.save_uuid(uuid=self.get_row_value('TaxonID'),
object_id=taxon_sass.id)
def process_row(self, row, index):
taxon_name = self.get_row_value('TaxonName')
taxon_name = taxon_name.split(' ')[0]
taxon_name = taxon_name.split('(')[0]
taxon_name = taxon_name.split('/')[0]
group = self.get_object_from_uuid(
'GroupID',
TaxonGroup
)
taxonomy = search_taxon_identifier(taxon_name)
if not taxonomy:
return
air_breather_value = self.get_row_value(
'AirBreather',
return_none_if_empty=True)
if not air_breather_value:
air_breather_value = 0
taxon_sass, created = SassTaxon.objects.get_or_create(
taxon=taxonomy,
taxon_sass_4=self.get_row_value('TaxonSASS4'),
score=self.get_row_value(
'Score',
return_none_if_empty=True
),
sass_5_score=self.get_row_value(
'SASS5Score',
return_none_if_empty=True
),
air_breather=air_breather_value,
display_order_sass_4=self.get_row_value(
'DisplayOrderSASS4',
return_none_if_empty=True),
display_order_sass_5=self.get_row_value(
'DisplayOrderSASS5',
return_none_if_empty=True
)
)
biobase_id = self.get_row_value('BioBaseID')
if biobase_id:
biobase_id = str(biobase_id)
lifestage = self.get_row_value('Lifestage')
if lifestage:
lifestage = str(lifestage)
taxon_sass.additional_data = {
'TaxonK': self.get_row_value('TaxonK'),
'TaxonOrder': self.get_row_value('TaxonOrder'),
'Lifestage': lifestage,
'BioBaseID': biobase_id,
}
taxon_sass.save()
if group:
group.taxonomies.add(taxonomy)
group.save()
self.save_uuid(
uuid=self.get_row_value('TaxonID'),
object_id=taxon_sass.id
)
def process_row(self, row, index):
current_species_rank = 0
original_species_name = self.get_row_value('OriginalSpeciesName')
abundance = self.get_row_value('Abundance')
if abundance:
abundance = float(abundance)
else:
abundance = None
present = self.get_row_value('Present')
biotope = self.get_object_from_uuid(column='BioBiotopeID',
model=Biotope)
site = self.get_object_from_uuid(column='BioSiteID',
model=LocationSite)
source_reference = self.get_object_from_uuid(
column='BioReferenceID', model=SourceReferenceBibliography)
month = self.get_row_value('Month')
if month == 13:
month = 1
year = self.get_row_value('Year')
date_string = '{year}-{month}-{day}'.format(day=1,
month=month,
year=year)
# Get taxonomy detail
taxonomy_unspecified = False
species = self.get_row_value(self.species_ranks[current_species_rank])
while (species.lower() == 'unspecified'
and current_species_rank < len(self.species_ranks)
and not taxonomy_unspecified):
current_species_rank += 1
try:
species = self.get_row_value(
self.species_ranks[current_species_rank])
except IndexError:
taxonomy_unspecified = True
if not taxonomy_unspecified:
taxonomy = search_taxon_identifier(species)
while (not taxonomy
and current_species_rank < len(self.species_ranks)
and not taxonomy_unspecified):
current_species_rank += 1
try:
species = self.get_row_value(
self.species_ranks[current_species_rank])
taxonomy = search_taxon_identifier(species)
except IndexError:
taxonomy = None
taxonomy_unspecified = True
else:
taxonomy = None
collection, created = BiologicalCollectionRecord.objects.get_or_create(
original_species_name=original_species_name,
site=site,
biotope=biotope,
present=present == 1,
abundance_number=abundance,
collection_date=datetime.strptime(date_string, '%Y-%m-%d'),
validated=True)
collection.additional_data = {
'BioBaseData': True,
'BioDate': self.get_row_value('BioDate'),
'Season': self.get_row_value('Season'),
'WarningType': self.get_row_value('WarningType'),
'WarningDescription': self.get_row_value('WarningDescription'),
'User': self.get_row_value('User')
}
collection.source_reference = source_reference
user_ctype = ContentType.objects.get_for_model(Profile)
owners = FbisUUID.objects.filter(uuid=self.get_row_value('User'),
content_type=user_ctype)
if owners.exists():
collection.owner = owners[0].content_object
collection.collector_user = owners[0].content_object
else:
superusers = get_user_model().objects.filter(is_superuser=True)
if superusers.exists():
collection.owner = superusers[0]
collection.collector_user = superusers[0]
collection.save()
self.save_uuid(
uuid=self.get_row_value('BioSiteVisitBioBiotopeTaxonID'),
object_id=collection.id)
def test_search_taxon_identifier(self):
species_name = 'Enteromius anoplus (Weber, 1897)'
species = search_taxon_identifier(species_name, False)
self.assertTrue(isinstance(species, Taxonomy))
self.assertEqual(species_name, species.scientific_name)