def handle(self, *args, **options): # Get collection without taxa record only_update_missing_class = options.get('only_update_missing_class') if only_update_missing_class: collections_non_taxa = BiologicalCollectionRecord.objects.filter( Q(taxon_gbif_id__taxon_class__isnull=True) | Q(taxon_gbif_id__taxon_class__exact='')) else: collections_non_taxa = BiologicalCollectionRecord.objects.filter( taxon_gbif_id__isnull=True) models.signals.post_save.disconnect( collection_post_save_update_cluster, ) models.signals.post_save.disconnect(collection_post_save_handler, ) for collection in collections_non_taxa: existed = BiologicalCollectionRecord.objects.filter( original_species_name=collection.original_species_name, taxon_gbif_id__isnull=False) if existed and not only_update_missing_class: collection.taxon_gbif_id = existed[0].taxon_gbif_id collection.save() else: update_collection_record(collection) models.signals.post_save.connect(collection_post_save_update_cluster, ) models.signals.post_save.connect(collection_post_save_handler, )
def handle(self, *args, **options): # Get collection without taxa record collections_non_taxa = BiologicalCollectionRecord.objects.filter( taxon_gbif_id__isnull=True) for collection in collections_non_taxa: existed = BiologicalCollectionRecord.objects.filter( original_species_name=collection.original_species_name, taxon_gbif_id__isnull=False) if existed: collection.taxon_gbif_id = existed[0].taxon_gbif_id collection.save() else: update_collection_record(collection)
def on_post_save(self): if not self.taxonomy: update_collection_record(self) if not self.source_collection: if preferences.SiteSetting.default_data_source: self.source_collection = ( preferences.SiteSetting.default_data_source ) self.save() if self.taxonomy and not self.module_group: # Get taxon group if exists taxonomies = [self.taxonomy.id] taxon_groups = TaxonGroup.objects.filter( taxonomies__in=taxonomies, category=TaxonomicGroupCategory.SPECIES_MODULE.name, ) if taxon_groups.exists(): self.module_group = taxon_groups[0] self.save()
def handle(self, *args, **options): models.signals.pre_save.disconnect( taxonomy_pre_save_handler, ) models.signals.post_save.disconnect( collection_post_save_handler, ) taxonomy_gbif_keys = Taxonomy.objects.values_list( 'gbif_key', flat=True ) taxa = Taxon.objects.all().exclude( gbif_id__in=taxonomy_gbif_keys ) for taxon in taxa: print('Migrate %s' % taxon.scientific_name) taxon_identifier = process_taxon_identifier(taxon.gbif_id) if taxon_identifier: taxon_identifier.iucn_data = taxon.iucn_data taxon_identifier.iucn_redlist_id = taxon.iucn_redlist_id taxon_identifier.author = taxon.author taxon_identifier.endemism = taxon.endemism taxon_identifier.save() if taxon.iucn_status: iucn_status = IUCNStatus.objects.get( pk=taxon.iucn_status.pk) taxonomy = Taxonomy.objects.get(id=taxon_identifier.id) taxonomy.iucn_status = iucn_status taxonomy.save() collections = BiologicalCollectionRecord.objects.filter( taxonomy__isnull=True, ) for collection in collections: update_collection_record(collection) models.signals.post_save.connect( collection_post_save_handler, )
def on_post_save(self): if not self.taxon_gbif_id: update_collection_record(self)
def form_valid(self, form): form.save(commit=True) collection_processed = { 'added': { 'count': 0, 'message': 'records added' }, 'duplicated': { 'count': 0, 'message': 'not accepted because duplicates' }, 'failed': { 'count': 0, 'message': 'failed' }, 'different_format': { 'count': 0, 'message': 'failed due to wrong format' } } # Read csv csv_file = form.instance.csv_file # disconnect post save handler of location sites # it is done from record signal models.signals.post_save.disconnect( location_site_post_save_handler, ) models.signals.post_save.disconnect( collection_post_save_update_cluster, ) location_sites = [] with open(csv_file.path, 'r') as csvfile: csv_reader = csv.DictReader(csvfile) for record in csv_reader: try: print('------------------------------------') print('Processing : %s' % record['species_name']) location_type, status = LocationType.objects.get_or_create( name='PointObservation', allowed_geometry='POINT' ) # Optional records for location site optional_site_records = {} # Optional fields and value optional_records = {} if sys.version_info > (3, 0): # Python 3 code in this block optional_fields_iter = self.additional_fields.items() else: # Python 2 code in this block optional_fields_iter = self.additional_fields.\ iteritems() for (opt_field, field_type) in optional_fields_iter: if opt_field in record: optional_record = self.parse_optional_record( record[opt_field], field_type ) if not optional_record: optional_record = '' if opt_field[:4] == 'site': optional_site_records[opt_field] = \ optional_record else: if optional_record: optional_records[opt_field] = \ optional_record record_point = Point( float(record['longitude']), float(record['latitude'])) try: location_site, status = LocationSite.objects.\ get_or_create( location_type=location_type, geometry_point=record_point, name=record['location_site'] ) except LocationSite.MultipleObjectsReturned: location_site = LocationSite.objects.filter( location_type=location_type, geometry_point=record_point, name=record['location_site'] )[0] if sys.version_info > (3, 0): optional_site_records_iter = \ optional_site_records.items() else: optional_site_records_iter = \ optional_site_records.iteritems() for opt_key, opt_val in optional_site_records_iter: setattr(location_site, opt_key, opt_val) location_site.save() location_sites.append(location_site) # Get existed taxon collections = self.collection_record.objects.filter( original_species_name=record['species_name'] ) taxon_gbif = None if collections: taxon_gbif = collections[0].taxon_gbif_id # custodian field if 'custodian' in record: optional_records['institution_id'] = \ record['custodian'] collection_records, created = self.collection_record.\ objects.\ get_or_create( site=location_site, original_species_name=record['species_name'], category=record['category'].lower(), collection_date=datetime.strptime( record['date'], '%Y-%m-%d'), collector=record['collector'], notes=record['notes'], taxon_gbif_id=taxon_gbif, owner=self.request.user, **optional_records ) if created: print('%s records added' % record['species_name']) collection_processed['added']['count'] += 1 else: collection_processed['duplicated']['count'] += 1 if not taxon_gbif: print('Update taxon gbif') update_collection_record(collection_records) except KeyError: collection_processed['different_format']['count'] += 1 except ValueError: collection_processed['failed']['count'] += 1 print('------------------------------------') csv_upload_message = '' for processed in collection_processed: if collection_processed[processed]['count'] > 0: csv_upload_message += '%s %s <br/>' % ( collection_processed[processed]['count'], collection_processed[processed]['message'] ) if collection_processed['added']['count'] > 0: csv_upload_message += 'Verify your records ' \ '<a target="_blank" ' \ 'href="/nonvalidated-user-list/">' \ 'here</a> <br/>' self.context_data['uploaded'] = csv_upload_message # reconnect post save handler of location sites models.signals.post_save.connect( location_site_post_save_handler, ) models.signals.post_save.connect( collection_post_save_update_cluster, ) return JsonResponse({ 'message': self.context_data['uploaded'] })
def form_valid(self, form): form.save(commit=True) collection_processed = { 'added': { 'count': 0, 'message': 'records added' }, 'duplicated': { 'count': 0, 'message': 'not accepted because duplicates' }, 'failed': { 'count': 0, 'message': 'failed' }, 'different_format': { 'count': 0, 'message': 'failed due to wrong format' } } # Read csv csv_file = form.instance.csv_file # disconnect post save handler of location sites # it is done from record signal models.signals.post_save.disconnect(location_site_post_save_handler, ) models.signals.post_save.disconnect( collection_post_save_update_cluster, ) location_sites = [] with open(csv_file.path, 'r') as csvfile: csv_reader = csv.DictReader(csvfile) for record in csv_reader: try: print('------------------------------------') print('Processing : %s' % record['species_name']) location_type, status = LocationType.objects.get_or_create( name='PointObservation', allowed_geometry='POINT') # Optional records for location site optional_site_records = {} # Optional fields and value optional_records = {} if sys.version_info > (3, 0): # Python 3 code in this block optional_fields_iter = self.additional_fields.items() else: # Python 2 code in this block optional_fields_iter = ( self.additional_fields.iteritems()) for (opt_field, field_type) in optional_fields_iter: if opt_field in record: optional_record = self.parse_optional_record( record[opt_field], field_type) if not optional_record: optional_record = '' if opt_field[:4] == 'site': optional_site_records[opt_field] = \ optional_record else: if optional_record: optional_records[opt_field] = \ optional_record record_point = Point(float(record['longitude']), float(record['latitude'])) try: location_site, status = LocationSite.objects. \ get_or_create( location_type=location_type, geometry_point=record_point, name=record['location_site'] ) except LocationSite.MultipleObjectsReturned: location_site = LocationSite.objects.filter( location_type=location_type, geometry_point=record_point, name=record['location_site'])[0] if sys.version_info > (3, 0): optional_site_records_iter = \ optional_site_records.items() else: optional_site_records_iter = \ optional_site_records.items() for opt_key, opt_val in optional_site_records_iter: setattr(location_site, opt_key, opt_val) location_site.save() location_sites.append(location_site) # Get existed taxon collections = self.collection_record.objects.filter( original_species_name=record['species_name']) # Endemism endemism = None if 'endemism' in record and record['endemism']: endemism, endemism_created = ( Endemism.objects.get_or_create( name=record['endemism'])) taxonomy = None if collections: taxonomy = collections[0].taxonomy if taxonomy: taxonomy.endemism = endemism taxonomy.save() # custodian field if 'custodian' in record: optional_records['institution_id'] = \ record['custodian'] category = '' if 'category' in record: category = record['category'].lower() if 'origin' in record: origin_choices = { v: k for k, v in BiologicalCollectionRecord.CATEGORY_CHOICES } category = origin_choices[record['origin']] if 'habitat' in record: habitat_choices = { v: k for k, v in BiologicalCollectionRecord.HABITAT_CHOICES } optional_records['collection_habitat'] = ( habitat_choices[record['habitat']]) # sampling method sampling_method = None if 'sampling_method' in record: if record['sampling_method'] != 'unspecified': sampling_method, sm_created = ( SamplingMethod.objects.get_or_create( sampling_method=record['sampling_method'])) optional_records['sampling_method'] = (sampling_method) # sampling effort if sampling_method and 'effort_area' in record: effort_area = record['effort_area'] if effort_area: sampling_method.effort_measure = (effort_area + ' m2') sampling_method.save() if sampling_method and 'effort_time' in record: effort_time = record['effort_time'] if effort_time: sampling_method.effort_measure = (effort_time + ' min') sampling_method.save() # river if 'river' in record and record['river']: river, river_created = River.objects.get_or_create( name=record['river']) location_site.river = river location_site.save() allocate_site_codes_from_river( location_id=location_site.id) if record['date'].lower() == 'unspecified': print('Unspecified date -> Next row') continue created = False collection_records = None if 'uuid' in record and record['uuid']: try: uuid_value = uuid.UUID(record['uuid']).hex collection_records = ( BiologicalCollectionRecord.objects.filter( uuid=uuid_value)) if collection_records.exists(): collection_records.update( site=location_site, original_species_name=record[ 'species_name'], collection_date=datetime.strptime( record['date'], '%Y-%m-%d'), taxonomy=taxonomy, ) collection_records = collection_records[0] else: optional_records['uuid'] = uuid_value except ValueError: print('Bad uuid format') if not collection_records: collection_records, created = ( BiologicalCollectionRecord.objects.get_or_create( site=location_site, original_species_name=record['species_name'], collection_date=datetime.strptime( record['date'], '%Y-%m-%d'), taxonomy=taxonomy, category=category, collector=record['collector'], )) # Additional data additional_data = {} if 'effort_number_throws' in record: additional_data['effort_number_throws'] = ( record['effort_number_throws']) if 'catch_per_number' in record: additional_data['catch_per_number'] = ( record['catch_per_number']) if 'catch_per_unit_effort' in record: additional_data['catch_per_unit_effort'] = ( record['catch_per_unit_effort']) if 'number_of_replicates' in record: additional_data['number_of_replicates'] = ( record['number_of_replicates']) if 'hydraulic_biotope' in record: additional_data['hydraulic_biotope'] = ( record['hydraulic_biotope']) if 'substratum' in record: additional_data['substratum'] = (record['substratum']) if 'depth_m' in record: additional_data['depth_m'] = (record['depth_m']) if 'near_bed_velocity' in record: additional_data['near_bed_velocity'] = ( record['near_bed_velocity']) if 'conductivity' in record: additional_data['conductivity'] = ( record['conductivity']) if 'ph' in record: additional_data['ph'] = (record['ph']) if 'dissolved_oxygen' in record: additional_data['dissolved_oxygen'] = ( record['dissolved_oxygen']) if 'temperature' in record: additional_data['temperature'] = ( record['temperature']) if 'turbidity' in record: additional_data['turbidity'] = (record['turbidity']) collection_records.notes = record['notes'] collection_records.owner = self.request.user collection_records.additional_data = additional_data collection_records.save() # update multiple fields BiologicalCollectionRecord.objects.filter( id=collection_records.id).update(**optional_records) if created: print('%s records added' % record['species_name']) collection_processed['added']['count'] += 1 else: collection_processed['duplicated']['count'] += 1 if not taxonomy: print('Update taxon gbif') update_collection_record(collection_records) collection_records.taxonomy.endemism = endemism collection_records.taxonomy.save() except KeyError: collection_processed['different_format']['count'] += 1 except ValueError: collection_processed['failed']['count'] += 1 print('------------------------------------') csv_upload_message = '' for processed in collection_processed: if collection_processed[processed]['count'] > 0: csv_upload_message += '%s %s <br/>' % ( collection_processed[processed]['count'], collection_processed[processed]['message']) if collection_processed['added']['count'] > 0: csv_upload_message += 'Verify your records ' \ '<a href="/nonvalidated-user-list/">' \ 'here</a> <br/>' self.context_data['uploaded'] = csv_upload_message # reconnect post save handler of location sites models.signals.post_save.connect(location_site_post_save_handler, ) models.signals.post_save.connect(collection_post_save_update_cluster, ) return JsonResponse({'message': self.context_data['uploaded']})
def handle(self, *args, **options): folder_name = 'data' source_collection = 'CFE Fish Data' file_path = os.path.join( os.path.abspath(os.path.dirname(__name__)), 'bims/static/{folder}/{filename}'.format(folder=folder_name, filename=self.file_name)) with open(file_path, 'r') as csvfile: csv_reader = csv.DictReader(csvfile) for index, record in enumerate(csv_reader): try: log('Processing : %s' % record[SPECIES_NAME]) location_type, status = LocationType.objects.get_or_create( name='PointObservation', allowed_geometry='POINT') # Optional records for location site optional_site_records = {} # Optional fields and value optional_records = {} if sys.version_info > (3, 0): # Python 3 code in this block optional_fields_iter = self.additional_fields.items() else: # Python 2 code in this block optional_fields_iter = ( self.additional_fields.iteritems()) for (opt_field, field_type) in optional_fields_iter: if opt_field in record: optional_record = self.parse_optional_record( record[opt_field], field_type) if not optional_record: optional_record = '' if opt_field[:4] == 'site': optional_site_records[opt_field] = \ optional_record else: if optional_record: optional_records[opt_field] = \ optional_record record_point = Point(float(record[LONGITUDE]), float(record[LATITUDE])) # Create or get location site try: location_site, status = LocationSite.objects. \ get_or_create( location_type=location_type, geometry_point=record_point, name=record[LOCATION_SITE] ) except LocationSite.MultipleObjectsReturned: location_site = LocationSite.objects.filter( location_type=location_type, geometry_point=record_point, name=record[LOCATION_SITE])[0] if sys.version_info > (3, 0): optional_site_records_iter = ( optional_site_records.items()) else: optional_site_records_iter = ( optional_site_records.iteritems()) for opt_key, opt_val in optional_site_records_iter: setattr(location_site, opt_key, opt_val) location_site.save() # Get existed taxon collections = BiologicalCollectionRecord.objects.filter( original_species_name=record[SPECIES_NAME]) # Endemism endemism = None if ENDEMISM in record and record[ENDEMISM]: endemism, endemism_created = ( Endemism.objects.get_or_create( name=record[ENDEMISM])) taxonomy = None if collections: taxonomy = collections[0].taxonomy if taxonomy: taxonomy.endemism = endemism taxonomy.save() # custodian field if CUSTODIAN in record: optional_records['institution_id'] = \ record[CUSTODIAN] # reference if REFERENCE in record: optional_records['reference'] = record[REFERENCE] if REFERENCE_CATEGORY in record: optional_records['reference_category'] = ( record[REFERENCE_CATEGORY]) if PRESENT in record: optional_records['present'] = bool(record[PRESENT]) category = '' if CATEGORY in record: category = record[CATEGORY].lower() if ORIGIN in record: origin_choices = { v: k for k, v in BiologicalCollectionRecord.CATEGORY_CHOICES } category = origin_choices[record[ORIGIN]] if HABITAT in record: habitat_choices = { v: k for k, v in BiologicalCollectionRecord.HABITAT_CHOICES } optional_records['collection_habitat'] = ( habitat_choices[record[HABITAT]]) # sampling method sampling_method = None if SAMPLING_METHOD in record: if record[SAMPLING_METHOD] != 'unspecified': sampling_method, sm_created = ( SamplingMethod.objects.get_or_create( sampling_method=record[SAMPLING_METHOD])) optional_records['sampling_method'] = (sampling_method) # sampling effort if sampling_method and EFFORT_AREA in record: effort_area = record[EFFORT_AREA] if effort_area: sampling_method.effort_measure = (effort_area + ' m2') sampling_method.save() if sampling_method and EFFORT_TIME in record: effort_time = record[EFFORT_TIME] if effort_time: sampling_method.effort_measure = (effort_time + ' min') sampling_method.save() if record[SAMPLING_DATE].lower() == 'unspecified': self.add_to_error_summary( 'Unspecified date -> Next row', index) continue created = False collection_records = None if UUID in record and record[UUID]: try: uuid_value = uuid.UUID(record[UUID]).hex collection_records = ( BiologicalCollectionRecord.objects.filter( uuid=uuid_value)) if collection_records.exists(): collection_records.update( site=location_site, original_species_name=record[SPECIES_NAME], collection_date=datetime.strptime( record[SAMPLING_DATE], '%Y/%m/%d'), taxonomy=taxonomy, ) collection_records = collection_records[0] else: optional_records['uuid'] = uuid_value except ValueError: print('Bad uuid format') if not collection_records: collection_records, created = ( BiologicalCollectionRecord.objects.get_or_create( site=location_site, original_species_name=record[SPECIES_NAME], collection_date=datetime.strptime( record[SAMPLING_DATE], '%Y/%m/%d'), taxonomy=taxonomy, category=category, collector=record[COLLECTOR], )) # Additional data additional_data = {} if EFFORT_NUMBER_THROWS in record: additional_data['effort_number_throws'] = ( record[EFFORT_NUMBER_THROWS]) if CATCH_NUMBER in record: additional_data['catch_per_number'] = ( record[CATCH_NUMBER]) if CATCH_PER_UNIT in record: additional_data['catch_per_unit_effort'] = ( record[CATCH_PER_UNIT]) if NUMBER_OF_REPLICATES in record: additional_data['number_of_replicates'] = ( record[NUMBER_OF_REPLICATES]) if HYDRAULIC_BIOTOPE in record: additional_data['hydraulic_biotope'] = ( record[HYDRAULIC_BIOTOPE]) if SUBSTRATUM in record: additional_data['substratum'] = (record[SUBSTRATUM]) if DEPTH_M in record: additional_data['depth_m'] = (record[DEPTH_M]) if NBV in record: additional_data['near_bed_velocity'] = (record[NBV]) if CONDUCTIVITY in record: additional_data['conductivity'] = ( record[CONDUCTIVITY]) if PH in record: additional_data['ph'] = (record[PH]) if DISSOLVED_OXYGEN_PERCENT in record: additional_data['dissolved_oxygen_percent'] = ( record[DISSOLVED_OXYGEN_PERCENT]) if DISSOLVED_OXYGEN_MG in record: additional_data['dissolved_oxygen_mg_l'] = ( record[DISSOLVED_OXYGEN_MG]) if TEMP in record: additional_data['temperature'] = (record[TEMP]) if TURBIDITY in record: additional_data['turbidity'] = (record[TURBIDITY]) collection_records.notes = record[NOTES] superusers = get_user_model().objects.filter( is_superuser=True) collection_records.owner = superusers[0] collection_records.additional_data = additional_data collection_records.source_collection = source_collection collection_records.save() # update multiple fields BiologicalCollectionRecord.objects.filter( id=collection_records.id).update(**optional_records) if not created: self.data_updated += 1 else: self.data_added += 1 if not taxonomy: log('Update taxon gbif') update_collection_record(collection_records) # If taxonomy still not found if not collection_records.taxonomy: self.add_to_error_summary( 'Taxonomy {} not found'.format( record[SPECIES_NAME]), index) collection_records.delete() continue collection_records.taxonomy.endemism = endemism collection_records.taxonomy.save() # Update common names if COMMON_NAME in record and record[COMMON_NAME]: common_name = record[COMMON_NAME] try: vernacular_name, vernacular_created = ( VernacularName.objects.get_or_create( name=common_name, language='eng')) except VernacularName.MultipleObjectsReturned: vernacular_name = VernacularName.objects.filter( name=common_name)[0] collection_records.taxonomy.vernacular_names.clear() collection_records.taxonomy.vernacular_names.add( vernacular_name) except KeyError as e: self.add_to_error_summary( 'KeyError : {}'.format(e.message), index) continue except ValueError as e: self.add_to_error_summary( 'ValueError : {}'.format(e.message), index) continue self.summary['data_added'] = self.data_added self.summary['data_duplicated'] = self.data_updated self.summary['data_failed'] = self.data_failed self.summary['total_processed_data'] = (self.data_added + self.data_updated + self.data_failed) self.summary['error_list'] = self.errors log(json.dumps(self.summary))
def form_valid(self, form): form.save(commit=True) collection_processed = {'added': 0, 'failed': 0} # Read csv csv_file = form.instance.csv_file # disconnect post save handler of location sites # it is done from record signal models.signals.post_save.disconnect(location_site_post_save_handler, ) models.signals.post_save.disconnect( collection_post_save_update_cluster, ) location_sites = [] with open(csv_file.path, 'r') as csvfile: csv_reader = csv.DictReader(csvfile) for record in csv_reader: try: print('------------------------------------') print('Processing : %s' % record['species_name']) location_type, status = LocationType.objects.get_or_create( name='PointObservation', allowed_geometry='POINT') record_point = Point(float(record['longitude']), float(record['latitude'])) location_site, status = LocationSite.objects.get_or_create( location_type=location_type, geometry_point=record_point, name=record['location_site'], ) location_sites.append(location_site) # Get existed taxon collections = self.collection_record.objects.filter( original_species_name=record['species_name']) taxon_gbif = None if collections: taxon_gbif = collections[0].taxon_gbif_id # Optional fields and value optional_records = {} if (sys.version_info > (3, 0)): # Python 3 code in this block optional_fields_iter = self.additional_fields.items() else: # Python 2 code in this block optional_fields_iter = self.additional_fields.\ iteritems() for (opt_field, field_type) in optional_fields_iter: if opt_field in record: if field_type == 'bool': record[opt_field] = record[opt_field] == '1' elif field_type == 'str': record[opt_field] = record[opt_field].lower() optional_records[opt_field] = record[opt_field] collection_records, created = self.collection_record.\ objects.\ get_or_create( site=location_site, original_species_name=record['species_name'], category=record['category'].lower(), collection_date=datetime.strptime( record['date'], '%Y-%m-%d'), collector=record['collector'], notes=record['notes'], taxon_gbif_id=taxon_gbif, owner=self.request.user, **optional_records ) if created: print('%s Added' % record['species_name']) collection_processed['added'] += 1 else: if not taxon_gbif: print('Update taxon gbif') update_collection_record(collection_records) except (ValueError, KeyError): collection_processed['failed'] += 1 print('------------------------------------') self.context_data['uploaded'] = 'Collection added ' + \ str(collection_processed['added']) # reconnect post save handler of location sites models.signals.post_save.connect(location_site_post_save_handler, ) models.signals.post_save.connect(collection_post_save_update_cluster, ) return JsonResponse({'message': self.context_data['uploaded']})