def setup_seed_importer():
bi = BulkDataImportHandler()
bi.add_mapping(SeedFeatures, {
'Seed/fruit type': 'seed_type',
'Shape 3D': 'shape_3d',
'Shape 2D': 'shape_2d',
'Shape detail': 'shape_detail',
'Length (mm)': 'length',
'Breadth (mm)': 'breadth',
'Thickness (mm)': 'thickness',
'Colour': 'colour',
'Reflection': 'reflection',
'Testa/endocarp thickness (mm)': 'testa_endocarp_thickness',
'Surface description (Outer)': 'surface_outer_texture',
'Surface description (Inner)': 'surface_inner_texture',
'Anatomy Transverse Section': 'anatomy_transverse_section',
'Anatomy Longitudinal Sections': 'anatomy_longitudinal_sections',
'Embryo and endosperm details': 'embryo_endosperm',
'Other identification information': 'other_identification_information',
'References and links': 'references_and_links',
'Notes': 'notes',
})
def link(accession, seed_details):
if hasattr(accession, 'seedfeatures'):
accession.seedfeatures.delete()
seed_details.accession = accession
seed_details.save()
accession.material = u"Seed"
accession.save()
bi.add_linking_function(link)
return bi
def test_missing_unique_field(self):
# Contains fields 'First Name', "Last Name', 'Age', 'ID'
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age'
}, 'PersonID', 'id')
with self.assertRaises(MissingUniqueHeaderException):
affected_records, stats = bi.process_spreadsheet(spreadsheet)
def test_missing_unique_field(self):
# Contains fields 'First Name', "Last Name', 'Age', 'ID'
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age'
}, 'PersonID', 'id')
with self.assertRaises(MissingUniqueHeaderException):
affected_records, stats = bi.process_spreadsheet(spreadsheet)
def test_unique_field(self):
# Contains fields 'First Name', "Last Name', 'Age', 'ID'
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age',
'ID': 'id'
}, 'ID', 'id')
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def upload_storage_locations_spreadsheet(request):
if request.method == 'POST':
form = BulkImportForm(request.POST, request.FILES)
if form.is_valid():
spreadsheet = form.files['spreadsheet']
updated_records = {'value': 0}
def update_storage_locations(headers, values):
try:
reg_num = values[headers.index('registration number')]
reg_num = int(reg_num)
except ValueError:
messages.add_message(request, messages.ERROR, "Registration number not a number: '%s'" % reg_num)
return
try:
mo = MuseumObject.objects.get(registration_number=reg_num)
mo.storage_row = str(values[headers.index('row')])
mo.storage_bay = str(values[headers.index('bay')])
mo.storage_shelf_drawer = str(values[headers.index('shelf/drawer')])
mo.save()
updated_records['value'] += 1
except MuseumObject.DoesNotExist:
messages.add_message(request, messages.ERROR, "Unable to find registration number: %s" % reg_num)
db.reset_queries()
if (updated_records['value'] % 100) == 0:
logger.info("Updated %s records, lasted update was: %s" % (updated_records['value'], reg_num))
db.transaction.commit()
bi = BulkDataImportHandler()
bi.add_function_mapping(update_storage_locations)
bi.process_spreadsheet(spreadsheet)
db.transaction.commit()
messages.add_message(request, messages.SUCCESS, "Updated %s records" % updated_records['value'])
else:
form = BulkImportForm()
return render(request, 'storage_bulkupdate.html', {
'form': form,
'title': 'Bulk update storage locations'
})
def test_read_simple_spreadsheet(self):
"""
Load in a simple spreadsheet
"""
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age'
})
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def test_read_spreadsheet_case_insensitive(self):
"""
Test the column names to be mapped are case insensitive
"""
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First name': 'first_name',
'Last NaMe': 'last_name',
'Age': 'age'
})
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def test_unique_field(self):
# Contains fields 'First Name', "Last Name', 'Age', 'ID'
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(
Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age',
'ID': 'id'
}, 'ID', 'id')
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def test_read_simple_spreadsheet(self):
"""
Load in a simple spreadsheet
"""
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First Name': 'first_name',
'Last Name': 'last_name',
'Age': 'age'
})
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def test_read_spreadsheet_case_insensitive(self):
"""
Test the column names to be mapped are case insensitive
"""
spreadsheet = 'bulkimport/testdata/names.xlsx'
bi = BulkDataImportHandler()
bi.add_mapping(Person, {
'First name': 'first_name',
'Last NaMe': 'last_name',
'Age': 'age'
})
affected_records, stats = bi.process_spreadsheet(spreadsheet)
self.assertEqual(3, len(affected_records))
self.assertEqual('Bob', affected_records[0][0].first_name)
self.assertEqual(50, affected_records[2][0].age)
def setup_accessions_importer():
bi = BulkDataImportHandler()
bi.add_mapping(Accession, {
'UQ Accession': 'uq_accession',
'Material': 'material',
'Source': 'source',
'State': 'preservation_state',
'Family': 'family',
'SUBFAM': 'subfam',
'TRIBE': 'tribe',
'Genus': 'genus',
'Species': 'species',
'AUTHOR': 'species_author',
'SSPNA': 'sspna',
'SSPAU': 'sspau',
'VARNA': 'varna',
'VARAU': 'varau',
'CULTIVAR': 'cultivar',
'Common Name': 'common_name',
'Biological Synonym': 'biological_synonym',
'DETNA': 'detna',
'DETDATE': 'detdate',
'Collector': 'collector',
'Collector Serial No.': 'collector_serial_no',
'Collection Date': 'collection_date',
'SOURCE': 'source',
'SOURCE NUMBER': 'source_number',
'id level flag': 'id_level_flag',
'Country': 'country',
'Site Name': 'site_name',
'Lat/Long.': 'lat_long',
'Altitude': 'altitude',
'Notes': 'location_notes',
'Related Accession': 'related_accession',
'GRIN & Seed Atlas?': 'weblinks',
})
bi.header_row = 1
bi.first_data_row = 2
return bi
def test_process_row_single(self):
mapping = {'one': 'one'}
save_mock = mock.Mock(return_value=None)
model = mock.Mock(MyModel)
model.return_value.save = save_mock
bdih = BulkDataImportHandler()
bdih.add_mapping(model, mapping)
headers = ['one', 'two']
vals = ['val1', 'spot']
affected_records, used_cols = bdih.process_row(headers, vals)
new_record = affected_records[0]
# make sure class was created
self.assertEqual(model.call_count, 1)
# now make sure save was called once
self.assertEqual(save_mock.call_count, 1)
self.assertEqual('val1', new_record.one)
def test_process_row_single(self):
mapping = {'one': 'one'}
save_mock = mock.Mock(return_value=None)
model = mock.Mock(MyModel)
model.return_value.save = save_mock
bdih = BulkDataImportHandler()
bdih.add_mapping(model, mapping)
headers = ['one', 'two']
vals = ['val1', 'spot']
affected_records, used_cols = bdih.process_row(headers, vals)
new_record = affected_records[0]
# make sure class was created
self.assertEqual(model.call_count, 1)
# now make sure save was called once
self.assertEqual(save_mock.call_count, 1)
self.assertEqual('val1', new_record.one)
def setup_accessions_importer():
bi = BulkDataImportHandler()
bi.add_mapping(Accession, {
'UQ Accession': 'uq_accession',
'Material': 'material',
'Source': 'source',
'State': 'state',
'Family': 'family',
'SUBFAM': 'subfam',
'TRIBE': 'tribe',
'Genus': 'genus',
'Species': 'species',
'AUTHOR': 'author',
'SSPNA': 'sspna',
'SSPAU': 'sspau',
'VARNA': 'varna',
'VARAU': 'varau',
'CULTIVAR': 'cultivar',
'Common Name': 'common_name',
'Biological Synonym': 'biological_synonym',
'FAMNO': 'famno',
'GENNO': 'genno',
'SPNO': 'spno',
'SSPNO': 'sspno',
'VARNO': 'varno',
'DETNA': 'detna',
'DETDATE': 'detdate',
'Collector': 'collector',
'Collector Serial No.': 'collector_serial_no',
'Collection Date': 'collection_date',
'SOURCE': 'source',
'SOURCE NUMBER': 'source_number',
'id level flag': 'id_level_flag',
'Country': 'country',
'Site Name': 'site_name',
'Lat/Long.': 'lat_long',
'Altitude': 'altitude',
'Notes': 'notes',
'Related Accession': 'related_accession',
'GRIN & Seed Atlas?': 'grin__seed_atlas',
})
bi.header_row = 1
bi.first_data_row = 2
return bi
def setup_bulk_importer():
# setup mappings
bi = BulkDataImportHandler()
bi.add_mapping(Species, {
'Class': 'class_name',
'Subclass': 'subclass',
'Order': 'order',
'Superfamily': 'superfamily',
'Family': 'family',
'Subfamily': 'subfamily',
'Genus': 'genus',
'Subgenus': 'subgenus',
'Species': 'species',
'Author Name / Year': 'authority',
'Synonyms': 'synonyms',
'Common Names': 'common_names',
'Geographic Range': 'geographic_range',
'Habitat': 'habitat',
'Shell Size': 'shell_size',
'Shell Sculpture': 'shell_sculpture',
'Shell Colour': 'shell_colour',
'References': 'references',
'NOTES': 'location_notes',
'Additional Information': 'additional_information',
})
bi.add_mapping(Specimen, {
'Specimen Collection Date': 'collection_date',
'Specimen Collection Location': 'collection_location',
'Specimen Collection Information': 'collection_information',
})
def link(first, specimen):
specimen.species = first
specimen.save()
bi.add_linking_function(link)
return bi
def test_process_row_multi(self):
mapping_1 = {'one': 'one'}
save_mock_1 = mock.Mock(return_value=None)
model_1 = mock.Mock(MyModel)
model_1.return_value.save = save_mock_1
mapping_2 = {'two': 'two'}
save_mock_2 = mock.Mock(return_value=None)
model_2 = mock.Mock(MyModel)
model_2.return_value.save = save_mock_2
linking_func = mock.Mock()
bdih = BulkDataImportHandler()
bdih.add_mapping(model_1, mapping_1)
bdih.add_mapping(model_2, mapping_2)
bdih.add_linking_function(linking_func)
headers = ['one', 'two']
vals = ['val1', 'spot']
affected_records, stats = bdih.process_row(headers, vals)
result_1, result_2 = affected_records
# make sure one class each was created
self.assertEqual(model_1.call_count, 1)
self.assertEqual(model_2.call_count, 1)
# now make sure each save was called once
self.assertEqual(save_mock_1.call_count, 1)
self.assertEqual(save_mock_2.call_count, 1)
# check values were saved onto each instance
self.assertEqual('val1', result_1.one)
self.assertEqual('spot', result_2.two)
# check the linking function was called
self.assertTrue(linking_func.called)
linking_func.assert_called_with(result_1, result_2)
def test_process_row_multi(self):
mapping_1 = {'one': 'one'}
save_mock_1 = mock.Mock(return_value=None)
model_1 = mock.Mock(MyModel)
model_1.return_value.save = save_mock_1
mapping_2 = {'two': 'two'}
save_mock_2 = mock.Mock(return_value=None)
model_2 = mock.Mock(MyModel)
model_2.return_value.save = save_mock_2
linking_func = mock.Mock()
bdih = BulkDataImportHandler()
bdih.add_mapping(model_1, mapping_1)
bdih.add_mapping(model_2, mapping_2)
bdih.add_linking_function(linking_func)
headers = ['one', 'two']
vals = ['val1', 'spot']
affected_records, stats = bdih.process_row(headers, vals)
result_1, result_2 = affected_records
# make sure one class each was created
self.assertEqual(model_1.call_count, 1)
self.assertEqual(model_2.call_count, 1)
# now make sure each save was called once
self.assertEqual(save_mock_1.call_count, 1)
self.assertEqual(save_mock_2.call_count, 1)
# check values were saved onto each instance
self.assertEqual('val1', result_1.one)
self.assertEqual('spot', result_2.two)
# check the linking function was called
self.assertTrue(linking_func.called)
linking_func.assert_called_with(result_1, result_2)
def setup_wood_importer():
bi = BulkDataImportHandler()
bi.add_mapping(Accession, {
'family': 'family',
'Genus': 'genus',
'Species': 'species',
'Common Name': 'common_name',
# 'Indigenous Name': 'indigenous_name',
'Accession': 'uq_accession',
'Specimen Collection Date': 'collection_date',
# 'Specimen Collection Location': 'specimen_collection_location',
# 'Specimen Collection Information': 'specimen_collection_information',
# 'Voucher Category': 'voucher_category',
# 'Geographic Range': 'geographic_range',
# 'Habitat': 'habitat',
# 'Plant part': 'plant_part',
'State': 'state',
# 'Type of Plant': 'type_of_plant',
})
bi.add_mapping(WoodFeatures, {
'aggregate rays': 'aggregate_rays',
'Australia': 'australia',
'axial canals': 'axial_canals',
'axial parenchyma arrangment': 'axial_parenchyma_arrangment',
'axial parenchyma bands': 'axial_parenchyma_bands',
'axial parenchyma present': 'axial_parenchyma_present',
'cambial variants': 'cambial_variants',
'druses': 'druses',
'family': 'family',
'fibre helical thickenings': 'fibre_helical_thickenings',
'fibre pits': 'fibre_pits',
'fibres wall thickeness': 'fibres_wall_thickeness',
'fusiform parenchyma cells': 'fusiform_parenchyma_cells',
'helical thickenings': 'helical_thickenings',
'included phloem': 'included_phloem',
'Indigenous name': 'indigenous_name',
'intervessels pits arrangment': 'intervessels_pits_arrangment',
'intervessels pits size': 'intervessels_pits_size',
'intervessels pits specific shapes': 'intervessels_pits_specific_shapes',
'lactifers tanniferous tubes': 'lactifers_tanniferous_tubes',
'New Caledonia': 'new_caledonia',
'NOTES': 'notes',
'parenchyma like fibres present': 'parenchyma_like_fibres_present',
'perforation plates types': 'perforation_plates_types',
'prismatic crystal': 'prismatic_crystal',
'radial secretory canals': 'radial_secretory_canals',
'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms',
'rays': 'rays',
'rays cellular composition': 'rays_cellular_composition',
'rays height': 'rays_height',
'rays sheat cells': 'rays_sheat_cells',
'RAYS STRUCTURE': 'rays_structure',
'rays width': 'rays_width',
'Reference Specimens': 'reference_specimens',
'silica': 'silica',
'solitary vessels with angular outline': 'solitary_vessels_with_angular_outline',
'Species': 'species',
'spetate fibres present': 'spetate_fibres_present',
'storied structure': 'storied_structure',
'tile cells': 'tile_cells',
'Turkey': 'turkey',
'vascular-vasicentric tracheids present': 'vascularvasicentric_tracheids_present',
'vessels': 'vessels',
'vessels arrangment': 'vessels_arrangment',
'vessels deposits': 'vessels_deposits',
'vessels grouping': 'vessels_grouping',
'vessels porosity': 'vessels_porosity',
'vessels rays pitting': 'vessels_rays_pitting',
'vessels tyloses': 'vessels_tyloses',
'walls': 'walls',
'Contributor': 'contributor',
'DATE': 'date',
})
def link(accession, wood_details):
wood_details.accession = accession
wood_details.save()
bi.add_linking_function(link)
return bi
def setup_wood_importer():
bi = BulkDataImportHandler()
bi.add_mapping(Accession, {
'family': 'family',
'genus': 'genus',
'Species': 'species',
'Common Name': 'common_name',
'Indigenous name': 'indigenous_name',
'UQM Accession Number': 'uqm_accession',
'Specimen Collection Date': 'collection_date',
'State': 'preservation_state',
'Contributor': 'contributor',
'DATE': 'date_contributed',
'notes': 'accession_notes',
'sample number': 'sample_number',
'unique identifier': 'unique_identifier'
}, 'unique identifier', 'unique_identifier')
bi.add_mapping(WoodFeatures, {
'aggregate rays': 'aggregate_rays',
'axial canals': 'axial_canals',
'axial parenchyma arrangement (1)': 'axial_parenchyma_arrangement1',
'axial parenchyma arrangement (2)': 'axial_parenchyma_arrangement2',
'axial parenchyma arrangement (3)': 'axial_parenchyma_arrangement3',
'axial parenchyma arrangement (4)': 'axial_parenchyma_arrangement4',
'axial parenchyma arrangement (5)': 'axial_parenchyma_arrangement5',
'axial parenchyma bands': 'axial_parenchyma_bands',
'axial parenchyma present': 'axial_parenchyma_present',
'cambial variants': 'cambial_variants',
'druses': 'druses',
'Early/Late wood transition': 'early_late_wood_transition',
'fibre helical thickenings': 'fibre_helical_thickenings',
'fibre pits': 'fibre_pits',
'fibre wall thickness': 'fibre_wall_thickness',
'fusiform parenchyma cells': 'fusiform_parenchyma_cells',
'helical thickenings': 'helical_thickenings',
'included phloem': 'included_phloem',
'Intervessel/tracheid pit arrangement': 'intervessel_pit_arrangement',
'intervessel pit size': 'intervessel_pit_size',
'Intervessel/tracheid pit shapes': 'intervessel_tracheid_pit_shapes',
'lactifer tanniferous tubes': 'lactifer_tanniferous_tubes',
'parenchyma like fibres present': 'parenchyma_like_fibres_present',
'perforation plates types': 'perforation_plates_types',
'prismatic crystals': 'prismatic_crystals',
'radial secretory canals': 'radial_secretory_canals',
'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms',
'rays': 'rays',
'rays cellular composition': 'rays_cellular_composition',
'ray height': 'ray_height',
'rays sheath cells': 'rays_sheath_cells',
'rays structure': 'rays_structure',
'ray width': 'ray_width',
'ray type': 'ray_type',
'Reference Specimens': 'reference_specimens',
'silica': 'silica',
'solitary vessels with angular outline': 'solitary_vessels_with_angular_outline',
'Species': 'species',
'spetate fibres present': 'spetate_fibres_present',
'storied structure': 'storied_structure',
'tile cells': 'tile_cells',
'tracheid diameter': 'tracheid_diameter',
'vascular-vasicentric tracheids present': 'vascularvasicentric_tracheids_present',
'vessels': 'vessels',
'vessel arrangement': 'vessel_arrangement',
'vessels deposits': 'vessels_deposits',
'vessel grouping': 'vessel_grouping',
'vessel porosity': 'vessel_porosity',
'vessels rays pitting': 'vessels_rays_pitting',
'vessels tyloses': 'vessels_tyloses',
'walls': 'walls',
'early late wood transition': 'early_late_wood_transition',
'axiel resin canals': 'axiel_resin_canals',
'epithelial cells': 'epithelial_cells',
'axial tracheid pits': 'axial_tracheid_pits',
'spiral thickenings': 'spiral_thickenings',
'crassulae': 'crassulae',
'nodular tangential ray walls': 'nodular_tangential_ray_walls',
'early wood ray pits': 'early_wood_ray_pits',
'late wood ray pits': 'late_wood_ray_pits',
})
def link(accession, wood_details):
if hasattr(accession, 'woodfeatures'):
accession.woodfeatures.delete()
wood_details.accession = accession
wood_details.save()
accession.material = u"Wood"
accession.save()
bi.add_linking_function(link)
return bi
