def setup_seed_importer(): bi = BulkDataImportHandler() bi.add_mapping(SeedFeatures, { 'Seed/fruit type': 'seed_type', 'Shape 3D': 'shape_3d', 'Shape 2D': 'shape_2d', 'Shape detail': 'shape_detail', 'Length (mm)': 'length', 'Breadth (mm)': 'breadth', 'Thickness (mm)': 'thickness', 'Colour': 'colour', 'Reflection': 'reflection', 'Testa/endocarp thickness (mm)': 'testa_endocarp_thickness', 'Surface description (Outer)': 'surface_outer_texture', 'Surface description (Inner)': 'surface_inner_texture', 'Anatomy Transverse Section': 'anatomy_transverse_section', 'Anatomy Longitudinal Sections': 'anatomy_longitudinal_sections', 'Embryo and endosperm details': 'embryo_endosperm', 'Other identification information': 'other_identification_information', 'References and links': 'references_and_links', 'Notes': 'notes', }) def link(accession, seed_details): if hasattr(accession, 'seedfeatures'): accession.seedfeatures.delete() seed_details.accession = accession seed_details.save() accession.material = u"Seed" accession.save() bi.add_linking_function(link) return bi
def test_missing_unique_field(self): # Contains fields 'First Name', "Last Name', 'Age', 'ID' spreadsheet = 'bulkimport/testdata/names.xlsx' bi = BulkDataImportHandler() bi.add_mapping(Person, { 'First Name': 'first_name', 'Last Name': 'last_name', 'Age': 'age' }, 'PersonID', 'id') with self.assertRaises(MissingUniqueHeaderException): affected_records, stats = bi.process_spreadsheet(spreadsheet)
def test_unique_field(self): # Contains fields 'First Name', "Last Name', 'Age', 'ID' spreadsheet = 'bulkimport/testdata/names.xlsx' bi = BulkDataImportHandler() bi.add_mapping(Person, { 'First Name': 'first_name', 'Last Name': 'last_name', 'Age': 'age', 'ID': 'id' }, 'ID', 'id') affected_records, stats = bi.process_spreadsheet(spreadsheet) self.assertEqual(3, len(affected_records)) self.assertEqual('Bob', affected_records[0][0].first_name) self.assertEqual(50, affected_records[2][0].age)
def upload_storage_locations_spreadsheet(request): if request.method == 'POST': form = BulkImportForm(request.POST, request.FILES) if form.is_valid(): spreadsheet = form.files['spreadsheet'] updated_records = {'value': 0} def update_storage_locations(headers, values): try: reg_num = values[headers.index('registration number')] reg_num = int(reg_num) except ValueError: messages.add_message(request, messages.ERROR, "Registration number not a number: '%s'" % reg_num) return try: mo = MuseumObject.objects.get(registration_number=reg_num) mo.storage_row = str(values[headers.index('row')]) mo.storage_bay = str(values[headers.index('bay')]) mo.storage_shelf_drawer = str(values[headers.index('shelf/drawer')]) mo.save() updated_records['value'] += 1 except MuseumObject.DoesNotExist: messages.add_message(request, messages.ERROR, "Unable to find registration number: %s" % reg_num) db.reset_queries() if (updated_records['value'] % 100) == 0: logger.info("Updated %s records, lasted update was: %s" % (updated_records['value'], reg_num)) db.transaction.commit() bi = BulkDataImportHandler() bi.add_function_mapping(update_storage_locations) bi.process_spreadsheet(spreadsheet) db.transaction.commit() messages.add_message(request, messages.SUCCESS, "Updated %s records" % updated_records['value']) else: form = BulkImportForm() return render(request, 'storage_bulkupdate.html', { 'form': form, 'title': 'Bulk update storage locations' })
def test_read_simple_spreadsheet(self): """ Load in a simple spreadsheet """ spreadsheet = 'bulkimport/testdata/names.xlsx' bi = BulkDataImportHandler() bi.add_mapping(Person, { 'First Name': 'first_name', 'Last Name': 'last_name', 'Age': 'age' }) affected_records, stats = bi.process_spreadsheet(spreadsheet) self.assertEqual(3, len(affected_records)) self.assertEqual('Bob', affected_records[0][0].first_name) self.assertEqual(50, affected_records[2][0].age)
def test_read_spreadsheet_case_insensitive(self): """ Test the column names to be mapped are case insensitive """ spreadsheet = 'bulkimport/testdata/names.xlsx' bi = BulkDataImportHandler() bi.add_mapping(Person, { 'First name': 'first_name', 'Last NaMe': 'last_name', 'Age': 'age' }) affected_records, stats = bi.process_spreadsheet(spreadsheet) self.assertEqual(3, len(affected_records)) self.assertEqual('Bob', affected_records[0][0].first_name) self.assertEqual(50, affected_records[2][0].age)
def test_unique_field(self): # Contains fields 'First Name', "Last Name', 'Age', 'ID' spreadsheet = 'bulkimport/testdata/names.xlsx' bi = BulkDataImportHandler() bi.add_mapping( Person, { 'First Name': 'first_name', 'Last Name': 'last_name', 'Age': 'age', 'ID': 'id' }, 'ID', 'id') affected_records, stats = bi.process_spreadsheet(spreadsheet) self.assertEqual(3, len(affected_records)) self.assertEqual('Bob', affected_records[0][0].first_name) self.assertEqual(50, affected_records[2][0].age)
def setup_accessions_importer(): bi = BulkDataImportHandler() bi.add_mapping(Accession, { 'UQ Accession': 'uq_accession', 'Material': 'material', 'Source': 'source', 'State': 'preservation_state', 'Family': 'family', 'SUBFAM': 'subfam', 'TRIBE': 'tribe', 'Genus': 'genus', 'Species': 'species', 'AUTHOR': 'species_author', 'SSPNA': 'sspna', 'SSPAU': 'sspau', 'VARNA': 'varna', 'VARAU': 'varau', 'CULTIVAR': 'cultivar', 'Common Name': 'common_name', 'Biological Synonym': 'biological_synonym', 'DETNA': 'detna', 'DETDATE': 'detdate', 'Collector': 'collector', 'Collector Serial No.': 'collector_serial_no', 'Collection Date': 'collection_date', 'SOURCE': 'source', 'SOURCE NUMBER': 'source_number', 'id level flag': 'id_level_flag', 'Country': 'country', 'Site Name': 'site_name', 'Lat/Long.': 'lat_long', 'Altitude': 'altitude', 'Notes': 'location_notes', 'Related Accession': 'related_accession', 'GRIN & Seed Atlas?': 'weblinks', }) bi.header_row = 1 bi.first_data_row = 2 return bi
def test_process_row_single(self): mapping = {'one': 'one'} save_mock = mock.Mock(return_value=None) model = mock.Mock(MyModel) model.return_value.save = save_mock bdih = BulkDataImportHandler() bdih.add_mapping(model, mapping) headers = ['one', 'two'] vals = ['val1', 'spot'] affected_records, used_cols = bdih.process_row(headers, vals) new_record = affected_records[0] # make sure class was created self.assertEqual(model.call_count, 1) # now make sure save was called once self.assertEqual(save_mock.call_count, 1) self.assertEqual('val1', new_record.one)
def setup_accessions_importer(): bi = BulkDataImportHandler() bi.add_mapping(Accession, { 'UQ Accession': 'uq_accession', 'Material': 'material', 'Source': 'source', 'State': 'state', 'Family': 'family', 'SUBFAM': 'subfam', 'TRIBE': 'tribe', 'Genus': 'genus', 'Species': 'species', 'AUTHOR': 'author', 'SSPNA': 'sspna', 'SSPAU': 'sspau', 'VARNA': 'varna', 'VARAU': 'varau', 'CULTIVAR': 'cultivar', 'Common Name': 'common_name', 'Biological Synonym': 'biological_synonym', 'FAMNO': 'famno', 'GENNO': 'genno', 'SPNO': 'spno', 'SSPNO': 'sspno', 'VARNO': 'varno', 'DETNA': 'detna', 'DETDATE': 'detdate', 'Collector': 'collector', 'Collector Serial No.': 'collector_serial_no', 'Collection Date': 'collection_date', 'SOURCE': 'source', 'SOURCE NUMBER': 'source_number', 'id level flag': 'id_level_flag', 'Country': 'country', 'Site Name': 'site_name', 'Lat/Long.': 'lat_long', 'Altitude': 'altitude', 'Notes': 'notes', 'Related Accession': 'related_accession', 'GRIN & Seed Atlas?': 'grin__seed_atlas', }) bi.header_row = 1 bi.first_data_row = 2 return bi
def setup_bulk_importer(): # setup mappings bi = BulkDataImportHandler() bi.add_mapping(Species, { 'Class': 'class_name', 'Subclass': 'subclass', 'Order': 'order', 'Superfamily': 'superfamily', 'Family': 'family', 'Subfamily': 'subfamily', 'Genus': 'genus', 'Subgenus': 'subgenus', 'Species': 'species', 'Author Name / Year': 'authority', 'Synonyms': 'synonyms', 'Common Names': 'common_names', 'Geographic Range': 'geographic_range', 'Habitat': 'habitat', 'Shell Size': 'shell_size', 'Shell Sculpture': 'shell_sculpture', 'Shell Colour': 'shell_colour', 'References': 'references', 'NOTES': 'location_notes', 'Additional Information': 'additional_information', }) bi.add_mapping(Specimen, { 'Specimen Collection Date': 'collection_date', 'Specimen Collection Location': 'collection_location', 'Specimen Collection Information': 'collection_information', }) def link(first, specimen): specimen.species = first specimen.save() bi.add_linking_function(link) return bi
def test_process_row_multi(self): mapping_1 = {'one': 'one'} save_mock_1 = mock.Mock(return_value=None) model_1 = mock.Mock(MyModel) model_1.return_value.save = save_mock_1 mapping_2 = {'two': 'two'} save_mock_2 = mock.Mock(return_value=None) model_2 = mock.Mock(MyModel) model_2.return_value.save = save_mock_2 linking_func = mock.Mock() bdih = BulkDataImportHandler() bdih.add_mapping(model_1, mapping_1) bdih.add_mapping(model_2, mapping_2) bdih.add_linking_function(linking_func) headers = ['one', 'two'] vals = ['val1', 'spot'] affected_records, stats = bdih.process_row(headers, vals) result_1, result_2 = affected_records # make sure one class each was created self.assertEqual(model_1.call_count, 1) self.assertEqual(model_2.call_count, 1) # now make sure each save was called once self.assertEqual(save_mock_1.call_count, 1) self.assertEqual(save_mock_2.call_count, 1) # check values were saved onto each instance self.assertEqual('val1', result_1.one) self.assertEqual('spot', result_2.two) # check the linking function was called self.assertTrue(linking_func.called) linking_func.assert_called_with(result_1, result_2)
def setup_wood_importer(): bi = BulkDataImportHandler() bi.add_mapping(Accession, { 'family': 'family', 'Genus': 'genus', 'Species': 'species', 'Common Name': 'common_name', # 'Indigenous Name': 'indigenous_name', 'Accession': 'uq_accession', 'Specimen Collection Date': 'collection_date', # 'Specimen Collection Location': 'specimen_collection_location', # 'Specimen Collection Information': 'specimen_collection_information', # 'Voucher Category': 'voucher_category', # 'Geographic Range': 'geographic_range', # 'Habitat': 'habitat', # 'Plant part': 'plant_part', 'State': 'state', # 'Type of Plant': 'type_of_plant', }) bi.add_mapping(WoodFeatures, { 'aggregate rays': 'aggregate_rays', 'Australia': 'australia', 'axial canals': 'axial_canals', 'axial parenchyma arrangment': 'axial_parenchyma_arrangment', 'axial parenchyma bands': 'axial_parenchyma_bands', 'axial parenchyma present': 'axial_parenchyma_present', 'cambial variants': 'cambial_variants', 'druses': 'druses', 'family': 'family', 'fibre helical thickenings': 'fibre_helical_thickenings', 'fibre pits': 'fibre_pits', 'fibres wall thickeness': 'fibres_wall_thickeness', 'fusiform parenchyma cells': 'fusiform_parenchyma_cells', 'helical thickenings': 'helical_thickenings', 'included phloem': 'included_phloem', 'Indigenous name': 'indigenous_name', 'intervessels pits arrangment': 'intervessels_pits_arrangment', 'intervessels pits size': 'intervessels_pits_size', 'intervessels pits specific shapes': 'intervessels_pits_specific_shapes', 'lactifers tanniferous tubes': 'lactifers_tanniferous_tubes', 'New Caledonia': 'new_caledonia', 'NOTES': 'notes', 'parenchyma like fibres present': 'parenchyma_like_fibres_present', 'perforation plates types': 'perforation_plates_types', 'prismatic crystal': 'prismatic_crystal', 'radial secretory canals': 'radial_secretory_canals', 'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms', 'rays': 'rays', 'rays cellular composition': 'rays_cellular_composition', 'rays height': 'rays_height', 'rays sheat cells': 'rays_sheat_cells', 'RAYS STRUCTURE': 'rays_structure', 'rays width': 'rays_width', 'Reference Specimens': 'reference_specimens', 'silica': 'silica', 'solitary vessels with angular outline': 'solitary_vessels_with_angular_outline', 'Species': 'species', 'spetate fibres present': 'spetate_fibres_present', 'storied structure': 'storied_structure', 'tile cells': 'tile_cells', 'Turkey': 'turkey', 'vascular-vasicentric tracheids present': 'vascularvasicentric_tracheids_present', 'vessels': 'vessels', 'vessels arrangment': 'vessels_arrangment', 'vessels deposits': 'vessels_deposits', 'vessels grouping': 'vessels_grouping', 'vessels porosity': 'vessels_porosity', 'vessels rays pitting': 'vessels_rays_pitting', 'vessels tyloses': 'vessels_tyloses', 'walls': 'walls', 'Contributor': 'contributor', 'DATE': 'date', }) def link(accession, wood_details): wood_details.accession = accession wood_details.save() bi.add_linking_function(link) return bi
def setup_wood_importer(): bi = BulkDataImportHandler() bi.add_mapping(Accession, { 'family': 'family', 'genus': 'genus', 'Species': 'species', 'Common Name': 'common_name', 'Indigenous name': 'indigenous_name', 'UQM Accession Number': 'uqm_accession', 'Specimen Collection Date': 'collection_date', 'State': 'preservation_state', 'Contributor': 'contributor', 'DATE': 'date_contributed', 'notes': 'accession_notes', 'sample number': 'sample_number', 'unique identifier': 'unique_identifier' }, 'unique identifier', 'unique_identifier') bi.add_mapping(WoodFeatures, { 'aggregate rays': 'aggregate_rays', 'axial canals': 'axial_canals', 'axial parenchyma arrangement (1)': 'axial_parenchyma_arrangement1', 'axial parenchyma arrangement (2)': 'axial_parenchyma_arrangement2', 'axial parenchyma arrangement (3)': 'axial_parenchyma_arrangement3', 'axial parenchyma arrangement (4)': 'axial_parenchyma_arrangement4', 'axial parenchyma arrangement (5)': 'axial_parenchyma_arrangement5', 'axial parenchyma bands': 'axial_parenchyma_bands', 'axial parenchyma present': 'axial_parenchyma_present', 'cambial variants': 'cambial_variants', 'druses': 'druses', 'Early/Late wood transition': 'early_late_wood_transition', 'fibre helical thickenings': 'fibre_helical_thickenings', 'fibre pits': 'fibre_pits', 'fibre wall thickness': 'fibre_wall_thickness', 'fusiform parenchyma cells': 'fusiform_parenchyma_cells', 'helical thickenings': 'helical_thickenings', 'included phloem': 'included_phloem', 'Intervessel/tracheid pit arrangement': 'intervessel_pit_arrangement', 'intervessel pit size': 'intervessel_pit_size', 'Intervessel/tracheid pit shapes': 'intervessel_tracheid_pit_shapes', 'lactifer tanniferous tubes': 'lactifer_tanniferous_tubes', 'parenchyma like fibres present': 'parenchyma_like_fibres_present', 'perforation plates types': 'perforation_plates_types', 'prismatic crystals': 'prismatic_crystals', 'radial secretory canals': 'radial_secretory_canals', 'radial tracheids for gymnosperms': 'radial_tracheids_for_gymnosperms', 'rays': 'rays', 'rays cellular composition': 'rays_cellular_composition', 'ray height': 'ray_height', 'rays sheath cells': 'rays_sheath_cells', 'rays structure': 'rays_structure', 'ray width': 'ray_width', 'ray type': 'ray_type', 'Reference Specimens': 'reference_specimens', 'silica': 'silica', 'solitary vessels with angular outline': 'solitary_vessels_with_angular_outline', 'Species': 'species', 'spetate fibres present': 'spetate_fibres_present', 'storied structure': 'storied_structure', 'tile cells': 'tile_cells', 'tracheid diameter': 'tracheid_diameter', 'vascular-vasicentric tracheids present': 'vascularvasicentric_tracheids_present', 'vessels': 'vessels', 'vessel arrangement': 'vessel_arrangement', 'vessels deposits': 'vessels_deposits', 'vessel grouping': 'vessel_grouping', 'vessel porosity': 'vessel_porosity', 'vessels rays pitting': 'vessels_rays_pitting', 'vessels tyloses': 'vessels_tyloses', 'walls': 'walls', 'early late wood transition': 'early_late_wood_transition', 'axiel resin canals': 'axiel_resin_canals', 'epithelial cells': 'epithelial_cells', 'axial tracheid pits': 'axial_tracheid_pits', 'spiral thickenings': 'spiral_thickenings', 'crassulae': 'crassulae', 'nodular tangential ray walls': 'nodular_tangential_ray_walls', 'early wood ray pits': 'early_wood_ray_pits', 'late wood ray pits': 'late_wood_ray_pits', }) def link(accession, wood_details): if hasattr(accession, 'woodfeatures'): accession.woodfeatures.delete() wood_details.accession = accession wood_details.save() accession.material = u"Wood" accession.save() bi.add_linking_function(link) return bi