def test_get_synonyms_map(self):
synonym_key = "synonym_key_1"
TaxonomyManager.register_synonym_key("synonym_key_1")
data = [
{
'id': 0,
'full_name': 'Family One',
'rank_name': 'One',
'rank': niamoto_db_meta.TaxonRankEnum.FAMILIA,
'parent_id': None,
'synonyms': {
synonym_key: 10,
},
'mptt_left': 0,
'mptt_right': 0,
'mptt_tree_id': 0,
'mptt_depth': 0,
},
{
'id': 1,
'full_name': 'Family Two',
'rank_name': 'Two',
'rank': niamoto_db_meta.TaxonRankEnum.FAMILIA,
'parent_id': None,
'synonyms': {
synonym_key: 20,
},
'mptt_left': 0,
'mptt_right': 0,
'mptt_tree_id': 0,
'mptt_depth': 0,
},
]
ins = niamoto_db_meta.taxon.insert().values(data)
with Connector.get_connection() as connection:
connection.execute(ins)
synonyms = TaxonomyManager.get_synonyms_for_key(synonym_key)
self.assertEqual(synonyms.loc[10], 0)
self.assertEqual(synonyms.loc[20], 1)
def map_provider_taxon_ids(self, dataframe):
"""
Map provider's taxon ids with Niamoto taxon ids when importing data.
:param dataframe: The dataframe where the mapping has to be done.
ids. The index must correspond to the provider's pk. The dataframe
corresponds to the provider's dataframe.
:return: A series with the same index, the niamoto corresponding
taxon id as values.
"""
m = "(provider_id='{}', synonym_key='{}'): " \
"Mapping provider's taxon ids..."
LOGGER.debug(
m.format(self.data_provider.db_id, self.data_provider.synonym_key))
synonyms = TaxonomyManager.get_synonyms_for_key(
self.data_provider.synonym_key)
dataframe["provider_taxon_id"] = dataframe["taxon_id"]
dataframe["taxon_id"] = dataframe["taxon_id"].map(synonyms)
m = "(provider_id='{}', synonym_key='{}'): {} taxon ids had " \
"been mapped."
LOGGER.debug(
m.format(self.data_provider.db_id, self.data_provider.synonym_key,
len(synonyms)))
def update_synonym_mapping(self, connection=None):
"""
Update the synonym mapping of an already stored dataframe.
To be called when a synonym had been defined or modified, but not
the occurrences.
:param connection: If passed, use an existing connection.
"""
# Log start
m = "(provider_id='{}', synonym_key='{}'): Updating synonym " \
"mapping..."
LOGGER.debug(
m.format(self.data_provider.db_id, self.data_provider.synonym_key))
close_after = False
if connection is None:
connection = Connector.get_engine().connect()
close_after = True
# Start
df = self.get_niamoto_occurrence_dataframe(connection)
if close_after:
connection.close()
synonyms = TaxonomyManager.get_synonyms_for_key(
self.data_provider.synonym_key)
mapping = df["provider_taxon_id"].map(synonyms)
if len(df) > 0:
df["taxon_id"] = mapping
df = df[['provider_id', 'provider_pk', 'taxon_id']]
s = io.StringIO()
df.where((pd.notnull(df)), None).rename(columns={
'provider_id': 'prov_id',
'provider_pk': 'prov_pk',
}).to_csv(s, columns=['taxon_id', 'prov_id', 'prov_pk'])
s.seek(0)
sql_create_temp = \
"""
DROP TABLE IF EXISTS {tmp};
CREATE TABLE {tmp} (
id float,
taxon_id float,
prov_id float,
prov_pk float
);
""".format(**{
'tmp': 'tmp_niamoto'
})
sql_copy_from = \
"""
COPY {tmp} FROM STDIN CSV HEADER DELIMITER ',';
""".format(**{
'tmp': 'tmp_niamoto'
})
sql_update = \
"""
UPDATE {occurrence_table}
SET taxon_id = {tmp}.taxon_id::int
FROM {tmp}
WHERE {occurrence_table}.provider_id = {tmp}.prov_id::int
AND {occurrence_table}.provider_pk = {tmp}.prov_pk::int;
DROP TABLE {tmp};
""".format(**{
'tmp': 'tmp_niamoto',
'occurrence_table': '{}.{}'.format(
settings.NIAMOTO_SCHEMA, occurrence.name
)
})
raw_connection = Connector.get_engine().raw_connection()
cur = raw_connection.cursor()
cur.execute(sql_create_temp)
cur.copy_expert(sql_copy_from, s)
cur.execute(sql_update)
cur.close()
raw_connection.commit()
raw_connection.close()
# Log end
m = "(provider_id='{}', synonym_key='{}'): {} synonym mapping had " \
"been updated."
LOGGER.debug(
m.format(self.data_provider.db_id, self.data_provider.synonym_key,
len(synonyms)))
return mapping, synonyms
def test_set_taxonomy(self):
result, synonyms = TaxonomyManager.set_taxonomy(
pd.DataFrame(
columns=['full_name', 'rank_name', 'rank', 'parent_id'], ))
self.assertEqual(result, 0)
data = pd.DataFrame.from_records([
{
'id': 0,
'full_name': 'Family One',
'rank_name': 'One',
'rank': niamoto_db_meta.TaxonRankEnum.FAMILIA,
'parent_id': None,
},
],
index='id')
result, synonyms = TaxonomyManager.set_taxonomy(data)
self.assertEqual(result, 1)
data = pd.DataFrame.from_records([
{
'id': 0,
'full_name': 'Family One',
'rank_name': 'One',
'rank': niamoto_db_meta.TaxonRankEnum.FAMILIA,
'parent_id': None,
'gbif': 5,
'taxref': 1,
},
{
'id': 1,
'full_name': 'Genus Two',
'rank_name': 'Two',
'rank': niamoto_db_meta.TaxonRankEnum.GENUS,
'parent_id': 0,
'gbif': 10,
'taxref': 2,
},
{
'id': 2,
'full_name': 'Species Three',
'rank_name': 'Three',
'rank': niamoto_db_meta.TaxonRankEnum.SPECIES,
'parent_id': None,
'gbif': 7,
'taxref': 3,
},
],
index='id')
result, synonyms = TaxonomyManager.set_taxonomy(data)
self.assertEqual(result, 3)
df = TaxonomyManager.get_raw_taxon_dataframe()
self.assertEqual(len(df), 3)
synonym_keys = TaxonomyManager.get_synonym_keys()
self.assertEqual(len(synonym_keys), 2)
identity_synonyms = TaxonomyManager.get_synonyms_for_key(
TaxonomyManager.IDENTITY_SYNONYM_KEY)
self.assertEqual(len(identity_synonyms), 3)
null_synonyms = TaxonomyManager.get_synonyms_for_key(None)
self.assertEqual(len(null_synonyms), 0)
gbif_synonyms = TaxonomyManager.get_synonyms_for_key("gbif")
self.assertEqual(len(gbif_synonyms), 3)
taxref_synonyms = TaxonomyManager.get_synonyms_for_key('taxref')
self.assertEqual(len(taxref_synonyms), 3)