def test_monitoring_check_es_db_counts(app, client, agent_idref_record):
"""Test monitoring check_es_db_counts."""
res = client.get(url_for('api_monitoring.check_es_db_counts'))
assert res.status_code == 200
assert json.loads(res.get_data(as_text=True)) == {
'data': {
'status': 'green'
}
}
pers = AgentIdrefRecord.create(data=agent_idref_record,
delete_pid=False,
dbcommit=True,
reindex=False)
AgentIdrefRecord.update_indexes()
res = client.get(url_for('api_monitoring.check_es_db_counts'))
assert res.status_code == 200
assert json.loads(res.get_data(as_text=True)) == {
'data': {
'status': 'red'
},
'errors': [{
'code': 'DB_ES_COUNTER_MISSMATCH',
'details': 'There are 1 items from aidref missing in ES.',
'id': 'DB_ES_COUNTER_MISSMATCH',
'links': {
'about': 'http://localhost/monitoring/check_es_db_counts',
'aidref': 'http://localhost/monitoring/missing_pids/aidref'
},
'title': "DB items counts don't match ES items count."
}]
}
# this view is only accessible by admin
res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref'))
assert res.status_code == 401
create_and_login_monitoring_user(app, client)
res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref'))
assert res.status_code == 200
assert json.loads(res.get_data(as_text=True)) == {
'data': {
'DB': [],
'ES': ['http://localhost/agents/idref/069774331'],
'ES duplicate': []
}
}
def test_view_agents_idref(client, agent_idref_record):
"""Test redirect IDREF."""
AgentIdrefRecord.create(
data=agent_idref_record,
delete_pid=False,
dbcommit=True,
reindex=True,
)
AgentIdrefRecord.update_indexes()
pid = agent_idref_record.get('pid')
url = url_for('api_agents_idref.redirect_list')
res = client.get(url)
assert res.status_code == 308
res = client.get(url, follow_redirects=True)
assert res.status_code == 200
assert json.loads(res.get_data(as_text=True))['aggregations'] == {
'agent_type': {
'buckets': [{
'doc_count': 1,
'key': 'bf:Person'
}],
'doc_count_error_upper_bound': 0,
'sum_other_doc_count': 0
},
'deleted': {
'doc_count': 0
}
}
url = url_for('api_agents_idref.redirect_item', pid=pid)
res = client.get(url)
assert res.status_code == 308
res = client.get(url, follow_redirects=True)
assert res.status_code == 200
data = json.loads(res.get_data(as_text=True))
assert data.get('id') == pid
assert data.get('metadata', {}).get('pid') == pid
url = url_for('api_agents_idref.redirect_item', pid='WRONG')
res = client.get(url, follow_redirects=True)
assert res.status_code == 404
assert json.loads(res.get_data(as_text=True)) == {
"status": 404,
"message": "PID does not exist."
}
def test_create_agent_record_no_viaf_links(app, agent_gnd_record,
agent_rero_record,
agent_idref_record):
"""Test create agent record without VIAF links."""
record, action, m_record, m_action, v_record, online = \
AgentGndRecord.create_or_update_agent_mef_viaf(
data=agent_gnd_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '12391664X'
assert m_action.name == 'CREATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {
'$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
},
'pid': '1',
}
record, action, m_record, m_action, v_record, online = \
AgentReroRecord.create_or_update_agent_mef_viaf(
data=agent_rero_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == 'A023655346'
assert m_action.name == 'CREATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'pid': '2',
'rero': {
'$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
},
}
record, action, m_record, m_action, v_record, online = \
AgentIdrefRecord.create_or_update_agent_mef_viaf(
data=agent_idref_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '069774331'
assert m_action.name == 'CREATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'idref': {
'$ref': 'https://mef.rero.ch/api/agents/idref/069774331'
},
'pid': '3',
}
def test_ref_resolvers(app, gnd_record, rero_record, viaf_record,
idref_record):
"""Test ref resolvers."""
# VIAF record
record, action = ViafRecord.create_or_update(data=viaf_record,
dbcommit=True,
reindex=True)
viaf_pid = record['pid']
# GND record
record, action, m_record, m_action, v_record, online = \
AgentGndRecord.create_or_update_agent_mef_viaf(
data=gnd_record,
dbcommit=True,
reindex=True,
online=False
)
gnd_pid = record.get('pid')
# RERO record
record, action, m_record, m_action, v_record, online = \
AgentReroRecord.create_or_update_agent_mef_viaf(
data=rero_record,
dbcommit=True,
reindex=True,
online=False
)
rero_pid = record.get('pid')
# IDREF record
record, action, m_record, m_action, v_record, online = \
AgentIdrefRecord.create_or_update_agent_mef_viaf(
data=idref_record,
dbcommit=True,
reindex=True,
online=False
)
idref_pid = record.get('pid')
# MEF record
mef_rec_resolved = MefRecord.get_mef_by_viaf_pid(viaf_pid=viaf_pid)
mef_rec_resolved = mef_rec_resolved.replace_refs()
assert mef_rec_resolved.get('gnd').get('pid') == gnd_pid
assert mef_rec_resolved.get('rero').get('pid') == rero_pid
assert mef_rec_resolved.get('idref').get('pid') == idref_pid
def test_update_agent_record_with_viaf_links(
app, viaf_record, gnd_record, rero_record, idref_record):
"""Test create agent record with viaf links."""
returned_record, action = AgentGndRecord.create_or_update(
gnd_record, dbcommit=True, reindex=True
)
assert action.name == 'UPDATE'
assert returned_record['pid'] == '12391664X'
returned_record, action = AgentReroRecord.create_or_update(
rero_record, dbcommit=True, reindex=True
)
assert action.name == 'UPDATE'
assert returned_record['pid'] == 'A023655346'
returned_record, action = AgentIdrefRecord.create_or_update(
idref_record, dbcommit=True, reindex=True
)
assert action.name == 'UPDATE'
assert returned_record['pid'] == '069774331'
def test_uptodate_agent_record_with_viaf_links_md5(app, agent_viaf_record,
agent_gnd_record,
agent_rero_record,
agent_idref_record):
"""Test create agent record with VIAF links."""
returned_record, action = AgentGndRecord.create_or_update(agent_gnd_record,
dbcommit=True,
reindex=True,
test_md5=True)
assert action.name == 'UPTODATE'
assert returned_record['pid'] == '12391664X'
returned_record, action = AgentReroRecord.create_or_update(
agent_rero_record, dbcommit=True, reindex=True, test_md5=True)
assert action.name == 'UPTODATE'
assert returned_record['pid'] == 'A023655346'
returned_record, action = AgentIdrefRecord.create_or_update(
agent_idref_record, dbcommit=True, reindex=True, test_md5=True)
assert action.name == 'UPTODATE'
assert returned_record['pid'] == '069774331'
def test_create_agent_updates(mock_get, app, gnd_record, rero_record,
idref_record):
"""Test create agent record with viaf links."""
# we have to mock the access to viaf
mock_get.return_value = {
'pid': '37268949',
'idref_pid': '069774331',
'gnd_pid': '100769527xxx'
}
# create first record no viaf and mef records exist
assert MefRecord.count() == 0
assert ViafRecord.count() == 0
record, action, m_record, m_action, v_record, online = \
AgentIdrefRecord.create_or_update_agent_mef_viaf(
data=idref_record,
dbcommit=True,
reindex=True,
online=True
)
assert action.name == 'CREATE'
assert record['pid'] == '069774331'
assert MefRecord.count() == 1
assert ViafRecord.count() == 1
assert online
# we should have a mef and viaf record now
mef_pid = list(MefRecord.get_all_pids())[-1]
mef_record = MefRecord.get_record_by_pid(mef_pid)
assert mef_record.get('pid') == '1'
assert mef_record.get('viaf_pid') == '37268949'
assert 'idref' in mef_record
viaf_pid = list(ViafRecord.get_all_pids())[-1]
viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
assert viaf_record.get('pid') == '37268949'
assert viaf_record.get('idref_pid') == '069774331'
# we have to mock the access to viaf
mock_get.return_value = {
'pid': '66739143',
'gnd_pid': '12391664X',
'idref_pid': '068979401',
'rero_pid': 'A023655346'
}
# create second record
record, action, m_record, m_action, v_record, online = \
AgentGndRecord.create_or_update_agent_mef_viaf(
data=gnd_record,
dbcommit=True,
reindex=True,
online=True
)
mef_pid = list(MefRecord.get_all_pids())[-1]
mef_record = MefRecord.get_record_by_pid(mef_pid)
assert mef_record.get('pid') == '2'
assert mef_record.get('viaf_pid') == '66739143'
assert 'gnd' in mef_record
viaf_pid = list(ViafRecord.get_all_pids())[-1]
viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
assert viaf_record.get('pid') == '66739143'
assert viaf_record.get('gnd_pid') == '12391664X'
record, action, m_record, m_action, v_record, online = \
AgentReroRecord.create_or_update_agent_mef_viaf(
data=rero_record,
dbcommit=True,
reindex=True,
online=True
)
mef_pid = list(MefRecord.get_all_pids())[-1]
mef_record = MefRecord.get_record_by_pid(mef_pid)
assert mef_record.get('pid') == '2'
assert mef_record.get('viaf_pid') == '66739143'
assert 'rero' in mef_record
viaf_pid = list(ViafRecord.get_all_pids())[-1]
viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
assert viaf_record.get('pid') == '66739143'
assert viaf_record.get('rero_pid') == 'A023655346'
mef_record = MefRecord.get_mef_by_viaf_pid(viaf_pid)
assert mef_record == {
'pid': '2',
'viaf_pid': '66739143',
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {
'$ref': 'https://mef.rero.ch/api/gnd/12391664X'
},
'rero': {
'$ref': 'https://mef.rero.ch/api/rero/A023655346'
}
}
new_viaf_pid = 'xxxxxxxx'
new_viaf_record = {
'pid': new_viaf_pid,
'gnd_pid': '12391664X',
'idref_pid': '069774331',
'rero_pid': 'A023655346'
}
mock_get.return_value = new_viaf_record
rec, msg = viaf_record.delete(dbcommit=True, delindex=True, online=True)
assert ViafRecord.get_record_by_pid(new_viaf_pid) == new_viaf_record
mef_record = MefRecord.get_mef_by_viaf_pid(new_viaf_pid)
assert mef_record == {
'viaf_pid': 'xxxxxxxx',
'gnd': {
'$ref': 'https://mef.rero.ch/api/gnd/12391664X'
},
'rero': {
'$ref': 'https://mef.rero.ch/api/rero/A023655346'
},
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'pid': '3'
}
def test_monitoring(app, agent_idref_record, script_info):
"""Test monitoring."""
cli_output = [
'DB - ES type count index count_es',
'----------------------------------------------------------------',
' 0 aggnd 0 agents_gnd 0',
' 0 agrero 0 agents_rero 0',
' 1 aidref 1 agents_idref 0',
' 0 comef 0 concepts_mef 0',
' 0 corero 0 concepts_rero 0',
' 0 mef 0 mef 0',
' 0 viaf 0 viaf 0'
]
mon = Monitoring()
assert mon.get_es_count('xxx') == 'No >>xxx<< in ES'
assert mon.get_db_count('xxx') == 'No >>xxx<< in DB'
idref = AgentIdrefRecord.create(data=agent_idref_record,
delete_pid=False,
dbcommit=True,
reindex=False)
idref_pid = idref.pid
assert mon.get_db_count('aidref') == 1
assert mon.get_es_count('agents_idref') == 0
assert mon.check() == {'aidref': {'db_es': 1}}
assert mon.missing('aidref') == {
'DB': [],
'ES': ['069774331'],
'ES duplicate': []
}
# not flushed by default
assert mon.info() == {
'aggnd': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'agents_gnd'
},
'agrero': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'agents_rero'
},
'aidref': {
'db': 1,
'db-es': 1,
'es': 0,
'index': 'agents_idref'
},
'comef': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'concepts_mef'
},
'corero': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'concepts_rero'
},
'mef': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'mef'
},
'viaf': {
'db': 0,
'db-es': 0,
'es': 0,
'index': 'viaf'
}
}
assert mon.__str__().split('\n') == cli_output + ['']
runner = CliRunner()
res = runner.invoke(es_db_missing_cli, ['aidref'], obj=script_info)
assert res.output == 'aidref: pids missing in ES:\n069774331\n'
runner = CliRunner()
res = runner.invoke(es_db_counts_cli, ['-m'], obj=script_info)
assert res.output.split(
'\n') == cli_output + ['aidref: pids missing in ES:', '069774331', '']
# we have to get the idref again because we lost the session after the use
# of the CliRunner
idref = AgentIdrefRecord.get_record_by_pid(idref_pid)
idref.reindex()
AgentIdrefRecord.update_indexes()
assert mon.get_es_count('agents_idref') == 1
assert mon.check() == {}
assert mon.missing('aidref') == {'DB': [], 'ES': [], 'ES duplicate': []}
idref.delete(dbcommit=True)
assert mon.get_db_count('aidref') == 0
assert mon.get_es_count('agents_idref') == 1
assert mon.check() == {'aidref': {'db_es': -1}}
assert mon.missing('aidref') == {
'DB': ['069774331'],
'ES': [],
'ES duplicate': []
}
def test_create_agent_record_with_viaf_links(
app, viaf_record, gnd_record, rero_record, idref_record):
"""Test create agent record with viaf links."""
returned_record, action = ViafRecord.create_or_update(
viaf_record, dbcommit=True, reindex=True
)
ViafRecord.update_indexes()
assert action.name == 'CREATE'
assert returned_record['pid'] == '66739143'
assert returned_record['gnd_pid'] == '12391664X'
assert returned_record['rero_pid'] == 'A023655346'
assert returned_record['idref_pid'] == '069774331'
record, action, m_record, m_action, v_record, online = \
AgentGndRecord.create_or_update_agent_mef_viaf(
data=gnd_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '12391664X'
assert m_action.name == 'CREATE'
assert m_record == {
'$schema':
'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
'pid': '1',
'viaf_pid': '66739143'
}
record, action, m_record, m_action, v_record, online = \
AgentReroRecord.create_or_update_agent_mef_viaf(
data=rero_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == 'A023655346'
assert m_action.name == 'UPDATE'
assert m_record == {
'$schema':
'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
'pid': '1',
'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'},
'viaf_pid': '66739143'
}
record, action, m_record, m_action, v_record, online = \
AgentIdrefRecord.create_or_update_agent_mef_viaf(
data=idref_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '069774331'
assert m_action.name == 'UPDATE'
assert m_record == {
'$schema':
'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
'idref': {'$ref': 'https://mef.rero.ch/api/idref/069774331'},
'pid': '1',
'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'},
'viaf_pid': '66739143'
}
def test_create_agent_record_with_viaf_links(app, agent_viaf_record,
agent_gnd_record,
agent_rero_record,
agent_idref_record):
"""Test create agent record with VIAF links."""
viaf_record, action = AgentViafRecord.create_or_update(agent_viaf_record,
dbcommit=True,
reindex=True)
AgentViafRecord.update_indexes()
assert action.name == 'CREATE'
assert viaf_record['pid'] == '66739143'
assert viaf_record['gnd_pid'] == '12391664X'
assert viaf_record['rero_pid'] == 'A023655346'
assert viaf_record['idref_pid'] == '069774331'
record, action, m_record, m_action, v_record, online = \
AgentGndRecord.create_or_update_agent_mef_viaf(
data=agent_gnd_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '12391664X'
assert m_action.name == 'CREATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {
'$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
},
'pid': '1',
'viaf_pid': '66739143'
}
record, action, m_record, m_action, v_record, online = \
AgentReroRecord.create_or_update_agent_mef_viaf(
data=agent_rero_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == 'A023655346'
assert m_action.name == 'UPDATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {
'$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
},
'pid': '1',
'rero': {
'$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
},
'viaf_pid': '66739143'
}
record, action, m_record, m_action, v_record, online = \
AgentIdrefRecord.create_or_update_agent_mef_viaf(
data=agent_idref_record,
dbcommit=True,
reindex=True,
online=False
)
assert action.name == 'CREATE'
assert record['pid'] == '069774331'
assert m_action.name == 'UPDATE'
assert m_record == {
'$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
'gnd': {
'$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
},
'idref': {
'$ref': 'https://mef.rero.ch/api/agents/idref/069774331'
},
'pid': '1',
'rero': {
'$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
},
'viaf_pid': '66739143'
}
assert m_record == AgentMefRecord.get_mef_by_entity_pid(
record.pid, 'idref')
assert m_record.pid == AgentMefRecord.get_mef_by_entity_pid(record.pid,
'idref',
pid_only=True)
assert AgentMefRecord.get_mef_by_viaf_pid(viaf_record.pid) == m_record
def test_reromefrecord_api(app, agent_idref_record):
"""Test ReroMefRecord api."""
idref = AgentIdrefRecord.create(
data=agent_idref_record,
delete_pid=False,
dbcommit=True,
reindex=True,
)
AgentIdrefRecord.update_indexes()
assert AgentIdrefRecord.count() == 1
assert AgentIdrefRecord.index_all() == 1
assert len(list(AgentIdrefRecord.get_all_records())) == len(
list(AgentIdrefRecord.get_all_records(limit=0)))
assert AgentIdrefRecord.get_metadata_identifier_names() == (
'agent_idref_metadata', 'agent_idref_id')
count = 0
for _ in AgentIdrefRecord.get_all_records():
count += 1
assert count == 1
_, agent_action = idref.update_test_md5(data=idref,
dbcommit=True,
reindex=True)
assert agent_action == Action.UPTODATE
idref['gender'] = 'female'
_, agent_action = idref.update_test_md5(data=idref,
dbcommit=True,
reindex=True)
assert agent_action == Action.UPDATE
assert AgentIdrefRecord.get_pid_by_id(idref.id) == idref.pid
AgentIdrefIndexer().bulk_index(list(AgentIdrefRecord.get_all_ids()))
process_bulk_queue(stats_only=True)