def test_monitoring_check_es_db_counts(app, client, agent_idref_record): """Test monitoring check_es_db_counts.""" res = client.get(url_for('api_monitoring.check_es_db_counts')) assert res.status_code == 200 assert json.loads(res.get_data(as_text=True)) == { 'data': { 'status': 'green' } } pers = AgentIdrefRecord.create(data=agent_idref_record, delete_pid=False, dbcommit=True, reindex=False) AgentIdrefRecord.update_indexes() res = client.get(url_for('api_monitoring.check_es_db_counts')) assert res.status_code == 200 assert json.loads(res.get_data(as_text=True)) == { 'data': { 'status': 'red' }, 'errors': [{ 'code': 'DB_ES_COUNTER_MISSMATCH', 'details': 'There are 1 items from aidref missing in ES.', 'id': 'DB_ES_COUNTER_MISSMATCH', 'links': { 'about': 'http://localhost/monitoring/check_es_db_counts', 'aidref': 'http://localhost/monitoring/missing_pids/aidref' }, 'title': "DB items counts don't match ES items count." }] } # this view is only accessible by admin res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref')) assert res.status_code == 401 create_and_login_monitoring_user(app, client) res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref')) assert res.status_code == 200 assert json.loads(res.get_data(as_text=True)) == { 'data': { 'DB': [], 'ES': ['http://localhost/agents/idref/069774331'], 'ES duplicate': [] } }
def test_view_agents_idref(client, agent_idref_record): """Test redirect IDREF.""" AgentIdrefRecord.create( data=agent_idref_record, delete_pid=False, dbcommit=True, reindex=True, ) AgentIdrefRecord.update_indexes() pid = agent_idref_record.get('pid') url = url_for('api_agents_idref.redirect_list') res = client.get(url) assert res.status_code == 308 res = client.get(url, follow_redirects=True) assert res.status_code == 200 assert json.loads(res.get_data(as_text=True))['aggregations'] == { 'agent_type': { 'buckets': [{ 'doc_count': 1, 'key': 'bf:Person' }], 'doc_count_error_upper_bound': 0, 'sum_other_doc_count': 0 }, 'deleted': { 'doc_count': 0 } } url = url_for('api_agents_idref.redirect_item', pid=pid) res = client.get(url) assert res.status_code == 308 res = client.get(url, follow_redirects=True) assert res.status_code == 200 data = json.loads(res.get_data(as_text=True)) assert data.get('id') == pid assert data.get('metadata', {}).get('pid') == pid url = url_for('api_agents_idref.redirect_item', pid='WRONG') res = client.get(url, follow_redirects=True) assert res.status_code == 404 assert json.loads(res.get_data(as_text=True)) == { "status": 404, "message": "PID does not exist." }
def test_create_agent_record_no_viaf_links(app, agent_gnd_record, agent_rero_record, agent_idref_record): """Test create agent record without VIAF links.""" record, action, m_record, m_action, v_record, online = \ AgentGndRecord.create_or_update_agent_mef_viaf( data=agent_gnd_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '12391664X' assert m_action.name == 'CREATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': { '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X' }, 'pid': '1', } record, action, m_record, m_action, v_record, online = \ AgentReroRecord.create_or_update_agent_mef_viaf( data=agent_rero_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == 'A023655346' assert m_action.name == 'CREATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'pid': '2', 'rero': { '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346' }, } record, action, m_record, m_action, v_record, online = \ AgentIdrefRecord.create_or_update_agent_mef_viaf( data=agent_idref_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '069774331' assert m_action.name == 'CREATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'idref': { '$ref': 'https://mef.rero.ch/api/agents/idref/069774331' }, 'pid': '3', }
def test_ref_resolvers(app, gnd_record, rero_record, viaf_record, idref_record): """Test ref resolvers.""" # VIAF record record, action = ViafRecord.create_or_update(data=viaf_record, dbcommit=True, reindex=True) viaf_pid = record['pid'] # GND record record, action, m_record, m_action, v_record, online = \ AgentGndRecord.create_or_update_agent_mef_viaf( data=gnd_record, dbcommit=True, reindex=True, online=False ) gnd_pid = record.get('pid') # RERO record record, action, m_record, m_action, v_record, online = \ AgentReroRecord.create_or_update_agent_mef_viaf( data=rero_record, dbcommit=True, reindex=True, online=False ) rero_pid = record.get('pid') # IDREF record record, action, m_record, m_action, v_record, online = \ AgentIdrefRecord.create_or_update_agent_mef_viaf( data=idref_record, dbcommit=True, reindex=True, online=False ) idref_pid = record.get('pid') # MEF record mef_rec_resolved = MefRecord.get_mef_by_viaf_pid(viaf_pid=viaf_pid) mef_rec_resolved = mef_rec_resolved.replace_refs() assert mef_rec_resolved.get('gnd').get('pid') == gnd_pid assert mef_rec_resolved.get('rero').get('pid') == rero_pid assert mef_rec_resolved.get('idref').get('pid') == idref_pid
def test_update_agent_record_with_viaf_links( app, viaf_record, gnd_record, rero_record, idref_record): """Test create agent record with viaf links.""" returned_record, action = AgentGndRecord.create_or_update( gnd_record, dbcommit=True, reindex=True ) assert action.name == 'UPDATE' assert returned_record['pid'] == '12391664X' returned_record, action = AgentReroRecord.create_or_update( rero_record, dbcommit=True, reindex=True ) assert action.name == 'UPDATE' assert returned_record['pid'] == 'A023655346' returned_record, action = AgentIdrefRecord.create_or_update( idref_record, dbcommit=True, reindex=True ) assert action.name == 'UPDATE' assert returned_record['pid'] == '069774331'
def test_uptodate_agent_record_with_viaf_links_md5(app, agent_viaf_record, agent_gnd_record, agent_rero_record, agent_idref_record): """Test create agent record with VIAF links.""" returned_record, action = AgentGndRecord.create_or_update(agent_gnd_record, dbcommit=True, reindex=True, test_md5=True) assert action.name == 'UPTODATE' assert returned_record['pid'] == '12391664X' returned_record, action = AgentReroRecord.create_or_update( agent_rero_record, dbcommit=True, reindex=True, test_md5=True) assert action.name == 'UPTODATE' assert returned_record['pid'] == 'A023655346' returned_record, action = AgentIdrefRecord.create_or_update( agent_idref_record, dbcommit=True, reindex=True, test_md5=True) assert action.name == 'UPTODATE' assert returned_record['pid'] == '069774331'
def test_create_agent_updates(mock_get, app, gnd_record, rero_record, idref_record): """Test create agent record with viaf links.""" # we have to mock the access to viaf mock_get.return_value = { 'pid': '37268949', 'idref_pid': '069774331', 'gnd_pid': '100769527xxx' } # create first record no viaf and mef records exist assert MefRecord.count() == 0 assert ViafRecord.count() == 0 record, action, m_record, m_action, v_record, online = \ AgentIdrefRecord.create_or_update_agent_mef_viaf( data=idref_record, dbcommit=True, reindex=True, online=True ) assert action.name == 'CREATE' assert record['pid'] == '069774331' assert MefRecord.count() == 1 assert ViafRecord.count() == 1 assert online # we should have a mef and viaf record now mef_pid = list(MefRecord.get_all_pids())[-1] mef_record = MefRecord.get_record_by_pid(mef_pid) assert mef_record.get('pid') == '1' assert mef_record.get('viaf_pid') == '37268949' assert 'idref' in mef_record viaf_pid = list(ViafRecord.get_all_pids())[-1] viaf_record = ViafRecord.get_record_by_pid(viaf_pid) assert viaf_record.get('pid') == '37268949' assert viaf_record.get('idref_pid') == '069774331' # we have to mock the access to viaf mock_get.return_value = { 'pid': '66739143', 'gnd_pid': '12391664X', 'idref_pid': '068979401', 'rero_pid': 'A023655346' } # create second record record, action, m_record, m_action, v_record, online = \ AgentGndRecord.create_or_update_agent_mef_viaf( data=gnd_record, dbcommit=True, reindex=True, online=True ) mef_pid = list(MefRecord.get_all_pids())[-1] mef_record = MefRecord.get_record_by_pid(mef_pid) assert mef_record.get('pid') == '2' assert mef_record.get('viaf_pid') == '66739143' assert 'gnd' in mef_record viaf_pid = list(ViafRecord.get_all_pids())[-1] viaf_record = ViafRecord.get_record_by_pid(viaf_pid) assert viaf_record.get('pid') == '66739143' assert viaf_record.get('gnd_pid') == '12391664X' record, action, m_record, m_action, v_record, online = \ AgentReroRecord.create_or_update_agent_mef_viaf( data=rero_record, dbcommit=True, reindex=True, online=True ) mef_pid = list(MefRecord.get_all_pids())[-1] mef_record = MefRecord.get_record_by_pid(mef_pid) assert mef_record.get('pid') == '2' assert mef_record.get('viaf_pid') == '66739143' assert 'rero' in mef_record viaf_pid = list(ViafRecord.get_all_pids())[-1] viaf_record = ViafRecord.get_record_by_pid(viaf_pid) assert viaf_record.get('pid') == '66739143' assert viaf_record.get('rero_pid') == 'A023655346' mef_record = MefRecord.get_mef_by_viaf_pid(viaf_pid) assert mef_record == { 'pid': '2', 'viaf_pid': '66739143', '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': { '$ref': 'https://mef.rero.ch/api/gnd/12391664X' }, 'rero': { '$ref': 'https://mef.rero.ch/api/rero/A023655346' } } new_viaf_pid = 'xxxxxxxx' new_viaf_record = { 'pid': new_viaf_pid, 'gnd_pid': '12391664X', 'idref_pid': '069774331', 'rero_pid': 'A023655346' } mock_get.return_value = new_viaf_record rec, msg = viaf_record.delete(dbcommit=True, delindex=True, online=True) assert ViafRecord.get_record_by_pid(new_viaf_pid) == new_viaf_record mef_record = MefRecord.get_mef_by_viaf_pid(new_viaf_pid) assert mef_record == { 'viaf_pid': 'xxxxxxxx', 'gnd': { '$ref': 'https://mef.rero.ch/api/gnd/12391664X' }, 'rero': { '$ref': 'https://mef.rero.ch/api/rero/A023655346' }, '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'pid': '3' }
def test_monitoring(app, agent_idref_record, script_info): """Test monitoring.""" cli_output = [ 'DB - ES type count index count_es', '----------------------------------------------------------------', ' 0 aggnd 0 agents_gnd 0', ' 0 agrero 0 agents_rero 0', ' 1 aidref 1 agents_idref 0', ' 0 comef 0 concepts_mef 0', ' 0 corero 0 concepts_rero 0', ' 0 mef 0 mef 0', ' 0 viaf 0 viaf 0' ] mon = Monitoring() assert mon.get_es_count('xxx') == 'No >>xxx<< in ES' assert mon.get_db_count('xxx') == 'No >>xxx<< in DB' idref = AgentIdrefRecord.create(data=agent_idref_record, delete_pid=False, dbcommit=True, reindex=False) idref_pid = idref.pid assert mon.get_db_count('aidref') == 1 assert mon.get_es_count('agents_idref') == 0 assert mon.check() == {'aidref': {'db_es': 1}} assert mon.missing('aidref') == { 'DB': [], 'ES': ['069774331'], 'ES duplicate': [] } # not flushed by default assert mon.info() == { 'aggnd': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'agents_gnd' }, 'agrero': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'agents_rero' }, 'aidref': { 'db': 1, 'db-es': 1, 'es': 0, 'index': 'agents_idref' }, 'comef': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'concepts_mef' }, 'corero': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'concepts_rero' }, 'mef': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'mef' }, 'viaf': { 'db': 0, 'db-es': 0, 'es': 0, 'index': 'viaf' } } assert mon.__str__().split('\n') == cli_output + [''] runner = CliRunner() res = runner.invoke(es_db_missing_cli, ['aidref'], obj=script_info) assert res.output == 'aidref: pids missing in ES:\n069774331\n' runner = CliRunner() res = runner.invoke(es_db_counts_cli, ['-m'], obj=script_info) assert res.output.split( '\n') == cli_output + ['aidref: pids missing in ES:', '069774331', ''] # we have to get the idref again because we lost the session after the use # of the CliRunner idref = AgentIdrefRecord.get_record_by_pid(idref_pid) idref.reindex() AgentIdrefRecord.update_indexes() assert mon.get_es_count('agents_idref') == 1 assert mon.check() == {} assert mon.missing('aidref') == {'DB': [], 'ES': [], 'ES duplicate': []} idref.delete(dbcommit=True) assert mon.get_db_count('aidref') == 0 assert mon.get_es_count('agents_idref') == 1 assert mon.check() == {'aidref': {'db_es': -1}} assert mon.missing('aidref') == { 'DB': ['069774331'], 'ES': [], 'ES duplicate': [] }
def test_create_agent_record_with_viaf_links( app, viaf_record, gnd_record, rero_record, idref_record): """Test create agent record with viaf links.""" returned_record, action = ViafRecord.create_or_update( viaf_record, dbcommit=True, reindex=True ) ViafRecord.update_indexes() assert action.name == 'CREATE' assert returned_record['pid'] == '66739143' assert returned_record['gnd_pid'] == '12391664X' assert returned_record['rero_pid'] == 'A023655346' assert returned_record['idref_pid'] == '069774331' record, action, m_record, m_action, v_record, online = \ AgentGndRecord.create_or_update_agent_mef_viaf( data=gnd_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '12391664X' assert m_action.name == 'CREATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'}, 'pid': '1', 'viaf_pid': '66739143' } record, action, m_record, m_action, v_record, online = \ AgentReroRecord.create_or_update_agent_mef_viaf( data=rero_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == 'A023655346' assert m_action.name == 'UPDATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'}, 'pid': '1', 'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'}, 'viaf_pid': '66739143' } record, action, m_record, m_action, v_record, online = \ AgentIdrefRecord.create_or_update_agent_mef_viaf( data=idref_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '069774331' assert m_action.name == 'UPDATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'}, 'idref': {'$ref': 'https://mef.rero.ch/api/idref/069774331'}, 'pid': '1', 'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'}, 'viaf_pid': '66739143' }
def test_create_agent_record_with_viaf_links(app, agent_viaf_record, agent_gnd_record, agent_rero_record, agent_idref_record): """Test create agent record with VIAF links.""" viaf_record, action = AgentViafRecord.create_or_update(agent_viaf_record, dbcommit=True, reindex=True) AgentViafRecord.update_indexes() assert action.name == 'CREATE' assert viaf_record['pid'] == '66739143' assert viaf_record['gnd_pid'] == '12391664X' assert viaf_record['rero_pid'] == 'A023655346' assert viaf_record['idref_pid'] == '069774331' record, action, m_record, m_action, v_record, online = \ AgentGndRecord.create_or_update_agent_mef_viaf( data=agent_gnd_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '12391664X' assert m_action.name == 'CREATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': { '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X' }, 'pid': '1', 'viaf_pid': '66739143' } record, action, m_record, m_action, v_record, online = \ AgentReroRecord.create_or_update_agent_mef_viaf( data=agent_rero_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == 'A023655346' assert m_action.name == 'UPDATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': { '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X' }, 'pid': '1', 'rero': { '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346' }, 'viaf_pid': '66739143' } record, action, m_record, m_action, v_record, online = \ AgentIdrefRecord.create_or_update_agent_mef_viaf( data=agent_idref_record, dbcommit=True, reindex=True, online=False ) assert action.name == 'CREATE' assert record['pid'] == '069774331' assert m_action.name == 'UPDATE' assert m_record == { '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json', 'gnd': { '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X' }, 'idref': { '$ref': 'https://mef.rero.ch/api/agents/idref/069774331' }, 'pid': '1', 'rero': { '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346' }, 'viaf_pid': '66739143' } assert m_record == AgentMefRecord.get_mef_by_entity_pid( record.pid, 'idref') assert m_record.pid == AgentMefRecord.get_mef_by_entity_pid(record.pid, 'idref', pid_only=True) assert AgentMefRecord.get_mef_by_viaf_pid(viaf_record.pid) == m_record
def test_reromefrecord_api(app, agent_idref_record): """Test ReroMefRecord api.""" idref = AgentIdrefRecord.create( data=agent_idref_record, delete_pid=False, dbcommit=True, reindex=True, ) AgentIdrefRecord.update_indexes() assert AgentIdrefRecord.count() == 1 assert AgentIdrefRecord.index_all() == 1 assert len(list(AgentIdrefRecord.get_all_records())) == len( list(AgentIdrefRecord.get_all_records(limit=0))) assert AgentIdrefRecord.get_metadata_identifier_names() == ( 'agent_idref_metadata', 'agent_idref_id') count = 0 for _ in AgentIdrefRecord.get_all_records(): count += 1 assert count == 1 _, agent_action = idref.update_test_md5(data=idref, dbcommit=True, reindex=True) assert agent_action == Action.UPTODATE idref['gender'] = 'female' _, agent_action = idref.update_test_md5(data=idref, dbcommit=True, reindex=True) assert agent_action == Action.UPDATE assert AgentIdrefRecord.get_pid_by_id(idref.id) == idref.pid AgentIdrefIndexer().bulk_index(list(AgentIdrefRecord.get_all_ids())) process_bulk_queue(stats_only=True)