コード例 #1
0
def test_monitoring_check_es_db_counts(app, client, agent_idref_record):
    """Test monitoring check_es_db_counts."""
    res = client.get(url_for('api_monitoring.check_es_db_counts'))
    assert res.status_code == 200
    assert json.loads(res.get_data(as_text=True)) == {
        'data': {
            'status': 'green'
        }
    }

    pers = AgentIdrefRecord.create(data=agent_idref_record,
                                   delete_pid=False,
                                   dbcommit=True,
                                   reindex=False)
    AgentIdrefRecord.update_indexes()
    res = client.get(url_for('api_monitoring.check_es_db_counts'))
    assert res.status_code == 200
    assert json.loads(res.get_data(as_text=True)) == {
        'data': {
            'status': 'red'
        },
        'errors': [{
            'code': 'DB_ES_COUNTER_MISSMATCH',
            'details': 'There are 1 items from aidref missing in ES.',
            'id': 'DB_ES_COUNTER_MISSMATCH',
            'links': {
                'about': 'http://localhost/monitoring/check_es_db_counts',
                'aidref': 'http://localhost/monitoring/missing_pids/aidref'
            },
            'title': "DB items counts don't match ES items count."
        }]
    }

    # this view is only accessible by admin
    res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref'))
    assert res.status_code == 401

    create_and_login_monitoring_user(app, client)

    res = client.get(url_for('api_monitoring.missing_pids', doc_type='aidref'))
    assert res.status_code == 200

    assert json.loads(res.get_data(as_text=True)) == {
        'data': {
            'DB': [],
            'ES': ['http://localhost/agents/idref/069774331'],
            'ES duplicate': []
        }
    }
コード例 #2
0
def test_view_agents_idref(client, agent_idref_record):
    """Test redirect IDREF."""

    AgentIdrefRecord.create(
        data=agent_idref_record,
        delete_pid=False,
        dbcommit=True,
        reindex=True,
    )
    AgentIdrefRecord.update_indexes()
    pid = agent_idref_record.get('pid')
    url = url_for('api_agents_idref.redirect_list')
    res = client.get(url)
    assert res.status_code == 308
    res = client.get(url, follow_redirects=True)
    assert res.status_code == 200
    assert json.loads(res.get_data(as_text=True))['aggregations'] == {
        'agent_type': {
            'buckets': [{
                'doc_count': 1,
                'key': 'bf:Person'
            }],
            'doc_count_error_upper_bound': 0,
            'sum_other_doc_count': 0
        },
        'deleted': {
            'doc_count': 0
        }
    }

    url = url_for('api_agents_idref.redirect_item', pid=pid)
    res = client.get(url)
    assert res.status_code == 308
    res = client.get(url, follow_redirects=True)
    assert res.status_code == 200
    data = json.loads(res.get_data(as_text=True))
    assert data.get('id') == pid
    assert data.get('metadata', {}).get('pid') == pid

    url = url_for('api_agents_idref.redirect_item', pid='WRONG')
    res = client.get(url, follow_redirects=True)
    assert res.status_code == 404
    assert json.loads(res.get_data(as_text=True)) == {
        "status": 404,
        "message": "PID does not exist."
    }
コード例 #3
0
def test_create_agent_record_no_viaf_links(app, agent_gnd_record,
                                           agent_rero_record,
                                           agent_idref_record):
    """Test create agent record without VIAF links."""
    record, action, m_record, m_action, v_record, online = \
        AgentGndRecord.create_or_update_agent_mef_viaf(
            data=agent_gnd_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == '12391664X'
    assert m_action.name == 'CREATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
        },
        'pid': '1',
    }

    record, action, m_record, m_action, v_record, online = \
        AgentReroRecord.create_or_update_agent_mef_viaf(
            data=agent_rero_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == 'A023655346'
    assert m_action.name == 'CREATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'pid': '2',
        'rero': {
            '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
        },
    }

    record, action, m_record, m_action, v_record, online = \
        AgentIdrefRecord.create_or_update_agent_mef_viaf(
            data=agent_idref_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == '069774331'
    assert m_action.name == 'CREATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'idref': {
            '$ref': 'https://mef.rero.ch/api/agents/idref/069774331'
        },
        'pid': '3',
    }
コード例 #4
0
def test_ref_resolvers(app, gnd_record, rero_record, viaf_record,
                       idref_record):
    """Test ref resolvers."""

    # VIAF record
    record, action = ViafRecord.create_or_update(data=viaf_record,
                                                 dbcommit=True,
                                                 reindex=True)
    viaf_pid = record['pid']

    # GND record
    record, action, m_record, m_action, v_record, online = \
        AgentGndRecord.create_or_update_agent_mef_viaf(
            data=gnd_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    gnd_pid = record.get('pid')

    # RERO record
    record, action, m_record, m_action, v_record, online = \
        AgentReroRecord.create_or_update_agent_mef_viaf(
            data=rero_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    rero_pid = record.get('pid')

    # IDREF record
    record, action, m_record, m_action, v_record, online = \
        AgentIdrefRecord.create_or_update_agent_mef_viaf(
            data=idref_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    idref_pid = record.get('pid')

    # MEF record
    mef_rec_resolved = MefRecord.get_mef_by_viaf_pid(viaf_pid=viaf_pid)
    mef_rec_resolved = mef_rec_resolved.replace_refs()
    assert mef_rec_resolved.get('gnd').get('pid') == gnd_pid
    assert mef_rec_resolved.get('rero').get('pid') == rero_pid
    assert mef_rec_resolved.get('idref').get('pid') == idref_pid
コード例 #5
0
def test_update_agent_record_with_viaf_links(
        app, viaf_record, gnd_record, rero_record, idref_record):
    """Test create agent record with viaf links."""
    returned_record, action = AgentGndRecord.create_or_update(
        gnd_record, dbcommit=True, reindex=True
    )
    assert action.name == 'UPDATE'
    assert returned_record['pid'] == '12391664X'

    returned_record, action = AgentReroRecord.create_or_update(
        rero_record, dbcommit=True, reindex=True
    )
    assert action.name == 'UPDATE'
    assert returned_record['pid'] == 'A023655346'

    returned_record, action = AgentIdrefRecord.create_or_update(
        idref_record, dbcommit=True, reindex=True
    )
    assert action.name == 'UPDATE'
    assert returned_record['pid'] == '069774331'
コード例 #6
0
def test_uptodate_agent_record_with_viaf_links_md5(app, agent_viaf_record,
                                                   agent_gnd_record,
                                                   agent_rero_record,
                                                   agent_idref_record):
    """Test create agent record with VIAF links."""
    returned_record, action = AgentGndRecord.create_or_update(agent_gnd_record,
                                                              dbcommit=True,
                                                              reindex=True,
                                                              test_md5=True)
    assert action.name == 'UPTODATE'
    assert returned_record['pid'] == '12391664X'

    returned_record, action = AgentReroRecord.create_or_update(
        agent_rero_record, dbcommit=True, reindex=True, test_md5=True)
    assert action.name == 'UPTODATE'
    assert returned_record['pid'] == 'A023655346'

    returned_record, action = AgentIdrefRecord.create_or_update(
        agent_idref_record, dbcommit=True, reindex=True, test_md5=True)
    assert action.name == 'UPTODATE'
    assert returned_record['pid'] == '069774331'
コード例 #7
0
def test_create_agent_updates(mock_get, app, gnd_record, rero_record,
                              idref_record):
    """Test create agent record with viaf links."""
    # we have to mock the access to viaf
    mock_get.return_value = {
        'pid': '37268949',
        'idref_pid': '069774331',
        'gnd_pid': '100769527xxx'
    }
    # create first record no viaf and mef records exist
    assert MefRecord.count() == 0
    assert ViafRecord.count() == 0
    record, action, m_record, m_action, v_record, online = \
        AgentIdrefRecord.create_or_update_agent_mef_viaf(
            data=idref_record,
            dbcommit=True,
            reindex=True,
            online=True
        )
    assert action.name == 'CREATE'
    assert record['pid'] == '069774331'
    assert MefRecord.count() == 1
    assert ViafRecord.count() == 1
    assert online

    # we should have a mef and viaf record now
    mef_pid = list(MefRecord.get_all_pids())[-1]
    mef_record = MefRecord.get_record_by_pid(mef_pid)
    assert mef_record.get('pid') == '1'
    assert mef_record.get('viaf_pid') == '37268949'
    assert 'idref' in mef_record
    viaf_pid = list(ViafRecord.get_all_pids())[-1]
    viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
    assert viaf_record.get('pid') == '37268949'
    assert viaf_record.get('idref_pid') == '069774331'

    # we have to mock the access to viaf
    mock_get.return_value = {
        'pid': '66739143',
        'gnd_pid': '12391664X',
        'idref_pid': '068979401',
        'rero_pid': 'A023655346'
    }
    # create second record
    record, action, m_record, m_action, v_record, online = \
        AgentGndRecord.create_or_update_agent_mef_viaf(
            data=gnd_record,
            dbcommit=True,
            reindex=True,
            online=True
        )
    mef_pid = list(MefRecord.get_all_pids())[-1]
    mef_record = MefRecord.get_record_by_pid(mef_pid)
    assert mef_record.get('pid') == '2'
    assert mef_record.get('viaf_pid') == '66739143'
    assert 'gnd' in mef_record
    viaf_pid = list(ViafRecord.get_all_pids())[-1]
    viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
    assert viaf_record.get('pid') == '66739143'
    assert viaf_record.get('gnd_pid') == '12391664X'

    record, action, m_record, m_action, v_record, online = \
        AgentReroRecord.create_or_update_agent_mef_viaf(
            data=rero_record,
            dbcommit=True,
            reindex=True,
            online=True
        )

    mef_pid = list(MefRecord.get_all_pids())[-1]
    mef_record = MefRecord.get_record_by_pid(mef_pid)
    assert mef_record.get('pid') == '2'
    assert mef_record.get('viaf_pid') == '66739143'
    assert 'rero' in mef_record
    viaf_pid = list(ViafRecord.get_all_pids())[-1]
    viaf_record = ViafRecord.get_record_by_pid(viaf_pid)
    assert viaf_record.get('pid') == '66739143'
    assert viaf_record.get('rero_pid') == 'A023655346'

    mef_record = MefRecord.get_mef_by_viaf_pid(viaf_pid)
    assert mef_record == {
        'pid': '2',
        'viaf_pid': '66739143',
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/gnd/12391664X'
        },
        'rero': {
            '$ref': 'https://mef.rero.ch/api/rero/A023655346'
        }
    }
    new_viaf_pid = 'xxxxxxxx'
    new_viaf_record = {
        'pid': new_viaf_pid,
        'gnd_pid': '12391664X',
        'idref_pid': '069774331',
        'rero_pid': 'A023655346'
    }
    mock_get.return_value = new_viaf_record
    rec, msg = viaf_record.delete(dbcommit=True, delindex=True, online=True)
    assert ViafRecord.get_record_by_pid(new_viaf_pid) == new_viaf_record
    mef_record = MefRecord.get_mef_by_viaf_pid(new_viaf_pid)
    assert mef_record == {
        'viaf_pid': 'xxxxxxxx',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/gnd/12391664X'
        },
        'rero': {
            '$ref': 'https://mef.rero.ch/api/rero/A023655346'
        },
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'pid': '3'
    }
コード例 #8
0
def test_monitoring(app, agent_idref_record, script_info):
    """Test monitoring."""

    cli_output = [
        'DB - ES    type      count                      index   count_es',
        '----------------------------------------------------------------',
        '      0   aggnd          0                 agents_gnd          0',
        '      0  agrero          0                agents_rero          0',
        '      1  aidref          1               agents_idref          0',
        '      0   comef          0               concepts_mef          0',
        '      0  corero          0              concepts_rero          0',
        '      0     mef          0                        mef          0',
        '      0    viaf          0                       viaf          0'
    ]
    mon = Monitoring()
    assert mon.get_es_count('xxx') == 'No >>xxx<< in ES'
    assert mon.get_db_count('xxx') == 'No >>xxx<< in DB'
    idref = AgentIdrefRecord.create(data=agent_idref_record,
                                    delete_pid=False,
                                    dbcommit=True,
                                    reindex=False)
    idref_pid = idref.pid
    assert mon.get_db_count('aidref') == 1
    assert mon.get_es_count('agents_idref') == 0
    assert mon.check() == {'aidref': {'db_es': 1}}
    assert mon.missing('aidref') == {
        'DB': [],
        'ES': ['069774331'],
        'ES duplicate': []
    }
    # not flushed by default
    assert mon.info() == {
        'aggnd': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'agents_gnd'
        },
        'agrero': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'agents_rero'
        },
        'aidref': {
            'db': 1,
            'db-es': 1,
            'es': 0,
            'index': 'agents_idref'
        },
        'comef': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'concepts_mef'
        },
        'corero': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'concepts_rero'
        },
        'mef': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'mef'
        },
        'viaf': {
            'db': 0,
            'db-es': 0,
            'es': 0,
            'index': 'viaf'
        }
    }
    assert mon.__str__().split('\n') == cli_output + ['']

    runner = CliRunner()
    res = runner.invoke(es_db_missing_cli, ['aidref'], obj=script_info)
    assert res.output == 'aidref: pids missing in ES:\n069774331\n'

    runner = CliRunner()
    res = runner.invoke(es_db_counts_cli, ['-m'], obj=script_info)
    assert res.output.split(
        '\n') == cli_output + ['aidref: pids missing in ES:', '069774331', '']

    # we have to get the idref again because we lost the session after the use
    # of the CliRunner
    idref = AgentIdrefRecord.get_record_by_pid(idref_pid)
    idref.reindex()
    AgentIdrefRecord.update_indexes()
    assert mon.get_es_count('agents_idref') == 1
    assert mon.check() == {}
    assert mon.missing('aidref') == {'DB': [], 'ES': [], 'ES duplicate': []}
    idref.delete(dbcommit=True)
    assert mon.get_db_count('aidref') == 0
    assert mon.get_es_count('agents_idref') == 1
    assert mon.check() == {'aidref': {'db_es': -1}}
    assert mon.missing('aidref') == {
        'DB': ['069774331'],
        'ES': [],
        'ES duplicate': []
    }
コード例 #9
0
def test_create_agent_record_with_viaf_links(
        app, viaf_record, gnd_record, rero_record, idref_record):
    """Test create agent record with viaf links."""
    returned_record, action = ViafRecord.create_or_update(
        viaf_record, dbcommit=True, reindex=True
    )
    ViafRecord.update_indexes()
    assert action.name == 'CREATE'
    assert returned_record['pid'] == '66739143'
    assert returned_record['gnd_pid'] == '12391664X'
    assert returned_record['rero_pid'] == 'A023655346'
    assert returned_record['idref_pid'] == '069774331'

    record, action, m_record, m_action, v_record, online = \
        AgentGndRecord.create_or_update_agent_mef_viaf(
            data=gnd_record,
            dbcommit=True,
            reindex=True,
            online=False
        )

    assert action.name == 'CREATE'
    assert record['pid'] == '12391664X'
    assert m_action.name == 'CREATE'
    assert m_record == {
        '$schema':
            'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
        'pid': '1',
        'viaf_pid': '66739143'
    }

    record, action, m_record, m_action, v_record, online = \
        AgentReroRecord.create_or_update_agent_mef_viaf(
            data=rero_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == 'A023655346'
    assert m_action.name == 'UPDATE'
    assert m_record == {
        '$schema':
            'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
        'pid': '1',
        'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'},
        'viaf_pid': '66739143'
    }

    record, action, m_record, m_action, v_record, online = \
        AgentIdrefRecord.create_or_update_agent_mef_viaf(
            data=idref_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == '069774331'
    assert m_action.name == 'UPDATE'
    assert m_record == {
        '$schema':
            'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {'$ref': 'https://mef.rero.ch/api/gnd/12391664X'},
        'idref': {'$ref': 'https://mef.rero.ch/api/idref/069774331'},
        'pid': '1',
        'rero': {'$ref': 'https://mef.rero.ch/api/rero/A023655346'},
        'viaf_pid': '66739143'
    }
コード例 #10
0
def test_create_agent_record_with_viaf_links(app, agent_viaf_record,
                                             agent_gnd_record,
                                             agent_rero_record,
                                             agent_idref_record):
    """Test create agent record with VIAF links."""
    viaf_record, action = AgentViafRecord.create_or_update(agent_viaf_record,
                                                           dbcommit=True,
                                                           reindex=True)
    AgentViafRecord.update_indexes()
    assert action.name == 'CREATE'
    assert viaf_record['pid'] == '66739143'
    assert viaf_record['gnd_pid'] == '12391664X'
    assert viaf_record['rero_pid'] == 'A023655346'
    assert viaf_record['idref_pid'] == '069774331'

    record, action, m_record, m_action, v_record, online = \
        AgentGndRecord.create_or_update_agent_mef_viaf(
            data=agent_gnd_record,
            dbcommit=True,
            reindex=True,
            online=False
        )

    assert action.name == 'CREATE'
    assert record['pid'] == '12391664X'
    assert m_action.name == 'CREATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
        },
        'pid': '1',
        'viaf_pid': '66739143'
    }

    record, action, m_record, m_action, v_record, online = \
        AgentReroRecord.create_or_update_agent_mef_viaf(
            data=agent_rero_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == 'A023655346'
    assert m_action.name == 'UPDATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
        },
        'pid': '1',
        'rero': {
            '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
        },
        'viaf_pid': '66739143'
    }

    record, action, m_record, m_action, v_record, online = \
        AgentIdrefRecord.create_or_update_agent_mef_viaf(
            data=agent_idref_record,
            dbcommit=True,
            reindex=True,
            online=False
        )
    assert action.name == 'CREATE'
    assert record['pid'] == '069774331'
    assert m_action.name == 'UPDATE'
    assert m_record == {
        '$schema': 'https://mef.rero.ch/schemas/mef/mef-v0.0.1.json',
        'gnd': {
            '$ref': 'https://mef.rero.ch/api/agents/gnd/12391664X'
        },
        'idref': {
            '$ref': 'https://mef.rero.ch/api/agents/idref/069774331'
        },
        'pid': '1',
        'rero': {
            '$ref': 'https://mef.rero.ch/api/agents/rero/A023655346'
        },
        'viaf_pid': '66739143'
    }

    assert m_record == AgentMefRecord.get_mef_by_entity_pid(
        record.pid, 'idref')
    assert m_record.pid == AgentMefRecord.get_mef_by_entity_pid(record.pid,
                                                                'idref',
                                                                pid_only=True)
    assert AgentMefRecord.get_mef_by_viaf_pid(viaf_record.pid) == m_record
コード例 #11
0
def test_reromefrecord_api(app, agent_idref_record):
    """Test ReroMefRecord api."""

    idref = AgentIdrefRecord.create(
        data=agent_idref_record,
        delete_pid=False,
        dbcommit=True,
        reindex=True,
    )
    AgentIdrefRecord.update_indexes()

    assert AgentIdrefRecord.count() == 1
    assert AgentIdrefRecord.index_all() == 1

    assert len(list(AgentIdrefRecord.get_all_records())) == len(
        list(AgentIdrefRecord.get_all_records(limit=0)))

    assert AgentIdrefRecord.get_metadata_identifier_names() == (
        'agent_idref_metadata', 'agent_idref_id')

    count = 0
    for _ in AgentIdrefRecord.get_all_records():
        count += 1
    assert count == 1

    _, agent_action = idref.update_test_md5(data=idref,
                                            dbcommit=True,
                                            reindex=True)
    assert agent_action == Action.UPTODATE

    idref['gender'] = 'female'
    _, agent_action = idref.update_test_md5(data=idref,
                                            dbcommit=True,
                                            reindex=True)
    assert agent_action == Action.UPDATE

    assert AgentIdrefRecord.get_pid_by_id(idref.id) == idref.pid

    AgentIdrefIndexer().bulk_index(list(AgentIdrefRecord.get_all_ids()))
    process_bulk_queue(stats_only=True)