def test_create_filter_dsl(): """Test request value extraction.""" app = Flask('testapp') kwargs = MultiDict([('a', '1')]) defs = dict( type=terms_filter('type.type'), subtype=terms_filter('type.subtype'), ) with app.test_request_context(u'?type=a&type=b&subtype=c&type=zażółcić'): filters, args = _create_filter_dsl(kwargs, defs) assert len(filters) == 2 assert args == MultiDict([ ('a', u'1'), ('type', u'a'), ('type', u'b'), ('subtype', u'c'), ('type', u'zażółcić') ]) kwargs = MultiDict([('a', '1')]) with app.test_request_context('?atype=a&atype=b'): filters, args = _create_filter_dsl(kwargs, defs) assert not filters assert args == kwargs
def test_default_facets_factory(app): """Test aggregations.""" defs = dict( aggs=dict( type=dict( terms=dict(field='upload_type'), ), subtype=dict( terms=dict(field='subtype'), ) ), filters=dict( subtype=terms_filter('subtype'), ), post_filters=dict( type=terms_filter('type'), ), ) app.config['RECORDS_REST_FACETS']['testidx'] = defs with app.test_request_context('?type=a&subtype=b'): search = Search().query(Q(query='value')) search, urlkwargs = default_facets_factory(search, 'testidx') assert search.to_dict()['aggs'] == defs['aggs'] assert 'post_filter' in search.to_dict() assert search.to_dict( )['query']['bool']['filter'][0]['terms']['subtype'] search = Search().query(Q(query='value')) search, urlkwargs = default_facets_factory(search, 'anotheridx') assert 'aggs' not in search.to_dict() assert 'post_filter' not in search.to_dict() assert 'bool' not in search.to_dict()['query']
def test_default_facets_factory(app, user_factory): """Test aggregations.""" defs = dict( aggs=dict( type=dict( terms=dict(field="upload_type"), ), subtype=dict( terms=dict(field="subtype"), ) ), filters=dict( subtype=terms_filter('subtype'), ), post_filters=dict( type=terms_filter('type'), ), ) app.config['RECORDS_REST_FACETS']['testidx'] = defs with app.test_request_context("?type=a&subtype=b"): q = Query("value") query, urlkwargs = default_facets_factory(q, 'testidx') assert query.body['aggs'] == defs['aggs'] assert 'post_filter' in query.body assert 'filtered' in query.body['query'] q = Query("value") query, urlkwargs = default_facets_factory(q, 'anotheridx') assert 'aggs' not in query.body assert 'post_filter' not in query.body assert 'filtered' not in query.body['query']
def test_query_filter(app, user_factory): """Test post filter.""" urlargs = MultiDict() defs = dict( type=terms_filter('type'), subtype=terms_filter('subtype'), ) with app.test_request_context("?type=test"): q = Query("value") body = q.body['query'] query, args = _query_filter(q, urlargs, defs) assert 'post_filter' not in query.body assert query.body['query']['filtered']['query'] == body assert query.body['query']['filtered']['filter'] == \ dict( bool=dict( filter=[dict(terms=dict(type=['test']))] ), ) assert args['type'] == 'test' with app.test_request_context("?anotertype=test"): q = Query("value") body = q.body['query'] query, args = _query_filter(q, urlargs, defs) assert query.body['query'] == body
def test_post_filter(app): """Test post filter.""" urlargs = MultiDict() defs = dict( type=terms_filter('type'), subtype=terms_filter('subtype'), ) with app.test_request_context('?type=test'): search = Search().query(Q(query='value')) search, args = _post_filter(search, urlargs, defs) assert 'post_filter' in search.to_dict() assert search.to_dict()['post_filter'] == dict( terms=dict(type=['test']) ) assert args['type'] == 'test' with app.test_request_context('?anotertype=test'): search = Search().query(Q(query='value')) search, args = _post_filter(search, urlargs, defs) assert 'post_filter' not in search.to_dict()
def test_create_filter_dsl(): """Test request value extraction.""" app = Flask('testapp') kwargs = MultiDict([('a', '1')]) defs = dict( type=terms_filter('type.type'), subtype=terms_filter('type.subtype'), ) with app.test_request_context("?type=a&type=b&subtype=c"): query, args = _create_filter_dsl(kwargs, defs) assert len(query['bool']['filter']) == 2 assert args == MultiDict([ ('a', '1'), ('type', 'a'), ('type', 'b'), ('subtype', 'c') ]) kwargs = MultiDict([('a', '1')]) with app.test_request_context("?atype=a&atype=b"): query, args = _create_filter_dsl(kwargs, defs) assert query is None assert args == kwargs
def test_query_filter(app): """Test post filter.""" urlargs = MultiDict() defs = dict( type=terms_filter('type'), subtype=terms_filter('subtype'), ) with app.test_request_context('?type=test'): search = Search().query(Q('multi_match', query='value')) body = search.to_dict() search, args = _query_filter(search, urlargs, defs) assert 'post_filter' not in search.to_dict() assert search.to_dict()['query']['bool']['must'][0] == body['query'] assert search.to_dict()['query']['bool']['filter'] == [ dict(terms=dict(type=['test'])) ] assert args['type'] == 'test' with app.test_request_context('?anotertype=test'): search = Search().query(Q(query='value')) body = search.to_dict() query, args = _query_filter(search, urlargs, defs) assert query.to_dict() == body
def subtest_search_aggregation_serialization(app, user_factory, expected): """Test the serialization of elasticsearch aggregations.""" with app.app_context(): # create the record using the internal API pid1, record1 = create_record(test_data) pid2, record2 = create_record(test_data2) pid3, record3 = create_record(test_data3) with user_factory('allowed') as allowed_user: # create one user allowed to delete the record allowed_user.read_access(True, str(record1.id)) allowed_user.read_access(True, str(record2.id)) allowed_user.read_access(True, str(record3.id)) allowed_login = allowed_user.login_function() db.session.commit() app.config['RECORDS_REST_FACETS'] = { 'invenio_records_rest_test_index': { 'aggs': { 'stars': {'terms': {'field': 'stars'}} }, 'post_filters': { 'type': terms_filter('type'), } } } es_index = app.config["RECORDS_REST_DEFAULT_SEARCH_INDEX"] current_search_client.indices.flush(wait_if_ongoing=True, force=True, index=es_index) with app.test_client() as client: allowed_login(client) headers = [('Accept', 'application/json')] res = client.get(url_for('invenio_records_rest.recid_list', q='the', sort='year'), headers=headers) assert res.status_code == 200 data = json.loads(res.get_data(as_text=True)) assert isinstance(data['hits']['hits'], list) assert data['hits']['total'] == 3 subtest_expected_hits(data['hits']['hits'], [ (pid3.pid_value, control_num(test_data3, 3)), (pid1.pid_value, control_num(test_data, 1)), (pid2.pid_value, control_num(test_data2, 2)), ], client) assert data['aggregations'] == expected
RECORDS_REST_DEFAULT_SORT = dict( records=dict( query='bestmatch', noquery='mostrecent', ) ) """Default sort option per index with/without query string.""" RECORDS_REST_FACETS = dict( records=dict( aggs=dict( type=dict(terms=dict(field='type')) ), post_filters=dict( type=terms_filter('type'), ) ) ) """Facets per index for the default facets factory. The structure of the dictionary is as follows:: { "<index or index alias>": { "aggs": { "<key>": <aggregation definition>, ... } "filters": { "<key>": <filter func>,
} # Default sorting. app.config['RECORDS_REST_DEFAULT_SORT'] = { index_name: { 'query': 'control_number', 'noquery': '-control_number', } } # Aggregations and filtering app.config['RECORDS_REST_FACETS'] = { index_name: { 'aggs': { 'type': {'terms': {'field': 'type'}} }, 'post_filters': { 'type': terms_filter('type'), }, 'filters': { 'typefilter': terms_filter('type'), } } } FlaskCLI(app) FlaskCeleryExt(app) InvenioDB(app) InvenioREST(app) InvenioPIDStore(app) InvenioRecords(app) search = InvenioSearch(app) search.register_mappings('testrecords', 'data') InvenioIndexer(app)
}, }, } #: Default sort for records REST API. RECORDS_REST_DEFAULT_SORT = { "scoap3-records-record": { 'query': '-date', 'noquery': '-date' }, } RECORDS_REST_FACETS = { "scoap3-records-record": { "filters": { "journal": terms_filter("publication_info.journal_title"), "country": terms_filter_with_must("authors.affiliations.country"), "collaboration": terms_filter("facet_collaboration"), "year": range_filter( 'year', format='yyyy', end_date_math='/y') }, "aggs": { "journal": { "terms": { "field": "publication_info.journal_title", "size": 20, "order": {"_term": "asc"} } },
('invenio_communities.serializers.json_v1_response'), }, search_serializers={ 'application/json': ('invenio_communities.serializers:json_v1_search'), }, record_loaders={ 'application/json': ('invenio_communities.loaders:json_v1'), }, indexer_class='invenio_communities.indexer:CommunityIndexer', default_media_type='application/json', read_permission_factory_imp=allow_all, create_permission_factory_imp=allow_logged_in, list_permission_factory_imp=allow_all, update_permission_factory_imp=allow_community_owner, delete_permission_factory_imp=allow_community_owner, ), ) COMMUNITIES_REST_FACETS = dict( communities=dict(aggs=dict(type=dict(terms=dict(field="type"), ), domain=dict(terms=dict(field="domain"), )), post_filters=dict( type=terms_filter('type'), domain=terms_filter('domain'), ))) COMMUNITIES_MEMBERSHIP_REQUESTS_CONFIRMLINK_EXPIRES_IN = 1000000 SUPPORT_EMAIL = '*****@*****.**' COMMUNITIES_RECORD_INDEX = 'records-record-v1.0.0'
} # Default sorting. app.config['RECORDS_REST_DEFAULT_SORT'] = { index_name: { 'query': 'control_number', 'noquery': '-control_number', } } # Aggregations and filtering app.config['RECORDS_REST_FACETS'] = { index_name: { 'aggs': { 'type': {'terms': {'field': 'type'}} }, 'post_filters': { 'type': terms_filter('type'), }, 'filters': { 'typefilter': terms_filter('type'), } } } app.url_map.converters['pid'] = PIDConverter FlaskCeleryExt(app) InvenioDB(app) InvenioREST(app) InvenioPIDStore(app) InvenioRecords(app) search = InvenioSearch(app) search.register_mappings('testrecords', 'data') InvenioIndexer(app)
RECORDS_REST_FACETS = { 'deposits': { 'aggs': { 'collections': { 'terms': { 'field': '_type', }, }, 'status': { 'terms': { 'field': '_deposit.status', }, }, }, 'post_filters': { 'collections': terms_filter('_type'), 'status': terms_filter('_deposit.status'), }, }, } # RECORDS_REST_FACETS.update(DEPOSIT_REST_FACETS) # #: Endpoints for displaying records. # RECORDS_UI_ENDPOINTS = dict( # recid=dict( # pid_type='recid', # route='/records/<pid_value>', # template='invenio_records_ui/detail.html', # record_class='invenio_records_files.api:Record' # ),
template="inspirehep_theme/format/record/Experiment_HTML_detailed.tpl", permission_factory_imp="invenio_records_rest.utils:allow_all", ), journals=dict( pid_type="journals", route="/journals/<pid_value>", template="inspirehep_theme/format/record/Journal_HTML_detailed.tpl", record_class="inspirehep.modules.records.wrappers:JournalsRecord", permission_factory_imp="invenio_records_rest.utils:allow_all", ), ) RECORDS_REST_FACETS = { "records-hep": { "filters": { "author": terms_filter("exactauthor.raw"), "subject": terms_filter("facet_inspire_subjects"), "doc_type": terms_filter("facet_inspire_doc_type"), "formulas": terms_filter("facet_formulas"), "experiment": terms_filter("accelerator_experiments.facet_experiment"), }, "aggs": { "subject": {"terms": {"field": "facet_inspire_subjects", "size": 20}}, "doc_type": {"terms": {"field": "facet_inspire_doc_type", "size": 20}}, "formulas": {"terms": {"field": "facet_formulas", "size": 20}}, "author": {"terms": {"field": "facet_authors", "size": 20}}, "experiment": {"terms": {"field": "accelerator_experiments.facet_experiment", "size": 20}}, "earliest_date": { "date_histogram": { "field": "earliest_date", "interval": "year",
records=dict( aggs=dict( type=dict( terms=dict(field="upload_type.type"), aggs=dict( subtype=dict( terms=dict(field="upload_type.subtype"), ) ) ), access_right=dict( terms=dict(field="access_right"), ), ), filters=dict( communities=terms_filter('communities'), ), post_filters=dict( access_right=terms_filter('access_right'), type=terms_filter('upload_type.type'), subtype=terms_filter('upload_type.subtype'), ) ) ) # REST # ==== REST_ENABLE_CORS = True # Accounts # ========
}, 'search_serializers': { 'application/json': 'invenio_records_rest.serializers' ':json_v1_search' }, 'list_route': '/records/', 'item_route': '/records/<pid_value>', } }, 'RECORDS_REST_FACETS': { 'invenio_records_rest_test_index': { 'aggs': { 'stars': {'terms': {'field': 'stars'}} }, 'post_filters': { 'type': terms_filter('type'), } } }, } })], indirect=['app']) def test_search_custom_aggregation_serialization(app, user_factory): """Test the elasticsearch aggregations with a custom formatter.""" subtest_search_aggregation_serialization(app, user_factory, { 'stars': { 'buckets': [ {'key': 4, 'doc_count': 2}, {'key': 3, 'doc_count': 1} ], 'sum_other_doc_count': 0, 'doc_count_error_upper_bound': 0,
route='/experiments/<pid_value>', template='inspirehep_theme/format/record/Experiment_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:ExperimentsRecord', ), journals=dict( pid_type='journals', route='/journals/<pid_value>', template='inspirehep_theme/format/record/Journal_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:JournalsRecord', ) ) RECORDS_REST_FACETS = { "records-hep": { "filters": { "author": terms_filter('exactauthor.raw'), "subject": terms_filter('facet_inspire_subjects'), "doc_type": terms_filter('facet_inspire_doc_type'), "formulas": terms_filter('facet_formulas'), "experiment": terms_filter( 'accelerator_experiments.facet_experiment'), }, "aggs": { "subject": { "terms": { "field": "facet_inspire_subjects", "size": 20 } }, "doc_type": { "terms": {
".object_type.keyword" }, "aggs": { "doc_count": { "reverse_nested": {} } } } } }, } }, }, 'post_filters': { 'type': terms_filter('_type'), 'cms_working_group': prefix_filter('cadi_id'), 'cadi_status': terms_filter('cadi_status'), 'physics_objects': nested_filter( 'main_measurements.signal_event_selection.physics_objects', 'main_measurements.signal_event_selection' '.physics_objects.object'), 'physics_objects_type': nested_filter( 'main_measurements.signal_event_selection.physics_objects', 'main_measurements.signal_event_selection.physics_objects' '.object_type.keyword') }
def app(request, search_class): """Flask application fixture. Note that RECORDS_REST_ENDPOINTS is used during application creation to create blueprints on the fly, hence once you have this fixture in a test, it's too late to customize the configuration variable. You can however customize it using parameterized tests: .. code-block:: python @pytest.mark.parametrize('app', [dict( endpoint=dict( search_class='conftest:TestSearch', ) def test_mytest(app, db, es): # ... This will parameterize the default 'recid' endpoint in RECORDS_REST_ENDPOINTS. Alternatively: .. code-block:: python @pytest.mark.parametrize('app', [dict( records_rest_endpoints=dict( recid=dict( search_class='conftest:TestSearch', ) ) def test_mytest(app, db, es): # ... This will fully parameterize RECORDS_REST_ENDPOINTS. """ instance_path = tempfile.mkdtemp() app = Flask('testapp', instance_path=instance_path) app.config.update( INDEXER_DEFAULT_DOC_TYPE='testrecord', INDEXER_DEFAULT_INDEX=search_class.Meta.index, RECORDS_REST_ENDPOINTS=copy.deepcopy(config.RECORDS_REST_ENDPOINTS), RECORDS_REST_DEFAULT_CREATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_SEARCH_INDEX=search_class.Meta.index, RECORDS_REST_FACETS={ search_class.Meta.index: { 'aggs': { 'stars': { 'terms': { 'field': 'stars' } } }, 'post_filters': { 'stars': terms_filter('stars'), } } }, RECORDS_REST_SORT_OPTIONS={ search_class.Meta.index: dict(year=dict(fields=['year'], )) }, SERVER_NAME='localhost:5000', SQLALCHEMY_DATABASE_URI=os.environ.get('SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'), SQLALCHEMY_TRACK_MODIFICATIONS=True, TESTING=True, ) app.config['RECORDS_REST_ENDPOINTS']['recid']['search_class'] = \ search_class # Parameterize application. if hasattr(request, 'param'): if 'endpoint' in request.param: app.config['RECORDS_REST_ENDPOINTS']['recid'].update( request.param['endpoint']) if 'records_rest_endpoints' in request.param: original_endpoint = app.config['RECORDS_REST_ENDPOINTS']['recid'] del app.config['RECORDS_REST_ENDPOINTS']['recid'] for new_endpoint_prefix, new_endpoint_value in \ request.param['records_rest_endpoints'].items(): new_endpoint = dict(original_endpoint) new_endpoint.update(new_endpoint_value) app.config['RECORDS_REST_ENDPOINTS'][new_endpoint_prefix] = \ new_endpoint app.url_map.converters['pid'] = PIDConverter InvenioDB(app) InvenioREST(app) InvenioRecords(app) InvenioPIDStore(app) search = InvenioSearch(app) search.register_mappings(search_class.Meta.index, 'mappings') InvenioRecordsREST(app) with app.app_context(): yield app # Teardown instance path. shutil.rmtree(instance_path)
}, 'task_transcode': { 'terms': { 'field': '_cds.state.file_transcode' }, }, 'task_extract_frames': { 'terms': { 'field': '_cds.state.file_video_extract_frames' }, }, 'created_by': created_by_me_aggs, }, 'filters': { 'project_status': terms_filter('_deposit.status'), 'category': terms_filter('category.untouched'), 'task_transcode': terms_filter('_cds.state.file_transcode'), 'task_extract_frames': terms_filter('_cds.state.file_video_extract_frames'), 'created_by': terms_filter('_deposit.created_by'), }, }, } RECORD_VIDEOS_FACETS = { 'records-videos-video': { 'aggs': {
'title_statement.title', ], title='Title', order=3, ), )) #: Default sort for records REST API. RECORDS_REST_DEFAULT_SORT = dict(records=dict(query='bestmatch', noquery='mostrecent'), ) #: Defined facets for records REST API. RECORDS_REST_FACETS = dict( records=dict(aggs=dict(author=dict(terms=dict( field="main_entry_personal_name.personal_name"), ), ), post_filters=dict(author=terms_filter( 'main_entry_personal_name.personal_name'), ))) # TODO: Remove once schema validation issue is fixed. INDEXER_DEFAULT_INDEX = 'marc21-bibliographic-bd-v1.0.0' INDEXER_DEFAULT_DOCTYPE = 'bd-v1.0.0' # OAI-PMH Server # ============== OAISERVER_RECORD_INDEX = 'marc21' # Celery configuration # ==================== #: Beat schedule CELERYBEAT_SCHEDULE = { 'indexer': { 'task': 'invenio_indexer.tasks.process_bulk_queue',
RECORDS_REST_DEFAULT_SORT = dict(records=dict(query='bestmatch', noquery='mostrecent'), ) #: Defined facets for records REST API. RECORDS_REST_FACETS = dict(marc21=dict( aggs=dict(identifier=dict(terms=dict( field='other_standard_identifier.source_of_number_or_code'), ), language=dict(terms=dict( field=('language_code.language_code_of_text_sound_track_or' '_separate_title'))), affiliation=dict(terms=dict( field='main_entry_personal_name.affiliation')), years=dict(date_histogram=dict( field='_created', interval='year', format='yyyy'))), post_filters=dict( identifier=terms_filter( 'other_standard_identifier.source_of_number_or_code'), language=terms_filter(( 'language_code.language_code_of_text_sound_track_or' '_separate_title')), affiliation=terms_filter('main_entry_personal_name.affiliation'), years=range_filter('_created', format='yyyy', end_date_math='/y'), ))) # TODO: Remove once schema validation issue is fixed. INDEXER_DEFAULT_INDEX = 'marc21-bibliographic-bd-v1.0.0' INDEXER_DEFAULT_DOCTYPE = 'bd-v1.0.0' # TODO: Remove me once the problem with email is solved in flask-security: # https://github.com/mattupstate/flask-security/issues/685 SECURITY_EMAIL_SENDER = 'no-reply@localhost'
grants=dict(query='bestmatch', noquery='bestmatch'), funders=dict(query='bestmatch', noquery='bestmatch'), ) OPENAIRE_REST_FACETS = dict( funders=dict( aggs=dict( country=dict( terms=dict(field='country'), ), type=dict( terms=dict(field='type'), ), ), filters=dict( country=terms_filter('country'), type=terms_filter('type'), ), ), grants=dict( aggs=dict( funder=dict( terms=dict(field='funder.acronyms'), ), ), filters=dict( funder=terms_filter('funder.acronyms'), ), ) )
'aggs': { 'doc_count': { 'reverse_nested': {} } }, }, }, }, }, }, }, 'post_filters': { 'type': regex_filter('_type'), 'type_version': terms_filter('_type'), 'cms_working_group': prefix_filter('basic_info.cadi_id'), 'cadi_status': terms_filter('cadi_info.status'), 'next_deadline_date': range_filter('analysis_context.next_deadline_date', format='yyyy', end_date_math='/y'), 'collision_system': terms_filter('basic_info.analysis_keywords.collision_system.keyword'), 'accelerator_parameters': terms_filter( 'basic_info.analysis_keywords.accelerator_parameters.keyword'), 'physics_theme': terms_filter('basic_info.analysis_keywords.physics_theme.keyword'),
), ), acq_vendors=dict( # VendorSearch.Meta.index name=dict(fields=["name"], title="Name", default_order="asc", order=1), bestmatch=dict( fields=["-_score"], title="Best match", default_order="asc", order=2, ), ), ) FACET_VENDOR_LIMIT = 5 RECORDS_REST_FACETS = dict(acq_orders=dict( # OrderSearch.Meta.index aggs=dict( status=dict(terms=dict(field="status")), vendor=dict( terms=dict(field="vendor.name.keyword", size=FACET_VENDOR_LIMIT)), payment_mode=dict(terms=dict(field="order_lines.payment_mode")), medium=dict(terms=dict(field="order_lines.medium")), ), post_filters=dict( status=terms_filter("status"), vendor=terms_filter("vendor.name.keyword"), payment_mode=terms_filter("order_lines.payment_mode"), medium=terms_filter("order_lines.medium"), ), ))
}, { 'key': '1000000--9999999', 'from': 1000000, 'to': 9999999 }, { 'key': '10000000--', 'from': 10000000 } ] } } ), 'post_filters': dict( experiment=terms_filter('experiment.keyword'), type=terms_filter('type.primary.keyword'), subtype=terms_filter('type.secondary.keyword'), year=terms_filter('date_created.keyword'), tags=terms_filter('tags.keyword'), keywords=terms_filter('keywords.keyword'), collision_type=terms_filter('collision_information.type.keyword'), collision_energy=terms_filter('collision_information.energy' '.keyword'), category=terms_filter('categories.primary.keyword'), subcategory=terms_filter('categories.secondary.keyword'), file_type=terms_filter('distribution.formats.keyword'), collections=terms_filter('collections.keyword'), availability=terms_filter('distribution.availability.keyword'), signature=terms_filter('signature.keyword'), event_number=range_filter('distribution.number_events')
terms=dict(field="resource_type.subtype"), ) ) ), access_right=dict( terms=dict(field="access_right"), ), file_type=dict( terms=dict(field="filetype"), ), keywords=dict( terms=dict(field="keywords"), ), ), filters=dict( communities=terms_filter('communities'), provisional_communities=terms_filter('provisional_communities'), ), post_filters=dict( access_right=terms_filter('access_right'), file_type=terms_filter('filetype'), keywords=terms_filter('keywords'), subtype=terms_filter('resource_type.subtype'), type=terms_filter('resource_type.type'), ) ) ) RECORDS_REST_FACETS.update(OPENAIRE_REST_FACETS) RECORDS_REST_FACETS.update(DEPOSIT_REST_FACETS) RECORDS_REST_ELASTICSEARCH_ERROR_HANDLERS = {
experiment=dict(terms=dict(field='experiment')), category=dict(terms=dict(field='collections.secondary')), type=dict(terms=dict(field='type.primary'), aggs=dict(subtype=dict(terms=dict( field="type.secondary")))), file_type=dict(terms=dict(field='distribution.formats')), year=dict(terms=dict(field='collections.year')), run=dict(terms=dict( field='production_publication_distribution_manufacture_and_' 'copyright_notice.' 'date_of_production_publication_distribution_' 'manufacture_or_copyright_notice' )), ), 'filters': dict( tags_pre=terms_filter('tags'), experiment_pre=terms_filter('experiment'), type_pre=terms_filter('type.primary'), subtype_pre=terms_filter('type.secondary'), ), 'post_filters': dict( experiment=terms_filter('experiment'), category=terms_filter('collections.secondary'), type=terms_filter('type.primary'), subtype=terms_filter('type.secondary'), year=terms_filter('collections.year'), tags=terms_filter('tags'), file_type=terms_filter('distribution.formats'), run=terms_filter( 'production_publication_distribution_manufacture_and_' 'copyright_notice.'
"arxiv_categories": { "terms": { "field": "facet_arxiv_categories", "size": 20 }, "meta": { "title": "arXiv Category", "order": 5, "type": "checkbox" }, }, }, }, "records-jobs": { "filters": { "field_of_interest": terms_filter("arxiv_categories"), "rank": terms_filter("ranks"), "region": terms_filter("regions"), }, "aggs": { "field_of_interest": { "terms": { "field": "arxiv_categories", "missing": "Other", "size": 500 }, "meta": { "order": 1, "type": "multiselect", "title": "Field of Interest", },
ELASTICSEARCH_LANGUAGE_TEMPLATES = { "*#subjectAll": { "type": "text", "copy_to": "subjectAll.*", "fields": { "raw": { "type": "keyword" } } } } FILTERS = { _('person'): terms_filter('person.keyword'), _('accessRights'): nested_filter( "accessRights", group_by_terms_filter( 'accessRights.title.en.raw', { "true": "open access", 1: "open access", True: "open access", "1": "open access", False: [ "embargoed access", "restricted access",
def test_terms_filter(): """Test terms filter.""" f = terms_filter("test") assert f(['a', 'b']) == dict(terms={'test': ['a', 'b']})
".physics_objects" ".object_type.keyword" }, "aggs": { "doc_count": { "reverse_nested": {} } } } } }, } }, }, 'post_filters': { 'type': terms_filter('_type'), 'status': terms_filter('status'), 'cadi_status': terms_filter('cadi_status'), 'publication_status': terms_filter('publication_status.keyword'), 'conference': terms_filter('conference'), 'physics_objects': nested_filter( 'main_measurements.signal_event_selection.physics_objects', 'main_measurements.signal_event_selection' '.physics_objects.object' ), 'physics_objects_type': nested_filter( 'main_measurements.signal_event_selection.physics_objects', 'main_measurements.signal_event_selection.physics_objects' '.object_type.keyword'), } }
record_class='inspirehep.modules.records.wrappers:ExperimentsRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ), journals=dict( pid_type='journals', route='/journals/<pid_value>', template='inspirehep_theme/format/record/Journal_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:JournalsRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ) ) RECORDS_REST_FACETS = { "records-hep": { "filters": { "author": terms_filter('exactauthor.raw'), "subject": terms_filter('facet_inspire_subjects'), "doc_type": terms_filter('facet_inspire_doc_type'), "formulas": terms_filter('facet_formulas'), "experiment": terms_filter( 'accelerator_experiments.facet_experiment'), }, "aggs": { "subject": { "terms": { "field": "facet_inspire_subjects", "size": 20 } }, "doc_type": { "terms": {
"""Result list template.""" PIDSTORE_RECID_FIELD = 'id' MY_SITE_ENDPOINTS_ENABLED = True """Enable/disable automatic endpoint registration.""" RECORDS_REST_FACETS = dict( records=dict( aggs=dict( type=dict(terms=dict(field='type')), keywords=dict(terms=dict(field='keywords')) ), post_filters=dict( type=terms_filter('type'), keywords=terms_filter('keywords'), ) ) ) """Introduce searching facets.""" RECORDS_REST_SORT_OPTIONS = dict( records=dict( bestmatch=dict( title=_('Best match'), fields=['_score'], default_order='desc', order=1, ),
} """Records UI for my-site.""" SEARCH_UI_JSTEMPLATE_RESULTS = 'templates/records/results.html' """Result list template.""" PIDSTORE_RECID_FIELD = 'control_number' MY_SITE_ENDPOINTS_ENABLED = True """Enable/disable automatic endpoint registration.""" RECORDS_REST_FACETS = dict( records=dict(aggs=dict(type=dict(terms=dict(field='type')), keywords=dict(terms=dict(field='keywords'))), post_filters=dict( type=terms_filter('type'), keywords=terms_filter('keywords'), ))) """Introduce searching facets.""" RECORDS_REST_SORT_OPTIONS = dict(records=dict( bestmatch=dict( title=_('Best match'), fields=['_score'], default_order='desc', order=1, ), mostrecent=dict( title=_('Most recent'), fields=['-_created'], default_order='asc',
class FilteredRecordsSearch(RecordsSearch): class Meta: index = RECORDS_SEARCH_INDEX doc_types = None default_filter = DefaultFilter(owner_permission_filter) def search_factory(*args, **kwargs): return default_search_factory(*args, query_parser=search_title, **kwargs) FILTERS = { 'title': language_aware_match_filter('title'), 'creator': terms_filter('creator.keyword'), 'owned': owned_filter('owners'), 'difficulty': terms_filter('difficulty'), 'license': terms_filter('license.keyword'), 'event.title.value.keyword': language_aware_terms_filter('event.title'), 'formats.title.value.keyword': language_aware_terms_filter('formats.title'), 'title.lang': nested_terms_filter('title', 'lang', lambda field: terms_filter(f'{field}')), }
'version': 'version' }), **{ 'from': nested_range_filter( 'from', 'History.LinkPublicationDate', op='gte') }, to=nested_range_filter('to', 'History.LinkPublicationDate', op='lte'), relation=enum_term_filter(label='relation', field='RelationshipType', choices={ 'isCitedBy': 'Cites', 'isSupplementedBy': 'IsSupplementTo', 'isRelatedTo': 'IsRelatedTo' }), type=terms_filter('Source.Type.Name'), ), )) # TODO: See if this actually works RECORDS_REST_SORT_OPTIONS = dict(relationships=dict(mostrecent=dict( fields=['Source.PublicationDate'], default_order='desc', ), ), ) RECORDS_REST_DEFAULT_SORT = {'relationships': {'noquery': 'mostrecent'}} APP_DEFAULT_SECURE_HEADERS['force_https'] = True APP_DEFAULT_SECURE_HEADERS['session_cookie_secure'] = True # Debug
# No permission checking RECORDS_REST_DEFAULT_CREATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY=None, DEPOSIT_SEARCH_API='/deposits', RECORDS_REST_FACETS=dict( deposits=dict( aggs=dict( status=dict(terms=dict( field='_deposit.status' )), ), post_filters=dict( status=terms_filter( '_deposit.status' ), ) ) ), RECORDS_UI_DEFAULT_PERMISSION_FACTORY=None, ) FlaskCLI(app) Babel(app) # Set jinja loader to first grab templates from the app's folder. app.jinja_loader = jinja2.ChoiceLoader([ jinja2.FileSystemLoader(join(dirname(__file__), "templates")), app.jinja_loader ])
RECORDS_REST_FACETS = dict( documents=dict( aggs=dict( # The organisation or library facet is defined # dynamically during the query (query.py) document_type=dict(terms=dict(field='type')), author__en=dict(terms=dict(field='facet_authors_en')), author__fr=dict(terms=dict(field='facet_authors_fr')), author__de=dict(terms=dict(field='facet_authors_de')), author__it=dict(terms=dict(field='facet_authors_it')), language=dict(terms=dict(field='language.value')), subject=dict(terms=dict(field='facet_subjects')), status=dict(terms=dict(field='holdings.items.status'))), filters={ _('document_type'): terms_filter('type'), _('organisation'): terms_filter('holdings.organisation.organisation_pid'), _('library'): terms_filter('holdings.organisation.library_pid'), _('author__en'): terms_filter('facet_authors_en'), _('author__fr'): terms_filter('facet_authors_fr'), _('author__de'): terms_filter('facet_authors_de'), _('author__it'): terms_filter('facet_authors_it'), _('language'): terms_filter('language.value'), _('subject'):
DEPOSIT_REST_SORT_OPTIONS = dict( deposits=dict(bestmatch=dict( fields=['-_score'], title='Best match', default_order='asc', order=2), mostrecent=dict(fields=['-_updated'], title='Most recent', default_order='asc', order=1))) DEPOSIT_REST_DEFAULT_SORT = dict( deposits=dict(query='bestmatch', noquery='mostrecent')) DEPOSIT_REST_FACETS = dict( deposits=dict(aggs=dict(status=dict(terms=dict( field='_deposit.status'), )), post_filters=dict(status=terms_filter('_deposit.status'), ))) DEPOSIT_RECORDS_UI_ENDPOINTS = dict(depid=dict( pid_type='depid', route='/deposit/<pid_value>', template='invenio_deposit/edit.html', record_class='invenio_deposit.api:Deposit', ), ) DEPOSIT_UI_INDEX_TEMPLATE = 'invenio_deposit/index.html' """Index template.""" DEPOSIT_UI_NEW_TEMPLATE = 'invenio_deposit/edit.html' """New deposit template.""" DEPOSIT_UI_JSTEMPLATE_ACTIONS = \
def app(request, search_class): """Flask application fixture. Note that RECORDS_REST_ENDPOINTS is used during application creation to create blueprints on the fly, hence once you have this fixture in a test, it's too late to customize the configuration variable. You can however customize it using parameterized tests: .. code-block:: python @pytest.mark.parametrize('app', [dict( endpoint=dict( search_class='conftest:TestSearch', ) def test_mytest(app, db, es): # ... This will parameterize the default 'recid' endpoint in RECORDS_REST_ENDPOINTS. """ instance_path = tempfile.mkdtemp() app = Flask('testapp', instance_path=instance_path) app.config.update( INDEXER_DEFAULT_DOC_TYPE='testrecord', INDEXER_DEFAULT_INDEX=search_class.Meta.index, RECORDS_REST_ENDPOINTS=config.RECORDS_REST_ENDPOINTS, RECORDS_REST_DEFAULT_CREATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY=None, RECORDS_REST_DEFAULT_SEARCH_INDEX=search_class.Meta.index, RECORDS_REST_FACETS={ search_class.Meta.index: { 'aggs': { 'stars': {'terms': {'field': 'stars'}} }, 'post_filters': { 'stars': terms_filter('stars'), } } }, RECORDS_REST_SORT_OPTIONS={ search_class.Meta.index: dict( year=dict( fields=['year'], ) ) }, SERVER_NAME='localhost:5000', SQLALCHEMY_DATABASE_URI=os.environ.get( 'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db' ), SQLALCHEMY_TRACK_MODIFICATIONS=True, TESTING=True, ) app.config['RECORDS_REST_ENDPOINTS']['recid']['search_class'] = \ search_class # Parameterize application. if hasattr(request, 'param'): if 'endpoint' in request.param: app.config['RECORDS_REST_ENDPOINTS']['recid'].update( request.param['endpoint']) app.url_map.converters['pid'] = PIDConverter FlaskCLI(app) InvenioDB(app) InvenioREST(app) InvenioRecords(app) InvenioPIDStore(app) search = InvenioSearch(app) search.register_mappings(search_class.Meta.index, 'mappings') InvenioRecordsREST(app) with app.app_context(): yield app # Teardown instance path. shutil.rmtree(instance_path)
".object_type.keyword" }, "aggs": { "doc_count": { "reverse_nested": {} } } } } }, } }, }, 'post_filters': { 'type': terms_filter('_type'), 'cms_working_group': prefix_filter('cadi_id'), 'publication_status': terms_filter('publication_status.keyword'), 'cadi_status': terms_filter('cadi_status'), 'conference': terms_filter('conference'), 'physics_objects': nested_filter( 'main_measurements.signal_event_selection.physics_objects', 'main_measurements.signal_event_selection' '.physics_objects.object'), 'physics_objects_type': nested_filter(
order=dict(_term='asc'))), keywords=dict(terms=dict( field='keywords.keyword', order=dict(_term='asc'))), collision_type=dict(terms=dict( field='collision_information.type.keyword', order=dict(_term='asc'))), collision_energy=dict(terms=dict( field='collision_information.energy.keyword', order=dict(_term='asc'))), topic_category=dict(terms=dict( field='topic.category.keyword', order=dict(_term='asc'))), ), 'post_filters': dict( experiment=terms_filter('experiment.keyword'), type=terms_filter('type.primary.keyword'), subtype=terms_filter('type.secondary.keyword'), year=terms_filter('date_created.keyword'), tags=terms_filter('tags.keyword'), keywords=terms_filter('keywords.keyword'), collision_type=terms_filter('collision_information.type.keyword'), collision_energy=terms_filter('collision_information.energy' '.keyword'), topic_category=terms_filter('topic.category.keyword'), file_type=terms_filter('distribution.formats.keyword'), collections=terms_filter('collections.keyword'), ), } }
order=dict(_term='asc'))), keywords=dict(terms=dict( field='keywords.keyword', order=dict(_term='asc'))), collision_type=dict(terms=dict( field='collision_information.type.keyword', order=dict(_term='asc'))), collision_energy=dict(terms=dict( field='collision_information.energy.keyword', order=dict(_term='asc'))), topic_category=dict(terms=dict( field='topic.category.keyword', order=dict(_term='asc'))), ), 'post_filters': dict( experiment=terms_filter('experiment.keyword'), type=terms_filter('type.primary.keyword'), subtype=terms_filter('type.secondary.keyword'), year=terms_filter('date_created.keyword'), tags=terms_filter('tags.keyword'), keywords=terms_filter('keywords.keyword'), collision_type=terms_filter('collision_information.type.keyword'), collision_energy=terms_filter('collision_information.energy' '.keyword'), topic_category=terms_filter('topic.category.keyword'), file_type=terms_filter('distribution.formats.keyword'), collections=terms_filter('collections'), ), } }
experiments=dict( pid_type='experiments', route='/experiments/<pid_value>', template='inspirehep_theme/format/record/Experiment_HTML_detailed.tpl' ), journals=dict( pid_type='journals', route='/journals/<pid_value>', template='inspirehep_theme/format/record/Journal_HTML_detailed.tpl' ) ) RECORDS_REST_FACETS = { "records-hep": { "filters": { "author": terms_filter('exactauthor.raw'), "subject": terms_filter('facet_inspire_subjects'), "doc_type": terms_filter('facet_inspire_doc_type'), "formulas": terms_filter('facet_formulas'), "experiment": terms_filter( 'accelerator_experiments.facet_experiment'), }, "aggs": { "subject": { "terms": { "field": "facet_inspire_subjects", "size": 20 } }, "doc_type": { "terms": {
access=dict(terms=dict(field="restricted")), tag=dict(terms=dict(field="tags", size=FACET_TAG_LIMIT)), language=dict(terms=dict(field="languages")), doctype=dict(terms=dict(field="document_type")), relation=dict(terms=dict(field="relation_types")), availability=dict(range=dict( field="circulation.has_items_for_loan", ranges=[{ "key": "available for loan", "from": 1 }], )), medium=dict(terms=dict(field="stock.mediums")), ), post_filters=dict( access=terms_filter("restricted"), doctype=terms_filter("document_type"), language=terms_filter("languages"), tag=terms_filter("tags"), availability=keyed_range_filter( "circulation.has_items_for_loan", {"available for loan": { "gt": 0 }}, ), relation=terms_filter("relation_types"), medium=terms_filter("stock.mediums"), ), ), document_requests=dict( # DocumentRequestSearch.Meta.index aggs=dict(
), ), RECORDS_REST_FACETS=dict( testrecords=dict( aggs=dict( authors=dict(terms=dict( field='added_entry_personal_name.personal_name')), languages=dict(terms=dict( field='language_code.language_code_of_text_' 'sound_track_or_separate_title')), topic=dict(terms=dict( field='subject_added_entry_topical_term.' 'topical_term_or_geographic_name_entry_element')), ), post_filters=dict( authors=terms_filter( 'added_entry_personal_name.personal_name'), languages=terms_filter( 'language_code.language_code_of_text_' 'sound_track_or_separate_title'), topic=terms_filter( 'subject_added_entry_topical_term.' 'topical_term_or_geographic_name_entry_element'), ) ) ), RECORDS_REST_SORT_OPTIONS=dict( testrecords=dict( bestmatch=dict( title='Best match', fields=['-_score'], default_order='asc',
testrecords=dict( aggs=dict( authors=dict(terms=dict( field='added_entry_personal_name.personal_name')), languages=dict(terms=dict( field='language_code.language_code_of_text_' 'sound_track_or_separate_title')), topic=dict(terms=dict( field='subject_added_entry_topical_term.' 'topical_term_or_geographic_name_entry_element')), years=dict(date_histogram=dict(field='imprint.complete_date', interval='year', format='yyyy')), ), post_filters=dict( authors=terms_filter( 'added_entry_personal_name.personal_name'), languages=terms_filter('language_code.language_code_of_text_' 'sound_track_or_separate_title'), topic=terms_filter( 'subject_added_entry_topical_term.' 'topical_term_or_geographic_name_entry_element'), years=range_filter('imprint.complete_date', format='yyyy', end_date_math='/y'), ))), RECORDS_REST_SORT_OPTIONS=dict( testrecords=dict(bestmatch=dict( title='Best match', fields=['-_score'], default_order='asc', order=1,
def test_terms_filter(): """Test terms filter.""" f = terms_filter('test') assert f(['a', 'b']).to_dict() == dict(terms={'test': ['a', 'b']})
access=dict(terms=dict(field="open_access")), tag=dict(terms=dict(field="tags", size=FACET_TAG_LIMIT)), language=dict(terms=dict(field="languages")), doctype=dict(terms=dict(field="document_type")), relation=dict(terms=dict(field="relation_types")), availability=dict(range=dict( field="circulation.has_items_for_loan", ranges=[{ "key": "available for loan", "from": 1 }], )), medium=dict(terms=dict(field="stock.mediums")), ), post_filters=dict( access=terms_filter("open_access"), doctype=terms_filter("document_type"), language=terms_filter("languages"), tag=terms_filter("tags"), availability=keyed_range_filter( "circulation.has_items_for_loan", {"available for loan": { "gt": 0 }}, ), relation=terms_filter("relation_types"), medium=terms_filter("stock.mediums"), ), ), document_requests=dict( # DocumentRequestSearch.Meta.index aggs=dict(
), enddate=dict( fields=['enddate'], title='End date', default_order='asc', order=2, ), )) #: Default sort for records REST API. OPENAIRE_REST_DEFAULT_SORT = dict( grants=dict(query='bestmatch', noquery='bestmatch'), funders=dict(query='bestmatch', noquery='bestmatch'), ) OPENAIRE_REST_FACETS = dict( funders=dict( aggs=dict( country=dict(terms=dict(field='country'), ), type=dict(terms=dict(field='type'), ), ), filters=dict( country=terms_filter('country'), type=terms_filter('type'), ), ), grants=dict( aggs=dict(funder=dict(terms=dict(field='funder.acronyms'), ), ), filters=dict(funder=terms_filter('funder.acronyms'), ), ))
template='inspirehep_theme/format/record/Institution_HTML_detailed.tpl' ), experiments=dict( pid_type='experiments', route='/experiments/<pid_value>', template='inspirehep_theme/format/record/Experiment_HTML_detailed.tpl' ), journals=dict( pid_type='journals', route='/journals/<pid_value>', template='inspirehep_theme/format/record/Journal_HTML_detailed.tpl')) RECORDS_REST_FACETS = { "records-hep": { "filters": { "author": terms_filter('exactauthor.raw'), "subject": terms_filter('facet_inspire_subjects'), "doc_type": terms_filter('facet_inspire_doc_type'), "formulas": terms_filter('facet_formulas'), "experiment": terms_filter('accelerator_experiments.facet_experiment'), }, "aggs": { "subject": { "terms": { "field": "facet_inspire_subjects", "size": 20 } }, "doc_type": { "terms": {
terms=dict(field="resource_type.subtype"), ) ) ), access_right=dict( terms=dict(field="access_right"), ), file_type=dict( terms=dict(field="files.type"), ), keywords=dict( terms=dict(field="keywords"), ), ), filters=dict( communities=terms_filter('communities'), provisional_communities=terms_filter('provisional_communities'), ), post_filters=dict( access_right=terms_filter('access_right'), type=terms_filter('resource_type.type'), subtype=terms_filter('resource_type.subtype'), file_type=terms_filter('files.type'), keywords=terms_filter('keywords'), ) ) ) RECORDS_REST_FACETS.update(OPENAIRE_REST_FACETS) # Previewer # =========
}, }, } #: Default sort for records REST API. RECORDS_REST_DEFAULT_SORT = { "records-record": { 'query': '-record_creation_date', 'noquery': '-record_creation_date' }, } RECORDS_REST_FACETS = { "records-record": { "filters": { "journal": terms_filter("publication_info.journal_title"), "country": terms_filter_with_must("authors.affiliations.country"), "collaboration": terms_filter("facet_collaboration"), "record_creation_date": range_filter( 'record_creation_date', format='yyyy', end_date_math='/y') }, "aggs": { "journal": { "terms": { "field": "publication_info.journal_title", "size": 20, "order": {"_term": "asc"} } },