def test_bulk(self): ''' Test the Bulk.load(). ''' self.set_up() idx = IDX['MARKER']['indexName'] elastic = Search(ElasticQuery(Query.match_all()), idx=idx) hits_total1 = elastic.get_count()['count'] json_data = '{"index": {"_index": "%s", "_type": "%s"}}\n' % \ (idx, 'marker') json_data += json.dumps({"alt": "G", "start": 946, "seqid": "1", "filter": ".", "ref": "A", "id": "rsXXXXX", "qual": ".", "info": "RS=XXXXX"}) resp = Bulk.load(idx, '', json_data) self.assertNotEquals(resp.status_code, 200) # note: needs a trailing line return to work Bulk.load(idx, '', json_data + '\n') Search.index_refresh(idx) hits_total2 = elastic.get_count()['count'] self.assertEquals(hits_total2, hits_total1+1, "contains documents") # produce errors updating doc id that doesn't exist json_data += '{"delete": {"_index": "%s", "_type": "%s", "_id": "%s"}}\n' % \ (idx, 'marker', 'XYZ') json_data += '{"update": {"_index": "%s", "_type": "%s", "_id": "%s"}}\n' % \ (idx, 'marker', 'XYZ') json_data += '{"doc": {"start": 100, "end": 200}}\n' resp = Bulk.load(idx, '', json_data) self.assertTrue('errors' in resp.json() and resp.json()['errors'])
def setUpModule(): ''' Change ini config (MY_INI_FILE) to use the test suffix when creating pipeline indices. ''' ini_file = os.path.join(os.path.dirname(__file__), 'test_download.ini') if os.path.isfile(MY_INI_FILE): return with open(MY_INI_FILE, 'w') as new_file: with open(ini_file) as old_file: for line in old_file: new_file.write(line.replace('auto_tests', IDX_SUFFIX)) '''load ensembl GTF and GENE_HISTORY''' INI_CONFIG = IniParser().read_ini(MY_INI_FILE) idx = INI_CONFIG['ENSEMBL_GENE_GTF']['index'] call_command('pipeline', '--steps', 'load', sections='GENE_HISTORY', dir=TEST_DATA_DIR, ini=MY_INI_FILE) call_command('pipeline', '--steps', 'stage', 'load', sections='ENSEMBL_GENE_GTF', dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) call_command('pipeline', '--steps', 'load', sections='GENE2ENSEMBL', dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx)
def test_region_idx_loader(self): ''' Test loader has created and populated indices. ''' key = 'PRIVATE_REGIONS_GFF' if key in IDX.keys(): idx = IDX[key]['indexName'] Search.index_refresh(idx) self.assertTrue(Search.index_exists(idx=idx), 'Index exists: '+idx) ndocs = Search(idx=idx).get_count()['count'] self.assertTrue(ndocs > 0, "Elastic count documents in " + idx + ": " + str(ndocs))
def test_pub_ini_file2(self): ''' Test publication pipeline with a list of PMIDs. ''' out = StringIO() call_command('publications', '--dir', TEST_DATA_DIR, '--steps', 'load', sections='DISEASE::TEST', ini=MY_PUB_INI_FILE, stdout=out) INI_CONFIG = IniParser().read_ini(MY_PUB_INI_FILE) idx = INI_CONFIG['DISEASE']['index'] Search.index_refresh(idx) query = ElasticQuery.query_string("test", fields=["tags.disease"]) elastic = Search(query, idx=idx) docs = elastic.search().docs self.assertGreater(len(docs), 1)
def test_update_doc(self): ''' Update with a partial document. ''' idx = IDX['MARKER']['indexName'] docs = Search(ElasticQuery(Query.term("id", "rs2476601"), sources=['id']), idx=idx).search().docs self.assertEquals(len(docs), 1, "rs2476601 document") update_field = {"doc": {"start": 100, "end": 200}} Update.update_doc(docs[0], update_field) Search.index_refresh(IDX['MARKER']['indexName']) docs = Search(ElasticQuery(Query.term("id", "rs2476601")), idx=idx).search().docs self.assertEquals(len(docs), 1, "rs2476601 document") self.assertEquals(getattr(docs[0], 'start'), 100, "rs2476601 start") self.assertEquals(getattr(docs[0], 'end'), 200, "rs2476601 end")
def test_region_idx_loader(self): ''' Test loader has created and populated indices. ''' key = 'PRIVATE_REGIONS_GFF' if key in IDX.keys(): idx = IDX[key]['indexName'] Search.index_refresh(idx) self.assertTrue(Search.index_exists(idx=idx), 'Index exists: ' + idx) ndocs = Search(idx=idx).get_count()['count'] self.assertTrue( ndocs > 0, "Elastic count documents in " + idx + ": " + str(ndocs))
def test_gene_history_loader(self): """ Test the gene history loading. """ call_command("pipeline", "--steps", "load", sections="GENE_HISTORY", dir=TEST_DATA_DIR, ini=MY_INI_FILE) INI_CONFIG = IniParser().read_ini(MY_INI_FILE) idx = INI_CONFIG["GENE_HISTORY"]["index"] idx_type = INI_CONFIG["GENE_HISTORY"]["index_type"] elastic = Search(idx=idx, idx_type=idx_type) Search.index_refresh(idx) self.assertTrue(elastic.get_count()["count"] > 1, "Count documents in the index") map1_props = Gene.gene_history_mapping(idx, idx_type, test_mode=True).mapping_properties map2_props = elastic.get_mapping() if idx not in map2_props: logger.error("MAPPING ERROR: " + json.dumps(map2_props)) self._cmpMappings(map2_props[idx]["mappings"], map1_props, idx_type)
def test_marker_pipeline(self): """ Test marker pipeline. """ call_command("pipeline", "--steps", "load", sections="DBSNP", dir=TEST_DATA_DIR, ini=MY_INI_FILE) INI_CONFIG = IniParser().read_ini(MY_INI_FILE) idx = INI_CONFIG["DBSNP"]["index"] idx_type = INI_CONFIG["DBSNP"]["index_type"] elastic = Search(idx=idx, idx_type=idx_type) Search.index_refresh(idx) self.assertGreater(elastic.get_count()["count"], 0) call_command("pipeline", "--steps", "load", sections="RSMERGEARCH", dir=TEST_DATA_DIR, ini=MY_INI_FILE) idx = INI_CONFIG["RSMERGEARCH"]["index"] idx_type = INI_CONFIG["RSMERGEARCH"]["index_type"] elastic = Search(idx=idx, idx_type=idx_type) Search.index_refresh(idx) self.assertGreater(elastic.get_count()["count"], 0)
def setUpModule(): ''' Run the index loading script to create test indices and create test repository ''' for idx_kwargs in IDX.values(): call_command('index_search', **idx_kwargs) # wait for the elastic load to finish for key in IDX: Search.index_refresh(IDX[key]['indexName']) for idx_kwargs in IDX_UPDATE.values(): call_command('index_search', **idx_kwargs) # create test repository call_command('repository', SnapshotTest.TEST_REPO, dir=SnapshotTest.TEST_REPO_DIR)
def test_delete_docs_by_query(self): ''' Test deleting docs using a query. ''' self.set_up() idx = IDX['MARKER']['indexName'] elastic = Search(ElasticQuery(Query.match_all()), idx=idx) hits_total1 = elastic.get_count()['count'] self.assertGreater(hits_total1, 0, "contains documents") # delete single doc Delete.docs_by_query(idx, query=Query.term("id", "rs2476601")) Search.index_refresh(idx) hits_total2 = elastic.get_count()['count'] self.assertEquals(hits_total2, hits_total1-1, "contains documents") # delete remaining docs Delete.docs_by_query(idx, 'marker') Search.index_refresh(idx) self.assertEquals(elastic.get_count()['count'], 0, "contains no documents")
def setupIdx(cls, idx_name_arr): ''' Setup indices in the given array of key names (e.g. ['GENE', 'DIISEASE', ...]). ''' idx_settings = { "settings": { "analysis": { "analyzer": { "full_name": {"filter": ["standard", "lowercase"], "tokenizer": "keyword"}} }, "number_of_shards": 1 } } IDX = PydginTestSettings.IDX for name in idx_name_arr: requests.put(ElasticSettings.url() + '/' + IDX[name]['indexName'], data=json.dumps(idx_settings)) call_command('index_search', **IDX[name]) for name in idx_name_arr: # wait for the elastic load to finish Search.index_refresh(IDX[name]['indexName'])
def test_doc_auth(self): ''' Test private documents are not returned in the search. ''' idx = PydginTestSettings.IDX['MARKER']['indexName'] docs = Search(ElasticQuery(Query.match_all(), sources=['id']), idx=idx, size=1).search().docs self.assertEquals(len(docs), 1, "MARKER document") marker_id = getattr(docs[0], 'id') url = reverse('search_page') resp = self.client.post(url+'?idx=ALL&query='+marker_id) nhits1 = resp.context['hits_total'] self.assertGreater(nhits1, 0, 'search hits > 0') # update document to be in DIL update_field = {"doc": {"group_name": "DIL"}} Update.update_doc(docs[0], update_field) Search.index_refresh(PydginTestSettings.IDX['MARKER']['indexName']) resp = self.client.post(url+'?idx=ALL&query='+marker_id) nhits2 = resp.context['hits_total'] self.assertEqual(nhits1-1, nhits2, 'private document hidden')
def test_doc_auth(self): idx = PydginTestSettings.IDX["STUDY_HITS"]["indexName"] docs = Search(ElasticQuery(Query.match_all(), sources=["chr_band", "marker"]), idx=idx, size=1).search().docs self.assertEquals(len(docs), 1, "STUDY_HITS document") marker_id = getattr(docs[0], "marker") url = reverse("search_page") resp = self.client.post(url + "?idx=ALL&query=" + marker_id) nhits1 = resp.context["hits_total"] self.assertGreater(nhits1, 0, "search hits > 0") # update document to be in DIL update_field = {"doc": {"group_name": "DIL"}} Update.update_doc(docs[0], update_field) Search.index_refresh(PydginTestSettings.IDX["STUDY_HITS"]["indexName"]) url = reverse("search_page") resp = self.client.post(url + "?idx=ALL&query=" + marker_id) nhits2 = resp.context["hits_total"] self.assertEqual(nhits1 - 1, nhits2, "private document hidden")
def test_mapping_parent_child(self): ''' Test creating mapping with parent child relationship. ''' gene_mapping = MappingProperties("gene") gene_mapping.add_property("symbol", "string", analyzer="full_name") inta_mapping = MappingProperties("publication", "gene") load = Loader() idx = "test__mapping__"+SEARCH_SUFFIX options = {"indexName": idx, "shards": 1} requests.delete(ElasticSettings.url() + '/' + idx) # add child mappings first status = load.mapping(inta_mapping, "publication", analyzer=Loader.KEYWORD_ANALYZER, **options) self.assertTrue(status, "mapping inteactions") status = load.mapping(gene_mapping, "gene", analyzer=Loader.KEYWORD_ANALYZER, **options) self.assertTrue(status, "mapping genes") ''' load docs and test has parent query''' json_data = '{"index": {"_index": "%s", "_type": "gene", "_id" : "1"}}\n' % idx json_data += json.dumps({"symbol": "PAX1"}) + '\n' json_data += '{"index": {"_index": "%s", "_type": "publication", "_id" : "2", "parent": "1"}}\n' % idx json_data += json.dumps({"pubmed": 1234}) + '\n' Bulk.load(idx, '', json_data) Search.index_refresh(idx) query = ElasticQuery.has_parent('gene', Query.match('symbol', 'PAX1')) elastic = Search(query, idx=idx, idx_type='publication', size=500) docs = elastic.search().docs self.assertEquals(len(docs), 1) self.assertEquals(getattr(docs[0], 'pubmed'), 1234) self.assertEquals(docs[0].parent(), '1') self.assertRaises(QueryError, ElasticQuery.has_parent, 'gene', 'xxxxx') ''' test has child query ''' query = ElasticQuery.has_child('publication', Query.match('pubmed', 1234)) elastic = Search(query, idx=idx, idx_type='gene', size=500) docs = elastic.search().docs self.assertEquals(len(docs), 1) self.assertEquals(getattr(docs[0], 'symbol'), 'PAX1') self.assertEquals(docs[0].parent(), None) requests.delete(ElasticSettings.url() + '/' + idx)
def test_create_restore_delete_snapshot(self): self.wait_for_running_snapshot() snapshot = 'test_' + ElasticSettings.getattr('TEST') repo = SnapshotTest.TEST_REPO # create a snapshot call_command('snapshot', snapshot, indices=IDX['MARKER']['indexName'], repo=repo) Snapshot.wait_for_snapshot(repo, snapshot) self.assertTrue(Snapshot.exists(repo, snapshot), "Created snapshot " + snapshot) # snapshot already exist so return false self.assertFalse( Snapshot.create_snapshot(repo, snapshot, IDX['MARKER']['indexName'])) # delete index requests.delete(ElasticSettings.url() + '/' + IDX['MARKER']['indexName']) self.assertFalse(Search.index_exists(IDX['MARKER']['indexName']), "Removed index") # restore from snapshot call_command('restore_snapshot', snapshot, repo=repo, indices=IDX['MARKER']['indexName']) Search.index_refresh(IDX['MARKER']['indexName']) self.assertTrue(Search.index_exists(IDX['MARKER']['indexName']), "Restored index exists") # remove snapshot call_command('snapshot', snapshot, delete=True, repo=repo) Snapshot.wait_for_snapshot(repo, snapshot, delete=True, count=10) self.assertFalse(Snapshot.exists(repo, snapshot), "Deleted snapshot " + snapshot)
def setUpModule(): ''' Load test indices (marker) ''' call_command('index_search', **IDX['MARKER']) call_command('index_search', **IDX['GFF_GENERIC']) call_command('index_search', **IDX['JSON_NESTED']) # wait for the elastic load to finish Search.index_refresh(IDX['MARKER']['indexName']) Search.index_refresh(IDX['GFF_GENERIC']['indexName']) Search.index_refresh(IDX['JSON_NESTED']['indexName'])
def test_elastic_group_name(self): ''' Testing the workflow defined in: https://killin.cimr.cam.ac.uk/nextgensite/2015/08/05/region-authorization/ Testing various elastic queries idx doc: "_source":{"attr": {"region_id": "803", "group_name": "[\"DIL\"]", "Name": "4q27"}, "seqid": "chr4", "source": "immunobase", "type": "region", "score": ".", "strand": ".", "phase": ".", "start": 122061159, "end": 122684373} idx_query: Private(in given group) OR Public -d '{"query":{"filtered":{"filter":{"bool": { "should": [ {"terms": {"group_name":["dil"]}}, { "missing": { "field": "group_name" }} ] }}}}}' Private(in given group): -d '{"query":{"filtered":{"filter":{"terms":{"group_name":["dil"]}}}}}' Public: -d {'query': {'filtered': {'filter': {'missing': {'field': 'group_name'}}, - 'query': {'term': {'match_all': '{}'}}}}} ''' # get the groups for the given user response = self.client.post('/accounts/login/', { 'username': '******', 'password': '******' }) self.assertTrue(response.status_code, "200") logged_in_user = User.objects.get( id=self.client.session['_auth_user_id']) if logged_in_user and logged_in_user.is_authenticated(): user_groups = get_user_groups(logged_in_user) self.assertTrue('READ' in user_groups, "user present in READ group") # make sure the user is not yet in DIL group self.assertFalse('DIL' in user_groups, "user not present in DIL group") group_names = get_user_groups(logged_in_user) if 'READ' in group_names: group_names.remove('READ') # @IgnorePep8 group_names = [x.lower() for x in group_names] self.assertTrue(len(group_names) == 0, "No group present") # Match all query, as there is no group we do a match all query = ElasticQuery(Query.match_all()) expected_query_string = {"query": {"match_all": {}}} self.assertJSONEqual(json.dumps(query.query), json.dumps(expected_query_string), "Query string matched") Search.index_refresh(self.index_name) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue( len(docs) == 12, "Elastic string query retrieved all public regions") # Filtered query for group names, add the user to DIL group and get the query string self.dil_group = Group.objects.create(name='DIL') logged_in_user.groups.add(self.dil_group) group_names = get_user_groups(logged_in_user) if 'READ' in group_names: group_names.remove('READ') # @IgnorePep8 group_names = [x.lower() for x in group_names] self.assertTrue(len(group_names) > 0, "More than 1 group present") self.assertTrue("dil" in group_names, "DIL group present") # retrieves all docs with missing field group_name - 11 docs terms_filter = TermsFilter.get_missing_terms_filter( "field", "attr.group_name") query = ElasticQuery.filtered(Query.match_all(), terms_filter) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue( len(docs) == 11, "Elastic string query retrieved all public regions") # build filtered boolean query to bring all public docs + private docs 11+1 = 12 docs query_bool = BoolQuery() query_bool.should(Query.missing_terms("field", "group_name")) \ .should(Query.terms("group_name", group_names).query_wrap()) query = ElasticQuery.filtered_bool(Query.match_all(), query_bool) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue( len(docs) == 12, "Elastic string query retrieved both public + private regions") terms_filter = TermsFilter.get_terms_filter("attr.group_name", group_names) query = ElasticQuery.filtered(Query.match_all(), terms_filter) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue( len(docs) == 1, "Elastic string query retrieved one private regions") self.assertEqual(docs[0].attr['Name'], "4q27", "type matched region") self.assertEqual(docs[0].attr['region_id'], "803", "type matched region") self.assertEqual(docs[0].attr['group_name'], "[\"DIL\"]", "type matched region")
def set_up(self): ''' Load test indices (marker) ''' call_command('index_search', **IDX['MARKER']) # wait for the elastic load to finish Search.index_refresh(IDX['MARKER']['indexName'])
def setUpModule(): ''' Load test indices (marker) ''' call_command('index_search', **IDX['MARKER']) Search.index_refresh(IDX['MARKER']['indexName'])
def test_gene_pipeline(self): """ Test gene pipeline. """ INI_CONFIG = IniParser().read_ini(MY_INI_FILE) idx = INI_CONFIG["ENSEMBL_GENE_GTF"]["index"] idx_type = INI_CONFIG["ENSEMBL_GENE_GTF"]["index_type"] """ 1. Test ensembl GTF loading. """ call_command( "pipeline", "--steps", "stage", "load", sections="ENSEMBL_GENE_GTF", dir=TEST_DATA_DIR, ini=MY_INI_FILE ) Search.index_refresh(idx) elastic = Search(idx=idx, idx_type=idx_type) self.assertGreaterEqual(elastic.get_count()["count"], 1, "Count documents in the index") map1_props = Gene.gene_mapping(idx, idx_type, test_mode=True).mapping_properties map2_props = elastic.get_mapping() if idx not in map2_props: logger.error("MAPPING ERROR: " + json.dumps(map2_props)) self._cmpMappings(map2_props[idx]["mappings"], map1_props, idx_type) """ 2. Test adding entrez ID to documents """ call_command("pipeline", "--steps", "load", sections="GENE2ENSEMBL", dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) query = ElasticQuery.query_string("PTPN22", fields=["symbol"]) elastic = Search(query, idx=idx) docs = elastic.search().docs self.assertEqual(len(docs), 1) self.assertTrue("entrez" in getattr(docs[0], "dbxrefs")) self.assertEqual(getattr(docs[0], "dbxrefs")["entrez"], "26191") """ 3. Add uniprot and fill in missing entrez fields. """ call_command( "pipeline", "--steps", "download", "load", sections="ENSMART_GENE", dir=TEST_DATA_DIR, ini=MY_INI_FILE ) Search.index_refresh(idx) query = ElasticQuery.query_string("DNMT3L", fields=["symbol"]) elastic = Search(query, idx=idx) docs = elastic.search().docs self.assertTrue("entrez" in getattr(docs[0], "dbxrefs")) self.assertTrue("swissprot" in getattr(docs[0], "dbxrefs")) """ 4. Add gene synonyms and dbxrefs. """ call_command("pipeline", "--steps", "load", sections="GENE_INFO", dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) query = ElasticQuery.query_string("PTPN22", fields=["symbol"]) elastic = Search(query, idx=idx) docs = elastic.search().docs self.assertTrue("PTPN8" in getattr(docs[0], "synonyms")) """ 5. Add PMIDs to gene docs. """ call_command("pipeline", "--steps", "load", sections="GENE_PUBS", dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) query = ElasticQuery.query_string("PTPN22", fields=["symbol"]) elastic = Search(query, idx=idx) docs = elastic.search().docs self.assertGreater(len(getattr(docs[0], "pmids")), 0) """ 6. Add ortholog data. """ call_command("pipeline", "--steps", "load", sections="ENSMART_HOMOLOG", dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) query = ElasticQuery.query_string("PTPN22", fields=["symbol"]) elastic = Search(query, idx=idx) docs = elastic.search().docs dbxrefs = getattr(docs[0], "dbxrefs") self.assertTrue("orthologs" in dbxrefs, dbxrefs) self.assertTrue("mmusculus" in dbxrefs["orthologs"], dbxrefs) self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"]) query = ElasticQuery.filtered( Query.match_all(), TermsFilter.get_terms_filter("dbxrefs.orthologs.mmusculus.ensembl", ["ENSMUSG00000027843"]), ) docs = Search(query, idx=idx, size=1).search().docs self.assertEqual(len(docs), 1) """ 7. Add mouse ortholog link to MGI """ call_command("pipeline", "--steps", "load", sections="ENSEMBL2MGI", dir=TEST_DATA_DIR, ini=MY_INI_FILE) Search.index_refresh(idx) docs = Search(query, idx=idx, size=1).search().docs dbxrefs = getattr(docs[0], "dbxrefs") self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"]) self.assertEqual("107170", dbxrefs["orthologs"]["mmusculus"]["MGI"])
"tier": tier, "species": species, "tags": {"weight": weight}, "locus_id": locus_id, "seqid": seqid, "hits": doc_ids } # "suggest": {"input": [disease+" "+regionName, regionName], "weight": weight} resp = requests.put(ElasticSettings.url()+'/' + idx+'/disease_locus/'+locus_id, data=json.dumps(data)) if resp.status_code != 201: print(str(resp.content)) print("Problem loading "+getattr(doc, "disease")+" "+regionName) else: print("Loaded "+locus_id+" - "+regionName) Search.index_refresh(idx) diseases_by_seqid = Agg('diseases_by_seqid', 'terms', {"size": 0, "field": "disease"}) disease_hits = Agg('disease_hits', 'reverse_nested', {}, sub_agg=diseases_by_seqid) seq_hits = Agg('seq_hits', 'terms', {'field': 'build_info.seqid', 'size': 0}, sub_agg=disease_hits) build_info = Agg('build_info', 'nested', {"path": 'build_info'}, sub_agg=[seq_hits]) qnested = ElasticQuery(Query.nested('build_info', Query.term("build_info.build", build))) elastic = Search(qnested, idx=idx, aggs=Aggs(build_info), search_type='count') resultObj = elastic.search() resultAggs = resultObj.aggs seq_hits = getattr(resultAggs['build_info'], 'seq_hits')['buckets'] for chr_bucket in seq_hits: seqid = chr_bucket['key'].upper() for disease_bucket in chr_bucket['disease_hits']['diseases_by_seqid']['buckets']:
def test_elastic_group_name(self): ''' Testing the workflow defined in: https://killin.cimr.cam.ac.uk/nextgensite/2015/08/05/region-authorization/ Testing various elastic queries idx doc: "_source":{"attr": {"region_id": "803", "group_name": "[\"DIL\"]", "Name": "4q27"}, "seqid": "chr4", "source": "immunobase", "type": "region", "score": ".", "strand": ".", "phase": ".", "start": 122061159, "end": 122684373} idx_query: Private(in given group) OR Public -d '{"query":{"filtered":{"filter":{"bool": { "should": [ {"terms": {"group_name":["dil"]}}, { "missing": { "field": "group_name" }} ] }}}}}' Private(in given group): -d '{"query":{"filtered":{"filter":{"terms":{"group_name":["dil"]}}}}}' Public: -d {'query': {'filtered': {'filter': {'missing': {'field': 'group_name'}}, - 'query': {'term': {'match_all': '{}'}}}}} ''' # get the groups for the given user response = self.client.post('/accounts/login/', {'username': '******', 'password': '******'}) self.assertTrue(response.status_code, "200") logged_in_user = User.objects.get(id=self.client.session['_auth_user_id']) if logged_in_user and logged_in_user.is_authenticated(): user_groups = get_user_groups(logged_in_user) self.assertTrue('READ' in user_groups, "user present in READ group") # make sure the user is not yet in DIL group self.assertFalse('DIL' in user_groups, "user not present in DIL group") group_names = get_user_groups(logged_in_user) if 'READ' in group_names : group_names.remove('READ') # @IgnorePep8 group_names = [x.lower() for x in group_names] self.assertTrue(len(group_names) == 0, "No group present") # Match all query, as there is no group we do a match all query = ElasticQuery(Query.match_all()) expected_query_string = {"query": {"match_all": {}}} self.assertJSONEqual(json.dumps(query.query), json.dumps(expected_query_string), "Query string matched") Search.index_refresh(self.index_name) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue(len(docs) == 12, "Elastic string query retrieved all public regions") # Filtered query for group names, add the user to DIL group and get the query string self.dil_group = Group.objects.create(name='DIL') logged_in_user.groups.add(self.dil_group) group_names = get_user_groups(logged_in_user) if 'READ' in group_names : group_names.remove('READ') # @IgnorePep8 group_names = [x.lower() for x in group_names] self.assertTrue(len(group_names) > 0, "More than 1 group present") self.assertTrue("dil" in group_names, "DIL group present") # retrieves all docs with missing field group_name - 11 docs terms_filter = TermsFilter.get_missing_terms_filter("field", "attr.group_name") query = ElasticQuery.filtered(Query.match_all(), terms_filter) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue(len(docs) == 11, "Elastic string query retrieved all public regions") # build filtered boolean query to bring all public docs + private docs 11+1 = 12 docs query_bool = BoolQuery() query_bool.should(Query.missing_terms("field", "group_name")) \ .should(Query.terms("group_name", group_names).query_wrap()) query = ElasticQuery.filtered_bool(Query.match_all(), query_bool) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue(len(docs) == 12, "Elastic string query retrieved both public + private regions") terms_filter = TermsFilter.get_terms_filter("attr.group_name", group_names) query = ElasticQuery.filtered(Query.match_all(), terms_filter) elastic = Search(query, idx=self.index_name) docs = elastic.search().docs self.assertTrue(len(docs) == 1, "Elastic string query retrieved one private regions") self.assertEqual(docs[0].attr['Name'], "4q27", "type matched region") self.assertEqual(docs[0].attr['region_id'], "803", "type matched region") self.assertEqual(docs[0].attr['group_name'], "[\"DIL\"]", "type matched region")
def setUpModule(): ''' Load test indices (marker) ''' call_command('index_search', **IDX['MARKER']) call_command('index_search', **IDX['MARKER_RS_HISTORY']) call_command('index_search', **IDX['JSON_MARKER_IC']) Search.index_refresh(IDX['MARKER']['indexName'])