def prov_test(self): iu = IndexerUtils(self.cfg) obj = dict() resp = iu._get_prov(obj) self.assertIn('prv_meth', resp) self.assertIsNone(resp['prv_meth']) prv = { 'service': 'bogus', 'method': 'bogus2', 'service_ver': '1.0', 'description': 'desc' } obj['provenance'] = [prv] resp = iu._get_prov(obj) self.assertEquals(resp['prv_mod'], 'bogus') self.assertEquals(resp['prv_meth'], 'bogus2') self.assertEquals(resp['prv_ver'], '1.0') self.assertEquals(resp['prv_cmt'], 'desc') # Test script obj['provenance'] = [{'script': 'bogus3', 'script_ver': '1.1'}] resp = iu._get_prov(obj) self.assertEquals(resp['prv_mod'], 'legacy_transform') self.assertEquals(resp['prv_meth'], 'bogus3') self.assertEquals(resp['prv_ver'], '1.1')
def index_request_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = self.wsinfo iu.ws.get_objects2.return_value = self.genobj iu.ws.get_object_info3.return_value = self.geninfo rv = {'docker_img_name': 'mock_indexer:latest'} iu.mr.catalog.get_module_version.return_value = rv ev = self.new_version_event.copy() ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = '3' self.reset() iu.process_event(ev) id = 'WS:1:3:3' res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) self.assertTrue(res['_source']['islast']) # Test with older version ev = self.new_version_event.copy() ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = '3' ev['ver'] = 2 self.reset() iu.process_event(ev) id = 'WS:1:3:2' res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) self.assertFalse(res['_source']['islast'])
def index_request_genome_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = self.wsinfo # iu.ws.get_objects2.return_value = self.genobj iu.ws.get_object_info3.return_value = self.geninfo rv = {'docker_img_name': 'test/kb_genomeindexer:latest'} iu.mr.catalog.get_module_version.return_value = rv ev = self.new_version_event.copy() ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = '2' ev['accgrp'] = 15792 ev['ver'] = 1 id = 'WS:15792:2:1' self.reset() iu.process_event(ev) res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) fid = 'WS:15792:2:1:L876_RS0116375' res = self.es.get(index=self._iname('genomefeature'), routing=id, doc_type='data', id=fid) self.assertIsNotNone(res) self.assertIn('ojson', res['_source']) self.assertTrue(res['_source']['islast'])
def rename_event_test(self): # RENAME_ALL_VERSIONS, ev = self.new_version_event.copy() ev['evtype'] = 'RENAME_ALL_VERSIONS' ev['objid'] = '3' iu = IndexerUtils(self.cfg) iu.process_event(ev)
def index_request_genome_test(self, mock_cat): iu = IndexerUtils(self.cfg) rv = {'docker_img_name': 'test/kb_genomeindexer:latest'} iu.method_runner.catalog.get_module_version.return_value = rv ev = self.new_version_event.copy() wsid = 15792 objid = 2 ver = 12 ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = str(objid) ev['wsid'] = wsid ev['ver'] = ver id = f'WS:{wsid:d}:{objid:d}:{ver:d}' self.reset() iu.process_event(ev) res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) fid = f'WS:{wsid:d}/{objid:d}/{ver:d}:feature/L876_RS0116375' res = self.es.get(index=self._iname('genomefeature'), routing=id, doc_type='data', id=fid) self.assertIsNotNone(res) self.assertIn('ojson', res['_source']) self.assertTrue(res['_source']['islast'])
def index_error_test(self): iu = IndexerUtils(self.cfg) if os.path.exists('error.log'): os.remove('error.log') iu._new_object_version_index = Mock(side_effect=KeyError()) iu._new_object_version_feature_index = Mock(side_effect=KeyError()) ev = self.new_version_event.copy() iu.process_event(ev) self.assertTrue(os.path.exists('error.log'))
def index_object_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = self.wsinfo # iu.ws.get_objects2.return_value = {'data': [self.narobj]} iu.ws.get_objects2.return_value = self.narobj rv = {'docker_img_name': 'mock_indexer:latest'} iu.mr.catalog.get_module_version.return_value = rv event = self.new_version_event.copy() event['upa'] = '1/2/3' res = iu.new_object_version(event) self.assertIsNotNone(res)
def skip_temp_test(self, wsa): iu = IndexerUtils(self.cfg) wsi = self.wsinfo wsi[8]['is_temporary'] = 'true' iu.ws.get_workspace_info.return_value = wsi # iu.ws.get_objects2.return_value = {'data': [self.narobj]} iu.ws.get_objects2.return_value = self.narobj res = iu._create_obj_rec('1/2/3') self.assertIsNone(res) res = iu._update_es_access('bogus', 1, 2, 3, '1/2/3') self.assertIsNone(res)
def delete_event_test(self, mock_ws): # DELETE_ALL_VERSIONS, # UNDELETE_ALL_VERSIONS, # DELETE_ACCESS_GROUP, self._init_es_genome() ev = self.new_version_event.copy() ev['evtype'] = 'DELETE_ALL_VERSIONS' ev['objid'] = '3' iu = IndexerUtils(self.cfg) iu.process_event(ev) id = 'WS:1:3:3' res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id, ignore=404) self.assertFalse(res['found']) # ev = self.new_version_event.copy() ev['evtype'] = 'UNDELETE_ALL_VERSIONS' iu = IndexerUtils(self.cfg)
def kafka_watcher(config): topic = config.get('kafka-topic', 'wsevents') indexer_topic = config.get('kafka-index-topic', 'idxevents') server = config.get('kafka-server', 'kafka') cgroup = config.get('kafka-clientgroup', 'search_indexer') config = config log = logging.getLogger('indexrunner') log.info("Initializing EventHandler") run_one = False if 'run_one' in config: run_one = True indexer = IndexerUtils(config) c = Consumer({ 'bootstrap.servers': server, 'group.id': cgroup, 'auto.offset.reset': 'earliest' }) log.info("Starting consumer") log.info("Server %s" % (server)) log.info("Group: %s" % (cgroup)) log.info("Topic: %s" % (topic)) log.info("Index Topic: %s" % (indexer_topic)) c.subscribe([topic, indexer_topic]) while True: msg = c.poll(0.5) data = None if msg is None: pass elif msg.error(): if msg.error().code() != KafkaError._PARTITION_EOF: err = str(msg.error()) _log_error('', err) log.error("Kafka error: " + err) else: try: data = json.loads(msg.value().decode('utf-8')) if data['strcde'] != 'WS': _log_error(data, 'Bad strcde') log.warning("Unreconginized strcde") else: indexer.process_event(data) except BaseException as e: _log_error(data, e) log.error('Uncaught exception: ' + str(e)) # This is just used in testing if run_one: break c.close()
def index_request_misc_test(self): iu = IndexerUtils(self.cfg) ev = self.new_version_event.copy() ev['objtype'] = 'KBaseGenomeAnnotations.Assembly' ev['objid'] = '31' ev['accgrp'] = 7998 ev['ver'] = 2 id = 'WS:7998:31:2' self.reset() iu.process_event(ev) res = self.es.get(index=self._iname('assembly'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res)
def reindex_event_test(self, mock_ep, mock_ws): ev = { "strcde": "WS", "accgrp": 1, "objid": None, "ver": None, "newname": None, "time": datetime.datetime.utcnow(), "evtype": "REINDEX_WORKSPACE", "objtype": None, "objtypever": None, "public": False } iu = IndexerUtils(self.cfg) iu.ws.list_objects.return_value = self.wslist iu.process_event(ev) iu.ep.index_objects.assert_called()
def copy_event_test(self, mock_ep, mock_ws): # COPY_ACCESS_GROUP; ev = { "strcde": "WS", "accgrp": 1, "objid": None, "ver": None, "newname": None, "time": datetime.datetime.utcnow(), "evtype": "COPY_ACCESS_GROUP", "objtype": None, "objtypever": None, "public": False } iu = IndexerUtils(self.cfg) iu.ws.list_objects.return_value = self.wslist iu.process_event(ev) iu.ep.index_objects.assert_called()
def get_indexes_test(self): iu = IndexerUtils(self.cfg) self.assertEqual(iu._get_indexes("KBaseNarrative.Narrative"), [{ 'index_method': 'NarrativeIndexer.index', 'index_name': 'ciraw.narrative' }]) self.assertEqual( iu._get_indexes("Foo.Bar"), [{ 'comment': 'Everything without a specific indexer', 'index_method': 'default_indexer', 'index_name': 'ciraw.objects' }]) self.assertEqual(iu._get_indexes("KBaseMatrices.ExpressionMatrix"), [{ 'index_method': 'KBaseMatrices.index', 'index_name': 'ciraw.KBaseMatrices' }])
def index_request_default_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = self.wsinfo obj = self.genobj iu.ws.get_objects2.return_value = obj iu.ws.get_object_info3.return_value = self.geninfo ev = self.new_version_event.copy() ev['objtype'] = 'Blah.Blah' ev['objid'] = '3' id = 'WS:1:3:3' self.reset() iu.process_event(ev) res = self.es.get(index=self._iname('objects'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) self.assertTrue(res['_source']['islast'])
def index_raw_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.mapping['KBaseGenomes.Genome'] = [{ 'index_method': 'bogus.genome_index', 'raw': True, 'index_name': 'ciraw.sketch' }] index = { 'schema': { 'upa': { 'type': 'string' }, 'mash': { 'type': 'binary' }, 'timestamp': { 'type': 'long' } }, 'data': { 'upa': '1/3/2', 'mash': 'blaslsadlfasdf', 'timestamp': '1234' } } iu.mr.run.return_value = [index] ev = self.new_version_event.copy() ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = '3' ev['ver'] = 2 self.reset() iu.process_event(ev) id = "WS:1:3:2" res = self.es.get(index=self._iname('sketch'), doc_type='data', id=id) self.assertIsNotNone(res) self.assertIn('mash', res['_source']) # Code path test... index and indexed object iu.process_event(ev) # Code path test ignore empty results ev['objtype'] = 'KBaseGenomes.Genome' ev['objid'] = '4' ev['ver'] = 2 iu.mr.run.return_value = [{}] iu.process_event(ev)
def index_new_all_test(self, mock_wsa, mock_cat): iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = self.wsinfo iu.ws.get_objects2.return_value = self.genobj iu.ws.get_object_info3.return_value = self.geninfo rv = {'docker_img_name': 'mock_indexer:latest'} iu.mr.catalog.get_module_version.return_value = rv ev = self.new_version_event.copy() ev['objtype'] = None ev['objid'] = '3' ev['evtype'] = 'NEW_ALL_VERSIONS' ev['ver'] = None id = 'WS:1:3:3' self.reset() iu.process_event(ev) res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertIsNotNone(res) self.assertIn('ojson', res['_source']) self.assertTrue(res['_source']['islast'])
def bogus_event_test(self): # RENAME_ALL_VERSIONS, ev = self.new_version_event.copy() ev['evtype'] = 'BOGUS' iu = IndexerUtils(self.cfg) iu.process_event(ev)
def publish_test(self, ws_mock): """ Publish and unpublish tests """ # PUBLISH_ALL_VERSIONS, # PUBLISH_ACCESS_GROUP, # UNPUBLISH_ALL_VERSIONS, # UNPUBLISH_ACCESS_GROUP, mwsi = [ 1, "auser:narrative_1485560571814", "auser", "2018-10-18T00:12:42+0000", 25, "a", "y", "unlocked", { "narrative_nice_name": "A Fancy Nasrrative", "is_temporary": "false", "data_palette_id": "22", "narrative": "23" } ] self._init_es_genome() ev = { "strcde": "WS", "accgrp": 1, "objid": None, "ver": None, "newname": None, "time": datetime.datetime.utcnow(), "evtype": "PUBLISH_ACCESS_GROUP", "objtype": None, "objtypever": None, "public": None } iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = mwsi iu.process_event(ev) # Check that accgrp changed id = 'WS:1:3:3' res = self.es.get(index=self._iname('genome'), routing=id, doc_type='access', id=id) self.assertIn(-1, res['_source']['groups']) res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertTrue(res['_source']['public']) # ev['evtype'] = "UNPUBLISH_ACCESS_GROUP" mwsi[6] = 'n' iu = IndexerUtils(self.cfg) iu.ws.get_workspace_info.return_value = mwsi iu.process_event(ev) # Check that accgrp changed id = 'WS:1:3:3' res = self.es.get(index=self._iname('genome'), routing=id, doc_type='access', id=id) self.assertNotIn(-1, res['_source']['groups']) res = self.es.get(index=self._iname('genome'), routing=id, doc_type='data', id=id) self.assertFalse(res['_source']['public'])
def acces_rec_test(self): iu = IndexerUtils(self.cfg) resp = iu._access_rec(1, 2, 3, public=True) self.assertIn(-1, resp['groups'])
def get_id_test(self): iu = IndexerUtils(self.cfg) with self.assertRaises(ValueError): iu._get_id('blah')