Ejemplo n.º 1
0
 def index_request_genome_test(self, mock_cat):
     iu = IndexerUtils(self.cfg)
     rv = {'docker_img_name': 'test/kb_genomeindexer:latest'}
     iu.method_runner.catalog.get_module_version.return_value = rv
     ev = self.new_version_event.copy()
     wsid = 15792
     objid = 2
     ver = 12
     ev['objtype'] = 'KBaseGenomes.Genome'
     ev['objid'] = str(objid)
     ev['wsid'] = wsid
     ev['ver'] = ver
     id = f'WS:{wsid:d}:{objid:d}:{ver:d}'
     self.reset()
     iu.process_event(ev)
     res = self.es.get(index=self._iname('genome'),
                       routing=id,
                       doc_type='data',
                       id=id)
     self.assertIsNotNone(res)
     fid = f'WS:{wsid:d}/{objid:d}/{ver:d}:feature/L876_RS0116375'
     res = self.es.get(index=self._iname('genomefeature'),
                       routing=id,
                       doc_type='data',
                       id=fid)
     self.assertIsNotNone(res)
     self.assertIn('ojson', res['_source'])
     self.assertTrue(res['_source']['islast'])
Ejemplo n.º 2
0
    def prov_test(self):
        iu = IndexerUtils(self.cfg)
        obj = dict()
        resp = iu._get_prov(obj)
        self.assertIn('prv_meth', resp)
        self.assertIsNone(resp['prv_meth'])
        prv = {
            'service': 'bogus',
            'method': 'bogus2',
            'service_ver': '1.0',
            'description': 'desc'
        }
        obj['provenance'] = [prv]
        resp = iu._get_prov(obj)
        self.assertEquals(resp['prv_mod'], 'bogus')
        self.assertEquals(resp['prv_meth'], 'bogus2')
        self.assertEquals(resp['prv_ver'], '1.0')
        self.assertEquals(resp['prv_cmt'], 'desc')

        # Test script
        obj['provenance'] = [{'script': 'bogus3', 'script_ver': '1.1'}]
        resp = iu._get_prov(obj)
        self.assertEquals(resp['prv_mod'], 'legacy_transform')
        self.assertEquals(resp['prv_meth'], 'bogus3')
        self.assertEquals(resp['prv_ver'], '1.1')
Ejemplo n.º 3
0
 def index_request_genome_test(self, mock_wsa, mock_cat):
     iu = IndexerUtils(self.cfg)
     iu.ws.get_workspace_info.return_value = self.wsinfo
     # iu.ws.get_objects2.return_value = self.genobj
     iu.ws.get_object_info3.return_value = self.geninfo
     rv = {'docker_img_name': 'test/kb_genomeindexer:latest'}
     iu.mr.catalog.get_module_version.return_value = rv
     ev = self.new_version_event.copy()
     ev['objtype'] = 'KBaseGenomes.Genome'
     ev['objid'] = '2'
     ev['accgrp'] = 15792
     ev['ver'] = 1
     id = 'WS:15792:2:1'
     self.reset()
     iu.process_event(ev)
     res = self.es.get(index=self._iname('genome'),
                       routing=id,
                       doc_type='data',
                       id=id)
     self.assertIsNotNone(res)
     fid = 'WS:15792:2:1:L876_RS0116375'
     res = self.es.get(index=self._iname('genomefeature'),
                       routing=id,
                       doc_type='data',
                       id=fid)
     self.assertIsNotNone(res)
     self.assertIn('ojson', res['_source'])
     self.assertTrue(res['_source']['islast'])
Ejemplo n.º 4
0
 def rename_event_test(self):
     # RENAME_ALL_VERSIONS,
     ev = self.new_version_event.copy()
     ev['evtype'] = 'RENAME_ALL_VERSIONS'
     ev['objid'] = '3'
     iu = IndexerUtils(self.cfg)
     iu.process_event(ev)
Ejemplo n.º 5
0
 def index_object_test(self, mock_wsa, mock_cat):
     iu = IndexerUtils(self.cfg)
     iu.ws.get_workspace_info.return_value = self.wsinfo
     # iu.ws.get_objects2.return_value = {'data': [self.narobj]}
     iu.ws.get_objects2.return_value = self.narobj
     rv = {'docker_img_name': 'mock_indexer:latest'}
     iu.mr.catalog.get_module_version.return_value = rv
     event = self.new_version_event.copy()
     event['upa'] = '1/2/3'
     res = iu.new_object_version(event)
     self.assertIsNotNone(res)
Ejemplo n.º 6
0
 def skip_temp_test(self, wsa):
     iu = IndexerUtils(self.cfg)
     wsi = self.wsinfo
     wsi[8]['is_temporary'] = 'true'
     iu.ws.get_workspace_info.return_value = wsi
     # iu.ws.get_objects2.return_value = {'data': [self.narobj]}
     iu.ws.get_objects2.return_value = self.narobj
     res = iu._create_obj_rec('1/2/3')
     self.assertIsNone(res)
     res = iu._update_es_access('bogus', 1, 2, 3, '1/2/3')
     self.assertIsNone(res)
Ejemplo n.º 7
0
def kafka_watcher(config):
    topic = config.get('kafka-topic', 'wsevents')
    indexer_topic = config.get('kafka-index-topic', 'idxevents')
    server = config.get('kafka-server', 'kafka')
    cgroup = config.get('kafka-clientgroup', 'search_indexer')
    config = config
    log = logging.getLogger('indexrunner')
    log.info("Initializing EventHandler")
    run_one = False
    if 'run_one' in config:
        run_one = True
    indexer = IndexerUtils(config)
    c = Consumer({
        'bootstrap.servers': server,
        'group.id': cgroup,
        'auto.offset.reset': 'earliest'
    })
    log.info("Starting consumer")
    log.info("Server %s" % (server))
    log.info("Group: %s" % (cgroup))
    log.info("Topic: %s" % (topic))
    log.info("Index Topic: %s" % (indexer_topic))

    c.subscribe([topic, indexer_topic])

    while True:
        msg = c.poll(0.5)

        data = None
        if msg is None:
            pass
        elif msg.error():
            if msg.error().code() != KafkaError._PARTITION_EOF:
                err = str(msg.error())
                _log_error('', err)
                log.error("Kafka error: " + err)
        else:
            try:
                data = json.loads(msg.value().decode('utf-8'))
                if data['strcde'] != 'WS':
                    _log_error(data, 'Bad strcde')
                    log.warning("Unreconginized strcde")
                else:
                    indexer.process_event(data)
            except BaseException as e:
                _log_error(data, e)
                log.error('Uncaught exception: ' + str(e))
            # This is just used in testing
        if run_one:
            break
    c.close()
Ejemplo n.º 8
0
 def index_request_misc_test(self):
     iu = IndexerUtils(self.cfg)
     ev = self.new_version_event.copy()
     ev['objtype'] = 'KBaseGenomeAnnotations.Assembly'
     ev['objid'] = '31'
     ev['accgrp'] = 7998
     ev['ver'] = 2
     id = 'WS:7998:31:2'
     self.reset()
     iu.process_event(ev)
     res = self.es.get(index=self._iname('assembly'),
                       routing=id,
                       doc_type='data',
                       id=id)
     self.assertIsNotNone(res)
Ejemplo n.º 9
0
 def reindex_event_test(self, mock_ep, mock_ws):
     ev = {
         "strcde": "WS",
         "accgrp": 1,
         "objid": None,
         "ver": None,
         "newname": None,
         "time": datetime.datetime.utcnow(),
         "evtype": "REINDEX_WORKSPACE",
         "objtype": None,
         "objtypever": None,
         "public": False
     }
     iu = IndexerUtils(self.cfg)
     iu.ws.list_objects.return_value = self.wslist
     iu.process_event(ev)
     iu.ep.index_objects.assert_called()
Ejemplo n.º 10
0
 def copy_event_test(self, mock_ep, mock_ws):
     # COPY_ACCESS_GROUP;
     ev = {
         "strcde": "WS",
         "accgrp": 1,
         "objid": None,
         "ver": None,
         "newname": None,
         "time": datetime.datetime.utcnow(),
         "evtype": "COPY_ACCESS_GROUP",
         "objtype": None,
         "objtypever": None,
         "public": False
     }
     iu = IndexerUtils(self.cfg)
     iu.ws.list_objects.return_value = self.wslist
     iu.process_event(ev)
     iu.ep.index_objects.assert_called()
Ejemplo n.º 11
0
 def get_indexes_test(self):
     iu = IndexerUtils(self.cfg)
     self.assertEqual(iu._get_indexes("KBaseNarrative.Narrative"),
                      [{
                          'index_method': 'NarrativeIndexer.index',
                          'index_name': 'ciraw.narrative'
                      }])
     self.assertEqual(
         iu._get_indexes("Foo.Bar"), [{
             'comment': 'Everything without a specific indexer',
             'index_method': 'default_indexer',
             'index_name': 'ciraw.objects'
         }])
     self.assertEqual(iu._get_indexes("KBaseMatrices.ExpressionMatrix"),
                      [{
                          'index_method': 'KBaseMatrices.index',
                          'index_name': 'ciraw.KBaseMatrices'
                      }])
Ejemplo n.º 12
0
 def index_request_default_test(self, mock_wsa, mock_cat):
     iu = IndexerUtils(self.cfg)
     iu.ws.get_workspace_info.return_value = self.wsinfo
     obj = self.genobj
     iu.ws.get_objects2.return_value = obj
     iu.ws.get_object_info3.return_value = self.geninfo
     ev = self.new_version_event.copy()
     ev['objtype'] = 'Blah.Blah'
     ev['objid'] = '3'
     id = 'WS:1:3:3'
     self.reset()
     iu.process_event(ev)
     res = self.es.get(index=self._iname('objects'),
                       routing=id,
                       doc_type='data',
                       id=id)
     self.assertIsNotNone(res)
     self.assertTrue(res['_source']['islast'])
Ejemplo n.º 13
0
    def index_request_test(self, mock_wsa, mock_cat):
        iu = IndexerUtils(self.cfg)
        iu.ws.get_workspace_info.return_value = self.wsinfo
        iu.ws.get_objects2.return_value = self.genobj
        iu.ws.get_object_info3.return_value = self.geninfo
        rv = {'docker_img_name': 'mock_indexer:latest'}
        iu.mr.catalog.get_module_version.return_value = rv
        ev = self.new_version_event.copy()
        ev['objtype'] = 'KBaseGenomes.Genome'
        ev['objid'] = '3'
        self.reset()
        iu.process_event(ev)
        id = 'WS:1:3:3'
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='data',
                          id=id)
        self.assertIsNotNone(res)
        self.assertTrue(res['_source']['islast'])

        # Test with older version
        ev = self.new_version_event.copy()
        ev['objtype'] = 'KBaseGenomes.Genome'
        ev['objid'] = '3'
        ev['ver'] = 2
        self.reset()
        iu.process_event(ev)
        id = 'WS:1:3:2'
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='data',
                          id=id)
        self.assertIsNotNone(res)
        self.assertFalse(res['_source']['islast'])
Ejemplo n.º 14
0
 def index_new_all_test(self, mock_wsa, mock_cat):
     iu = IndexerUtils(self.cfg)
     iu.ws.get_workspace_info.return_value = self.wsinfo
     iu.ws.get_objects2.return_value = self.genobj
     iu.ws.get_object_info3.return_value = self.geninfo
     rv = {'docker_img_name': 'mock_indexer:latest'}
     iu.mr.catalog.get_module_version.return_value = rv
     ev = self.new_version_event.copy()
     ev['objtype'] = None
     ev['objid'] = '3'
     ev['evtype'] = 'NEW_ALL_VERSIONS'
     ev['ver'] = None
     id = 'WS:1:3:3'
     self.reset()
     iu.process_event(ev)
     res = self.es.get(index=self._iname('genome'),
                       routing=id,
                       doc_type='data',
                       id=id)
     self.assertIsNotNone(res)
     self.assertIn('ojson', res['_source'])
     self.assertTrue(res['_source']['islast'])
Ejemplo n.º 15
0
 def index_error_test(self):
     iu = IndexerUtils(self.cfg)
     if os.path.exists('error.log'):
         os.remove('error.log')
     iu._new_object_version_index = Mock(side_effect=KeyError())
     iu._new_object_version_feature_index = Mock(side_effect=KeyError())
     ev = self.new_version_event.copy()
     iu.process_event(ev)
     self.assertTrue(os.path.exists('error.log'))
Ejemplo n.º 16
0
 def index_raw_test(self, mock_wsa, mock_cat):
     iu = IndexerUtils(self.cfg)
     iu.mapping['KBaseGenomes.Genome'] = [{
         'index_method': 'bogus.genome_index',
         'raw': True,
         'index_name': 'ciraw.sketch'
     }]
     index = {
         'schema': {
             'upa': {
                 'type': 'string'
             },
             'mash': {
                 'type': 'binary'
             },
             'timestamp': {
                 'type': 'long'
             }
         },
         'data': {
             'upa': '1/3/2',
             'mash': 'blaslsadlfasdf',
             'timestamp': '1234'
         }
     }
     iu.mr.run.return_value = [index]
     ev = self.new_version_event.copy()
     ev['objtype'] = 'KBaseGenomes.Genome'
     ev['objid'] = '3'
     ev['ver'] = 2
     self.reset()
     iu.process_event(ev)
     id = "WS:1:3:2"
     res = self.es.get(index=self._iname('sketch'), doc_type='data', id=id)
     self.assertIsNotNone(res)
     self.assertIn('mash', res['_source'])
     # Code path test... index and indexed object
     iu.process_event(ev)
     # Code path test ignore empty results
     ev['objtype'] = 'KBaseGenomes.Genome'
     ev['objid'] = '4'
     ev['ver'] = 2
     iu.mr.run.return_value = [{}]
     iu.process_event(ev)
Ejemplo n.º 17
0
 def delete_event_test(self, mock_ws):
     # DELETE_ALL_VERSIONS,
     # UNDELETE_ALL_VERSIONS,
     # DELETE_ACCESS_GROUP,
     self._init_es_genome()
     ev = self.new_version_event.copy()
     ev['evtype'] = 'DELETE_ALL_VERSIONS'
     ev['objid'] = '3'
     iu = IndexerUtils(self.cfg)
     iu.process_event(ev)
     id = 'WS:1:3:3'
     res = self.es.get(index=self._iname('genome'),
                       routing=id,
                       doc_type='data',
                       id=id,
                       ignore=404)
     self.assertFalse(res['found'])
     #
     ev = self.new_version_event.copy()
     ev['evtype'] = 'UNDELETE_ALL_VERSIONS'
     iu = IndexerUtils(self.cfg)
Ejemplo n.º 18
0
 def bogus_event_test(self):
     # RENAME_ALL_VERSIONS,
     ev = self.new_version_event.copy()
     ev['evtype'] = 'BOGUS'
     iu = IndexerUtils(self.cfg)
     iu.process_event(ev)
Ejemplo n.º 19
0
    def publish_test(self, ws_mock):
        """
        Publish and unpublish tests
        """
        # PUBLISH_ALL_VERSIONS,
        # PUBLISH_ACCESS_GROUP,
        # UNPUBLISH_ALL_VERSIONS,
        # UNPUBLISH_ACCESS_GROUP,
        mwsi = [
            1, "auser:narrative_1485560571814", "auser",
            "2018-10-18T00:12:42+0000", 25, "a", "y", "unlocked", {
                "narrative_nice_name": "A Fancy Nasrrative",
                "is_temporary": "false",
                "data_palette_id": "22",
                "narrative": "23"
            }
        ]

        self._init_es_genome()
        ev = {
            "strcde": "WS",
            "accgrp": 1,
            "objid": None,
            "ver": None,
            "newname": None,
            "time": datetime.datetime.utcnow(),
            "evtype": "PUBLISH_ACCESS_GROUP",
            "objtype": None,
            "objtypever": None,
            "public": None
        }
        iu = IndexerUtils(self.cfg)
        iu.ws.get_workspace_info.return_value = mwsi
        iu.process_event(ev)
        # Check that accgrp changed
        id = 'WS:1:3:3'
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='access',
                          id=id)
        self.assertIn(-1, res['_source']['groups'])
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='data',
                          id=id)
        self.assertTrue(res['_source']['public'])
        #
        ev['evtype'] = "UNPUBLISH_ACCESS_GROUP"
        mwsi[6] = 'n'
        iu = IndexerUtils(self.cfg)
        iu.ws.get_workspace_info.return_value = mwsi
        iu.process_event(ev)
        # Check that accgrp changed
        id = 'WS:1:3:3'
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='access',
                          id=id)
        self.assertNotIn(-1, res['_source']['groups'])
        res = self.es.get(index=self._iname('genome'),
                          routing=id,
                          doc_type='data',
                          id=id)
        self.assertFalse(res['_source']['public'])
Ejemplo n.º 20
0
 def acces_rec_test(self):
     iu = IndexerUtils(self.cfg)
     resp = iu._access_rec(1, 2, 3, public=True)
     self.assertIn(-1, resp['groups'])
Ejemplo n.º 21
0
 def get_id_test(self):
     iu = IndexerUtils(self.cfg)
     with self.assertRaises(ValueError):
         iu._get_id('blah')