def index_request_genome_test(self, mock_cat):
iu = IndexerUtils(self.cfg)
rv = {'docker_img_name': 'test/kb_genomeindexer:latest'}
iu.method_runner.catalog.get_module_version.return_value = rv
ev = self.new_version_event.copy()
wsid = 15792
objid = 2
ver = 12
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = str(objid)
ev['wsid'] = wsid
ev['ver'] = ver
id = f'WS:{wsid:d}:{objid:d}:{ver:d}'
self.reset()
iu.process_event(ev)
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
fid = f'WS:{wsid:d}/{objid:d}/{ver:d}:feature/L876_RS0116375'
res = self.es.get(index=self._iname('genomefeature'),
routing=id,
doc_type='data',
id=fid)
self.assertIsNotNone(res)
self.assertIn('ojson', res['_source'])
self.assertTrue(res['_source']['islast'])
def prov_test(self):
iu = IndexerUtils(self.cfg)
obj = dict()
resp = iu._get_prov(obj)
self.assertIn('prv_meth', resp)
self.assertIsNone(resp['prv_meth'])
prv = {
'service': 'bogus',
'method': 'bogus2',
'service_ver': '1.0',
'description': 'desc'
}
obj['provenance'] = [prv]
resp = iu._get_prov(obj)
self.assertEquals(resp['prv_mod'], 'bogus')
self.assertEquals(resp['prv_meth'], 'bogus2')
self.assertEquals(resp['prv_ver'], '1.0')
self.assertEquals(resp['prv_cmt'], 'desc')
# Test script
obj['provenance'] = [{'script': 'bogus3', 'script_ver': '1.1'}]
resp = iu._get_prov(obj)
self.assertEquals(resp['prv_mod'], 'legacy_transform')
self.assertEquals(resp['prv_meth'], 'bogus3')
self.assertEquals(resp['prv_ver'], '1.1')
def index_request_genome_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = self.wsinfo
# iu.ws.get_objects2.return_value = self.genobj
iu.ws.get_object_info3.return_value = self.geninfo
rv = {'docker_img_name': 'test/kb_genomeindexer:latest'}
iu.mr.catalog.get_module_version.return_value = rv
ev = self.new_version_event.copy()
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = '2'
ev['accgrp'] = 15792
ev['ver'] = 1
id = 'WS:15792:2:1'
self.reset()
iu.process_event(ev)
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
fid = 'WS:15792:2:1:L876_RS0116375'
res = self.es.get(index=self._iname('genomefeature'),
routing=id,
doc_type='data',
id=fid)
self.assertIsNotNone(res)
self.assertIn('ojson', res['_source'])
self.assertTrue(res['_source']['islast'])
def rename_event_test(self):
# RENAME_ALL_VERSIONS,
ev = self.new_version_event.copy()
ev['evtype'] = 'RENAME_ALL_VERSIONS'
ev['objid'] = '3'
iu = IndexerUtils(self.cfg)
iu.process_event(ev)
def index_object_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = self.wsinfo
# iu.ws.get_objects2.return_value = {'data': [self.narobj]}
iu.ws.get_objects2.return_value = self.narobj
rv = {'docker_img_name': 'mock_indexer:latest'}
iu.mr.catalog.get_module_version.return_value = rv
event = self.new_version_event.copy()
event['upa'] = '1/2/3'
res = iu.new_object_version(event)
self.assertIsNotNone(res)
def skip_temp_test(self, wsa):
iu = IndexerUtils(self.cfg)
wsi = self.wsinfo
wsi[8]['is_temporary'] = 'true'
iu.ws.get_workspace_info.return_value = wsi
# iu.ws.get_objects2.return_value = {'data': [self.narobj]}
iu.ws.get_objects2.return_value = self.narobj
res = iu._create_obj_rec('1/2/3')
self.assertIsNone(res)
res = iu._update_es_access('bogus', 1, 2, 3, '1/2/3')
self.assertIsNone(res)
def kafka_watcher(config):
topic = config.get('kafka-topic', 'wsevents')
indexer_topic = config.get('kafka-index-topic', 'idxevents')
server = config.get('kafka-server', 'kafka')
cgroup = config.get('kafka-clientgroup', 'search_indexer')
config = config
log = logging.getLogger('indexrunner')
log.info("Initializing EventHandler")
run_one = False
if 'run_one' in config:
run_one = True
indexer = IndexerUtils(config)
c = Consumer({
'bootstrap.servers': server,
'group.id': cgroup,
'auto.offset.reset': 'earliest'
})
log.info("Starting consumer")
log.info("Server %s" % (server))
log.info("Group: %s" % (cgroup))
log.info("Topic: %s" % (topic))
log.info("Index Topic: %s" % (indexer_topic))
c.subscribe([topic, indexer_topic])
while True:
msg = c.poll(0.5)
data = None
if msg is None:
pass
elif msg.error():
if msg.error().code() != KafkaError._PARTITION_EOF:
err = str(msg.error())
_log_error('', err)
log.error("Kafka error: " + err)
else:
try:
data = json.loads(msg.value().decode('utf-8'))
if data['strcde'] != 'WS':
_log_error(data, 'Bad strcde')
log.warning("Unreconginized strcde")
else:
indexer.process_event(data)
except BaseException as e:
_log_error(data, e)
log.error('Uncaught exception: ' + str(e))
# This is just used in testing
if run_one:
break
c.close()
def index_request_misc_test(self):
iu = IndexerUtils(self.cfg)
ev = self.new_version_event.copy()
ev['objtype'] = 'KBaseGenomeAnnotations.Assembly'
ev['objid'] = '31'
ev['accgrp'] = 7998
ev['ver'] = 2
id = 'WS:7998:31:2'
self.reset()
iu.process_event(ev)
res = self.es.get(index=self._iname('assembly'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
def reindex_event_test(self, mock_ep, mock_ws):
ev = {
"strcde": "WS",
"accgrp": 1,
"objid": None,
"ver": None,
"newname": None,
"time": datetime.datetime.utcnow(),
"evtype": "REINDEX_WORKSPACE",
"objtype": None,
"objtypever": None,
"public": False
}
iu = IndexerUtils(self.cfg)
iu.ws.list_objects.return_value = self.wslist
iu.process_event(ev)
iu.ep.index_objects.assert_called()
def copy_event_test(self, mock_ep, mock_ws):
# COPY_ACCESS_GROUP;
ev = {
"strcde": "WS",
"accgrp": 1,
"objid": None,
"ver": None,
"newname": None,
"time": datetime.datetime.utcnow(),
"evtype": "COPY_ACCESS_GROUP",
"objtype": None,
"objtypever": None,
"public": False
}
iu = IndexerUtils(self.cfg)
iu.ws.list_objects.return_value = self.wslist
iu.process_event(ev)
iu.ep.index_objects.assert_called()
def get_indexes_test(self):
iu = IndexerUtils(self.cfg)
self.assertEqual(iu._get_indexes("KBaseNarrative.Narrative"),
[{
'index_method': 'NarrativeIndexer.index',
'index_name': 'ciraw.narrative'
}])
self.assertEqual(
iu._get_indexes("Foo.Bar"), [{
'comment': 'Everything without a specific indexer',
'index_method': 'default_indexer',
'index_name': 'ciraw.objects'
}])
self.assertEqual(iu._get_indexes("KBaseMatrices.ExpressionMatrix"),
[{
'index_method': 'KBaseMatrices.index',
'index_name': 'ciraw.KBaseMatrices'
}])
def index_request_default_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = self.wsinfo
obj = self.genobj
iu.ws.get_objects2.return_value = obj
iu.ws.get_object_info3.return_value = self.geninfo
ev = self.new_version_event.copy()
ev['objtype'] = 'Blah.Blah'
ev['objid'] = '3'
id = 'WS:1:3:3'
self.reset()
iu.process_event(ev)
res = self.es.get(index=self._iname('objects'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
self.assertTrue(res['_source']['islast'])
def index_request_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = self.wsinfo
iu.ws.get_objects2.return_value = self.genobj
iu.ws.get_object_info3.return_value = self.geninfo
rv = {'docker_img_name': 'mock_indexer:latest'}
iu.mr.catalog.get_module_version.return_value = rv
ev = self.new_version_event.copy()
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = '3'
self.reset()
iu.process_event(ev)
id = 'WS:1:3:3'
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
self.assertTrue(res['_source']['islast'])
# Test with older version
ev = self.new_version_event.copy()
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = '3'
ev['ver'] = 2
self.reset()
iu.process_event(ev)
id = 'WS:1:3:2'
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
self.assertFalse(res['_source']['islast'])
def index_new_all_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = self.wsinfo
iu.ws.get_objects2.return_value = self.genobj
iu.ws.get_object_info3.return_value = self.geninfo
rv = {'docker_img_name': 'mock_indexer:latest'}
iu.mr.catalog.get_module_version.return_value = rv
ev = self.new_version_event.copy()
ev['objtype'] = None
ev['objid'] = '3'
ev['evtype'] = 'NEW_ALL_VERSIONS'
ev['ver'] = None
id = 'WS:1:3:3'
self.reset()
iu.process_event(ev)
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertIsNotNone(res)
self.assertIn('ojson', res['_source'])
self.assertTrue(res['_source']['islast'])
def index_error_test(self):
iu = IndexerUtils(self.cfg)
if os.path.exists('error.log'):
os.remove('error.log')
iu._new_object_version_index = Mock(side_effect=KeyError())
iu._new_object_version_feature_index = Mock(side_effect=KeyError())
ev = self.new_version_event.copy()
iu.process_event(ev)
self.assertTrue(os.path.exists('error.log'))
def index_raw_test(self, mock_wsa, mock_cat):
iu = IndexerUtils(self.cfg)
iu.mapping['KBaseGenomes.Genome'] = [{
'index_method': 'bogus.genome_index',
'raw': True,
'index_name': 'ciraw.sketch'
}]
index = {
'schema': {
'upa': {
'type': 'string'
},
'mash': {
'type': 'binary'
},
'timestamp': {
'type': 'long'
}
},
'data': {
'upa': '1/3/2',
'mash': 'blaslsadlfasdf',
'timestamp': '1234'
}
}
iu.mr.run.return_value = [index]
ev = self.new_version_event.copy()
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = '3'
ev['ver'] = 2
self.reset()
iu.process_event(ev)
id = "WS:1:3:2"
res = self.es.get(index=self._iname('sketch'), doc_type='data', id=id)
self.assertIsNotNone(res)
self.assertIn('mash', res['_source'])
# Code path test... index and indexed object
iu.process_event(ev)
# Code path test ignore empty results
ev['objtype'] = 'KBaseGenomes.Genome'
ev['objid'] = '4'
ev['ver'] = 2
iu.mr.run.return_value = [{}]
iu.process_event(ev)
def delete_event_test(self, mock_ws):
# DELETE_ALL_VERSIONS,
# UNDELETE_ALL_VERSIONS,
# DELETE_ACCESS_GROUP,
self._init_es_genome()
ev = self.new_version_event.copy()
ev['evtype'] = 'DELETE_ALL_VERSIONS'
ev['objid'] = '3'
iu = IndexerUtils(self.cfg)
iu.process_event(ev)
id = 'WS:1:3:3'
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id,
ignore=404)
self.assertFalse(res['found'])
#
ev = self.new_version_event.copy()
ev['evtype'] = 'UNDELETE_ALL_VERSIONS'
iu = IndexerUtils(self.cfg)
def bogus_event_test(self):
# RENAME_ALL_VERSIONS,
ev = self.new_version_event.copy()
ev['evtype'] = 'BOGUS'
iu = IndexerUtils(self.cfg)
iu.process_event(ev)
def publish_test(self, ws_mock):
"""
Publish and unpublish tests
"""
# PUBLISH_ALL_VERSIONS,
# PUBLISH_ACCESS_GROUP,
# UNPUBLISH_ALL_VERSIONS,
# UNPUBLISH_ACCESS_GROUP,
mwsi = [
1, "auser:narrative_1485560571814", "auser",
"2018-10-18T00:12:42+0000", 25, "a", "y", "unlocked", {
"narrative_nice_name": "A Fancy Nasrrative",
"is_temporary": "false",
"data_palette_id": "22",
"narrative": "23"
}
]
self._init_es_genome()
ev = {
"strcde": "WS",
"accgrp": 1,
"objid": None,
"ver": None,
"newname": None,
"time": datetime.datetime.utcnow(),
"evtype": "PUBLISH_ACCESS_GROUP",
"objtype": None,
"objtypever": None,
"public": None
}
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = mwsi
iu.process_event(ev)
# Check that accgrp changed
id = 'WS:1:3:3'
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='access',
id=id)
self.assertIn(-1, res['_source']['groups'])
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertTrue(res['_source']['public'])
#
ev['evtype'] = "UNPUBLISH_ACCESS_GROUP"
mwsi[6] = 'n'
iu = IndexerUtils(self.cfg)
iu.ws.get_workspace_info.return_value = mwsi
iu.process_event(ev)
# Check that accgrp changed
id = 'WS:1:3:3'
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='access',
id=id)
self.assertNotIn(-1, res['_source']['groups'])
res = self.es.get(index=self._iname('genome'),
routing=id,
doc_type='data',
id=id)
self.assertFalse(res['_source']['public'])
def acces_rec_test(self):
iu = IndexerUtils(self.cfg)
resp = iu._access_rec(1, 2, 3, public=True)
self.assertIn(-1, resp['groups'])
def get_id_test(self):
iu = IndexerUtils(self.cfg)
with self.assertRaises(ValueError):
iu._get_id('blah')