def main():
print("starting algorithm runner")
conf = config.Config
alg = Algorunner()
alg.loadAlgorithm(conf.algorithm)
job = alg.connectToWorker(conf.socket)
job.join()
def xtest_exit():
with patch('sys.exit') as exit_mock:
def doExit(a):
status['exit'] = True
def invokeExit():
algorunner._exit(None)
def isServingTrue():
return True
def isServingFalse():
return False
algorunner = Algorunner()
algorunner.loadAlgorithmCallbacks(startCallback)
algorunner.connectToWorker(config)
sleep(1)
status = {'exit': False}
algorunner.loadAlgorithmCallbacks(startCallback, exit=doExit)
algorunner._dataServer.isServing = isServingTrue
Thread(target=invokeExit).start()
sleep(1)
assert status['exit'] == False
algorunner._dataServer.isServing = isServingFalse
sleep(1)
assert status['exit'] == True
assert exit_mock.called
def xtest_load_algorithm():
alg = Algorithm()
alg.algorithm = {
"path": "test_alg",
"entryPoint": "main.py"
}
cwd = os.getcwd()
os.chdir(cwd + '/tests')
algorunner = Algorunner()
algorunner.loadAlgorithm(alg)
# os.chdir(cwd)
result1 = algorunner._originalAlgorithm['start']({'input': mockdata.initData}, None)
result2 = startCallback({'input': mockdata.initData})
assert result1 == result2
def test_connect_to_worker():
config.discovery.update({"port": "9021"})
algorunner = Algorunner()
algorunner.loadAlgorithmCallbacks(startCallback2, options=config)
algorunner.connectToWorker(config)
time.sleep(2)
assert algorunner._connected == True
assert algorunner._input == mockdata.initData
algorunner.close()
def test_load_algorithm_callbacks():
algorunner = Algorunner()
algorunner.loadAlgorithmCallbacks(startCallback, options=config)
result1 = algorunner._originalAlgorithm['start']({'input': mockdata.initData}, None)
result2 = startCallback({'input': mockdata.initData})
assert result1 == result2
algorunner.close()
def test_load_algorithm_callbacks(mocker):
interval = 500
config.discovery["servingReportInterval"] = interval
config.discovery["enable"] = True
algorunner = Algorunner()
algorunner._job = Job({"kind": "batch"})
spy = mocker.spy(algorunner, '_reportServingStatus')
algorunner._initDataServer(config)
sleep(0.8)
assert spy.call_count == 2
algorunner.close()
def test_failed_load_algorithm():
alg = Algorithm()
alg.algorithm = {
"path": "no_such_path",
"entryPoint": "main.py"
}
algorunner = Algorunner()
algorunner.loadAlgorithm(alg)
assert "No module named" in algorunner._loadAlgorithmError
assert "no_such_path" in algorunner._loadAlgorithmError
algorunner.close()
def test_callback():
sm = StorageManager(config.storage)
algorunner = Algorunner()
algorunner.loadAlgorithmCallbacks(start, options=config)
algorunner.connectToWorker(config)
origGet = algorunner._dataAdapter.tryGetDataFromPeerOrStorage
def mockStorageGet(response):
return storageMock.get(response.get('storageInfo').get('path'))
algorunner._dataAdapter.tryGetDataFromPeerOrStorage = mockStorageGet
time.sleep(3)
(header, payload) = sm.storage.get({"path": "local-hkube/jobId/taskId"})
algorunner._dataAdapter.tryGetDataFromPeerOrStorage = origGet
decoded = encoding.decode(header=header, value=payload)
assert decoded[0][0] == inp1
assert decoded[1] == outp2
time.sleep(2)
Tracer.instance.close()
algorunner.close()
def main():
print("starting algorithm runner")
Algorunner.Run()
def main():
print("starting algorithm runner")
Algorunner.Run(start=algorithm.start, stop=algorithm.stop)
def main():
print("starting algorithm runner")
alg = Algorunner()
alg.loadAlgorithmCallbacks(algorithm.start)
job = alg.connectToWorker(Config.socket)
job.join()
def test_load_algorithm_streaming_then_batch():
algorunner = Algorunner()
algorunner.loadAlgorithmCallbacks(startCallback, options=config)
algorunner.streamingManager = StreamingManager()
algorunner._hkubeApi = HKubeApi(None, algorunner, None, None,algorunner.streamingManager)
algorunner._init(mockdata.streamingInitData)
thrd = Thread(target=algorunner._originalAlgorithm['start'], args=[{'input': mockdata.streamingInitData}, algorunner._hkubeApi])
thrd.start()
algorunner._stopAlgorithm(mockdata.initData)
result1 = algorunner._originalAlgorithm['start']({'input': mockdata.initData}, algorunner._hkubeApi)
result2 = startCallback({'input': mockdata.initData})
assert result1 == result2
algorunner.close()
def test_entryPoint(test_input,expected):
actual = Algorunner._getEntryPoint(test_input)
assert actual == expected
def main():
alg = Algorunner()
alg.loadAlgorithmCallbacks(start)
job = alg.connectToWorker(socket)
job.join()
from typing import Dict
import concurrent.futures
from hkube_python_wrapper import HKubeApi
from hkube_python_wrapper import Algorunner
import time
def start(args, hkubeapi: HKubeApi):
input=args['input']
params=input[0] if len(input) else {}
if(isinstance(params, dict)):
print('params: '+repr(params))
size=params.get('size',0)
batch=params.get('batch',0)
sleep=params.get('sleep',0)
alg=params.get('alg',None)
if (sleep):
time.sleep(sleep)
if (alg):
print('got alg: '+alg)
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
res=list(executor.map(lambda i: hkubeapi.start_algorithm(alg,input=[i]),range(10)))
return('got {res} from algo'.format(res=res))
if (size):
return ['d'*size]*batch
return 10
if(isinstance(params, str)):
print('data length='+str(len(params)))
return params
if __name__ == "__main__":
Algorunner.Run(start=start)
def main():
print("starting algorithm runner")
alg = Algorunner(config.Config)