def _decommission(): misc_config = config_loader.get(worker_misc_config_path) client1, trans = get_client(host=misc_config['host'], port=misc_config['port']) trans.open() resp1 = client1.decommission() assert(resp1.status == 'Success') trans.close()
def _decommission(): misc_config = config_loader.get(worker_misc_config_path) client1, trans = get_client(host=misc_config['host'], port=misc_config['port']) trans.open() resp1 = client1.decommission() assert (resp1.status == 'Success') trans.close()
def _commission(): misc_config = config_loader.get(worker_misc_config_path) client1, trans = get_client(host=misc_config['host'], port=misc_config['port']) ci = CommissionInput() ci.modelId = 'model-1' ci.modelPathsJson = '' trans.open() resp1 = client1.commission(ci) trans.close() assert(resp1.status == 'Success')
def _commission(): misc_config = config_loader.get(worker_misc_config_path) client1, trans = get_client(host=misc_config['host'], port=misc_config['port']) ci = CommissionInput() ci.modelId = 'model-1' ci.modelPathsJson = '' trans.open() resp1 = client1.commission(ci) trans.close() assert (resp1.status == 'Success')
def test_get_config(self): self.assertEquals(len(config_loader.get('../../resources/test_config_1.yaml').keys()), 2) self.assertEquals(set(config_loader.get('../../resources/test_config_1.yaml').keys()), set(['key_1', 'key_2'])) self.assertEquals(config_loader.get('../../resources/test_config_1.yaml').get('key_1'), 1) self.assertEquals(config_loader.get('../../resources/test_config_1.yaml').get('key_2'), 2) self.assertRaises(IOError, lambda : config_loader.get('../../resources/invalid_config_path.yaml')) self.assertRaises(TypeError, lambda : config_loader.get(None))
def __get_file_from_s3(self, s3_key_name, local_path): """ Fetches file from S3 to local machine Args: s3_key_name: local_path: Returns: """ config = config_loader.get(sys.argv[2]) s3_loader = S3Loader(config["s3"]["host"], config["s3"]["access_key"], config["s3"]["secret_key"]) s3_loader.fetch_file(config["s3"]["bucket_name"], s3_key_name, local_path)
def run(self): """ Starts thrift server in a Daemon Returns: """ self.logger = logging.getLogger(__name__) config = config_loader.get(sys.argv[2]) handler = WorkerHandler(sys.argv[2], self.rundir) proc = Worker.Processor(handler) trans_svr = TSocket.TServerSocket(port=int(sys.argv[3]) + int(config["port"]["start"])) trans_fac = TTransport.TFramedTransportFactory() proto_fac = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadPoolServer(proc, trans_svr, trans_fac, proto_fac) self.logger.info("Starting CaffeWorker Daemon") server.serve()
def __init__(self, config_file_path, rundir_path): """ Constructor for Worker Thrift Handler Args: config_file_path: Returns: """ config = config_loader.get(config_file_path) self.rundir_path = rundir_path self.blob_dir = config["local-store"]["blob"] if not os.path.isabs(self.blob_dir): self.blob_dir = self.rundir_path + '/' + self.blob_dir self.prototxt_dir = config["local-store"]["prototxt"] if not os.path.isabs(self.prototxt_dir): self.prototxt_dir = self.rundir_path + '/' + self.prototxt_dir self.logger = logging.getLogger(__name__) self.logger.info("WorkerHandler is being initialized")
def test_get_config(self): self.assertEquals( len( config_loader.get( '../../resources/test_config_1.yaml').keys()), 2) self.assertEquals( set( config_loader.get( '../../resources/test_config_1.yaml').keys()), set(['key_1', 'key_2'])) self.assertEquals( config_loader.get('../../resources/test_config_1.yaml').get( 'key_1'), 1) self.assertEquals( config_loader.get('../../resources/test_config_1.yaml').get( 'key_2'), 2) self.assertRaises( IOError, lambda: config_loader.get( '../../resources/invalid_config_path.yaml')) self.assertRaises(TypeError, lambda: config_loader.get(None))
class ServerDeamon(Daemon): def run(self): handler = KnnServiceHandler() processor = KnnThriftService.Processor(handler) transport = TSocket.TServerSocket(port=misc_config['port']) tfactory = TTransport.TBufferedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadedServer(processor, transport, tfactory, pfactory) print "Starting python server..." server.serve() if __name__ == "__main__": if len(sys.argv) == 4: misc_config = config_loader.get(sys.argv[2]) worker_name = "knn-worker-" + sys.argv[3] pid_file = '/tmp/' + worker_name + '.pid' daemon = ServerDeamon(pid_file, stdout="/dev/null", stderr=worker_name + ".stderr") if 'start' == sys.argv[1]: daemon.start() elif 'stop' == sys.argv[1]: daemon.stop() elif 'restart' == sys.argv[1]: daemon.restart() elif 'status' == sys.argv[1]: status = daemon.status() print(status) sys.exit(0)
import requests from commons.src.config import config_loader worker_misc_config_path = "conf/knnaas_config.yaml" misc_config = config_loader.get(worker_misc_config_path) import json for i in range(3): d = { "modelId": "model-1", "operation": "SearchByVector", "vector": [float(i + 1), float(i + 2), float(i + 1), float(i + 3)], "tags": ["fdp-ml-test-tags"] } r = requests.post('http://localhost:8000/v1/knn_model/predict', data=json.dumps(d)) assert (r.status_code == 200) assert (len(r.json()['result']) == misc_config['k'])
from knnaas.router.knn_worker_client import KnnWorkerClient from commons.src.config import config_loader worker_misc_config_path = "conf/knnaas_config.yaml" misc_config = config_loader.get(worker_misc_config_path) from commons.src.load_balancer.worker_load_balancer import WorkerLoadBalancer import unittest
self.logger = logging.getLogger(__name__) config = config_loader.get(sys.argv[2]) handler = WorkerHandler(sys.argv[2], self.rundir) proc = Worker.Processor(handler) trans_svr = TSocket.TServerSocket(port=int(sys.argv[3]) + int(config["port"]["start"])) trans_fac = TTransport.TFramedTransportFactory() proto_fac = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadPoolServer(proc, trans_svr, trans_fac, proto_fac) self.logger.info("Starting CaffeWorker Daemon") server.serve() if __name__ == "__main__": if len(sys.argv) == 4: setup_logging() logger = logging.getLogger(__name__) config = config_loader.get(sys.argv[2]) sys.argv[2] = os.path.abspath(sys.argv[2]) os.environ['GLOG_minloglevel'] = '3' sys.path.append(config["caffe"]["path"]) import caffe caffe.set_mode_cpu() from commons.src.s3.s3_loader import S3Loader from commons.src.system.daemon import Daemon worker_name = "caffe-worker-" + sys.argv[3] daemon = CaffeWorker("/tmp/" + worker_name + ".pid", stderr= worker_name + ".stderr") if 'start' == sys.argv[1]: