def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory
)
def setUp(self):
self.defaultGroupId = 42
self.ports = [54321, 53421, 52431]
self.connectionCounts = {}
self.multiChannelListener = MultiChannelListener.MultiChannelListener(
callbackScheduler, self.ports)
self.channelFactory = TcpChannelFactory.TcpStringChannelFactory(
callbackScheduler)
def createCumulusGateway(self, callbackScheduler, vdm=None):
if vdm is None:
vdm = VectorDataManager.constructVDM(callbackScheduler)
vdm.setDropUnreferencedPagesWhenFull(True)
viewFactory = self.getViewFactory()
return CumulusGatewayRemote.RemoteGateway(
self.callbackScheduler, vdm,
TcpChannelFactory.TcpStringChannelFactory(self.callbackScheduler),
CumulusActiveMachines.CumulusActiveMachines(viewFactory),
viewFactory)
def createService(args):
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
callbackScheduler = callbackSchedulerFactory.createScheduler('ufora-worker', 1)
channelListener = MultiChannelListener(callbackScheduler,
[args.base_port, args.base_port + 1])
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackSchedulerFactory.createScheduler('SharedState', 1),
args.manager_address,
int(args.manager_port)
)
channelFactory = TcpChannelFactory.TcpStringChannelFactory(callbackScheduler)
diagnostics_dir = os.getenv("UFORA_WORKER_DIAGNOSTICS_DIR")
eventHandler = diagnostics_dir and createEventHandler(
diagnostics_dir,
callbackSchedulerFactory.createScheduler("ufora-worker-event-handler", 1)
)
own_address = args.own_address or get_own_ip()
print "Listening on", own_address, "ports:", args.base_port, "and", args.base_port+1
config = Setup.config()
print "RAM cache of %d / %d MB and %d threads. Track tcmalloc: %s" % (
config.cumulusVectorRamCacheMB,
config.cumulusMaxRamCacheMB,
config.cumulusServiceThreadCount,
config.cumulusTrackTcmalloc
)
print "Ufora store at %s:%s" % (args.manager_address, args.manager_port)
s3InterfaceFactory = ActualS3Interface.ActualS3InterfaceFactory()
print "PythonIoTasks threads: %d. Out of process: %s" % (
config.externalDatasetLoaderServiceThreads,
s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
)
return CumulusService.CumulusService(
own_address,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnostics_dir,
Setup.config(),
viewFactory=sharedStateViewFactory,
s3InterfaceFactory=s3InterfaceFactory,
objectStore=NullObjectStore.NullObjectStore()
)
def test_socket_channel_shutdown(self):
done = threading.Event()
listener = ChannelListener.SocketListener(self.port)
listener.registerConnectCallback(lambda sock, address: done.set())
try:
thread = ManagedThread.ManagedThread(target=listener.start)
thread.start()
listener.blockUntilReady()
TcpChannelFactory.TcpStringChannelFactory(
callbackScheduler).createChannel(('localhost', self.port))
self.assertTrue(done.wait(2))
finally:
listener.stop()
thread.join()
def test_socket_listener(self):
server = ChannelEchoServer(self.port)
try:
thread = ManagedThread.ManagedThread(target=server.start)
thread.start()
server.blockUntilReady()
stringChannelFactory = TcpChannelFactory.TcpStringChannelFactory(
callbackScheduler)
channel = stringChannelFactory.createChannel(
('localhost', self.port))
channel = channel.makeQueuelike(callbackScheduler)
toSend = "Hi There!"
channel.write(toSend)
self.assertEquals(toSend, channel.get())
finally:
try:
server.teardown()
except UserWarning:
pass
thread.join()
def __init__(self,
socketFd,
sharedStateAddress):
self.socketFd = socketFd
self.callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory()
self.scheduler = self.callbackSchedulerFactory.createScheduler(
"BackendGatewayRequestHandler",
1
)
sharedStateHost, sharedStatePort = sharedStateAddress.split(':')
sharedStateViewFactory = ViewFactory.ViewFactory.TcpViewFactory(
self.callbackSchedulerFactory.createScheduler('SharedState', 1),
sharedStateHost,
int(sharedStatePort)
)
self.subscribableHandler = ConnectionHandler.ConnectionHandler(
self.scheduler,
sharedStateViewFactory,
lambda: TcpChannelFactory.TcpStringChannelFactory(self.scheduler)
)
self.sock = socket.fromfd(socketFd, socket.AF_INET, socket.SOCK_STREAM)