def main(reactor, description): endpoint = endpoints.clientFromString(reactor, description) client = JSONRPCClientFactory(endpoint, reactor=reactor) try: r = yield client.callRemote('bar.foo') except JSONRPCClientError as e: print e r = yield client.callRemote('bar.add', 1, 2) print "add result: %s" % str(r) r = yield client.callRemote('bar.whoami') print "whoami result: %s" % str(r)
def setUp(self): self.reactor = task.Clock() self.endpoint = FakeEndpoint() self.factory = JSONRPCClientFactory( self.endpoint, reactor=self.reactor)
class ClientTestCase(TXJasonTestCase): """ Tests for JSONRPCClientFactory. """ def setUp(self): self.reactor = task.Clock() self.endpoint = FakeEndpoint() self.factory = JSONRPCClientFactory( self.endpoint, reactor=self.reactor) def test_callRemote(self): """ callRemote sends data and returns a Deferred that fires with the result from over the wire. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), {'params': [], 'jsonrpc': '2.0', 'method': 'spam', 'id': 1}) self.endpoint.proto.stringReceived(json.dumps( {'jsonrpc': '2.0', 'id': 1, 'result': 'eggs'})) self.assertEqual(self.successResultOf(d), 'eggs') def test_callRemote_error_response(self): """ callRemote's Deferred can also errback if an error comes over the wire. """ d = self.factory.callRemote('spam') self.endpoint.proto.stringReceived(json.dumps( {'jsonrpc': '2.0', 'id': 1, 'error': { 'message': 'error', 'code': -19}})) self.failureResultOf(d, client.JSONRPCClientError) def test_notifyRemote(self): """ notifyRemote sends data but and returns a Deferred, but does not expect a response. """ self.assertFalse(self.endpoint.connected) d = self.factory.notifyRemote('spam') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), {'params': [], 'jsonrpc': '2.0', 'method': 'spam'}) self.successResultOf(d) def test_callRemote_connection_failure(self): """ Connection failures get propagated as an errback on callRemote's Deferred. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d = self.factory.callRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.failureResultOf(d, FakeError) def test_notifyRemote_connection_failure(self): """ Connection failures get propagated as an errback on notifyRemote's Deferred. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d = self.factory.notifyRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.failureResultOf(d, FakeError) def test_notifyRemote_two_connection_failures(self): """ In the case of two synchronous connection failures, both notifyRemote calls errback with the connection failure. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d1 = self.factory.notifyRemote('spam') d2 = self.factory.notifyRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 2) self.successResultOf(defer.gatherResults([ self.assertFailure(d1, FakeError), self.assertFailure(d2, FakeError), ])) def test_notifyRemote_two_pending_connection_failures(self): """ In the case of two notifyRemotes both waiting on the same connection, and the connection fails, both Deferreds returned by notifyRemote will errback. """ self.assertFalse(self.endpoint.connected) self.endpoint.deferred = defer.Deferred() d1 = self.factory.notifyRemote('spam') d2 = self.factory.notifyRemote('spam') self.endpoint.deferred.errback(FakeError()) self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.successResultOf(defer.gatherResults([ self.assertFailure(d1, FakeError), self.assertFailure(d2, FakeError), ])) def test_callRemote_cancellation_during_connection(self): """ The Deferred returned by callRemote can be cancelled during the connection attempt. """ self.assertFalse(self.endpoint.connected) canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.callRemote('spam') d.cancel() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_callRemote_cancellation_during_request(self): """ The Deferred returned by callRemote can be cancelled while waiting on a response. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') d.cancel() self.failureResultOf(d, defer.CancelledError) def test_notifyRemote_cancellation_during_connection(self): """ The Deferred returned by notifyRemote can be cancelled during the connection attempt. """ self.assertFalse(self.endpoint.connected) canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.notifyRemote('spam') d.cancel() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_reconnection(self): """ A new connection is established if the connection is lost between notifyRemote calls. """ self.assertFalse(self.endpoint.connected) self.factory.notifyRemote('spam') self.assert_(self.endpoint.connected) self.endpoint.disconnect(FakeDisconnectedError()) self.assertFalse(self.endpoint.connected) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) self.factory.notifyRemote('eggs') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), {'params': [], 'jsonrpc': '2.0', 'method': 'eggs'}) def test_callRemote_timeout(self): """ A timeout causes the Deferred returned by callRemote to errback with CancelledError. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') self.reactor.advance(10) self.failureResultOf(d, defer.CancelledError) def test_disconnect(self): """ The disconnect method drops the current connection. """ d = self.factory.notifyRemote('spam') self.successResultOf(d) self.assertIsNot(self.endpoint.transport, None) self.factory.disconnect() self.assertIs(self.endpoint.transport, None) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_disconnect_connection_cancellation(self): """ The disconnect method cancels pending connections. """ canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.notifyRemote('spam') self.factory.disconnect() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_disconnect_callRemote_cancellation(self): """ The disconnect method cancels pending callRemote Deferreds. """ d = self.factory.callRemote('spam') self.assertIsNot(self.endpoint.transport, None) self.factory.disconnect() self.assertIs(self.endpoint.transport, None) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) self.failureResultOf(d, defer.CancelledError) def test_connect(self): """ The connect method returns a Deferred that fires when the connection is established. """ self.endpoint.deferred = defer.Deferred() d = self.factory.connect() self.assertNoResult(d) self.endpoint.deferred.callback(self.factory.buildProtocol(None)) self.successResultOf(d) def test_connect_cancellation(self): """ Cancelling the connect method's Deferred cancels the connection's Deferred. """ canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.connect() self.assertNoResult(d) d.cancel() self.assert_(canceled) self.failureResultOf(d, defer.CancelledError) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) def test_connect_failure(self): """ The connect method's Deferred errbacks if the connection itself errbacks. """ self.endpoint.deferred = defer.Deferred() d = self.factory.connect() self.assertNoResult(d) self.endpoint.deferred.errback(FakeError()) self.failureResultOf(d, FakeError) self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) def test_notifyDisconnect(self): """ The notifyDisconnect method returns a Deferred that fires when the client has disconnected. """ d = self.factory.notifyDisconnect() self.successResultOf(self.factory.connect()) self.assertNoResult(d) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_notifyDisconnect_after_connection(self): """ The notifyDisconnect method returns a Deferred that fires when the client has disconnected even if it's called after a connection is established. """ self.successResultOf(self.factory.connect()) d = self.factory.notifyDisconnect() self.assertNoResult(d) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_notifyDisconnect_after_disconnection(self): """ notifyDisconnect doesn't return a fired Deferred if it's called after disconnection but instead returns a Deferred that fires after the next disconnection. """ self.successResultOf(self.factory.connect()) self.endpoint.disconnect(FakeDisconnectedError()) d = self.factory.notifyDisconnect() self.assertNoResult(d) self.successResultOf(self.factory.connect()) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 2)
def __init__(self, *args, **kwargs): super(Worker, self).__init__() self.reactor = kwargs['reactor'] self.client = JSONRPCClientFactory(*args, **kwargs)
class Worker(object): def __init__(self, *args, **kwargs): super(Worker, self).__init__() self.reactor = kwargs['reactor'] self.client = JSONRPCClientFactory(*args, **kwargs) def tell(self, *args, **kwargs): call = self.client.callRemote(*args, **kwargs) call.addCallback(json.loads) return call @defer.inlineCallbacks def sync(self, transaction): local, remote = transaction new_path = get_root(self.config, local) if not os.path.exists(new_path): os.mkdir(new_path) with open(local, 'ab') as f: remote_size = yield self.tell('cache.get size', remote) cache_size = os.stat(local).st_size while cache_size < remote_size[remote]: data = yield self.tell('cache.get file', remote, cache_size) data = data[data.keys()[0]] data = base64.standard_b64decode(data) f.seek(cache_size) f.write(data) cache_size = os.stat(local).st_size @defer.inlineCallbacks def sync_static(self, job): filename = job['static_file'] cache = get_cache(self.config, filename) storage = get_storage(self.config, filename) yield self.sync((cache, storage)) yield job return def mkdir_p(self, path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise @defer.inlineCallbacks def clean_data(self, job): print "Cleaning data..." path = self.config['master']['cache'] data_path = path + '/'.join(job['data'][0].split('/')[:-1]) all_files = sorted(glob('{:s}/*.nc'.format(data_path))) if all_files: files = job['data'] idx = all_files.index(path + files[0]) end = idx - 840 if idx > 840 else 0 map(os.remove, all_files[0:end]) yield job return @defer.inlineCallbacks def sync_data(self, job): print "Synchronizing data..." job['data'].sort() data_path = '{:s}{:s}'.format(self.config['master']['cache'], '/'.join(job['data'][0].split('/')[:-1])) self.mkdir_p(data_path) translate = lambda m: map(lambda f: m(self.config, f), job['data']) cache = translate(get_cache) remote = translate(get_storage) for transaction in zip(cache, remote): yield self.sync(transaction) yield job return @defer.inlineCallbacks def work_in(self, job): t_job = deepcopy(job) cached = lambda f: self.config['master']['cache'] + f t_job['data'] = map(cached, job['data']) t_job['static_file'] = cached(job['static_file']) t_job['product'] = None descriptor = JobDescription(**(t_job)) elapsed, output = descriptor.run() result = { "time": output.time[-1].tolist(), "cloudindex": output.cloudindex[-1, :].tolist(), "globalindex": output.globalradiation[-1, :].tolist(), } yield result return @defer.inlineCallbacks def work(self, ip): ask_job = self.tell('manager.no job', ip) job = yield ask_job yield self.sync_static(job) yield self.sync_data(job) yield self.clean_data(job) result = yield self.work_in(job) job['result'] = result ready_job = self.tell('manager.job ready', json.dumps(job))
class ClientTestCase(TXJasonTestCase): """ Tests for JSONRPCClientFactory. """ def setUp(self): self.reactor = task.Clock() self.endpoint = FakeEndpoint() self.factory = JSONRPCClientFactory(self.endpoint, reactor=self.reactor) def test_callRemote(self): """ callRemote sends data and returns a Deferred that fires with the result from over the wire. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), { 'params': [], 'jsonrpc': '2.0', 'method': 'spam', 'id': 1 }) self.endpoint.proto.stringReceived( json.dumps({ 'jsonrpc': '2.0', 'id': 1, 'result': 'eggs' })) self.assertEqual(self.successResultOf(d), 'eggs') def test_callRemote_error_response(self): """ callRemote's Deferred can also errback if an error comes over the wire. """ d = self.factory.callRemote('spam') self.endpoint.proto.stringReceived( json.dumps({ 'jsonrpc': '2.0', 'id': 1, 'error': { 'message': 'error', 'code': -19 } })) self.failureResultOf(d, client.JSONRPCClientError) def test_notifyRemote(self): """ notifyRemote sends data but and returns a Deferred, but does not expect a response. """ self.assertFalse(self.endpoint.connected) d = self.factory.notifyRemote('spam') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), { 'params': [], 'jsonrpc': '2.0', 'method': 'spam' }) self.successResultOf(d) def test_callRemote_connection_failure(self): """ Connection failures get propagated as an errback on callRemote's Deferred. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d = self.factory.callRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.failureResultOf(d, FakeError) def test_notifyRemote_connection_failure(self): """ Connection failures get propagated as an errback on notifyRemote's Deferred. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d = self.factory.notifyRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.failureResultOf(d, FakeError) def test_notifyRemote_two_connection_failures(self): """ In the case of two synchronous connection failures, both notifyRemote calls errback with the connection failure. """ self.assertFalse(self.endpoint.connected) self.endpoint.fail = True d1 = self.factory.notifyRemote('spam') d2 = self.factory.notifyRemote('spam') self.assertEqual(len(self.flushLoggedErrors(FakeError)), 2) self.successResultOf( defer.gatherResults([ self.assertFailure(d1, FakeError), self.assertFailure(d2, FakeError), ])) def test_notifyRemote_two_pending_connection_failures(self): """ In the case of two notifyRemotes both waiting on the same connection, and the connection fails, both Deferreds returned by notifyRemote will errback. """ self.assertFalse(self.endpoint.connected) self.endpoint.deferred = defer.Deferred() d1 = self.factory.notifyRemote('spam') d2 = self.factory.notifyRemote('spam') self.endpoint.deferred.errback(FakeError()) self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) self.successResultOf( defer.gatherResults([ self.assertFailure(d1, FakeError), self.assertFailure(d2, FakeError), ])) def test_callRemote_cancellation_during_connection(self): """ The Deferred returned by callRemote can be cancelled during the connection attempt. """ self.assertFalse(self.endpoint.connected) canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.callRemote('spam') d.cancel() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_callRemote_cancellation_during_request(self): """ The Deferred returned by callRemote can be cancelled while waiting on a response. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') d.cancel() self.failureResultOf(d, defer.CancelledError) def test_notifyRemote_cancellation_during_connection(self): """ The Deferred returned by notifyRemote can be cancelled during the connection attempt. """ self.assertFalse(self.endpoint.connected) canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.notifyRemote('spam') d.cancel() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_reconnection(self): """ A new connection is established if the connection is lost between notifyRemote calls. """ self.assertFalse(self.endpoint.connected) self.factory.notifyRemote('spam') self.assert_(self.endpoint.connected) self.endpoint.disconnect(FakeDisconnectedError()) self.assertFalse(self.endpoint.connected) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) self.factory.notifyRemote('eggs') self.assert_(self.endpoint.connected) self.assertEqual( json.loads(readNetstring(self.endpoint.transport.value())), { 'params': [], 'jsonrpc': '2.0', 'method': 'eggs' }) def test_callRemote_timeout(self): """ A timeout causes the Deferred returned by callRemote to errback with CancelledError. """ self.assertFalse(self.endpoint.connected) d = self.factory.callRemote('spam') self.reactor.advance(10) self.failureResultOf(d, defer.CancelledError) def test_disconnect(self): """ The disconnect method drops the current connection. """ d = self.factory.notifyRemote('spam') self.successResultOf(d) self.assertIsNot(self.endpoint.transport, None) self.factory.disconnect() self.assertIs(self.endpoint.transport, None) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_disconnect_connection_cancellation(self): """ The disconnect method cancels pending connections. """ canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.notifyRemote('spam') self.factory.disconnect() self.assert_(canceled) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) self.failureResultOf(d, defer.CancelledError) def test_disconnect_callRemote_cancellation(self): """ The disconnect method cancels pending callRemote Deferreds. """ d = self.factory.callRemote('spam') self.assertIsNot(self.endpoint.transport, None) self.factory.disconnect() self.assertIs(self.endpoint.transport, None) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) self.failureResultOf(d, defer.CancelledError) def test_connect(self): """ The connect method returns a Deferred that fires when the connection is established. """ self.endpoint.deferred = defer.Deferred() d = self.factory.connect() self.assertNoResult(d) self.endpoint.deferred.callback(self.factory.buildProtocol(None)) self.successResultOf(d) def test_connect_cancellation(self): """ Cancelling the connect method's Deferred cancels the connection's Deferred. """ canceled = [] self.endpoint.deferred = defer.Deferred(canceled.append) d = self.factory.connect() self.assertNoResult(d) d.cancel() self.assert_(canceled) self.failureResultOf(d, defer.CancelledError) self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1) def test_connect_failure(self): """ The connect method's Deferred errbacks if the connection itself errbacks. """ self.endpoint.deferred = defer.Deferred() d = self.factory.connect() self.assertNoResult(d) self.endpoint.deferred.errback(FakeError()) self.failureResultOf(d, FakeError) self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1) def test_notifyDisconnect(self): """ The notifyDisconnect method returns a Deferred that fires when the client has disconnected. """ d = self.factory.notifyDisconnect() self.successResultOf(self.factory.connect()) self.assertNoResult(d) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_notifyDisconnect_after_connection(self): """ The notifyDisconnect method returns a Deferred that fires when the client has disconnected even if it's called after a connection is established. """ self.successResultOf(self.factory.connect()) d = self.factory.notifyDisconnect() self.assertNoResult(d) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1) def test_notifyDisconnect_after_disconnection(self): """ notifyDisconnect doesn't return a fired Deferred if it's called after disconnection but instead returns a Deferred that fires after the next disconnection. """ self.successResultOf(self.factory.connect()) self.endpoint.disconnect(FakeDisconnectedError()) d = self.factory.notifyDisconnect() self.assertNoResult(d) self.successResultOf(self.factory.connect()) self.endpoint.disconnect(FakeDisconnectedError()) self.failureResultOf(d, FakeDisconnectedError) self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 2)
def setUp(self): self.reactor = task.Clock() self.endpoint = FakeEndpoint() self.factory = JSONRPCClientFactory(self.endpoint, reactor=self.reactor)