def main(reactor, description):
endpoint = endpoints.clientFromString(reactor, description)
client = JSONRPCClientFactory(endpoint, reactor=reactor)
try:
r = yield client.callRemote('bar.foo')
except JSONRPCClientError as e:
print e
r = yield client.callRemote('bar.add', 1, 2)
print "add result: %s" % str(r)
r = yield client.callRemote('bar.whoami')
print "whoami result: %s" % str(r)
def setUp(self):
self.reactor = task.Clock()
self.endpoint = FakeEndpoint()
self.factory = JSONRPCClientFactory(
self.endpoint, reactor=self.reactor)
class ClientTestCase(TXJasonTestCase):
"""
Tests for JSONRPCClientFactory.
"""
def setUp(self):
self.reactor = task.Clock()
self.endpoint = FakeEndpoint()
self.factory = JSONRPCClientFactory(
self.endpoint, reactor=self.reactor)
def test_callRemote(self):
"""
callRemote sends data and returns a Deferred that fires with the result
from over the wire.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())),
{'params': [], 'jsonrpc': '2.0', 'method': 'spam', 'id': 1})
self.endpoint.proto.stringReceived(json.dumps(
{'jsonrpc': '2.0', 'id': 1, 'result': 'eggs'}))
self.assertEqual(self.successResultOf(d), 'eggs')
def test_callRemote_error_response(self):
"""
callRemote's Deferred can also errback if an error comes over the wire.
"""
d = self.factory.callRemote('spam')
self.endpoint.proto.stringReceived(json.dumps(
{'jsonrpc': '2.0', 'id': 1, 'error': {
'message': 'error', 'code': -19}}))
self.failureResultOf(d, client.JSONRPCClientError)
def test_notifyRemote(self):
"""
notifyRemote sends data but and returns a Deferred, but does not expect
a response.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.notifyRemote('spam')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())),
{'params': [], 'jsonrpc': '2.0', 'method': 'spam'})
self.successResultOf(d)
def test_callRemote_connection_failure(self):
"""
Connection failures get propagated as an errback on callRemote's
Deferred.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d = self.factory.callRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.failureResultOf(d, FakeError)
def test_notifyRemote_connection_failure(self):
"""
Connection failures get propagated as an errback on notifyRemote's
Deferred.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d = self.factory.notifyRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.failureResultOf(d, FakeError)
def test_notifyRemote_two_connection_failures(self):
"""
In the case of two synchronous connection failures, both notifyRemote
calls errback with the connection failure.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d1 = self.factory.notifyRemote('spam')
d2 = self.factory.notifyRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 2)
self.successResultOf(defer.gatherResults([
self.assertFailure(d1, FakeError),
self.assertFailure(d2, FakeError),
]))
def test_notifyRemote_two_pending_connection_failures(self):
"""
In the case of two notifyRemotes both waiting on the same connection,
and the connection fails, both Deferreds returned by notifyRemote will
errback.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.deferred = defer.Deferred()
d1 = self.factory.notifyRemote('spam')
d2 = self.factory.notifyRemote('spam')
self.endpoint.deferred.errback(FakeError())
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.successResultOf(defer.gatherResults([
self.assertFailure(d1, FakeError),
self.assertFailure(d2, FakeError),
]))
def test_callRemote_cancellation_during_connection(self):
"""
The Deferred returned by callRemote can be cancelled during the
connection attempt.
"""
self.assertFalse(self.endpoint.connected)
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.callRemote('spam')
d.cancel()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_callRemote_cancellation_during_request(self):
"""
The Deferred returned by callRemote can be cancelled while waiting on a
response.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
d.cancel()
self.failureResultOf(d, defer.CancelledError)
def test_notifyRemote_cancellation_during_connection(self):
"""
The Deferred returned by notifyRemote can be cancelled during the
connection attempt.
"""
self.assertFalse(self.endpoint.connected)
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.notifyRemote('spam')
d.cancel()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_reconnection(self):
"""
A new connection is established if the connection is lost between
notifyRemote calls.
"""
self.assertFalse(self.endpoint.connected)
self.factory.notifyRemote('spam')
self.assert_(self.endpoint.connected)
self.endpoint.disconnect(FakeDisconnectedError())
self.assertFalse(self.endpoint.connected)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
self.factory.notifyRemote('eggs')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())),
{'params': [], 'jsonrpc': '2.0', 'method': 'eggs'})
def test_callRemote_timeout(self):
"""
A timeout causes the Deferred returned by callRemote to errback with
CancelledError.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
self.reactor.advance(10)
self.failureResultOf(d, defer.CancelledError)
def test_disconnect(self):
"""
The disconnect method drops the current connection.
"""
d = self.factory.notifyRemote('spam')
self.successResultOf(d)
self.assertIsNot(self.endpoint.transport, None)
self.factory.disconnect()
self.assertIs(self.endpoint.transport, None)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_disconnect_connection_cancellation(self):
"""
The disconnect method cancels pending connections.
"""
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.notifyRemote('spam')
self.factory.disconnect()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_disconnect_callRemote_cancellation(self):
"""
The disconnect method cancels pending callRemote Deferreds.
"""
d = self.factory.callRemote('spam')
self.assertIsNot(self.endpoint.transport, None)
self.factory.disconnect()
self.assertIs(self.endpoint.transport, None)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_connect(self):
"""
The connect method returns a Deferred that fires when the connection is
established.
"""
self.endpoint.deferred = defer.Deferred()
d = self.factory.connect()
self.assertNoResult(d)
self.endpoint.deferred.callback(self.factory.buildProtocol(None))
self.successResultOf(d)
def test_connect_cancellation(self):
"""
Cancelling the connect method's Deferred cancels the connection's
Deferred.
"""
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.connect()
self.assertNoResult(d)
d.cancel()
self.assert_(canceled)
self.failureResultOf(d, defer.CancelledError)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
def test_connect_failure(self):
"""
The connect method's Deferred errbacks if the connection itself
errbacks.
"""
self.endpoint.deferred = defer.Deferred()
d = self.factory.connect()
self.assertNoResult(d)
self.endpoint.deferred.errback(FakeError())
self.failureResultOf(d, FakeError)
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
def test_notifyDisconnect(self):
"""
The notifyDisconnect method returns a Deferred that fires when the
client has disconnected.
"""
d = self.factory.notifyDisconnect()
self.successResultOf(self.factory.connect())
self.assertNoResult(d)
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_notifyDisconnect_after_connection(self):
"""
The notifyDisconnect method returns a Deferred that fires when the
client has disconnected even if it's called after a connection is
established.
"""
self.successResultOf(self.factory.connect())
d = self.factory.notifyDisconnect()
self.assertNoResult(d)
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_notifyDisconnect_after_disconnection(self):
"""
notifyDisconnect doesn't return a fired Deferred if it's called after
disconnection but instead returns a Deferred that fires after the next
disconnection.
"""
self.successResultOf(self.factory.connect())
self.endpoint.disconnect(FakeDisconnectedError())
d = self.factory.notifyDisconnect()
self.assertNoResult(d)
self.successResultOf(self.factory.connect())
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 2)
def __init__(self, *args, **kwargs):
super(Worker, self).__init__()
self.reactor = kwargs['reactor']
self.client = JSONRPCClientFactory(*args, **kwargs)
class Worker(object):
def __init__(self, *args, **kwargs):
super(Worker, self).__init__()
self.reactor = kwargs['reactor']
self.client = JSONRPCClientFactory(*args, **kwargs)
def tell(self, *args, **kwargs):
call = self.client.callRemote(*args, **kwargs)
call.addCallback(json.loads)
return call
@defer.inlineCallbacks
def sync(self, transaction):
local, remote = transaction
new_path = get_root(self.config, local)
if not os.path.exists(new_path):
os.mkdir(new_path)
with open(local, 'ab') as f:
remote_size = yield self.tell('cache.get size', remote)
cache_size = os.stat(local).st_size
while cache_size < remote_size[remote]:
data = yield self.tell('cache.get file', remote, cache_size)
data = data[data.keys()[0]]
data = base64.standard_b64decode(data)
f.seek(cache_size)
f.write(data)
cache_size = os.stat(local).st_size
@defer.inlineCallbacks
def sync_static(self, job):
filename = job['static_file']
cache = get_cache(self.config, filename)
storage = get_storage(self.config, filename)
yield self.sync((cache, storage))
yield job
return
def mkdir_p(self, path):
try:
os.makedirs(path)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else: raise
@defer.inlineCallbacks
def clean_data(self, job):
print "Cleaning data..."
path = self.config['master']['cache']
data_path = path + '/'.join(job['data'][0].split('/')[:-1])
all_files = sorted(glob('{:s}/*.nc'.format(data_path)))
if all_files:
files = job['data']
idx = all_files.index(path + files[0])
end = idx - 840 if idx > 840 else 0
map(os.remove, all_files[0:end])
yield job
return
@defer.inlineCallbacks
def sync_data(self, job):
print "Synchronizing data..."
job['data'].sort()
data_path = '{:s}{:s}'.format(self.config['master']['cache'],
'/'.join(job['data'][0].split('/')[:-1]))
self.mkdir_p(data_path)
translate = lambda m: map(lambda f: m(self.config, f), job['data'])
cache = translate(get_cache)
remote = translate(get_storage)
for transaction in zip(cache, remote):
yield self.sync(transaction)
yield job
return
@defer.inlineCallbacks
def work_in(self, job):
t_job = deepcopy(job)
cached = lambda f: self.config['master']['cache'] + f
t_job['data'] = map(cached, job['data'])
t_job['static_file'] = cached(job['static_file'])
t_job['product'] = None
descriptor = JobDescription(**(t_job))
elapsed, output = descriptor.run()
result = {
"time": output.time[-1].tolist(),
"cloudindex": output.cloudindex[-1, :].tolist(),
"globalindex": output.globalradiation[-1, :].tolist(),
}
yield result
return
@defer.inlineCallbacks
def work(self, ip):
ask_job = self.tell('manager.no job', ip)
job = yield ask_job
yield self.sync_static(job)
yield self.sync_data(job)
yield self.clean_data(job)
result = yield self.work_in(job)
job['result'] = result
ready_job = self.tell('manager.job ready', json.dumps(job))
class ClientTestCase(TXJasonTestCase):
"""
Tests for JSONRPCClientFactory.
"""
def setUp(self):
self.reactor = task.Clock()
self.endpoint = FakeEndpoint()
self.factory = JSONRPCClientFactory(self.endpoint,
reactor=self.reactor)
def test_callRemote(self):
"""
callRemote sends data and returns a Deferred that fires with the result
from over the wire.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())), {
'params': [],
'jsonrpc': '2.0',
'method': 'spam',
'id': 1
})
self.endpoint.proto.stringReceived(
json.dumps({
'jsonrpc': '2.0',
'id': 1,
'result': 'eggs'
}))
self.assertEqual(self.successResultOf(d), 'eggs')
def test_callRemote_error_response(self):
"""
callRemote's Deferred can also errback if an error comes over the wire.
"""
d = self.factory.callRemote('spam')
self.endpoint.proto.stringReceived(
json.dumps({
'jsonrpc': '2.0',
'id': 1,
'error': {
'message': 'error',
'code': -19
}
}))
self.failureResultOf(d, client.JSONRPCClientError)
def test_notifyRemote(self):
"""
notifyRemote sends data but and returns a Deferred, but does not expect
a response.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.notifyRemote('spam')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())), {
'params': [],
'jsonrpc': '2.0',
'method': 'spam'
})
self.successResultOf(d)
def test_callRemote_connection_failure(self):
"""
Connection failures get propagated as an errback on callRemote's
Deferred.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d = self.factory.callRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.failureResultOf(d, FakeError)
def test_notifyRemote_connection_failure(self):
"""
Connection failures get propagated as an errback on notifyRemote's
Deferred.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d = self.factory.notifyRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.failureResultOf(d, FakeError)
def test_notifyRemote_two_connection_failures(self):
"""
In the case of two synchronous connection failures, both notifyRemote
calls errback with the connection failure.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.fail = True
d1 = self.factory.notifyRemote('spam')
d2 = self.factory.notifyRemote('spam')
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 2)
self.successResultOf(
defer.gatherResults([
self.assertFailure(d1, FakeError),
self.assertFailure(d2, FakeError),
]))
def test_notifyRemote_two_pending_connection_failures(self):
"""
In the case of two notifyRemotes both waiting on the same connection,
and the connection fails, both Deferreds returned by notifyRemote will
errback.
"""
self.assertFalse(self.endpoint.connected)
self.endpoint.deferred = defer.Deferred()
d1 = self.factory.notifyRemote('spam')
d2 = self.factory.notifyRemote('spam')
self.endpoint.deferred.errback(FakeError())
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
self.successResultOf(
defer.gatherResults([
self.assertFailure(d1, FakeError),
self.assertFailure(d2, FakeError),
]))
def test_callRemote_cancellation_during_connection(self):
"""
The Deferred returned by callRemote can be cancelled during the
connection attempt.
"""
self.assertFalse(self.endpoint.connected)
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.callRemote('spam')
d.cancel()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_callRemote_cancellation_during_request(self):
"""
The Deferred returned by callRemote can be cancelled while waiting on a
response.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
d.cancel()
self.failureResultOf(d, defer.CancelledError)
def test_notifyRemote_cancellation_during_connection(self):
"""
The Deferred returned by notifyRemote can be cancelled during the
connection attempt.
"""
self.assertFalse(self.endpoint.connected)
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.notifyRemote('spam')
d.cancel()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_reconnection(self):
"""
A new connection is established if the connection is lost between
notifyRemote calls.
"""
self.assertFalse(self.endpoint.connected)
self.factory.notifyRemote('spam')
self.assert_(self.endpoint.connected)
self.endpoint.disconnect(FakeDisconnectedError())
self.assertFalse(self.endpoint.connected)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
self.factory.notifyRemote('eggs')
self.assert_(self.endpoint.connected)
self.assertEqual(
json.loads(readNetstring(self.endpoint.transport.value())), {
'params': [],
'jsonrpc': '2.0',
'method': 'eggs'
})
def test_callRemote_timeout(self):
"""
A timeout causes the Deferred returned by callRemote to errback with
CancelledError.
"""
self.assertFalse(self.endpoint.connected)
d = self.factory.callRemote('spam')
self.reactor.advance(10)
self.failureResultOf(d, defer.CancelledError)
def test_disconnect(self):
"""
The disconnect method drops the current connection.
"""
d = self.factory.notifyRemote('spam')
self.successResultOf(d)
self.assertIsNot(self.endpoint.transport, None)
self.factory.disconnect()
self.assertIs(self.endpoint.transport, None)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_disconnect_connection_cancellation(self):
"""
The disconnect method cancels pending connections.
"""
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.notifyRemote('spam')
self.factory.disconnect()
self.assert_(canceled)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_disconnect_callRemote_cancellation(self):
"""
The disconnect method cancels pending callRemote Deferreds.
"""
d = self.factory.callRemote('spam')
self.assertIsNot(self.endpoint.transport, None)
self.factory.disconnect()
self.assertIs(self.endpoint.transport, None)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
self.failureResultOf(d, defer.CancelledError)
def test_connect(self):
"""
The connect method returns a Deferred that fires when the connection is
established.
"""
self.endpoint.deferred = defer.Deferred()
d = self.factory.connect()
self.assertNoResult(d)
self.endpoint.deferred.callback(self.factory.buildProtocol(None))
self.successResultOf(d)
def test_connect_cancellation(self):
"""
Cancelling the connect method's Deferred cancels the connection's
Deferred.
"""
canceled = []
self.endpoint.deferred = defer.Deferred(canceled.append)
d = self.factory.connect()
self.assertNoResult(d)
d.cancel()
self.assert_(canceled)
self.failureResultOf(d, defer.CancelledError)
self.assertEqual(len(self.flushLoggedErrors(defer.CancelledError)), 1)
def test_connect_failure(self):
"""
The connect method's Deferred errbacks if the connection itself
errbacks.
"""
self.endpoint.deferred = defer.Deferred()
d = self.factory.connect()
self.assertNoResult(d)
self.endpoint.deferred.errback(FakeError())
self.failureResultOf(d, FakeError)
self.assertEqual(len(self.flushLoggedErrors(FakeError)), 1)
def test_notifyDisconnect(self):
"""
The notifyDisconnect method returns a Deferred that fires when the
client has disconnected.
"""
d = self.factory.notifyDisconnect()
self.successResultOf(self.factory.connect())
self.assertNoResult(d)
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_notifyDisconnect_after_connection(self):
"""
The notifyDisconnect method returns a Deferred that fires when the
client has disconnected even if it's called after a connection is
established.
"""
self.successResultOf(self.factory.connect())
d = self.factory.notifyDisconnect()
self.assertNoResult(d)
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 1)
def test_notifyDisconnect_after_disconnection(self):
"""
notifyDisconnect doesn't return a fired Deferred if it's called after
disconnection but instead returns a Deferred that fires after the next
disconnection.
"""
self.successResultOf(self.factory.connect())
self.endpoint.disconnect(FakeDisconnectedError())
d = self.factory.notifyDisconnect()
self.assertNoResult(d)
self.successResultOf(self.factory.connect())
self.endpoint.disconnect(FakeDisconnectedError())
self.failureResultOf(d, FakeDisconnectedError)
self.assertEqual(len(self.flushLoggedErrors(FakeDisconnectedError)), 2)
def setUp(self):
self.reactor = task.Clock()
self.endpoint = FakeEndpoint()
self.factory = JSONRPCClientFactory(self.endpoint,
reactor=self.reactor)
