def create(self, state, partner_class, recp,
allocation_id=None, role=None, substitute=None,
options=None):
options = options or dict()
found = self.find(recp)
if found:
self.info('We already are in partnership with recipient '
'%r, instance: %r.', recp, found)
return fiber.succeed(found)
factory = self.query_handler(partner_class, role)
partner = factory(recp, allocation_id, role)
self.debug(
'Registering partner %r (lookup (%r, %r)) for recipient: %r',
factory, partner_class, role, recp)
if substitute:
self.debug('It will substitute: %r', substitute)
f = fiber.succeed()
if allocation_id:
f.add_callback(fiber.drop_param,
state.agent.get_allocation,
allocation_id)
f.add_callback(fiber.drop_param, self.initiate_partner, partner,
substitute, **options)
return f
def testUtils(self):
def check_callback(param, expected):
self.assertEqual(param, expected)
return param
def check_errback(failure, expected):
self.assertTrue(failure.check(expected))
def check_true(value=True):
self.assertTrue(value)
return value
def check_false(value=False):
self.assertFalse(value)
return value
def unexpected(_param):
self.fail("Unexpected")
deferreds = []
# Test fiber.succeed()
f = fiber.succeed(1)
f.add_callbacks(check_callback, unexpected, cbargs=(1, ))
deferreds.append(f.start())
# Test fiber.fail()
f = fiber.fail(ValueError())
f.add_callbacks(unexpected, check_errback, ebargs=(ValueError, ))
deferreds.append(f.start())
# Test fiber.drop_param
f = fiber.succeed(False)
f.add_callback(fiber.drop_param, check_true)
deferreds.append(f.start())
# Test fiber.override_result
f = fiber.succeed(False)
f.add_callback(fiber.override_result, True)
f.add_callback(check_true)
deferreds.append(f.start())
# Test fiber.bridge_param
f = fiber.succeed(False)
f.add_callback(fiber.bridge_param, check_true)
f.add_callback(check_false)
deferreds.append(f.start())
return defer.DeferredList(deferreds)
def establish_partnership(self, state, recp, allocation_id=None,
partner_allocation_id=None,
partner_role=None, our_role=None,
substitute=None, allow_double=False,
max_retries=0):
f = fiber.succeed()
found = state.partners.find(recp)
default_role = getattr(self.partners_class, 'default_role', None)
our_role = our_role or default_role
if not allow_double and found:
msg = ('establish_partnership() called for %r which is already '
'our partner with the class %r.' % (recp, type(found), ))
self.debug(msg)
if substitute:
f.add_callback(fiber.drop_param, state.partners.remove,
substitute)
f.chain(fiber.fail(partners.DoublePartnership(msg)))
return f
# f.add_callback(fiber.drop_param, self.initiate_protocol,
# requester.Propose, recp, allocation_id,
# partner_allocation_id,
# our_role, partner_role, substitute)
factory = retrying.RetryingProtocolFactory(requester.Propose,
max_retries=max_retries)
f.add_callback(fiber.drop_param, self.initiate_protocol,
factory, recp, allocation_id,
partner_allocation_id,
our_role, partner_role, substitute)
f.add_callback(fiber.call_param, "notify_finish")
return f
def wait_for_bids(self, state):
f = fiber.succeed()
f.add_callback(fiber.drop_param,
state.medium.wait_for_state,
ContractState.closed, ContractState.expired)
f.add_callback(lambda _: state.medium.get_bids())
return f
def add_document(self, state, doc_id):
if doc_id in state.documents:
return fiber.succeed(copy.deepcopy(state.documents[doc_id]))
f = self._refresh_document(doc_id)
if state.view_factory is None:
f.add_callback(defer.bridge_param, state.agent.register_change_listener, doc_id, self._document_changed)
return f
def wait_for_empty_import_entries(self):
f = fiber.succeed()
desc = self.get_descriptor()
if desc.import_entries:
f.add_callback(fiber.drop_param,
self.wait_for_event, 'import_entries_empty')
return f
def wait_for_bids(self, state):
#FIXME: It would be better to use an agent notifier
f = fiber.succeed()
f.add_callback(fiber.drop_param, state.medium.wait_for_state,
ContractState.closed, ContractState.expired)
f.add_callback(fiber.drop_param, state.medium.get_bids)
return f
def fetch_nested_bids(self, state, recipients, original_announcement,
keep_sender=False):
recipients = recipient.IRecipients(recipients)
sender = original_announcement.reply_to
max_distance = original_announcement.max_distance
if sender in recipients and not keep_sender:
self.log("Removing sender from list of recipients to nest")
recipients.remove(sender)
if len(recipients) == 0:
self.log("Empty list to nest to, will not nest")
return fiber.succeed(list())
elif max_distance is not None and \
original_announcement.level + 1 > max_distance:
self.log("Reached max distance for nesting of %d, returning empy "
"list.", max_distance)
return list()
else:
self.log("Will nest contract to %d contractors.", len(recipients))
announcement = original_announcement.duplicate()
announcement.level += 1
announcement.expiration_time = self._get_time_window(
announcement.expiration_time)
current_time = state.agent.get_time()
time_left = announcement.expiration_time - current_time
state.nested_manager = state.agent.initiate_protocol(
NestedManagerFactory(self.protocol_id, time_left),
recipients, announcement)
f = fiber.Fiber()
f.add_callback(fiber.drop_param,
state.nested_manager.wait_for_bids)
return f.succeed()
def _on_restarted(self, state, partner, new_address, sender):
old = partner.recipient if new_address != partner.recipient else None
f = fiber.succeed()
partner.recipient = recipient.IRecipient(new_address)
f.add_callback(fiber.drop_param, self._call_next_cb_broken,
partner, 'on_restarted', True, old_recipient=old)
return f
def wait_for_complete(self, state):
#FIXME: It would be better to use an agent notifier
f = fiber.succeed()
f.add_callback(fiber.drop_param, state.medium.wait_for_state,
ContractState.completed, ContractState.expired)
f.add_callback(fiber.override_result, None)
return f
def register_dns_mapping(self, state):
'''
Registers to dns and stores the current address in descriptor.
If we already have ip in descriptor, this means that we have been
restarted. In this case we remove the old DNS entry.
'''
def update_ip(desc, ip):
desc.ip = ip
def dns_error(failure):
self.raise_alert("Failed to register dns entry!",
alert.Severity.medium)
self.error('Error registering dns entry. %r', failure)
desc = self.get_descriptor()
ip = state.medium.get_ip()
f = fiber.succeed()
if desc.ip is not None:
self.info("Removing old dns mapping for ip: %s", desc.ip)
f.add_callback(dns.remove_mapping, self, state.dns_prefix,
desc.ip)
f.add_callback(fiber.drop_param, self.update_descriptor,
update_ip, None)
f.add_callback(fiber.drop_param,
dns.add_mapping, self, state.dns_prefix, ip)
f.add_callback(fiber.drop_param, self.update_descriptor,
update_ip, ip)
f.add_errback(dns_error)
return f
def call_mro_ex(self, method_name, keywords, raise_on_unconsumed=True):
f = fiber.succeed()
call_list = self._get_mro_call_list(
method_name, keywords, raise_on_unconsumed)
for method, kwargs in call_list:
f.add_callback(fiber.drop_param, method, self, **kwargs)
return f
def on_goodbye(self, agent):
f = fiber.succeed()
f.add_both(fiber.drop_param, agent.become_king)
f.add_both(fiber.drop_param, agent.look_for_neighbours)
f.add_both(fiber.drop_param, agent.call_next,
agent.on_neighbour_gone, self.recipient)
return f
def get_host_list(agent):
shard_recp = agent.query_partners("shard")
if not shard_recp:
agent.warning("get_host_list() called, but agent doesn't have shard partner, " "returning empty list")
return fiber.succeed(list())
else:
return agent.call_remote(shard_recp, "get_host_list", _timeout=1)
def initiate(self, state, host, port, exchange, exchange_type):
state.connection = self.dependency(
IAMQPClientFactory, self, exchange, port=port,
exchange_type=exchange_type)
f = fiber.succeed()
f.add_callback(fiber.drop_param, state.connection.connect)
return f
def initiate(self, state, port=None, ns_ttl=None, aa_ttl=None,
ns=None, suffix=None):
config = state.medium.get_configuration()
state.port = port or config.port
state.ns_ttl = ns_ttl or config.ns_ttl
state.aa_ttl = aa_ttl or config.aa_ttl
state.ns = ns or config.ns or self._lookup_ns()
state.suffix = suffix or config.suffix or self._lookup_suffix()
state.notify_cfg = config.notify
self.debug("Initializing DNS agent with: port=%d, ns_ttl=%d, "
"aa_ttl=%d, ns=%s, suffix=%s", state.port, state.ns_ttl,
state.aa_ttl, state.ns, state.suffix)
state.resolver = Resolver(state.suffix, state.ns, state.notify_cfg,
self._get_ip(), state.ns_ttl)
state.labour = self.dependency(IDNSServerLabourFactory,
self, state.resolver,
state.notify_cfg.slaves,
state.suffix)
ami = state.medium.register_interest(AddMappingContractor)
rmi = state.medium.register_interest(RemoveMappingContractor)
muc = state.medium.register_interest(MappingUpdatesCollector)
ami.bind_to_lobby()
rmi.bind_to_lobby()
muc.bind_to_lobby()
f = fiber.succeed()
f.add_callback(fiber.drop_param, state.labour.initiate)
return f
def notify_finish(self):
if self._failed:
return fiber.fail(self._result)
elif self._finished:
return fiber.succeed(self._result)
else:
return fiber.wrap_defer(self._notifier.wait, "finished")
def _release_allocation(self, state):
f = fiber.succeed()
if state.allocation_id:
return f.add_callback(fiber.drop_param,
state.agent.release_resource,
state.allocation_id)
return f
def get_list(self, state):
if not state.partners.connections:
return fiber.succeed(dict())
prot = self.initiate_protocol(
InspectManager, state.partners.connections)
return prot.notify_finish()
def _restart_yourself(self, state):
f = fiber.succeed()
f.add_callback(fiber.drop_param,
host.start_agent_in_shard,
state.agent, state.descriptor, state.descriptor.shard)
f.add_callbacks(self._send_restarted_notifications,
self._local_restart_failed)
return f
def get_shard_partner(self, state):
partner = state.partners.shard
self.log('In get_shard_partner(). Current result is: %r', partner)
if partner:
return fiber.succeed(partner)
f = self.wait_for_event('joined_to_shard')
f.add_callback(fiber.drop_param, self.get_shard_partner)
return f
def requested(self, state, request):
fun_id = request.payload['fun_id']
args = request.payload['args']
kwargs = request.payload['kwargs']
f = fiber.succeed(fun_id, state.medium.get_canceller())
f.add_callback(state.agent.call_local, *args, **kwargs)
f.add_callbacks(callback=self.got_result, errback=self.got_failure)
return f
def _starting_failed(self, state, fail):
error.handle_failure(self, fail, 'Starting failed, cancelling')
msg = message.Cancellation(reason=str(fail))
f = fiber.succeed()
f.add_callback(fiber.drop_param, self._release_allocation)
f.add_callback(fiber.drop_param, state.medium.defect,
msg)
return f
def fixup_host_role(self, state, hosted, partner):
has_role = partner.role == "host"
f = fiber.succeed()
if hosted and not has_role:
f.add_callback(fiber.drop_param, self.add_host_role, partner)
elif not hosted and has_role:
f.add_callback(fiber.drop_param, self.remove_host_role, partner)
return f
def remove(self, state, partner):
# FIXME: Two subsequent updates of descriptor.
f = fiber.succeed()
f.add_callback(fiber.drop_param, state.agent.update_descriptor,
self._remove_partner, partner)
if partner.allocation_id:
f.add_callback(fiber.drop_param, state.agent.release_resource,
partner.allocation_id)
return f
def _release_unused_allocations(self, state, *_):
# FIXME: possible n HTTP requests
f = fiber.succeed()
for alloc_id in state.allocations:
if not state.agent.allocation_used(alloc_id):
f.add_callback(fiber.drop_param,
state.agent.release_resource,
alloc_id)
return f
def announced(self, state, announce):
state.agent.got('announce')
to_nest = state.agent.get_from_state('to_nest')
f = fiber.succeed()
if to_nest:
f.add_callback(fiber.drop_param, self.fetch_nested_bids,
[to_nest], announce)
f.add_callback(self._send_bid)
return f
def initiate(self, agent):
f = fiber.succeed()
if self.allocation_id is None:
f.add_callback(fiber.drop_param,
agent.allocate_resource, neighbours=1)
f.add_callback(self._store_alloc_id)
f.add_callback(fiber.drop_param, agent.call_next,
agent.on_new_neighbour, self.recipient)
return f
def _handle_export(self, state, blackbox):
mig_recp = state.migration.get_migration_agent()
if mig_recp is None:
return fiber.succeed()
cmd = spec.HandleImport(agent_type=state.descriptor.type_name, blackbox=blackbox)
req = state.agent.initiate_protocol(protocol.Requester, mig_recp, cmd)
f = req.notify_finish()
f.add_callback(fiber.override_result, None)
return f
def check_if_agency_hosts(self, state, recp):
'''
Called after partner has been restarted. It checks our agency
is in charge of this agent. If not it removes the partnership.
'''
f = fiber.succeed(recp.key)
f.add_callback(state.medium.check_if_hosted)
f.add_callback(self._got_if_hosted, recp)
return f
