def process_next(self):
if not self._is_processing:
self._is_processing = True
d = defer.Deferred()
d.addCallback(self._process_next)
d.addBoth(self._finish_processing)
time.callLater(0, d.callback, None)
def reply(self, reply):
reply = reply.duplicate()
self.debug("Sending reply: %r", reply)
self._send_message(reply, self.request.expiration_time)
self._cancel_expiration_call()
self._set_state(RequestState.closed)
time.callLater(0, self._terminate, None)
def handover(self, bid):
bid = bid.clone()
self.debug('Sending bid of the nested contractor: %r.', bid)
assert isinstance(bid, message.Bid)
self._ensure_state(ContractState.announced)
self._set_state(ContractState.delegated)
self.bid = self._handover_message(bid)
time.callLater(0, self._terminate, None)
return self.bid
def handover(self, bid):
new_bid = bid.duplicate()
new_bid.reply_to = bid.reply_to
self.debug("Sending bid of the nested contractor: %r.", new_bid)
assert isinstance(new_bid, message.Bid)
self._ensure_state(ContractState.announced)
self._set_state(ContractState.delegated)
self.bid = self._handover_message(new_bid)
time.callLater(0, self._terminate, None)
return self.bid
def reconnect(self):
# ping database to figure trigger changing state to connected
if self.is_connected():
return defer.succeed(self)
if self.disconnected:
return
if self.reconnector is None or not self.reconnector.active():
self.retry += 1
wait = min(2**(self.retry - 1), 300)
if self.retry > 1:
self.debug('CouchDB refused connection for %d time. '
'This indicates misconfiguration or temporary '
'network problem. Will try to reconnect in '
'%d seconds.', self.retry, wait)
d = defer.Deferred()
d.addCallback(defer.drop_param, self.couchdb_call,
self.couchdb.get, '/')
d.addCallback(self._set_version)
d.addCallback(defer.drop_param, self._setup_notifiers)
d.addErrback(failure.Failure.trap, NotConnectedError)
d.addErrback(failure.Failure.trap, defer.CancelledError)
self.reconnector = time.callLater(wait, d.callback, None)
return d
else:
return self.wait_connected()
def on_master_missing(self):
'''
Tries to spawn a master agency if the slave agency failed to connect
for several times. To avoid several slave agencies spawning the master
agency a file lock is used
@param master: Remote reference to the broker object
'''
self.info("We could not contact the master agency, starting a new one")
if self._starting_master:
self.info("Master already starting, waiting for it")
return
if self._shutdown_task is not None:
self.info("Not spwaning master because we are about to terminate "
"ourselves")
return
if self._startup_task is not None:
raise error.FeatError("Standalone started without a previous "
"master agency already running, terminating "
"it")
# Try the get an exclusive lock on the master agency startup
if self._acquire_lock():
self._starting_master = True
# Allow restarting a master if we didn't succeed after 10 seconds
self._release_lock_cl = time.callLater(10, self._release_lock)
return self._spawn_agency('master')
def testRessetingCall(self):
d = defer.Deferred()
time.scale(0.09)
call = time.callLater(10, d.callback, None)
call.reset(1)
return d
def call_later_ex(self, time_left, method,
args=None, kwargs=None, busy=True):
args = args or []
kwargs = kwargs or {}
call = time.callLater(time_left, self._call, method,
*args, **kwargs)
call_id = str(uuid.uuid1())
self._store_delayed_call(call_id, call, busy)
return call_id
def testScaledCalls(self):
d = defer.Deferred()
time.scale(0.09)
call = time.callLater(1, d.callback, None)
self.assertIsInstance(call, time.ScaledDelayedCall)
fire_time = call.getTime()
left = fire_time - time.time()
self.assertApproximates(1, left, 0.01)
self.assertTrue(0.9 < left <= 1)
self.assertTrue(call.active())
return d
def reconnect(self):
# ping database to figure trigger changing state to connected
self.retry += 1
wait = min(2**(self.retry - 1), 300)
if self.retry > 1:
self.debug('CouchDB refused connection for %d time. '
'This indicates misconfiguration or temporary '
'network problem. Will try to reconnect in %d seconds.',
self.retry, wait)
if self.reconnector is None or not self.reconnector.active():
d = defer.Deferred()
d.addCallback(defer.drop_param, self._paisley_call,
None, self.paisley.listDB)
d.addErrback(failure.Failure.trap, NotConnectedError)
self.reconnector = time.callLater(wait, d.callback, None)
return d
else:
return self.wait_connected()
def _setup_expiration_call(self, expire_time, state,
method, *args, **kwargs):
self.log('Setting expiration call of method: %r.%r',
self.__class__.__name__, method.__name__)
time_left = time.left(expire_time)
if time_left < 0:
raise RuntimeError('Tried to call method in the past! ETA: %r' %
(time_left, ))
def to_call(callback):
if state:
self._set_state(state)
self.log('Calling method: %r with args: %r', method, args)
d = self._call(method, *args, **kwargs)
d.chainDeferred(callback)
result = defer.Deferred()
self._expiration_call = time.callLater(
time_left, to_call, result)
return result
def connectionLost(self, reason):
if reason.check(error.ConnectionDone):
# expected just pass
return
elif reason.check(ResponseDone):
self.debug("CouchDB closed the notification listener. This might "
"indicate missconfiguration. Take look at it")
return
elif reason.check(error.ConnectionRefusedError):
self.retry += 1
wait = min(2**(self.retry - 1), 300)
self.debug('CouchDB refused connection for %d time. '
'This indicates missconfiguration or temporary '
'network problem. Will try to reconnect in %d seconds.',
self.retry, wait)
self.reconnector = time.callLater(wait, self._setup_notifier)
self._on_disconnected()
return
else:
# FIXME handle disconnection when network is down
self._on_disconnected()
self.warning('Connection to db lost with reason: %r', reason)
return self._setup_notifier()
def _setup_snapshoter(self):
self._snapshot_task = time.callLater(300, self._trigger_snapshot)
def _error(self, arg):
self._error_handler(arg)
self._set_state(TaskState.error)
time.callLater(0, self._terminate, arg)
def _completed(self, arg):
if arg != NOT_DONE_YET or not self._cmp_state(TaskState.performing):
self._set_state(TaskState.completed)
time.callLater(0, self._terminate, arg)
def _schedule_protocol(self, _):
self._delayed_call = time.callLater(self.period, self._start_protocol)
def _host_restart_failed(self, failure):
error.handle_failure(self, failure, "Failure during host restart")
self.debug("Retrying in %d seconds",
HOST_RESTART_RETRY_INTERVAL)
time.callLater(HOST_RESTART_RETRY_INTERVAL,
self._start_host_agent_if_necessary)
def delay(value, delay):
"""Returns a deferred whose callback will be triggered
after the specified delay with the specified value."""
d = defer.Deferred()
time.callLater(delay, d.callback, value)
return d
def unregister_agent(self, medium):
agency.Agency.unregister_agent(self, medium)
time.callLater(1, self._shutdown, stop_process=True)
def render_resource(self, request, response, location):
# Only the master agency knows the other ones
self._ensure_master(request)
# Force mime-type to html
response.set_mime_type("text/html")
response.set_header("Cache-Control", "no-store")
response.set_header("connection", "close")
if request.method == http.Methods.POST:
if self.enable_actions and self.model.is_master():
data = "\n".join(request.readlines())
params = http.urldecode(data, request.encoding)
if 'full_shutdown' in params:
time.callLater(0, self.model.full_shutdown)
doc = ["<HTML><HEAD>"
"<TITLE>F3AT Gateway</TITLE></HEAD><BODY>"
"<H2>Offline</H2>"
"</BODY></HTML>"]
response.writelines(doc)
return
if 'start_agent' in params:
agent_type = params['agent_type'][0]
if agent_type in self.startable_agents:
d = self.model.spawn_agent(agent_type)
d.addCallback(defer.drop_param,
self.redirect, request.path)
return d
self.redirect(request.path)
doc = []
self.render_header(doc)
doc.extend(["<H2>Agencies</H2>"
"<TABLE border='0'>"])
for agency_id in self.model.iter_agency_ids():
agency_url = self.create_url(request, agency_id)
doc.extend(["<TR><TD><A href='", agency_url, "'>",
agency_id, "</A>"
"</TD></TR>"])
doc.extend(["</TABLE>"])
if self.enable_actions:
if self.startable_agents and self.model.is_master():
doc.extend(["</TABLE>"
"<H2>Actions</H2>"
"<TABLE>"
"<TR>"
"<TD valign='top'><B>Start Agent:</B></TD>"
"<TD>"
"<FORM method='post'>"
"<DIV>"
"<SELECT name='agent_type'>"
"<option value='' selected>"
"(select type)</option>"])
startable = self.startable_agents.items()
startable.sort(key=operator.itemgetter(0))
for k, v in startable:
doc.extend(["<option value='", k, "'>", v, "</option>"])
doc.extend(["</SELECT>"
"<INPUT type='submit' name='start_agent'"
" value='Start'>"
"</DIV>"
"</FORM>"
"</TD>"
"</TR>"
"<TR>"
"<TD>"
"<FORM method='post'>"
"<INPUT type='submit' name='full_shutdown'"
" value='Full Shutdown'>"
"</FORM>"
"</TD>"
"</TR>"
"</TABLE>"])
self.render_footer(doc)
response.writelines(doc)
def bind():
self._reporter_call = time.callLater(frequency, send_report)
def render_resource(self, request, response, location):
# Force mime-type to html
response.set_mime_type("text/html")
response.set_header("Cache-Control", "no-store")
response.set_header("connection", "close")
if self.enable_actions and request.method == http.Methods.POST:
data = "\n".join(request.readlines())
params = http.urldecode(data, request.encoding)
if 'set_logging_filter' in params:
filter = params["filter"][0]
self.model.set_logging_filter(filter)
if 'shutdown_agency' in params:
time.callLater(1, self.model.shutdown_agency)
return self._redirect_to_top(request)
if 'terminate_agency' in params:
time.callLater(1, self.model.terminate_agency)
return self._redirect_to_top(request)
if 'kill_agency' in params:
time.callLater(1, self.model.kill_agency)
return self._redirect_to_top(request)
agents_url = "/agents"
doc = []
self.render_header(doc)
doc.extend(["<H2>Agency</H2>"
"<TABLE>"
"<TR>"
"<TD><B>Identifier:</B></TD>"
"<TD>", self.model.agency_id, "</TD>"
"</TR>",
"<TR>"
"<TD><B>Role:</B></TD>"
"<TD>", self.model.role.name, "</TD>"
"</TR>"
"</TABLE>"
"<UL>"
"<LI><H4><A href='", agents_url, "'>Agents</A></H4></LI>"
"</UL>"])
if self.enable_actions:
dbg = self.model.get_logging_filter()
doc.extend(["</TABLE>"
"<H2>Actions</H2>"
"<TABLE>"
"<TR>"
"<TD valign='top'><B>Global Logging Filter:</TD>"
"<TD colspan='2'>"
"<DIV>"
"<FORM method='post'>"
"<INPUT type='text' name='filter' value='", dbg, "'>"
"<INPUT type='submit' name='set_logging_filter'"
" value='Update'>"
"</FORM>"
"</DIV>"
"</TD>"
"</TR>"
"<TR>"
"<TD>"
"<FORM method='post'>"
"<INPUT type='submit' name='shutdown_agency'"
" value='Shutdown'>"
"</FORM>"
"</TD>"
"<TD>"
"<FORM method='post'>"
"<INPUT type='submit' name='terminate_agency'"
" value='Terminate'>"
"</FORM>"
"</TD>"
"<TD>"
"<FORM method='post'>"
"<INPUT type='submit' name='kill_agency'"
" value='Kill'>"
"</FORM>"
"</TD>"
"</TR>"
"</TABLE>"])
self.render_footer(doc)
response.writelines(doc)
def missing_master(self):
self._reconn_dc = time.callLater(5, self.initiate_broker)
return self.spawn_missing_master()
def initiate(self):
self.medium.external_counter += 1
time.callLater(2, self.finish.callback, self)
return task.NOT_DONE_YET
def _schedule_flush(self):
if self._flush_task is None:
self._flush_task = time.callLater(0, self._flush)
def call_later_ex(self, _time, _method, args=None, kwargs=None, busy=True):
args = args or ()
kwargs = kwargs or {}
return time.callLater(_time, _method, *args, **kwargs)
def delay_errback(failure, delay):
"""Returns a deferred whose errback will be triggered
after the specified delay with the specified value."""
d = defer.Deferred()
time.callLater(delay, d.errback, failure)
return d
def _start_host_agent_if_necessary(self):
'''
This method starts saves the host agent descriptor and runs it.
To make this happen following conditions needs to be fulfilled:
- it is a master agency,
- we are not starting a host agent already,
- we are not terminating,
- and last but not least, we dont have a host agent running.
'''
def set_flag(value):
self._starting_host = value
if self.role != BrokerRole.master:
self.log('Not starting host agent, because we are not the '
'master agency')
return
if self._shutting_down:
self.log('Not starting host agent, because the agency '
'is about to terminate itself')
return
if self._get_host_agent():
self.log('Not starting host agent, because we already '
' have one')
return
if self._starting_host:
self.log('Not starting host agent, because we are already '
'starting one.')
return
def handle_error_on_get(fail, connection, doc_id):
fail.trap(NotFoundError)
desc = host.Descriptor(shard=u'lobby', doc_id=doc_id)
self.log("Host Agent descriptor not found in database, "
"creating a brand new instance.")
return connection.save_document(desc)
def handle_success_on_get(desc):
if not self._broker.shared_state['enable_host_restart']:
msg = ("Descriptor of host agent has been found in "
"database (hostname: %s). This should not happen "
"on the first run. If you really want to restart "
"the host agent include --force-host-restart "
"options to feat script." % desc.doc_id)
self.error(msg)
d = self.full_shutdown(stop_process=True)
d.addBoth(defer.raise_error, RuntimeError, msg)
return d
self.log("Host Agent descriptor found in database, will restart.")
return desc
set_flag(True)
self.info('Starting host agent.')
conn = self._database.get_connection()
doc_id = self.get_hostname()
d = defer.Deferred()
d.addCallback(defer.drop_param, self.wait_connected)
d.addCallback(defer.drop_param, conn.get_document, doc_id)
d.addCallbacks(handle_success_on_get, handle_error_on_get,
errbackArgs=(conn, doc_id, ))
d.addCallback(self.start_agent, hostdef=self._hostdef)
d.addBoth(defer.bridge_param, set_flag, False)
d.addCallback(defer.drop_param, self._broker.shared_state.__setitem__,
'enable_host_restart', True)
d.addCallback(defer.drop_param, self._flush_agents_to_spawn)
d.addErrback(self._host_restart_failed)
time.callLater(0, d.callback, None)
