def initialize(self, config):
""" Initial setup """
self.logger = logging.getLogger()
self.config = config
# provides the global statistics
self.statistics = StatisticsProcessing(config)
# stopped circuits tracker
self.stopped_track = StoppedTracker()
# started circuits tracker
self.started_track = StartedTracker(self.config.max_command_time)
# mainloop locker
self.lock_start = DeferredLock()
# bundles tracking container
self.bundles = BundleReferences(config)
# dispatching the batchs of start
self.loop_starter = LoopingStarter(dispatcher=self,
emitting_period=config.emitting_period)
# launchers driving and slots detect
self.groups = [net for (net, mask) in self.config.preferred_network]
self.launchers_networks = dict([(launcher,[n[0] for n in net_and_mask])
for (launcher,net_and_mask) in self.config.launchers_networks.items()])
self.logger.info("preferred networks by launchers: %s" % str(self.launchers_networks))
self.launchers = self.config.launchers_uri
# FIXME - main default launcher
temp_launcher = self.launchers.keys()[0]
default_slots = dict([(name, line["slots"]) \
for (name, line) in self.config.launchers.items()])
self.launchers_provider = RemoteCallProxy(self.config.launchers_uri,
default_slots,
temp_launcher)
return self._get_all_slots()
class MscContainer (object):
__metaclass__ = SingletonN
"""
Main database of circuits and access methods.
All circuits to run are stocked here.
"""
slots = {}
# All the workflow circuits are stocked here
_circuits = []
# A lookup to refer all phases to use
installed_phases = []
# list of all networks
groups = []
# commands id to process a cleanup when a command expires
# {cmd_id: fully_expired}
# fully_expired :
# - True if expired and ready to cleanup
# - False if contains some unprocessed commands_on_host
# This cmd_id is removed after the cleanup
candidats_to_cleanup = {}
# a provider to collect the commands statistics
statistics = None
kill_all = False
@property
def ready_candidats_to_cleanup(self):
""" Shortcut to expired circuits according to cleanup conditions """
return [cmd_id for (cmd_id, expired) in self.candidats_to_cleanup.items() if expired]
def checkout_command(self, cmd_id):
"""
Called on circuit releasing to hold a commands.id for cleanup
@param cmd_id: id of commands record
@type cmd_id: int
"""
if cmd_id not in self.candidats_to_cleanup :
self.candidats_to_cleanup[cmd_id] = False
def set_ready_to_cleanup(self, cmd_id):
"""
Corrensponding released circuits are ready to cleanup check
@param cmd_id: id of commands record
@type cmd_id: int
"""
self.candidats_to_cleanup[cmd_id] = True
@property
def circuits(self):
"""Shortcut to all active circuits"""
return [c for c in self._circuits if c.status == CC_STATUS.ACTIVE]
@property
def waiting_circuits(self):
"""Shortcut to all waiting circuits"""
return [c for c in self._circuits if c.status == CC_STATUS.WAITING]
def remove_circuit(self, circuit):
"""
Release the circuit from container
@param circuit: circuit to remove
@type circuit: Circuit
"""
self.checkout_command(circuit.cmd_id)
self._circuits.remove(circuit)
def cancel(self):
""" Stops all the running loops and other traitements """
self.logger.info("Stopping the dispatcher...")
self.loop_starter.cancel()
self.kill_all = True
@property
def max_slots(self):
""" All slots from all detected launchers """
return reduce(lambda x, y: (x + y), self.slots.values())
@property
def free_slots(self):
""" Free slots to use """
return self.max_slots - len(self.get_running_circuits())
def has_unstarted_circuits(self):
"""Checks the unstarted circuits to avoid a next heavy query execution"""
return len([c for c in self.circuits if not c.is_running]) > 0
def _in_waitings(self, id):
"""
Test if a circuit is waiting.
@param id: commands_on_host id
@type id: int
@return: True if command_on_host in container
@rtype: bool
"""
return id in [wf.id for wf in self.waiting_circuits]
def __contains__(self, id):
"""
Test if a circuit is already running,
that means not released yet or added.
@param id: commands_on_host id
@type id: int
@return: True if command_on_host in container
@rtype: bool
"""
return id in [wf.id for wf in self.circuits]
def initialize(self, config):
""" Initial setup """
self.logger = logging.getLogger()
self.config = config
# provides the global statistics
self.statistics = StatisticsProcessing(config)
# stopped circuits tracker
self.stopped_track = StoppedTracker()
# started circuits tracker
self.started_track = StartedTracker(self.config.max_command_time)
# mainloop locker
self.lock_start = DeferredLock()
# bundles tracking container
self.bundles = BundleReferences(config)
# dispatching the batchs of start
self.loop_starter = LoopingStarter(dispatcher=self,
emitting_period=config.emitting_period)
# launchers driving and slots detect
self.groups = [net for (net, mask) in self.config.preferred_network]
self.launchers_networks = dict([(launcher,[n[0] for n in net_and_mask])
for (launcher,net_and_mask) in self.config.launchers_networks.items()])
self.logger.info("preferred networks by launchers: %s" % str(self.launchers_networks))
self.launchers = self.config.launchers_uri
# FIXME - main default launcher
temp_launcher = self.launchers.keys()[0]
default_slots = dict([(name, line["slots"]) \
for (name, line) in self.config.launchers.items()])
self.launchers_provider = RemoteCallProxy(self.config.launchers_uri,
default_slots,
temp_launcher)
return self._get_all_slots()
def _get_all_slots(self):
"""
Detects the total of slots from all launchers.
@return: total of slots per launcher
@rtype: dict
"""
d = self.launchers_provider.get_all_slots()
d.addCallback(self._set_slots)
d.addCallback(self._slots_info)
@d.addErrback
def _eb(failure):
self.logger.error("An error occured when getting the slots: %s" % failure)
return d
def _set_slots(self, slots):
"""
Sets the detected slots from launchers
@param slots: total of slots per launcher
@type slots: dict
"""
self.slots = slots
return slots
def _slots_info(self, result):
"""A little log stuff on start"""
self.logger.info("Detected slots SUM from all launchers: %d" % self.max_slots)
return result
def has_free_slots(self):
""" Checks if at least one slot is free"""
return len(self.get_running_circuits()) < self.max_slots
def get(self, id):
"""
Get the circuit if exists.
@param id: commands_on_host id
@type id: int
@return: requested circuit
@rtype: Circuit object
"""
matches = [wf for wf in self._circuits if wf.id == id]
if len(matches) > 0 :
return matches[0]
else :
self.logger.debug("Circuit #%s: not exists" % id)
return None
def _release(self, id, suspend_to_waitings=False):
"""
Circuit releasing from the main container.
Called typicaly when last phase ends or overtimed.
A reference of this method is passed on each running phase to call
when finished or overtimed.
@param id: commands_on_host id
@type id: int
"""
if any([True for c in self._circuits if c.id == id]):
self.logger.debug("circuit #%d finished" % id)
circuit = self.get(id)
if suspend_to_waitings :
circuit.status = CC_STATUS.WAITING
self.logger.info("Circuit #%d: failed and queued" % id)
else :
self.remove_circuit(circuit)
self.stopped_track.remove(circuit.id)
self.logger.info("Remaining content: %d circuits (+%d waitings)" % ((len(self.circuits)),len(self.waiting_circuits)))
return True
return False
