コード例 #1
0
    def __init__(self, _id, installed_phases, config, pull=False):
        """
        @param id: CommandOnHost id
        @type id: int

        @param installed_phases: all possible phases classes to use
        @type installed_phases: dict

        @param config: scheduler's configuration container
        @type config: SchedulerConfig

        @param pull: True if pull mode
        @type pull: bool
        """
        self.logger = logging.getLogger()
        self.id = _id
        self.config = config

        self.cohq = CoHQuery(int(_id))
        self.cmd_id = self.cohq.cmd.id

        self.installed_phases = installed_phases
        self.pull = pull

        self.__phases_list = []
コード例 #2
0
    def perform(self):
        """
            Handle first Phase: upload time
        """
        # fullfil used proxy (if we can)
        if self.cmd.hasToUseProxy():
            cohq = CoHQuery(self.coh.id)
            #d = defer.maybeDeferred(Analyses().localProxyUploadStatus, cohq)
            d = defer.maybeDeferred(self.dispatcher.local_proxy_upload_status, cohq)
            d.addCallback(self._cbChooseUploadMode)
            return d

        return self._chooseUploadMode()
コード例 #3
0
ファイル: remote.py プロジェクト: resonancellc/mmc-1
 def apply_initial_rules(self):
     ret = self._apply_initial_rules()
     if not self.cmd.hasSomethingToExecute():
         self.logger.info("Circuit #%s: Nothing to execute" % self.coh.id)
         self.phase.set_done()
         return self.next()
     if self.cmd.hasToUseProxy():
         cohq = CoHQuery(self.coh.id)
         if not self.dispatcher.local_proxy_may_continue(cohq):
             self.logger.info(
                 "Circuit #%s: execution postponed, waiting for some clients"
                 % self.coh.id)
             if not self.isStateStopped():
                 self.coh.setStateScheduled()
     if ret not in (DIRECTIVE.NEXT, DIRECTIVE.GIVE_UP, DIRECTIVE.KILLED,
                    DIRECTIVE.STOPPED, DIRECTIVE.OVER_TIMED):
         return self._switch_on()
     return ret
コード例 #4
0
    def parsePullError(self, reason, error_code = PULSE2_UNKNOWN_ERROR):
        """
           error_code : by default we consider un unknwo error was raised (PULSE2_UNKNOWN_ERROR)
        """
        # something goes really wrong: immediately give up
        self.logger.warn("Circuit #%s: pull failed, unattented reason: %s" %
                (self.coh.id, reason.getErrorMessage()))

        proxy_coh_id = self.coh.getUsedProxy()
        if proxy_coh_id:
            proxy = CoHQuery(proxy_coh_id)
            proxy_uuid = proxy.target.getUUID()
            # see if we can unload a proxy
            # no ret val
            LocalProxiesUsageTracking().untake(proxy_uuid, self.cmd.getId())
            self.logger.debug("scheduler %s: coh #%s used coh #%s as local proxy, releasing one slot (%d left)" % (self.config.name,
                    self.coh.id, proxy_coh_id,
                    LocalProxiesUsageTracking().how_much_left_for(proxy_uuid,
                        self.cmd.getId())))

        self.update_history_failed(error_code, '', reason.getErrorMessage())
        return self.switch_phase_failed()
コード例 #5
0
    def phase_process(self, result):
        """
        A callback to recursive phase processing.
        Can be called as an ordinnary routine (i.e. on start)

        @param result: returned result from initial phase tests
        @type result: str

        @return: recursive workflow routine
        @rtype: func
        """
        self.cohq = CoHQuery(self.id)
        self.update_stats()
	self.dispatcher.started_track.update(self.id)
	if self.id in self.dispatcher.stopped_track :
	    self.release()
        # if give-up - actual phase is probably running - do not move - wait...
        if result == DIRECTIVE.GIVE_UP or result == None :
            return lambda : DIRECTIVE.GIVE_UP
        elif result == DIRECTIVE.FAILED :
            self.logger.info("Circuit #%s: failed - releasing" % self.id)
            self.release()
            return
        elif result in (DIRECTIVE.KILLED, DIRECTIVE.OVER_TIMED) :
            self.logger.info("Circuit #%s: releasing" % self.id)
            if result == DIRECTIVE.OVER_TIMED :
                self.schedule_last_stats()
            self.release()
            return
        elif result == DIRECTIVE.STOPPED:
            self.logger.info("Circuit #%s: stopping" % self.id)
            self.cohq.coh.setStateStopped()
            self.release()
            return
        else :
            return self.phase_step()
コード例 #6
0
ファイル: types.py プロジェクト: pavelpromin/mmc
class CircuitBase(object):
    """ 
    Data container of circuit.
    
    Provides the base operations with phases and builds all the needed
    references between phases and dispatcher.
    """
    # commands_on_host id
    id = None
    # commands id
    cmd_id = None



    status = CC_STATUS.ACTIVE
    # Main container of selected phases
    phases = None
 
    # methods called by scheduler-proxy
    _proxy_methods = {}
    # list of phases to refer phase objects 
    installed_phases = {}
    # Main container of selected phases
    _phases = None
    # msc data persistence model
    cohq = None
    # running phase reference
    running_phase = None
    # detected IP address of target
    host = None
    # detected network address of target
    network_address = None
    # first initialisation flag 
    initialized = False
    # A callable to self-releasing from the container 
    releaser = None
    # last activity timestamp
    last_activity_time = None
    # 
    launcher = None
    launchers_provider = None

    def __init__(self, _id, installed_phases, config, pull=False):
        """
        @param id: CommandOnHost id
        @type id: int

        @param installed_phases: all possible phases classes to use
        @type installed_phases: dict

        @param config: scheduler's configuration container
        @type config: SchedulerConfig

        @param pull: True if pull mode
        @type pull: bool
        """
        self.logger = logging.getLogger()
        self.id = _id
        self.config = config

        self.cohq = CoHQuery(int(_id))
        self.cmd_id = self.cohq.cmd.id

        self.installed_phases = installed_phases
        self.pull = pull

    @property 
    def is_running(self):
        return isinstance(self.running_phase, Phase)

    def setup(self, recurrent=False):
        """
        Post-init - detecting the networking info of target. 

        @param recurrent: if True, the circuit is on pull mode
        @type recurrent: bool
        """
        self.recurrent = recurrent

        if not self.initialized :
            d = maybeDeferred(self._flow_create)

            if not recurrent :
                d.addCallback(self._chooseClientNetwork)
                d.addCallback(self._host_detect) 
                d.addCallback(self._network_detect)

            d.addCallback(self._init_end)
            d.addErrback(self._init_failed)

            return d
        else :
            return Deferred()
     
    def _flow_create(self):
        """ Builds the workflow of circuit """
        
        phases = []
        selected = self.cohq.get_phases()
        if self.pull:
            d_mode = "pull"
        else :
            d_mode = "push"

        self.logger.debug("Circuit #%s: started on %s mode" %(self.id, d_mode))

        for phase_name in selected :
            matches = [p for p in self.installed_phases[d_mode] if p.name==phase_name]
            if len(matches) == 1:
                phases.append(matches[0])
            else :
                # TODO - log it and process something .. ?
                raise KeyError


        self.phases = phases
        return True

    @property
    def phases(self):
        """Gets the phases iterator"""
        return self._phases


    @phases.setter # pyflakes.ignore 
    def phases(self, value):
        """
        Phases property set processing.

        - Initial verifications of list of phases
        - converting the _phases attribute to iterator
        """
        if isinstance(value, list) and all(p for p in value if issubclass(p, Phase)):
            self._phases = iter(value)
        else :
            raise TypeError("All elements must be <Phase> type")

        self.__phases_list = value
 
    def on_last_phase(self):
        try:
            last_phase_names = [p.name for p in self.__phases_list[-2:]]
            return self.running_phase.name in last_phase_names
        except Exception, e:

            self.logger.error("\033[32mlast phase failed: %s\033[0m" % str(e))
コード例 #7
0
class CircuitBase(object):
    """
    Data container of circuit.

    Provides the base operations with phases and builds all the needed
    references between phases and dispatcher.
    """
    # commands_on_host id
    id = None
    # commands id
    cmd_id = None



    status = CC_STATUS.ACTIVE

    # methods called by scheduler-proxy
    _proxy_methods = {}
    # list of phases to refer phase objects
    installed_phases = {}
    # Main container of selected phases
    _phases = None
    # msc data persistence model
    cohq = None
    # running phase reference
    running_phase = None
    # detected IP address of target
    host = None
    # detected network address of target
    network_address = None
    # first initialisation flag
    initialized = False
    # A callable to self-releasing from the container
    releaser = None
    # last activity timestamp
    last_activity_time = None
    #
    launcher = None
    launchers_provider = None

    def __init__(self, _id, installed_phases, config, pull=False):
        """
        @param id: CommandOnHost id
        @type id: int

        @param installed_phases: all possible phases classes to use
        @type installed_phases: dict

        @param config: scheduler's configuration container
        @type config: SchedulerConfig

        @param pull: True if pull mode
        @type pull: bool
        """
        self.logger = logging.getLogger()
        self.id = _id
        self.config = config

        self.cohq = CoHQuery(int(_id))
        self.cmd_id = self.cohq.cmd.id

        self.installed_phases = installed_phases
        self.pull = pull

        self.__phases_list = []


    @property
    def is_running(self):
        return isinstance(self.running_phase, Phase)

    def setup(self, recurrent=False):
        """
        Post-init - detecting the networking info of target.

        @param recurrent: if True, the circuit is on pull mode
        @type recurrent: bool
        """
        self.recurrent = recurrent

        if not self.initialized :
            d = maybeDeferred(self._flow_create)

            if not recurrent :
                d.addCallback(self._set_host)

            d.addCallback(self._init_end)
            d.addErrback(self._init_failed)

            return d
        else :
            return Deferred()

    def _set_host(self, reason):
        """Sets the detected IP address from CM"""

        if self.cohq.target.last_update and self.cohq.target.target_ipaddr:
            self.host = self.cohq.target.target_ipaddr

            ntw = NetworkDetect(self.cohq.target.target_ipaddr,
                                self.cohq.target.target_network)
            self.network_address = ntw.network

            return True
        else:
            self.logger.debug("Circuit #%s: IP address not updated yet" % (self.id))
            return False



    def _flow_create(self):
        """ Builds the workflow of circuit """

        phases = []
        selected = self.cohq.get_phases()
        if self.pull:
            d_mode = "pull"
        else :
            d_mode = "push"

        self.logger.debug("Circuit #%s: started on %s mode" %(self.id, d_mode))

        for phase_name in selected :
            matches = [p for p in self.installed_phases[d_mode] if p.name==phase_name]
            if len(matches) == 1:
                phases.append(matches[0])
            else :
                # TODO - log it and process something .. ?
                raise KeyError


        self.set_phases(phases)
        return True

    @property
    def phases(self):
        """
	Main container of selected phases.
	Gets the phases iterator.
	"""
        return self._phases


    def set_phases(self, value):
        """
        Phases property set processing.

        - Initial verifications of list of phases
        - converting the _phases attribute to iterator
        """
        if isinstance(value, list) and all(p for p in value if issubclass(p, Phase)):
            self._phases = iter(value)
        else :
            raise TypeError("All elements must be <Phase> type")

        self.__phases_list = value

    def on_last_phase(self):
        try:
            last_phase_names = [p.name for p in self.__phases_list[-2:]]
            return self.running_phase.name in last_phase_names
        except Exception, e:

            self.logger.error("\033[32mlast phase failed: %s\033[0m" % str(e))
コード例 #8
0
    def _runProxyClientPhase(self, client):
        # fulfill protocol
        client['protocol'] = 'rsyncproxy'

        proxyCoH = CoHQuery(self.coh.getUsedProxy())
        # get informations about our proxy
        if proxyCoH == None:
            return defer.fail(Exception("Cant access to CoH")).addErrback(self.parsePushError).addErrback(self.got_error_in_error)

        #proxy = self.get_client("transfert")

        proxy = {'host': chooseClientInfo(proxyCoH.target),
                 'uuid': proxyCoH.target.getUUID(),
                 'maxbw': proxyCoH.cmd.maxbw,
                 'client_check': getClientCheck(proxyCoH.target),
                 'server_check': getServerCheck(proxyCoH.target),
                 'action': getAnnounceCheck('transfert'),
                 #'group': getClientGroup(proxyCoH.target)} # TODO - get correct network address
                 'group': ""} # TODO - get from launchers select


        if not proxy['host']: # We couldn't get an IP address for the target host
            return defer.fail(Exception("Can't get proxy IP address")).addErrback(self.parsePushError).addErrback(self.got_error_in_error)
        # and fill struct
        # only proxy['host'] used until now
        client['proxy'] = {'command_id': self.coh.getUsedProxy(),
                           'host': proxy['host'],
                           'uuid': proxy['uuid']
        }

        # build file list
        files_list = []
        for file in self.cmd.files.split("\n"):
            fname = file.split('##')[1]
            if re_abs_path.search(fname):
                fname = re_basename.search(fname).group(1) # keeps last compontent of path
            files_list.append(fname)

        # prepare deffereds
        if self.config.mode == 'sync':
            self.update_history_in_progress()
            mydeffered = self.launchers_provider.sync_remote_pull(self.coh.getId(),
                                                             client,
                                                             files_list,
                                                             self.config.max_upload_time
                                                            )
            mydeffered.\
                addCallback(self.parsePushResult).\
                addErrback(self.parsePushError).\
                addErrback(self.got_error_in_error)
        elif self.config.mode == 'async':
            # 'server_check': {'IP': '192.168.0.16', 'MAC': 'abbcd'}
            mydeffered = self.launchers_provider.async_remote_pull(self.coh.getId(),
                                                              client,
                                                              files_list,
                                                              self.config.max_upload_time
                                                             )

            mydeffered.\
                addCallback(self.parsePushOrder).\
                addErrback(self.parsePushError).\
                addErrback(self.got_error_in_error)
        else:
            mydeffered = None

        return mydeffered
コード例 #9
0
        else: # failure: immediately give up
            self.logger.info("Circuit #%s: (%s on %s) push failed (exitcode != 0)" %
                    (self.coh.id,
                     self.cmd.title,
                     self.target.target_name))

            self.update_history_failed(exitcode, stdout, stderr)
            return self.switch_phase_failed()


    @launcher_proxymethod("completed_pull")
    def parsePullResult(self, (exitcode, stdout, stderr), id=None):

        proxy_coh_id = self.coh.getUsedProxy()
        if proxy_coh_id:
            proxy = CoHQuery(proxy_coh_id)
            proxy_uuid = proxy.target.getUUID()
            # see if we can unload a proxy
            # no ret val
            LocalProxiesUsageTracking().untake(proxy_uuid, self.cmd.getId())
            self.logger.debug("scheduler %s: coh #%s used coh #%s as local proxy, releasing one slot (%d left)" % (self.config.name,
                    self.coh.id, proxy_coh_id, LocalProxiesUsageTracking().how_much_left_for(proxy_uuid, self.cmd.getId())))

        if exitcode == PULSE2_SUCCESS_ERROR: # success
            self.logger.info("Circuit #%s: (%s on %s) pull done (exitcode == 0)" %
                    (self.coh.id,
                     self.cmd.title,
                     self.target.target_name))
            self.update_history_done(exitcode, stdout, stderr)
            if self.phase.switch_to_done():
                return self.next()