Exemplo n.º 1
0
    def startup(self, kwargs):
        self.logger.info("HTC Manager starting up")
        self.logger.info(str(kwargs))
        if self.state != self.S_INIT and self.state != self.S_STOPPED:
            vals = { 'curstate': self.state, 'action': 'startup' }
            return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
        if 'cloud' in kwargs:
            try:
                self._init_cloud(kwargs['cloud'])
            except Exception:
                return HttpErrorResponse(
                    "A cloud named '%s' could not be found" % kwargs['cloud'])        
        #self.logger.info('Get service TaskFarm')
        try:
            self.service = TaskFarm(kwargs['mode'], kwargs['type'])
        except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e:
            return HttpErrorResponse({ 'error': e.__str__() })
        #self.logger.info('Got service TaskFarm, delete some kwargs entries')
        self.state = self.S_PROLOGUE
        del kwargs['type']
        del kwargs['mode']
        #del kwargs['m_type']
        #self.logger.info('Show leftover kwargs entries')
        self.logger.info(str(kwargs))
        #self.logger.info('Starting Thread for startup')
        Thread(target=self._do_startup, kwargs=kwargs).start()
        
        self.logger.info(str(self.service))

        return HttpJsonResponse({ 'state': self.state })
Exemplo n.º 2
0
    def startup(self, kwargs):
        self.logger.info("HTC Manager starting up")
        self.logger.info("... with kwargs = " + str(kwargs))
        if self.state != self.S_INIT and self.state != self.S_STOPPED:
            vals = {"curstate": self.state, "action": "startup"}
            return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
        if "cloud" in kwargs:
            try:
                self._init_cloud(kwargs["cloud"])
            except Exception:
                return HttpErrorResponse("A cloud named '%s' could not be found" % kwargs["cloud"])
        # self.logger.info('Get service TaskFarm')
        if "mode" not in kwargs and "type" not in kwargs:
            return HttpErrorResponse("mode and type not defined")
        if "mode" not in kwargs:
            return HttpErrorResponse("mode not defined")
        if "type" not in kwargs:
            return HttpErrorResponse("type not defined")
        try:
            self.service = TaskFarm(kwargs["mode"], kwargs["type"])
        except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e:
            print e.__str__()
            sys.stdout.flush()
            sys.stderr.flush()
            return HttpErrorResponse(e.__str__())
            return HttpErrorResponse({"error": e.__str__()})
        except Exception as e:
            return HttpErrorResponse({"error": e.__str__()})
        # self.logger.info('Got service TaskFarm, delete some kwargs entries')
        self.state = self.S_PROLOGUE
        del kwargs["type"]
        del kwargs["mode"]
        # del kwargs['m_type']
        # self.logger.info('Show leftover kwargs entries')
        self.logger.info(str(kwargs))
        # self.logger.info('Starting Thread for startup')
        Thread(target=self._do_startup, kwargs=kwargs).start()

        self.logger.info(str(self.service))

        return HttpJsonResponse({"state": self.state})
Exemplo n.º 3
0
class HTCManager(BaseManager):
    """Manager class with the following exposed methods:

    shutdown() -- POST
    add_nodes(count) -- POST
    remove_nodes(count) -- POST
    list_nodes() -- GET
    get_service_info() -- GET
    get_node_info(serviceNodeId) -- GET
    """

    RWCT = "RemoteWallClockTime"
    MT = "MATCH_EXP_MachineCloudMachineType"

    def __init__(self, config_parser, **kwargs):
        """Initialize a HTC Manager. 

        'config_parser' represents the manager config file. 
        **kwargs holds anything that can't be sent in config_parser."""
        BaseManager.__init__(self, config_parser)

        self.nodes = []

        # Setup the clouds' controller
        self.controller.generate_context("htc")

        self.con = True
        self.hub_ip = None
        types = []
        costs = []
        limits = []

        types.append(self.config_parser.get("iaas", "INST_TYPE"))
        cpt = self.config_parser.get("iaas", "COST_PER_TIME")
        i = cpt.index("/")
        s = cpt.index("$")
        c = cpt[s + 2 : i]
        costs.append(float(c))
        limits.append(int(self.config_parser.get("iaas", "MAX_VMS")))

        for cloud in self.controller.get_clouds():
            types.append(self.config_parser.get(cloud.cloud_name, "INST_TYPE"))
            cpt = self.config_parser.get(cloud.cloud_name, "COST_PER_TIME")
            i = cpt.index("/")
            s = cpt.index("$")
            c = cpt[s + 2 : i]
            costs.append(float(c))
            limits.append(int(self.config_parser.get(cloud.cloud_name, "MAX_VMS")))
        self.configuration = Configuration(types, costs, limits)
        self.logger.info(self.configuration.costs)
        self.logger.info(self.configuration.keys)
        self.logger.info(self.configuration.limits)
        #        random.seed()
        #        for t in types:
        #            self.configuration.set_average(t, 2 * random.uniform(0,1))
        #        self.configuration.relevant_time_unit(20)
        #        self.configuration.compute_throughput()
        #        self.configuration.dynamic_configuration()
        self.pool = {}
        self.files = []

    @expose("POST")
    def startup(self, kwargs):
        self.logger.info("HTC Manager starting up")
        self.logger.info("... with kwargs = " + str(kwargs))
        if self.state != self.S_INIT and self.state != self.S_STOPPED:
            vals = {"curstate": self.state, "action": "startup"}
            return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
        if "cloud" in kwargs:
            try:
                self._init_cloud(kwargs["cloud"])
            except Exception:
                return HttpErrorResponse("A cloud named '%s' could not be found" % kwargs["cloud"])
        # self.logger.info('Get service TaskFarm')
        if "mode" not in kwargs and "type" not in kwargs:
            return HttpErrorResponse("mode and type not defined")
        if "mode" not in kwargs:
            return HttpErrorResponse("mode not defined")
        if "type" not in kwargs:
            return HttpErrorResponse("type not defined")
        try:
            self.service = TaskFarm(kwargs["mode"], kwargs["type"])
        except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e:
            print e.__str__()
            sys.stdout.flush()
            sys.stderr.flush()
            return HttpErrorResponse(e.__str__())
            return HttpErrorResponse({"error": e.__str__()})
        except Exception as e:
            return HttpErrorResponse({"error": e.__str__()})
        # self.logger.info('Got service TaskFarm, delete some kwargs entries')
        self.state = self.S_PROLOGUE
        del kwargs["type"]
        del kwargs["mode"]
        # del kwargs['m_type']
        # self.logger.info('Show leftover kwargs entries')
        self.logger.info(str(kwargs))
        # self.logger.info('Starting Thread for startup')
        Thread(target=self._do_startup, kwargs=kwargs).start()

        self.logger.info(str(self.service))

        return HttpJsonResponse({"state": self.state})

    # def _do_startup(self, cloud, m_type):
    def _do_startup(self, cloud):
        """Start up the service. The first node will be an agent running a
        HTC Hub and a HTC Node."""

        # self.logger.info("_do_startup(%s)" % cloud)
        startCloud = self._init_cloud(cloud)
        m_type = self.config_parser.get(
            startCloud.cloud_name, "INST_TYPE"
        )  # 'default' may have been replaced by 'iaas'
        # self.logger.info("_do_startup(%s)" % cloud)
        self.logger.info(str(self.controller.get_clouds()))
        vals = {"action": "_do_startup", "count": 1}
        self.logger.debug(self.ACTION_REQUESTING_NODES % vals)

        try:
            nodes = self.controller.create_nodes(1, client.check_agent_process, self.AGENT_PORT, startCloud)

            hub_node = nodes[0]

            # The first agent is a HTC Hub and a HTC Node
            client.create_hub(hub_node.ip, self.AGENT_PORT)

            client.create_node(hub_node.ip, self.AGENT_PORT, hub_node.ip)
            self.logger.info("Added node %s: %s " % (hub_node.id, hub_node.ip))
            node_info.add_node_info("/etc/hosts", hub_node.ip, hub_node.id)

            node = hub_node
            worker = Worker(node.id, node.ip, node.private_ip, node.cloud_name, m_type)
            self.service.add_worker(worker, int(node.id))

            self.hub_ip = hub_node.ip

            # Extend the nodes list with the newly created one
            self.nodes += nodes
            if m_type in self.pool:
                self.pool[m_type] += 1
            else:
                self.pool[m_type] = 1
            self.state = self.S_RUNNING
        except Exception, err:
            self.logger.exception("_do_startup: Failed to create hub: %s" % err)
            self.state = self.S_ERROR
Exemplo n.º 4
0
class HTCManager(BaseManager):
    """Manager class with the following exposed methods:

    shutdown() -- POST
    add_nodes(count) -- POST
    remove_nodes(count) -- POST
    list_nodes() -- GET
    get_service_info() -- GET
    get_node_info(serviceNodeId) -- GET
    """
    RWCT = 'RemoteWallClockTime'
    MT = 'MATCH_EXP_MachineCloudMachineType'

    def __init__(self, config_parser, **kwargs):
        """Initialize a HTC Manager. 

        'config_parser' represents the manager config file. 
        **kwargs holds anything that can't be sent in config_parser."""
        BaseManager.__init__(self, config_parser)

        self.nodes = []

        # Setup the clouds' controller
        self.controller.generate_context('htc')

        self.con = True
        self.hub_ip = None
        types = []
        costs = []
        limits = []
	
        types.append(self.config_parser.get('iaas', 'INST_TYPE'))
        cpt = self.config_parser.get('iaas', 'COST_PER_TIME')
        i = cpt.index('/')
        s = cpt.index('$')
        c = cpt[s+2:i]
        costs.append(float(c))
        limits.append(int(self.config_parser.get('iaas', 'MAX_VMS')))
	      

        for cloud in self.controller.get_clouds():
            types.append(self.config_parser.get(cloud.cloud_name, 'INST_TYPE'))
            cpt = self.config_parser.get(cloud.cloud_name, 'COST_PER_TIME')
            i = cpt.index('/')
            s = cpt.index('$')
            c = cpt[s+2:i]
            costs.append(float(c))
            limits.append(int(self.config_parser.get(cloud.cloud_name, 'MAX_VMS')))
        self.configuration = Configuration(types,costs,limits)
        self.logger.info(self.configuration.costs)
        self.logger.info(self.configuration.keys)
        self.logger.info(self.configuration.limits)
#        random.seed()
#        for t in types:
#            self.configuration.set_average(t, 2 * random.uniform(0,1))        
#        self.configuration.relevant_time_unit(20)
#        self.configuration.compute_throughput()
#        self.configuration.dynamic_configuration()
	self.pool={}
        self.files=[]


    @expose('POST')
    def startup(self, kwargs):
        self.logger.info("HTC Manager starting up")
        self.logger.info(str(kwargs))
        if self.state != self.S_INIT and self.state != self.S_STOPPED:
            vals = { 'curstate': self.state, 'action': 'startup' }
            return HttpErrorResponse(self.WRONG_STATE_MSG % vals)
        if 'cloud' in kwargs:
            try:
                self._init_cloud(kwargs['cloud'])
            except Exception:
                return HttpErrorResponse(
                    "A cloud named '%s' could not be found" % kwargs['cloud'])        
        #self.logger.info('Get service TaskFarm')
        try:
            self.service = TaskFarm(kwargs['mode'], kwargs['type'])
        except (UnknownHtcTypeError, UnimplementedHtcCombinationError) as e:
            return HttpErrorResponse({ 'error': e.__str__() })
        #self.logger.info('Got service TaskFarm, delete some kwargs entries')
        self.state = self.S_PROLOGUE
        del kwargs['type']
        del kwargs['mode']
        #del kwargs['m_type']
        #self.logger.info('Show leftover kwargs entries')
        self.logger.info(str(kwargs))
        #self.logger.info('Starting Thread for startup')
        Thread(target=self._do_startup, kwargs=kwargs).start()
        
        self.logger.info(str(self.service))

        return HttpJsonResponse({ 'state': self.state })


    #def _do_startup(self, cloud, m_type):
    def _do_startup(self, cloud):
        """Start up the service. The first node will be an agent running a
        HTC Hub and a HTC Node."""

        #self.logger.info("_do_startup(%s)" % cloud)
        startCloud = self._init_cloud(cloud)
        m_type = self.config_parser.get(startCloud.cloud_name, 'INST_TYPE')  # 'default' may have been replaced by 'iaas'
        #self.logger.info("_do_startup(%s)" % cloud)
        self.logger.info(str(self.controller.get_clouds()))
        vals = { 'action': '_do_startup', 'count': 1 }
        self.logger.debug(self.ACTION_REQUESTING_NODES % vals)

        try:
            nodes = self.controller.create_nodes(1,
                client.check_agent_process, self.AGENT_PORT, startCloud)

            hub_node = nodes[0]

            # The first agent is a HTC Hub and a HTC Node
            client.create_hub(hub_node.ip, self.AGENT_PORT)

            client.create_node(hub_node.ip, self.AGENT_PORT, hub_node.ip)
            self.logger.info("Added node %s: %s " % (hub_node.id, hub_node.ip))
            node_info.add_node_info('/etc/hosts', hub_node.ip, hub_node.id)

            node = hub_node
            worker = Worker(node.id, node.ip, node.private_ip, node.cloud_name, m_type)
            self.service.add_worker(worker, int(node.id))

            self.hub_ip = hub_node.ip

            # Extend the nodes list with the newly created one
            self.nodes += nodes
	    if m_type in self.pool:
		    self.pool[m_type]+=1
            else:
		    self.pool[m_type]=1
            self.state = self.S_RUNNING
        except Exception, err:
            self.logger.exception('_do_startup: Failed to create hub: %s' % err)
            self.state = self.S_ERROR