Exemple #1
0
 def release_queues():
     APIContainer.log_debug("active apis: %r", APIContainer.API_CONTAINERS)
     for api_name in APIQueue.QUEUES.keys():
         APIContainer.log_debug("checking queue for %s", api_name)
         if api_name not in APIContainer.API_CONTAINERS:
             APIConnector.release_connectors(api_name)
             APIQueue.release_queue(api_name)
Exemple #2
0
 def release_queues():
     APIContainer.log_debug("active apis: %r", APIContainer.API_CONTAINERS)
     for api_name in APIQueue.QUEUES.keys():
         APIContainer.log_debug("checking queue for %s", api_name)
         if api_name not in APIContainer.API_CONTAINERS:
             APIConnector.release_connectors(api_name)
             APIQueue.release_queue(api_name)
Exemple #3
0
    def create_new(api_name):
        container_name = APIContainer.unique_container_name(api_name)
        queue = APIQueue.get_queue(api_name)
        env = {
            "JBAPI_LOGLEVEL": "1",
            "JBAPI_NAME": api_name,
            "JBAPI_QUEUE": queue.get_endpoint_out(),
            "JBAPI_CMD": queue.get_command(),
            "JBAPI_CID": container_name
        }
        image_name = queue.get_image_name()
        if image_name is None:
            image_name = APIContainer.DCKR_IMAGE

        hostcfg = docker.utils.create_host_config(
            mem_limit=APIContainer.MEM_LIMIT)

        jsonobj = APIContainer.DCKR.create_container(
            image_name,
            detach=True,
            host_config=hostcfg,
            cpu_shares=APIContainer.CPU_LIMIT,
            environment=env,
            hostname='juliabox',
            name=container_name)
        dockid = jsonobj["Id"]
        cont = APIContainer(dockid)
        APIContainer.log_info("Created " + cont.debug_str())
        cont.start()
        return cont
Exemple #4
0
    def calc_desired_container_counts():
        for api_name in APIContainer.API_CONTAINERS:
            queue = APIQueue.get_queue(api_name, alloc=False)
            if queue is None:
                APIContainer.log_debug("no queue found for api: %s", api_name)
                APIContainer.DESIRED_CONTAINER_COUNTS[api_name] = 0

            desired = APIContainer.DESIRED_CONTAINER_COUNTS[api_name]
            APIContainer.log_debug(
                "re-calculating desired capacity with %s. now %d.",
                queue.debug_str(), desired)
            if queue.mean_outstanding > 1:
                incr = int(queue.mean_outstanding)
                desired = min(desired + incr,
                              APIContainer.MAX_PER_API_CONTAINERS)
            elif queue.mean_outstanding < 0.01:  # approx 5 polls where 0 q length was found
                desired = 0
            elif queue.mean_outstanding < 0.5 and desired > 1:  # nothing is queued when mean is 1/3
                desired -= 1
            else:
                desired = max(desired, 1)
            APIContainer.DESIRED_CONTAINER_COUNTS[api_name] = desired
            APIContainer.log_debug(
                "calculated desired capacity with %s to %d.",
                queue.debug_str(), desired)

            if queue.num_outstanding == 0:
                queue.incr_outstanding(
                    0)  # recalculate mean if no requests are coming
Exemple #5
0
    def create_new(api_name):
        container_name = APIContainer.unique_container_name(api_name)
        queue = APIQueue.get_queue(api_name)
        env = {
            "JBAPI_LOGLEVEL": "1",
            "JBAPI_NAME": api_name,
            "JBAPI_QUEUE": queue.get_endpoint_out(),
            "JBAPI_CMD": queue.get_command(),
            "JBAPI_CID": container_name
        }
        image_name = queue.get_image_name()
        if image_name is None:
            image_name = APIContainer.DCKR_IMAGE

        hostcfg = docker.utils.create_host_config(mem_limit=APIContainer.MEM_LIMIT)

        jsonobj = APIContainer.DCKR.create_container(image_name,
                                                     detach=True,
                                                     host_config=hostcfg,
                                                     cpu_shares=APIContainer.CPU_LIMIT,
                                                     environment=env,
                                                     hostname='juliabox',
                                                     name=container_name)
        dockid = jsonobj["Id"]
        cont = APIContainer(dockid)
        APIContainer.log_info("Created " + cont.debug_str())
        cont.start()
        return cont
Exemple #6
0
    def __init__(self, api_name):
        self.queue = APIQueue.get_queue(api_name)
        ctx = zmq.Context.instance()
        self.api_name = api_name
        self.sock = ctx.socket(zmq.REQ)
        self.sock.connect(self.queue.get_endpoint_in())
        self.has_errors = False
        self.timeout_callback = None
        self.timeout = self.queue.get_timeout()

        if api_name in APIConnector.CONNS:
            APIConnector.CONNS[api_name].append(self)
        else:
            APIConnector.CONNS[api_name] = [self]

        self.log_debug("%s: created", self.debug_str())
    def __init__(self, api_name):
        self.queue = APIQueue.get_queue(api_name)
        ctx = zmq.Context.instance()
        self.api_name = api_name
        self.sock = ctx.socket(zmq.REQ)
        self.sock.connect(self.queue.get_endpoint_in())
        self.has_errors = False
        self.timeout_callback = None
        self.timeout = self.queue.get_timeout()

        if api_name in APIConnector.CONNS:
            APIConnector.CONNS[api_name].append(self)
        else:
            APIConnector.CONNS[api_name] = [self]

        self.log_debug("%s: created", self.debug_str())
Exemple #8
0
    def create_new(api_name):
        print("The api_name is %s", api_name)
        print("The volume is %s", APIContainer.VOLUMES[0])
	vols = {
            APIContainer.HOST_LOG_FOLDER: {
                'bind': APIContainer.VOLUMES[0],
                'ro': False
            }
        }

        container_name = APIContainer.unique_container_name(api_name)
        queue = APIQueue.get_queue(api_name)
        env = {
            "JBAPI_NAME": api_name,
            "JBAPI_QUEUE": queue.get_endpoint_out(),
            "JBAPI_CMD": queue.get_command(),
            "JBAPI_CID": container_name
        }
        print("env ", env)
        image_name = queue.get_image_name()
        if image_name is None:
            image_name = APIContainer.DCKR_IMAGE

        hostcfg = docker.utils.create_host_config(binds=vols, mem_limit=APIContainer.MEM_LIMIT)
        print("hostcfg ", hostcfg)

        jsonobj = APIContainer.DCKR.create_container(image_name,
                                                     detach=True,
                                                     host_config=hostcfg,
                                                     cpu_shares=APIContainer.CPU_LIMIT,
                                                     volumes=APIContainer.VOLUMES,
                                                     environment=env,
                                                     hostname='juliabox',
                                                     name=container_name)
        print("The jsonobj is %s", jsonobj)
        dockid = jsonobj["Id"]
        cont = APIContainer(dockid)
        APIContainer.log_info("Created " + cont.debug_str())
        cont.start()
        return cont
Exemple #9
0
    def calc_desired_container_counts():
        for api_name in APIContainer.API_CONTAINERS:
            queue = APIQueue.get_queue(api_name, alloc=False)
            if queue is None:
                APIContainer.log_debug("no queue found for api: %s", api_name)
                APIContainer.DESIRED_CONTAINER_COUNTS[api_name] = 0

            desired = APIContainer.DESIRED_CONTAINER_COUNTS[api_name]
            APIContainer.log_debug("re-calculating desired capacity with %s. now %d.", queue.debug_str(), desired)
            if queue.mean_outstanding > 1:
                incr = int(queue.mean_outstanding)
                desired = min(desired + incr, APIContainer.MAX_PER_API_CONTAINERS)
            elif queue.mean_outstanding < 0.01:  # approx 5 polls where 0 q length was found
                desired = 0
            elif queue.mean_outstanding < 0.5 and desired > 1:  # nothing is queued when mean is 1/3
                desired -= 1
            else:
                desired = max(desired, 1)
            APIContainer.DESIRED_CONTAINER_COUNTS[api_name] = desired
            APIContainer.log_debug("calculated desired capacity with %s to %d.", queue.debug_str(), desired)

            if queue.num_outstanding == 0:
                queue.incr_outstanding(0)   # recalculate mean if no requests are coming