Пример #1
0
    def test_create_mesos_task_config(self):
        dynamic_env_master = {"APP": "hostmgr"}
        config_file = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            "..",
            "config",
            "default.yaml",
        )
        config = load_config(config_file)
        got = create_mesos_task_config(
            config,
            module="peloton",
            dynamic_env=dynamic_env_master,
            image_path="myregistry/peloton:0.1.0",
        )

        expected = task.TaskConfig(
            resource=task.ResourceConfig(cpuLimit=2.0,
                                         memLimitMb=4096,
                                         diskLimitMb=2048),
            ports=[
                task.PortConfig(name="HTTP_PORT", envName="HTTP_PORT"),
                task.PortConfig(name="GRPC_PORT", envName="GRPC_PORT"),
            ],
            container=mesos.ContainerInfo(
                type="MESOS",
                mesos=mesos.ContainerInfo.MesosInfo(image=mesos.Image(
                    type="DOCKER",
                    docker=mesos.Image.Docker(name="myregistry/peloton:0.1.0"),
                )),
            ),
            command=mesos.CommandInfo(
                uris=[
                    mesos.CommandInfo.URI(
                        value="https://gist.githubusercontent.com/scy0208/"
                        "08a66afe3a7837e5e1c1528d16b47e6f/raw/"
                        "2119f0fe20b7a1e827e4e43b288545799d6b4e5e/"
                        "hostmgr_mesos_secret",
                        executable=False,
                        cache=False,
                        output_file="hostmgr_mesos_secret",
                    )
                ],
                shell=True,
                value="bash /bin/entrypoint.sh",
                environment=mesos.Environment(variables=[
                    mesos.Environment.Variable(name="CONFIG_DIR",
                                               value="config"),
                    mesos.Environment.Variable(name="AUTO_MIGRATE",
                                               value="true"),
                    mesos.Environment.Variable(
                        name="MESOS_SECRET_FILE",
                        value="/mnt/mesos/sandbox/hostmgr_mesos_secret",
                    ),
                    mesos.Environment.Variable(name="APP", value="hostmgr"),
                ]),
            ),
        )

        self.assertEqual(got, expected)
Пример #2
0
def test__acquire_release_host_offers():
    resource_constraint = v0hostmgr.ResourceConstraint(
        minimum=task.ResourceConfig(cpuLimit=3.0))
    host_filter = v0hostmgr.HostFilter(
        resourceConstraint=resource_constraint,
        quantity=v0hostmgr.QuantityControl(maxHosts=2),
    )
    request = v0hostmgr.AcquireHostOffersRequest(filter=host_filter)
    client = with_private_stubs(Client())

    resp = client.hostmgr_svc.AcquireHostOffers(
        request, metadata=client.hostmgr_metadata, timeout=20)

    # max hosts is 2, we should expect 2 host offers
    assert len(resp.hostOffers) == 2
    for offer in resp.hostOffers:
        assert (offer.hostname in MESOS_AGENTS)

    # release offers to hostmgr
    resp = client.hostmgr_svc.ReleaseHostOffers(
        request=v0hostmgr.ReleaseHostOffersRequest(hostOffers=resp.hostOffers),
        metadata=client.hostmgr_metadata,
        timeout=20)

    assert resp.HasField("error") is False
Пример #3
0
 def create_pod_config(self, sleep_time, dynamic_factor):
     return task.TaskConfig(
         resource=task.ResourceConfig(
             cpuLimit=0.1, memLimitMb=32, diskLimitMb=32
         ),
         command=mesos.CommandInfo(
             shell=True,
             value="echo %s && sleep %s"
             % (str(dynamic_factor), str(sleep_time)),
         ),
     )
Пример #4
0
def test__acquire_return_offers_errors():
    resource_constraint = v0hostmgr.ResourceConstraint(
        minimum=task.ResourceConfig(cpuLimit=14.0))
    host_filter = v0hostmgr.HostFilter(resourceConstraint=resource_constraint)
    request = v0hostmgr.AcquireHostOffersRequest(filter=host_filter)
    # decorate the client to add peloton private API stubs
    client = with_private_stubs(Client())

    # ask is 14 cpus, so no hosts should match this
    resp = client.hostmgr_svc.AcquireHostOffers(
        request, metadata=client.hostmgr_metadata, timeout=20)
    assert len(resp.hostOffers) == 0

    # release offers to hostmgr with a invalid offer ID
    resp = client.hostmgr_svc.ReleaseHostOffers(
        request=v0hostmgr.ReleaseHostOffersRequest(hostOffers=[
            v0hostmgr.HostOffer(id=peloton.HostOfferID(value="invalid_id"))
        ]),
        metadata=client.hostmgr_metadata,
        timeout=20)

    assert resp.error is not None
Пример #5
0
def create_mesos_task_config(
    config, module, dynamic_env, version=None, image_path=None
):
    resource_config = config.get(module).get("resource")
    ports = config.get(module).get("ports", [])
    ports_config = []
    for port in ports:
        ports_config.append(task.PortConfig(name=port, envName=port))

    environments = []
    start_cmd = config.get(module).get("start_command", "")
    static_envs = config.get(module).get("static_env", [])
    fetch_files = config.get(module).get("fetch_files", [])

    for static_env in static_envs:
        environments.append(
            mesos.Environment.Variable(
                name=static_env["name"], value=static_env["value"]
            )
        )

    for dyn_env_name, dyn_env_value in dynamic_env.iteritems():
        environments.append(
            mesos.Environment.Variable(name=dyn_env_name, value=dyn_env_value)
        )

    if image_path:
        docker_image = image_path
    else:
        docker_registry = config.get(module).get("docker_registry")
        image_path = config.get(module).get("image_path")
        if not version:
            version = config.get(module).get("version")

        docker_image = (
            os.path.join(docker_registry, image_path) + ":" + version
        )

    return task.TaskConfig(
        resource=task.ResourceConfig(**resource_config),
        ports=ports_config,
        container=mesos.ContainerInfo(
            type="MESOS",
            mesos=mesos.ContainerInfo.MesosInfo(
                image=mesos.Image(
                    type="DOCKER", docker=mesos.Image.Docker(name=docker_image)
                )
            ),
        ),
        command=mesos.CommandInfo(
            uris=[
                mesos.CommandInfo.URI(
                    value=fetch_file["source"],
                    output_file=fetch_file["name"],
                    cache=bool(fetch_file.get("cache", False)),
                    executable=bool(fetch_file.get("executable", False)),
                )
                for fetch_file in fetch_files
            ],
            shell=True,
            value=start_cmd,
            environment=mesos.Environment(variables=environments),
        ),
    )
Пример #6
0
def test__launch_kill():
    client = with_private_stubs(Client())

    # acquire 1 host offer
    resource_constraint = v0hostmgr.ResourceConstraint(
        minimum=task.ResourceConfig(cpuLimit=3.0))
    host_filter = v0hostmgr.HostFilter(
        resourceConstraint=resource_constraint,
        quantity=v0hostmgr.QuantityControl(maxHosts=1),
    )
    request = v0hostmgr.AcquireHostOffersRequest(filter=host_filter, )

    resp = client.hostmgr_svc.AcquireHostOffers(
        request, metadata=client.hostmgr_metadata, timeout=20)

    assert len(resp.hostOffers) == 1

    # launch a test task using this offer
    cmd = "echo 'succeeded instance task' & sleep 100"
    tc = task.TaskConfig(
        command=mesos.CommandInfo(shell=True, value=cmd),
        name="task_name",
        resource=task.ResourceConfig(cpuLimit=1.0),
    )
    tid = mesos.TaskID(value=str(uuid.uuid4()) + '-1-1')
    t = v0hostmgr.LaunchableTask(
        taskId=tid,
        config=tc,
    )

    # Test 1
    # launch task using invalid offer
    req = v0hostmgr.LaunchTasksRequest(
        hostname=resp.hostOffers[0].hostname,
        agentId=resp.hostOffers[0].agentId,
        tasks=[t],
        id=peloton.HostOfferID(value=str(uuid.uuid4())))
    try:
        resp = client.hostmgr_svc.LaunchTasks(req,
                                              metadata=client.hostmgr_metadata,
                                              timeout=20)
        assert False, 'LaunchTasks should have failed'
    except:
        pass

    # Test 2
    # launch task using valid offer
    req = v0hostmgr.LaunchTasksRequest(hostname=resp.hostOffers[0].hostname,
                                       agentId=resp.hostOffers[0].agentId,
                                       tasks=[t],
                                       id=resp.hostOffers[0].id)
    resp = client.hostmgr_svc.LaunchTasks(req,
                                          metadata=client.hostmgr_metadata,
                                          timeout=20)
    assert resp.HasField("error") is False

    # Test 3
    # kill with empty TaskIDs list
    resp = client.hostmgr_svc.KillTasks(v0hostmgr.KillTasksRequest(taskIds=[]),
                                        metadata=client.hostmgr_metadata,
                                        timeout=20)
    assert resp.HasField("error") is True

    # Test 4
    # kill valid TaskID
    resp = client.hostmgr_svc.KillTasks(
        v0hostmgr.KillTasksRequest(taskIds=[tid]),
        metadata=client.hostmgr_metadata,
        timeout=20)
    assert resp.HasField("error") is False
Пример #7
0
def create_mesos_task_config(config, module, dynamic_env,
                             version=None, image_path=None):
    resource_config = config.get(module).get('resource')
    ports = config.get(module).get('ports', [])
    ports_config = []
    for port in ports:
        ports_config.append(
            task.PortConfig(
                name=port,
                envName=port,
            )
        )

    environments = []
    start_cmd = config.get(module).get('start_command', '')
    static_envs = config.get(module).get('static_env', [])
    fetch_files = config.get(module).get('fetch_files', [])

    for static_env in static_envs:
        environments.append(
            mesos.Environment.Variable(
                name=static_env['name'],
                value=static_env['value'],
            )
        )

    for dyn_env_name, dyn_env_value in dynamic_env.iteritems():
        environments.append(
            mesos.Environment.Variable(
                name=dyn_env_name,
                value=dyn_env_value,
            )
        )

    if image_path:
        docker_image = image_path
    else:
        docker_registry = config.get(module).get('docker_registry')
        image_path = config.get(module).get('image_path')
        if not version:
            version = config.get(module).get('version')

        docker_image = os.path.join(docker_registry, image_path) + ':' + \
            version

    return task.TaskConfig(
        resource=task.ResourceConfig(**resource_config),
        ports=ports_config,
        container=mesos.ContainerInfo(
            type='MESOS',
            mesos=mesos.ContainerInfo.MesosInfo(
                image=mesos.Image(
                    type='DOCKER',
                    docker=mesos.Image.Docker(
                        name=docker_image,

                    )
                )
            ),
        ),
        command=mesos.CommandInfo(
            uris=[mesos.CommandInfo.URI(
                value=fetch_file['source'],
                output_file=fetch_file['name'],
                cache=bool(fetch_file.get('cache', False)),
                executable=bool(fetch_file.get('executable', False)),
            ) for fetch_file in fetch_files],
            shell=True,
            value=start_cmd,
            environment=mesos.Environment(
                variables=environments
            )
        ),

    )
Пример #8
0
    def test_create_mesos_task_config(self):
        dynamic_env_master = {
            'APP': 'hostmgr',
        }
        config_file = os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            '..', 'config', 'default.yaml')
        config = load_config(config_file)
        got = create_mesos_task_config(
            config,
            module='peloton',
            dynamic_env=dynamic_env_master,
            image_path='myregistry/peloton:0.1.0')

        expected = task.TaskConfig(
            resource=task.ResourceConfig(
                cpuLimit=2.0,
                memLimitMb=4096,
                diskLimitMb=2048,
            ),
            ports=[task.PortConfig(
                    name='HTTP_PORT',
                    envName='HTTP_PORT',
                ), task.PortConfig(
                    name='GRPC_PORT',
                    envName='GRPC_PORT')],
            container=mesos.ContainerInfo(
                type='MESOS',
                mesos=mesos.ContainerInfo.MesosInfo(
                    image=mesos.Image(
                        type='DOCKER',
                        docker=mesos.Image.Docker(
                            name='myregistry/peloton:0.1.0',
                        )
                    )
                ),
            ),
            command=mesos.CommandInfo(
                uris=[mesos.CommandInfo.URI(
                    value='https://gist.githubusercontent.com/scy0208/'
                          '08a66afe3a7837e5e1c1528d16b47e6f/raw/'
                          '2119f0fe20b7a1e827e4e43b288545799d6b4e5e/'
                          'hostmgr_mesos_secret',
                    executable=False,
                    cache=False,
                    output_file='hostmgr_mesos_secret',
                )],
                shell=True,
                value='bash /bin/entrypoint.sh',
                environment=mesos.Environment(
                    variables=[mesos.Environment.Variable(
                        name='CONFIG_DIR',
                        value='config',
                    ), mesos.Environment.Variable(
                        name='AUTO_MIGRATE',
                        value='true',
                    ), mesos.Environment.Variable(
                        name='MESOS_SECRET_FILE',
                        value='/mnt/mesos/sandbox/hostmgr_mesos_secret',
                    ), mesos.Environment.Variable(
                        name='APP',
                        value='hostmgr',
                    )]
                )
            ),
        )

        self.assertEqual(got, expected)