Example #1
0
def list_app_env(name):
    app = App.get_by_name(name)
    if not app:
        logger.error('app not found, env set ignored')
        raise EruAbortException(code.HTTP_BAD_REQUEST)

    return {'r': 0, 'msg': 'ok', 'data': app.list_resource_config()}
Example #2
0
def register_app_version():
    data = request.get_json()
    version = data['version']

    appyaml = data['appyaml']
    if isinstance(appyaml, basestring):
        try:
            appyaml = yaml.load(appyaml)
        except yaml.error.YAMLError:
            abort(400, 'Error in app.yaml')

    try:
        verify_appconfig(appyaml)
    except (ValueError, KeyError) as e:
        abort(400, e.message)

    name = appyaml['appname']

    app = App.get_or_create(name, data['git'])

    v = app.add_version(version)
    if not v:
        _log.error('Version create failed. (name=%s, version=%s)', name, version[:7])
        abort(400, 'Version %s create failed' % version[:7])

    appconfig = v.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    _log.info('App-Version created. (name=%s, version=%s)', name, version[:7])
    return 201, DEFAULT_RETURN_VALUE
Example #3
0
def create_local_test_data(private=False):
    appyaml = {
        'appname': 'blueberry',
        'entrypoints': {
            'web': {
                'cmd': 'python app.py',
                'ports': ['5000/tcp'],
            },
            'daemon': {
                'cmd': 'python daemon.py',
            },
            'service': {
                'cmd': 'python service.py'
            },
        },
        'build': 'pip install -r ./requirements.txt',
    }
    app = App.get_or_create('blueberry', 'http://git.hunantv.com/tonic/blueberry.git', 'token')
    version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
    appconfig = version.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    group = Group.create('group', 'group')
    pod = Pod.create('pod', 'pod')
    pod.assigned_to_group(group)

    c = docker.Client(**kwargs_from_env(assert_hostname=False))
    r = c.info()
    host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])

    if private:
        host.assigned_to_group(group)

    return app, version, group, pod, host
Example #4
0
def test_container_release_cores(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
    v = a.add_version(random_sha1())
    p = Pod.create('pod', 'pod', 10, -1)
    host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0)

    for core in host.cores:
        assert core.host_id == host.id
        assert core.remain == 10

    containers = []
    
    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        used_cores = {'full': fcores, 'part': pcores}
        host.occupy_cores(used_cores, 5)
        c = Container.create(random_sha1(), host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=5)
        containers.append(c)

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 0
        for core in pcores:
            assert core.remain == 5

    for c in containers:
        c.delete()

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 10
        for core in pcores:
            assert core.remain == 10
Example #5
0
def alloc_resource(name, env, res_name, res_alias):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(code.HTTP_NOT_FOUND)

    r = RESOURCES.get(res_name)
    if not r:
        raise EruAbortException(code.HTTP_NOT_FOUND, '%s doesn\'t exist' % res_name)

    envconfig = app.get_resource_config(env)
    if envconfig.get(res_alias):
        raise EruAbortException(code.HTTP_CONFLICT, '%s already in env' % res_alias)

    try:
        mod = import_string(r)
        args = inspect.getargspec(mod.alloc)
        data = request.get_json()
        if set(data.iterkeys()) >= set(args.args[1:]):
            raise Exception()
        result = mod.alloc(**data)
        envconfig[res_alias] = result
        envconfig.save()
    except Exception, e:
        current_app.logger.exception(e)
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Error in creating %s' % res_name)
Example #6
0
def get_app_env(name):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env list ignored" % name)

    envconfig = app.get_resource_config(request.args["env"])
    return {"r": 0, "msg": "ok", "data": envconfig.to_env_dict()}
Example #7
0
def register_app_version():
    data = request.get_json()
    version = data['version']

    appyaml = data['appyaml']
    try:
        verify_appconfig(appyaml)
    except (ValueError, KeyError) as e:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, e.message)

    name = appyaml['appname']

    app = App.get_or_create(name, data['git'], data['token'])
    if not app:
        current_app.logger.error('App create failed. (name=%s, version=%s)', name, version[:7])
        raise EruAbortException(consts.HTTP_BAD_REQUEST,
                'App %s create failed, maybe token duplicated' % name)

    v = app.add_version(version)
    if not v:
        current_app.logger.error('Version create failed. (name=%s, version=%s)', name, version[:7])
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])

    appconfig = v.appconfig
    appconfig.update(**appyaml)
    appconfig.save()
    current_app.logger.info('App-Version created. (name=%s, version=%s)', name, version[:7])
    return {'r': 0, 'msg': 'ok'}
Example #8
0
def register_app_version():
    data = request.get_json()
    version = data['version']

    appyaml = data['appyaml']
    if isinstance(appyaml, basestring):
        try:
            appyaml = yaml.load(appyaml)
        except yaml.error.YAMLError:
            abort(400, 'Error in app.yaml')

    try:
        verify_appconfig(appyaml)
    except (ValueError, KeyError) as e:
        abort(400, e.message)

    name = appyaml['appname']

    app = App.get_or_create(name, data['git'])

    v = app.add_version(version)
    if not v:
        _log.error('Version create failed. (name=%s, version=%s)', name,
                   version[:7])
        abort(400, 'Version %s create failed' % version[:7])

    appconfig = v.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    _log.info('App-Version created. (name=%s, version=%s)', name, version[:7])
    return 201, DEFAULT_RETURN_VALUE
Example #9
0
def get_app_env(name):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)

    envconfig = app.get_resource_config(request.args['env'])
    return {'r': 0, 'msg': 'ok', 'data': envconfig.to_env_dict()}
Example #10
0
def list_app_containers(name):
    app = App.get_by_name(name)
    if not app:
        logger.error('app not found, env list ignored')
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
    containers = app.containers.all()
    return {'r': 0, 'msg': 'ok', 'containers': containers}
Example #11
0
def create_local_test_data(private=False):
    appyaml = {
        'appname': 'blueberry',
        'entrypoints': {
            'web': {
                'cmd': 'python app.py',
                'ports': ['5000/tcp'],
            },
            'daemon': {
                'cmd': 'python daemon.py',
            },
            'service': {
                'cmd': 'python service.py'
            },
        },
        'build': 'pip install -r ./requirements.txt',
    }
    app = App.get_or_create('blueberry',
                            'http://git.hunantv.com/tonic/blueberry.git')
    version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
    appconfig = version.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    pod = Pod.create('pod', 'pod')

    c = docker.Client(**kwargs_from_env(assert_hostname=False))
    r = c.info()
    host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'],
                       r['NCPU'], r['MemTotal'])

    if not private:
        host.set_public()

    return app, version, pod, host
Example #12
0
def list_version_tasks(name, version):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
    v = app.get_version(version)
    if not v:
        raise EruAbortException(consts.HTTP_NOT_FOUND, 'Version %s not found' % version)
    return {'r': 0, 'msg': 'ok', 'tasks': v.list_tasks(g.start, g.limit)}
Example #13
0
def list_version_tasks(name, version):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env list ignored" % name)
    v = app.get_version(version)
    if not v:
        raise EruAbortException(consts.HTTP_NOT_FOUND, "Version %s not found" % version)
    return {"r": 0, "msg": "ok", "tasks": v.list_tasks(g.start, g.limit)}
Example #14
0
def list_version_containers(name, version):
    app = App.get_by_name(name)
    if not app:
        logger.error('app not found, env list ignored')
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
    v = app.get_version(version)
    if not v:
        raise EruAbortException(code.HTTP_NOT_FOUND, 'Version %s not found' % version)
    return {'r': 0, 'msg': 'ok', 'containers': v.list_containers(g.start, g.limit)}
Example #15
0
def _get_app_and_version(appname, version, **kwargs):
    app = App.get_by_name(appname)
    if not app:
        abort(400, 'App `%s` not found' % appname)

    version = app.get_version(version)
    if not version:
        abort(400, 'Version `%s` not found' % version)
    return app, version
Example #16
0
def get_version(name, version):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_NOT_FOUND, 'App %s not found' % name)

    v = app.get_version(version)
    if not v:
        raise EruAbortException(consts.HTTP_NOT_FOUND, 'Version %s not found' % version)
    return v
Example #17
0
def set_app_env(name):
    app = App.get_by_name(name)
    if not app:
        logger.error('app not found, env set ignored')
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env set ignored' % name)

    data = request.get_json()
    env = data.pop('env')
    envconfig = app.get_resource_config(env)
    envconfig.update(**data)
    envconfig.save()
    return {'r': 0, 'msg': 'ok'}
Example #18
0
def test_container_transform(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
    assert a is not None

    v = a.add_version(random_sha1())
    v2 = a.add_version(random_sha1())
    assert v is not None
    assert v.app.id == a.id
    assert v.name == a.name
    assert len(v.containers.all()) == 0
    assert len(v.tasks.all()) == 0

    g = Group.create('group', 'group')
    p = Pod.create('pod', 'pod')
    assert p.assigned_to_group(g)
    hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'),
        random_uuid(), 4, 4096) for i in range(6)]

    for host in hosts[:3]:
        host.assigned_to_group(g)

    assert g.get_max_containers(p, 3, 0) == 3
    host_cores = g.get_free_cores(p, 3, 3, 0)
    assert len(host_cores) == 3

    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]}
            c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
            assert c is not None
            containers.append(c)
        host.occupy_cores(cores, 0)

    for host in g.private_hosts.all():
        assert len(host.get_free_cores()[0]) == 1
        assert len(host.containers.all()) == 1
        assert host.count == 1

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    cids = [c.container_id for c in containers]
    for c in containers:
        host = c.host
        cid = c.container_id
        c.transform(v2, random_sha1(), random_string())
        assert c.container_id != cid

    new_cids = [c.container_id for c in containers]
    assert new_cids != cids
Example #19
0
def set_app_env(name):
    data = request.get_json()
    env = data.pop("env")

    app = App.get_by_name(name)
    if not app:
        current_app.logger.error("App (name=%s) not found, env (env=%s) set ignored.", name, env)
        raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env set ignored" % name)

    envconfig = app.get_resource_config(env)
    envconfig.update(**data)
    envconfig.save()
    current_app.logger.error("App (name=%s) set env (env=%s) values done", name, env)
    return {"r": 0, "msg": "ok"}
Example #20
0
def set_app_env(name):
    data = request.get_json()
    env = data.pop('env')

    app = App.get_by_name(name)
    if not app:
        current_app.logger.error('App (name=%s) not found, env (env=%s) set ignored.', name, env)
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env set ignored' % name)

    envconfig = app.get_resource_config(env)
    envconfig.update(**data)
    envconfig.save()
    current_app.logger.error('App (name=%s) set env (env=%s) values done', name, env)
    return {'r': 0, 'msg': 'ok'}
Example #21
0
def touch_version_scale_info(name, version):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(code.HTTP_NOT_FOUND, 'App %s not found' % name)
    v = app.get_version(version)
    if not v:
        raise EruAbortException(code.HTTP_NOT_FOUND, 'Version %s not found' % version)
    containers = v.containers.limit(1).all()
    if not containers:
        raise EruAbortException(code.HTTP_NOT_FOUND, 'Not deployed')
    container = containers[0]
    return {
        'group': container.host.group.name,
        'pod': container.host.pod.name,
        'ncore': len(container.cores.all())
    }
Example #22
0
def test_container_release_cores(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
    v = a.add_version(random_sha1())
    p = Pod.create('pod', 'pod', 10, -1)
    host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200,
                       0)

    for core in host.cores:
        assert core.host_id == host.id
        assert core.remain == 10

    containers = []

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10),
                              chunked(cores[100:], 10)):
        used_cores = {'full': fcores, 'part': pcores}
        host.occupy_cores(used_cores, 5)
        c = Container.create(random_sha1(),
                             host,
                             v,
                             random_string(),
                             'entrypoint',
                             used_cores,
                             'env',
                             nshare=5)
        containers.append(c)

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10),
                              chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 0
        for core in pcores:
            assert core.remain == 5

    for c in containers:
        c.delete()

    cores = sorted(host.cores, key=operator.attrgetter('label'))
    for fcores, pcores in zip(chunked(cores[:100], 10),
                              chunked(cores[100:], 10)):
        for core in fcores:
            assert core.remain == 10
        for core in pcores:
            assert core.remain == 10
Example #23
0
def create_test_suite():
    appyaml = {
        'appname': 'app',
        'entrypoints': {
            'web': {
                'cmd': 'python app.py',
                'ports': ['5000/tcp'],
            },
            'daemon': {
                'cmd': 'python daemon.py',
            },
            'service': {
                'cmd': 'python service.py'
            },
        },
        'build': 'pip install -r ./requirements.txt',
    }
    app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
    version = app.add_version(random_sha1())
    appconfig = version.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    pod = Pod.create('pod', 'pod')

    hosts = [
        Host.create(pod, random_ipv4(), random_string(prefix='host'),
                    random_uuid(), 4, 4096) for i in range(4)
    ]

    containers = []
    for (host, count), cores in centralized_schedule(pod, 4, 4, 0).iteritems():
        cores_per_container = len(cores) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {
                'full':
                cores['full'][i * cores_per_container:(i + 1) *
                              cores_per_container]
            }
            c = Container.create(cid, host, version, random_string(), 'web',
                                 used_cores, 'env')
            containers.append(c)
        host.occupy_cores(cores, 0)
    return app, version, pod, hosts, containers
Example #24
0
def validate_instance(group_name, pod_name, appname, version):
    group = Group.get_by_name(group_name)
    if not group:
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Group `%s` not found' % group_name)

    pod = Pod.get_by_name(pod_name)
    if not pod:
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Pod `%s` not found' % pod_name)

    application = App.get_by_name(appname)
    if not application:
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App `%s` not found' % appname)

    version = application.get_version(version)
    if not version:
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version `%s` not found' % version)

    return group, pod, application, version
Example #25
0
def validate_instance(group_name, pod_name, appname, version):
    group = Group.get_by_name(group_name)
    if not group:
        abort(400, 'Group `%s` not found' % group_name)

    pod = Pod.get_by_name(pod_name)
    if not pod:
        abort(400, 'Pod `%s` not found' % pod_name)

    app = App.get_by_name(appname)
    if not app:
        abort(400, 'App `%s` not found' % appname)

    version = app.get_version(version)
    if not version:
        abort(400, 'Version `%s` not found' % version)

    return group, pod, app, version
Example #26
0
def create_test_suite():
    appyaml = {
        'appname': 'app',
        'entrypoints': {
            'web': {
                'cmd': 'python app.py',
                'ports': ['5000/tcp'],
            },
            'daemon': {
                'cmd': 'python daemon.py',
            },
            'service': {
                'cmd': 'python service.py'
            },
        },
        'build': 'pip install -r ./requirements.txt',
    }
    app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', 'token')
    version = app.add_version(random_sha1())
    appconfig = version.appconfig
    appconfig.update(**appyaml)
    appconfig.save()

    group = Group.create('group', 'group')
    pod = Pod.create('pod', 'pod')
    pod.assigned_to_group(group)

    hosts = [Host.create(pod, random_ipv4(), random_string(prefix='host'),
        random_uuid(), 4, 4096) for i in range(4)]

    for host in hosts:
        host.assigned_to_group(group)

    containers = []
    for (host, count), cores in group.get_free_cores(pod, 4, 4, 0).iteritems():
        cores_per_container = len(cores) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = cores['full'][i*cores_per_container:(i+1)*cores_per_container]
            c = Container.create(cid, host, version, random_string(), 'entrypoint', used_cores, 'env')
            containers.append(c)
        host.occupy_cores(cores, 0)
    return app, version, group, pod, hosts, containers
Example #27
0
def register_app_version():
    data = request.get_json()
    name = data['name']
    version = data['version']

    app = App.get_or_create(name, data['git'], data['token'])
    if not app:
        logger.error('app create failed')
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s create failed' % name)

    v = app.add_version(version)
    if not v:
        logger.error('version create failed')
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])

    appconfig = v.appconfig
    appconfig.update(**data['appyaml'])
    appconfig.save()
    logger.info('app version successfully created')
    return {'r': 0, 'msg': 'ok'}
Example #28
0
def clean_app(app_name):
    app = App.get_by_name(app_name)
    if not app:
        print 'app %s not found' % app_name
        return

    containers = app.list_containers(limit=None)
    version_dict = {}
    for c in containers:
        if not c:
            continue
        version_dict.setdefault((c.version, c.host), []).append(c)
    for (version, host), cs in version_dict.iteritems():
        cids = [c.id for c in cs]
        task_props = {'container_ids': cids}
        task = Task.create(TASK_REMOVE, version, host, task_props)
        remove_containers.apply_async(args=(task.id, cids, False),
                                      task_id='task:%s' % task.id)
        print task
    print 'done, waiting...'
Example #29
0
def clean_app(app_name):
    app = App.get_by_name(app_name)
    if not app:
        print 'app %s not found' % app_name
        return

    containers = app.list_containers(limit=None)
    version_dict = {}
    for c in containers:
        if not c:
            continue
        version_dict.setdefault((c.version, c.host), []).append(c)
    for (version, host), cs in version_dict.iteritems():
        cids = [c.id for c in cs]
        task_props = {'container_ids': cids}
        task = Task.create(TASK_REMOVE, version, host, task_props)
        remove_containers.apply_async(
            args=(task.id, cids, False), task_id='task:%s' % task.id
        )
        print task
    print 'done, waiting...'
Example #30
0
def fillup_data():
    app, _ = create_app_with_celery()
    with app.app_context():
        db.drop_all()
        db.create_all()

        group = Group.create('test-group', 'test-group')
        pod = Pod.create('test-pod', 'test-pod')
        pod.assigned_to_group(group)

        r = requests.get('http://192.168.59.103:2375/info').json()
        host = Host.create(pod, '192.168.59.103:2375', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])
        host.assigned_to_group(group)

        app = App.get_or_create('nbetest', 'http://git.hunantv.com/platform/nbetest.git', 'token')
        app.add_version('96cbf8c68ed214f105d9f79fa4f22f0e80e75cf3')
        app.assigned_to_group(group)
        version = app.get_version('96cbf8')

        host_cores = group.get_free_cores(pod, 2, 2)
        cores = []
        for (host, cn), coress in host_cores.iteritems():
            print host, cn, coress
            cores = coress
        print cores
        ports = host.get_free_ports(2)
        print ports
        props = {
            'entrypoint': 'web',
            'ncontainer': 2,
            'env': 'PROD',
            'cores': [c.id for c in cores],
            'ports': [p.id for p in ports],
        }
        task = Task.create(1, version, host, props)
        print task.props
        host.occupy_cores(cores)
        host.occupy_ports(ports)
        print group.get_free_cores(pod, 1, 1)
Example #31
0
def register_app_version():
    data = request.get_json()
    name = data["name"]

    if data.get("raw", "") and "version" not in data["appyaml"]:
        version = consts.RAW_VERSION_PLACEHOLDER
    else:
        version = data["version"]

    app = App.get_or_create(name, data["git"], data["token"])
    if not app:
        current_app.logger.error("App create failed. (name=%s, version=%s)", name, version[:7])
        raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s create failed" % name)

    v = app.add_version(version)
    if not v:
        current_app.logger.error("Version create failed. (name=%s, version=%s)", name, version[:7])
        raise EruAbortException(consts.HTTP_BAD_REQUEST, "Version %s create failed" % version[:7])

    appconfig = v.appconfig
    appconfig.update(**data["appyaml"])
    appconfig.save()
    current_app.logger.info("App-Version created. (name=%s, version=%s)", name, version[:7])
    return {"r": 0, "msg": "ok"}
Example #32
0
def register_app_version():
    data = request.get_json()
    name = data['name']

    if data.get('raw', ''):
        version = code.RAW_VERSION_PLACEHOLDER
    else:
        version = data['version']

    app = App.get_or_create(name, data['git'], data['token'])
    if not app:
        current_app.logger.error('App create failed. (name=%s, version=%s)', name, version[:7])
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s create failed' % name)

    v = app.add_version(version)
    if not v:
        current_app.logger.error('Version create failed. (name=%s, version=%s)', name, version[:7])
        raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])

    appconfig = v.appconfig
    appconfig.update(**data['appyaml'])
    appconfig.save()
    current_app.logger.info('App-Version created. (name=%s, version=%s)', name, version[:7])
    return {'r': 0, 'msg': 'ok'}
Example #33
0
def _get_app_by_name(name):
    app = App.get_by_name(name)
    if not app:
        abort(404, 'App %s not found' % name)
    return app
Example #34
0
def test_container(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
    assert a is not None
    assert a.id == a.user_id

    v = a.add_version(random_sha1())
    assert v is not None
    assert v.app.id == a.id
    assert v.name == a.name
    assert len(v.containers.all()) == 0
    assert len(v.tasks.all()) == 0

    g = Group.create('group', 'group')
    p = Pod.create('pod', 'pod', 10, -1)
    assert p.assigned_to_group(g)
    hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'),
        random_uuid(), 4, 4096) for i in range(6)]

    for host in hosts[:3]:
        host.assigned_to_group(g)
    host_ids1 = {h.id for h in hosts[:3]}
    host_ids2 = {h.id for h in hosts[3:]}

    host_cores = g.get_free_cores(p, 3, 3, 0)

    #测试没有碎片核的情况
    #获取核
    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores['full']) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]}
            # not using a port
            c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
            assert c is not None
            containers.append(c)
        host.occupy_cores(cores, 0)

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 1
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 1
        assert host.count == 1

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores.all()) == 3
        assert len(c.part_cores.all()) == 0
        all_core_labels = sorted(['0', '1', '2', '3', ])
        used_full_core_labels = [core.label for core in c.full_cores.all()]
        used_part_core_labels = [core.label for core in c.part_cores.all()]
        free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
        assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels)

    #释放核
    for c in containers:
        c.delete()

    assert len(v.containers.all()) == 0
    assert g.get_max_containers(p, 3, 0) == 3
    host_cores = g.get_free_cores(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 0
        assert host.count == 0

    #测试有碎片的情况
    #获取核
    host_cores = g.get_free_cores(p, 3, 3, 4)
    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores['full']) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {'full':  cores['full'][i*cores_per_container:(i+1)*cores_per_container],
                    'part': cores['part']}
            # not using a port
            c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
            assert c is not None
            containers.append(c)
        host.occupy_cores(cores, 4)

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 0
        assert len(part_cores) == 1
        assert part_cores[0].used == 4
        assert len(host.containers.all()) == 1
        assert host.count == 1

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores.all()) == 3
        assert len(c.part_cores.all()) == 1
        all_core_labels = sorted(['0', '1', '2', '3', ])
        used_full_core_labels = [core.label for core in c.full_cores.all()]
        used_part_core_labels = [core.label for core in c.part_cores.all()]
        free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
        assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels)


    #释放核
    for c in containers:
        c.delete(4)

    assert len(v.containers.all()) == 0
    assert g.get_max_containers(p, 3, 0) == 3
    host_cores = g.get_free_cores(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(host.containers.all()) == 0
        assert host.count == 0

    #获取
    host_cores = g.get_free_cores(p, 6, 1, 5)
    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores['full']) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {'full':  cores['full'][i*cores_per_container:(i+1)*cores_per_container],
                    'part': cores['part'][i:i+1]}
            # not using a port
            c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
            assert c is not None
            containers.append(c)
            host.occupy_cores(used_cores, 5)

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 1
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 2
        assert host.count == 2

    assert len(containers) == 6
    assert len(v.containers.all()) == 6

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores.all()) == 1
        assert len(c.part_cores.all()) == 1

    ##释放核
    for c in containers:
        c.delete(5)

    assert len(v.containers.all()) == 0
    assert g.get_max_containers(p, 3, 0) == 3
    host_cores = g.get_free_cores(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in g.private_hosts.all():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 0
        assert host.count == 0
#!/usr/bin/env python
# encoding: utf-8

from eru.app import create_app_with_celery
from eru.models import db, Group, Pod, Host, App
from tests.utils import random_ipv4, random_string, random_uuid, random_sha1

host_number = int(raw_input("how many hosts: "))

app, _ = create_app_with_celery
app.config['TESTING'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:@localhost:3306/erutest'

with app.app_context():
    db.create_all()
    
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
    v = a.add_version(random_sha1())
    g = Group.create('group', 'group')
    p = Pod.create('pod', 'pod', 10, -1)
    p.assigned_to_group(g)
    hosts = [Host.create(p, random_ipv4(), random_string(), random_uuid(), 24, 4096) for i in range(host_number)]
    
    for host in hosts:
        host.assigned_to_group(g)
Example #36
0
def list_app_env(name):
    app = App.get_by_name(name)
    if not app:
        raise EruAbortException(consts.HTTP_BAD_REQUEST)
    return {'r': 0, 'msg': 'ok', 'data': app.list_resource_config()}
Example #37
0
def list_all_apps():
    return App.list_all(g.start, g.limit)
Example #38
0
def test_container(test_db):
    a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
    assert a is not None
    assert a.id == a.user_id

    v = a.add_version(random_sha1())
    assert v is not None
    assert v.app.id == a.id
    assert v.name == a.name
    assert len(v.containers.all()) == 0
    assert len(v.tasks.all()) == 0

    p = Pod.create('pod', 'pod', 10, -1)
    hosts = [
        Host.create(p, random_ipv4(), random_string(prefix='host'),
                    random_uuid(), 4, 4096) for i in range(6)
    ]

    for host in hosts[3:]:
        host.set_public()
    host_ids1 = {h.id for h in hosts[:3]}
    host_ids2 = {h.id for h in hosts[3:]}

    host_cores = centralized_schedule(p, 3, 3, 0)

    #测试没有碎片核的情况
    #获取核
    containers = []
    for (host, count), cores in host_cores.iteritems():
        host.occupy_cores(cores, 0)
        cores_per_container = len(cores['full']) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {
                'full':
                cores['full'][i * cores_per_container:(i + 1) *
                              cores_per_container]
            }
            c = Container.create(cid,
                                 host,
                                 v,
                                 random_string(),
                                 'entrypoint',
                                 used_cores,
                                 'env',
                                 nshare=0)
            assert c is not None
            containers.append(c)

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 1
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 1
        assert host.count == 1

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores) == 3
        assert len(c.part_cores) == 0
        all_core_labels = sorted([
            '0',
            '1',
            '2',
            '3',
        ])
        used_full_core_labels = [core.label for core in c.full_cores]
        used_part_core_labels = [core.label for core in c.part_cores]
        free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
        assert all_core_labels == sorted(used_full_core_labels +
                                         used_part_core_labels +
                                         free_core_labels)

    #释放核
    for c in containers:
        c.delete()

    assert len(v.containers.all()) == 0
    assert get_max_container_count(p, 3, 0) == 3
    host_cores = centralized_schedule(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 0
        assert host.count == 4

    #测试有碎片的情况
    #获取核
    host_cores = centralized_schedule(p, 3, 3, 4)
    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores['full']) / count
        host.occupy_cores(cores, 4)
        for i in range(count):
            cid = random_sha1()
            used_cores = {
                'full':
                cores['full'][i * cores_per_container:(i + 1) *
                              cores_per_container],
                'part':
                cores['part']
            }
            # not using a port
            c = Container.create(cid,
                                 host,
                                 v,
                                 random_string(),
                                 'entrypoint',
                                 used_cores,
                                 'env',
                                 nshare=4)
            assert c is not None
            containers.append(c)

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 0
        assert len(part_cores) == 1
        assert part_cores[0].remain == 6
        assert len(host.containers.all()) == 1
        assert host.count == D('0.6')

    assert len(containers) == 3
    assert len(v.containers.all()) == 3

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores) == 3
        assert len(c.part_cores) == 1
        all_core_labels = sorted([
            '0',
            '1',
            '2',
            '3',
        ])
        used_full_core_labels = [core.label for core in c.full_cores]
        used_part_core_labels = [core.label for core in c.part_cores]
        free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
        assert all_core_labels == sorted(used_full_core_labels +
                                         used_part_core_labels +
                                         free_core_labels)

    #释放核
    for c in containers:
        c.delete()

    assert len(v.containers.all()) == 0
    assert get_max_container_count(p, 3, 0) == 3
    host_cores = centralized_schedule(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(host.containers.all()) == 0
        assert host.count == 4

    #获取
    host_cores = centralized_schedule(p, 6, 1, 5)
    containers = []
    for (host, count), cores in host_cores.iteritems():
        cores_per_container = len(cores['full']) / count
        for i in range(count):
            cid = random_sha1()
            used_cores = {
                'full':
                cores['full'][i * cores_per_container:(i + 1) *
                              cores_per_container],
                'part':
                cores['part'][i:i + 1],
            }
            host.occupy_cores(used_cores, 5)
            # not using a port
            c = Container.create(cid,
                                 host,
                                 v,
                                 random_string(),
                                 'entrypoint',
                                 used_cores,
                                 'env',
                                 nshare=5)
            assert c is not None
            containers.append(c)

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 1
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 2
        assert host.count == D('1')

    assert len(containers) == 6
    assert len(v.containers.all()) == 6

    for c in containers:
        assert c.host_id in host_ids1
        assert c.host_id not in host_ids2
        assert c.app.id == a.id
        assert c.version.id == v.id
        assert c.is_alive
        assert len(c.full_cores) == 1
        assert len(c.part_cores) == 1

    ##释放核
    for c in containers:
        c.delete()

    assert len(v.containers.all()) == 0
    assert get_max_container_count(p, 3, 0) == 3
    host_cores = centralized_schedule(p, 3, 3, 0)
    assert len(host_cores) == 3

    for host in p.get_private_hosts():
        full_cores, part_cores = host.get_free_cores()
        assert len(full_cores) == 4
        assert len(part_cores) == 0
        assert len(host.containers.all()) == 0
        assert host.count == 4