def list_app_env(name):
app = App.get_by_name(name)
if not app:
logger.error('app not found, env set ignored')
raise EruAbortException(code.HTTP_BAD_REQUEST)
return {'r': 0, 'msg': 'ok', 'data': app.list_resource_config()}
def register_app_version():
data = request.get_json()
version = data['version']
appyaml = data['appyaml']
if isinstance(appyaml, basestring):
try:
appyaml = yaml.load(appyaml)
except yaml.error.YAMLError:
abort(400, 'Error in app.yaml')
try:
verify_appconfig(appyaml)
except (ValueError, KeyError) as e:
abort(400, e.message)
name = appyaml['appname']
app = App.get_or_create(name, data['git'])
v = app.add_version(version)
if not v:
_log.error('Version create failed. (name=%s, version=%s)', name, version[:7])
abort(400, 'Version %s create failed' % version[:7])
appconfig = v.appconfig
appconfig.update(**appyaml)
appconfig.save()
_log.info('App-Version created. (name=%s, version=%s)', name, version[:7])
return 201, DEFAULT_RETURN_VALUE
def create_local_test_data(private=False):
appyaml = {
'appname': 'blueberry',
'entrypoints': {
'web': {
'cmd': 'python app.py',
'ports': ['5000/tcp'],
},
'daemon': {
'cmd': 'python daemon.py',
},
'service': {
'cmd': 'python service.py'
},
},
'build': 'pip install -r ./requirements.txt',
}
app = App.get_or_create('blueberry', 'http://git.hunantv.com/tonic/blueberry.git', 'token')
version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
appconfig = version.appconfig
appconfig.update(**appyaml)
appconfig.save()
group = Group.create('group', 'group')
pod = Pod.create('pod', 'pod')
pod.assigned_to_group(group)
c = docker.Client(**kwargs_from_env(assert_hostname=False))
r = c.info()
host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])
if private:
host.assigned_to_group(group)
return app, version, group, pod, host
def test_container_release_cores(test_db):
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
v = a.add_version(random_sha1())
p = Pod.create('pod', 'pod', 10, -1)
host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0)
for core in host.cores:
assert core.host_id == host.id
assert core.remain == 10
containers = []
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
used_cores = {'full': fcores, 'part': pcores}
host.occupy_cores(used_cores, 5)
c = Container.create(random_sha1(), host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=5)
containers.append(c)
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
for core in fcores:
assert core.remain == 0
for core in pcores:
assert core.remain == 5
for c in containers:
c.delete()
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)):
for core in fcores:
assert core.remain == 10
for core in pcores:
assert core.remain == 10
def alloc_resource(name, env, res_name, res_alias):
app = App.get_by_name(name)
if not app:
raise EruAbortException(code.HTTP_NOT_FOUND)
r = RESOURCES.get(res_name)
if not r:
raise EruAbortException(code.HTTP_NOT_FOUND, '%s doesn\'t exist' % res_name)
envconfig = app.get_resource_config(env)
if envconfig.get(res_alias):
raise EruAbortException(code.HTTP_CONFLICT, '%s already in env' % res_alias)
try:
mod = import_string(r)
args = inspect.getargspec(mod.alloc)
data = request.get_json()
if set(data.iterkeys()) >= set(args.args[1:]):
raise Exception()
result = mod.alloc(**data)
envconfig[res_alias] = result
envconfig.save()
except Exception, e:
current_app.logger.exception(e)
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Error in creating %s' % res_name)
def get_app_env(name):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env list ignored" % name)
envconfig = app.get_resource_config(request.args["env"])
return {"r": 0, "msg": "ok", "data": envconfig.to_env_dict()}
def register_app_version():
data = request.get_json()
version = data['version']
appyaml = data['appyaml']
try:
verify_appconfig(appyaml)
except (ValueError, KeyError) as e:
raise EruAbortException(consts.HTTP_BAD_REQUEST, e.message)
name = appyaml['appname']
app = App.get_or_create(name, data['git'], data['token'])
if not app:
current_app.logger.error('App create failed. (name=%s, version=%s)', name, version[:7])
raise EruAbortException(consts.HTTP_BAD_REQUEST,
'App %s create failed, maybe token duplicated' % name)
v = app.add_version(version)
if not v:
current_app.logger.error('Version create failed. (name=%s, version=%s)', name, version[:7])
raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])
appconfig = v.appconfig
appconfig.update(**appyaml)
appconfig.save()
current_app.logger.info('App-Version created. (name=%s, version=%s)', name, version[:7])
return {'r': 0, 'msg': 'ok'}
def register_app_version():
data = request.get_json()
version = data['version']
appyaml = data['appyaml']
if isinstance(appyaml, basestring):
try:
appyaml = yaml.load(appyaml)
except yaml.error.YAMLError:
abort(400, 'Error in app.yaml')
try:
verify_appconfig(appyaml)
except (ValueError, KeyError) as e:
abort(400, e.message)
name = appyaml['appname']
app = App.get_or_create(name, data['git'])
v = app.add_version(version)
if not v:
_log.error('Version create failed. (name=%s, version=%s)', name,
version[:7])
abort(400, 'Version %s create failed' % version[:7])
appconfig = v.appconfig
appconfig.update(**appyaml)
appconfig.save()
_log.info('App-Version created. (name=%s, version=%s)', name, version[:7])
return 201, DEFAULT_RETURN_VALUE
def get_app_env(name):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
envconfig = app.get_resource_config(request.args['env'])
return {'r': 0, 'msg': 'ok', 'data': envconfig.to_env_dict()}
def list_app_containers(name):
app = App.get_by_name(name)
if not app:
logger.error('app not found, env list ignored')
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
containers = app.containers.all()
return {'r': 0, 'msg': 'ok', 'containers': containers}
def create_local_test_data(private=False):
appyaml = {
'appname': 'blueberry',
'entrypoints': {
'web': {
'cmd': 'python app.py',
'ports': ['5000/tcp'],
},
'daemon': {
'cmd': 'python daemon.py',
},
'service': {
'cmd': 'python service.py'
},
},
'build': 'pip install -r ./requirements.txt',
}
app = App.get_or_create('blueberry',
'http://git.hunantv.com/tonic/blueberry.git')
version = app.add_version('abe23812aeb50a17a2509c02a28423462161d306')
appconfig = version.appconfig
appconfig.update(**appyaml)
appconfig.save()
pod = Pod.create('pod', 'pod')
c = docker.Client(**kwargs_from_env(assert_hostname=False))
r = c.info()
host = Host.create(pod, '192.168.59.103:2376', r['Name'], r['ID'],
r['NCPU'], r['MemTotal'])
if not private:
host.set_public()
return app, version, pod, host
def list_version_tasks(name, version):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
v = app.get_version(version)
if not v:
raise EruAbortException(consts.HTTP_NOT_FOUND, 'Version %s not found' % version)
return {'r': 0, 'msg': 'ok', 'tasks': v.list_tasks(g.start, g.limit)}
def list_version_tasks(name, version):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env list ignored" % name)
v = app.get_version(version)
if not v:
raise EruAbortException(consts.HTTP_NOT_FOUND, "Version %s not found" % version)
return {"r": 0, "msg": "ok", "tasks": v.list_tasks(g.start, g.limit)}
def list_version_containers(name, version):
app = App.get_by_name(name)
if not app:
logger.error('app not found, env list ignored')
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env list ignored' % name)
v = app.get_version(version)
if not v:
raise EruAbortException(code.HTTP_NOT_FOUND, 'Version %s not found' % version)
return {'r': 0, 'msg': 'ok', 'containers': v.list_containers(g.start, g.limit)}
def _get_app_and_version(appname, version, **kwargs):
app = App.get_by_name(appname)
if not app:
abort(400, 'App `%s` not found' % appname)
version = app.get_version(version)
if not version:
abort(400, 'Version `%s` not found' % version)
return app, version
def get_version(name, version):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_NOT_FOUND, 'App %s not found' % name)
v = app.get_version(version)
if not v:
raise EruAbortException(consts.HTTP_NOT_FOUND, 'Version %s not found' % version)
return v
def set_app_env(name):
app = App.get_by_name(name)
if not app:
logger.error('app not found, env set ignored')
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s not found, env set ignored' % name)
data = request.get_json()
env = data.pop('env')
envconfig = app.get_resource_config(env)
envconfig.update(**data)
envconfig.save()
return {'r': 0, 'msg': 'ok'}
def test_container_transform(test_db):
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
assert a is not None
v = a.add_version(random_sha1())
v2 = a.add_version(random_sha1())
assert v is not None
assert v.app.id == a.id
assert v.name == a.name
assert len(v.containers.all()) == 0
assert len(v.tasks.all()) == 0
g = Group.create('group', 'group')
p = Pod.create('pod', 'pod')
assert p.assigned_to_group(g)
hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'),
random_uuid(), 4, 4096) for i in range(6)]
for host in hosts[:3]:
host.assigned_to_group(g)
assert g.get_max_containers(p, 3, 0) == 3
host_cores = g.get_free_cores(p, 3, 3, 0)
assert len(host_cores) == 3
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores) / count
for i in range(count):
cid = random_sha1()
used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]}
c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
assert c is not None
containers.append(c)
host.occupy_cores(cores, 0)
for host in g.private_hosts.all():
assert len(host.get_free_cores()[0]) == 1
assert len(host.containers.all()) == 1
assert host.count == 1
assert len(containers) == 3
assert len(v.containers.all()) == 3
cids = [c.container_id for c in containers]
for c in containers:
host = c.host
cid = c.container_id
c.transform(v2, random_sha1(), random_string())
assert c.container_id != cid
new_cids = [c.container_id for c in containers]
assert new_cids != cids
def set_app_env(name):
data = request.get_json()
env = data.pop("env")
app = App.get_by_name(name)
if not app:
current_app.logger.error("App (name=%s) not found, env (env=%s) set ignored.", name, env)
raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s not found, env set ignored" % name)
envconfig = app.get_resource_config(env)
envconfig.update(**data)
envconfig.save()
current_app.logger.error("App (name=%s) set env (env=%s) values done", name, env)
return {"r": 0, "msg": "ok"}
def set_app_env(name):
data = request.get_json()
env = data.pop('env')
app = App.get_by_name(name)
if not app:
current_app.logger.error('App (name=%s) not found, env (env=%s) set ignored.', name, env)
raise EruAbortException(consts.HTTP_BAD_REQUEST, 'App %s not found, env set ignored' % name)
envconfig = app.get_resource_config(env)
envconfig.update(**data)
envconfig.save()
current_app.logger.error('App (name=%s) set env (env=%s) values done', name, env)
return {'r': 0, 'msg': 'ok'}
def touch_version_scale_info(name, version):
app = App.get_by_name(name)
if not app:
raise EruAbortException(code.HTTP_NOT_FOUND, 'App %s not found' % name)
v = app.get_version(version)
if not v:
raise EruAbortException(code.HTTP_NOT_FOUND, 'Version %s not found' % version)
containers = v.containers.limit(1).all()
if not containers:
raise EruAbortException(code.HTTP_NOT_FOUND, 'Not deployed')
container = containers[0]
return {
'group': container.host.group.name,
'pod': container.host.pod.name,
'ncore': len(container.cores.all())
}
def test_container_release_cores(test_db):
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
v = a.add_version(random_sha1())
p = Pod.create('pod', 'pod', 10, -1)
host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200,
0)
for core in host.cores:
assert core.host_id == host.id
assert core.remain == 10
containers = []
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10),
chunked(cores[100:], 10)):
used_cores = {'full': fcores, 'part': pcores}
host.occupy_cores(used_cores, 5)
c = Container.create(random_sha1(),
host,
v,
random_string(),
'entrypoint',
used_cores,
'env',
nshare=5)
containers.append(c)
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10),
chunked(cores[100:], 10)):
for core in fcores:
assert core.remain == 0
for core in pcores:
assert core.remain == 5
for c in containers:
c.delete()
cores = sorted(host.cores, key=operator.attrgetter('label'))
for fcores, pcores in zip(chunked(cores[:100], 10),
chunked(cores[100:], 10)):
for core in fcores:
assert core.remain == 10
for core in pcores:
assert core.remain == 10
def create_test_suite():
appyaml = {
'appname': 'app',
'entrypoints': {
'web': {
'cmd': 'python app.py',
'ports': ['5000/tcp'],
},
'daemon': {
'cmd': 'python daemon.py',
},
'service': {
'cmd': 'python service.py'
},
},
'build': 'pip install -r ./requirements.txt',
}
app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
version = app.add_version(random_sha1())
appconfig = version.appconfig
appconfig.update(**appyaml)
appconfig.save()
pod = Pod.create('pod', 'pod')
hosts = [
Host.create(pod, random_ipv4(), random_string(prefix='host'),
random_uuid(), 4, 4096) for i in range(4)
]
containers = []
for (host, count), cores in centralized_schedule(pod, 4, 4, 0).iteritems():
cores_per_container = len(cores) / count
for i in range(count):
cid = random_sha1()
used_cores = {
'full':
cores['full'][i * cores_per_container:(i + 1) *
cores_per_container]
}
c = Container.create(cid, host, version, random_string(), 'web',
used_cores, 'env')
containers.append(c)
host.occupy_cores(cores, 0)
return app, version, pod, hosts, containers
def validate_instance(group_name, pod_name, appname, version):
group = Group.get_by_name(group_name)
if not group:
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Group `%s` not found' % group_name)
pod = Pod.get_by_name(pod_name)
if not pod:
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Pod `%s` not found' % pod_name)
application = App.get_by_name(appname)
if not application:
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App `%s` not found' % appname)
version = application.get_version(version)
if not version:
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version `%s` not found' % version)
return group, pod, application, version
def validate_instance(group_name, pod_name, appname, version):
group = Group.get_by_name(group_name)
if not group:
abort(400, 'Group `%s` not found' % group_name)
pod = Pod.get_by_name(pod_name)
if not pod:
abort(400, 'Pod `%s` not found' % pod_name)
app = App.get_by_name(appname)
if not app:
abort(400, 'App `%s` not found' % appname)
version = app.get_version(version)
if not version:
abort(400, 'Version `%s` not found' % version)
return group, pod, app, version
def create_test_suite():
appyaml = {
'appname': 'app',
'entrypoints': {
'web': {
'cmd': 'python app.py',
'ports': ['5000/tcp'],
},
'daemon': {
'cmd': 'python daemon.py',
},
'service': {
'cmd': 'python service.py'
},
},
'build': 'pip install -r ./requirements.txt',
}
app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', 'token')
version = app.add_version(random_sha1())
appconfig = version.appconfig
appconfig.update(**appyaml)
appconfig.save()
group = Group.create('group', 'group')
pod = Pod.create('pod', 'pod')
pod.assigned_to_group(group)
hosts = [Host.create(pod, random_ipv4(), random_string(prefix='host'),
random_uuid(), 4, 4096) for i in range(4)]
for host in hosts:
host.assigned_to_group(group)
containers = []
for (host, count), cores in group.get_free_cores(pod, 4, 4, 0).iteritems():
cores_per_container = len(cores) / count
for i in range(count):
cid = random_sha1()
used_cores = cores['full'][i*cores_per_container:(i+1)*cores_per_container]
c = Container.create(cid, host, version, random_string(), 'entrypoint', used_cores, 'env')
containers.append(c)
host.occupy_cores(cores, 0)
return app, version, group, pod, hosts, containers
def register_app_version():
data = request.get_json()
name = data['name']
version = data['version']
app = App.get_or_create(name, data['git'], data['token'])
if not app:
logger.error('app create failed')
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s create failed' % name)
v = app.add_version(version)
if not v:
logger.error('version create failed')
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])
appconfig = v.appconfig
appconfig.update(**data['appyaml'])
appconfig.save()
logger.info('app version successfully created')
return {'r': 0, 'msg': 'ok'}
def clean_app(app_name):
app = App.get_by_name(app_name)
if not app:
print 'app %s not found' % app_name
return
containers = app.list_containers(limit=None)
version_dict = {}
for c in containers:
if not c:
continue
version_dict.setdefault((c.version, c.host), []).append(c)
for (version, host), cs in version_dict.iteritems():
cids = [c.id for c in cs]
task_props = {'container_ids': cids}
task = Task.create(TASK_REMOVE, version, host, task_props)
remove_containers.apply_async(args=(task.id, cids, False),
task_id='task:%s' % task.id)
print task
print 'done, waiting...'
def clean_app(app_name):
app = App.get_by_name(app_name)
if not app:
print 'app %s not found' % app_name
return
containers = app.list_containers(limit=None)
version_dict = {}
for c in containers:
if not c:
continue
version_dict.setdefault((c.version, c.host), []).append(c)
for (version, host), cs in version_dict.iteritems():
cids = [c.id for c in cs]
task_props = {'container_ids': cids}
task = Task.create(TASK_REMOVE, version, host, task_props)
remove_containers.apply_async(
args=(task.id, cids, False), task_id='task:%s' % task.id
)
print task
print 'done, waiting...'
def fillup_data():
app, _ = create_app_with_celery()
with app.app_context():
db.drop_all()
db.create_all()
group = Group.create('test-group', 'test-group')
pod = Pod.create('test-pod', 'test-pod')
pod.assigned_to_group(group)
r = requests.get('http://192.168.59.103:2375/info').json()
host = Host.create(pod, '192.168.59.103:2375', r['Name'], r['ID'], r['NCPU'], r['MemTotal'])
host.assigned_to_group(group)
app = App.get_or_create('nbetest', 'http://git.hunantv.com/platform/nbetest.git', 'token')
app.add_version('96cbf8c68ed214f105d9f79fa4f22f0e80e75cf3')
app.assigned_to_group(group)
version = app.get_version('96cbf8')
host_cores = group.get_free_cores(pod, 2, 2)
cores = []
for (host, cn), coress in host_cores.iteritems():
print host, cn, coress
cores = coress
print cores
ports = host.get_free_ports(2)
print ports
props = {
'entrypoint': 'web',
'ncontainer': 2,
'env': 'PROD',
'cores': [c.id for c in cores],
'ports': [p.id for p in ports],
}
task = Task.create(1, version, host, props)
print task.props
host.occupy_cores(cores)
host.occupy_ports(ports)
print group.get_free_cores(pod, 1, 1)
def register_app_version():
data = request.get_json()
name = data["name"]
if data.get("raw", "") and "version" not in data["appyaml"]:
version = consts.RAW_VERSION_PLACEHOLDER
else:
version = data["version"]
app = App.get_or_create(name, data["git"], data["token"])
if not app:
current_app.logger.error("App create failed. (name=%s, version=%s)", name, version[:7])
raise EruAbortException(consts.HTTP_BAD_REQUEST, "App %s create failed" % name)
v = app.add_version(version)
if not v:
current_app.logger.error("Version create failed. (name=%s, version=%s)", name, version[:7])
raise EruAbortException(consts.HTTP_BAD_REQUEST, "Version %s create failed" % version[:7])
appconfig = v.appconfig
appconfig.update(**data["appyaml"])
appconfig.save()
current_app.logger.info("App-Version created. (name=%s, version=%s)", name, version[:7])
return {"r": 0, "msg": "ok"}
def register_app_version():
data = request.get_json()
name = data['name']
if data.get('raw', ''):
version = code.RAW_VERSION_PLACEHOLDER
else:
version = data['version']
app = App.get_or_create(name, data['git'], data['token'])
if not app:
current_app.logger.error('App create failed. (name=%s, version=%s)', name, version[:7])
raise EruAbortException(code.HTTP_BAD_REQUEST, 'App %s create failed' % name)
v = app.add_version(version)
if not v:
current_app.logger.error('Version create failed. (name=%s, version=%s)', name, version[:7])
raise EruAbortException(code.HTTP_BAD_REQUEST, 'Version %s create failed' % version[:7])
appconfig = v.appconfig
appconfig.update(**data['appyaml'])
appconfig.save()
current_app.logger.info('App-Version created. (name=%s, version=%s)', name, version[:7])
return {'r': 0, 'msg': 'ok'}
def _get_app_by_name(name):
app = App.get_by_name(name)
if not app:
abort(404, 'App %s not found' % name)
return app
def test_container(test_db):
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
assert a is not None
assert a.id == a.user_id
v = a.add_version(random_sha1())
assert v is not None
assert v.app.id == a.id
assert v.name == a.name
assert len(v.containers.all()) == 0
assert len(v.tasks.all()) == 0
g = Group.create('group', 'group')
p = Pod.create('pod', 'pod', 10, -1)
assert p.assigned_to_group(g)
hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'),
random_uuid(), 4, 4096) for i in range(6)]
for host in hosts[:3]:
host.assigned_to_group(g)
host_ids1 = {h.id for h in hosts[:3]}
host_ids2 = {h.id for h in hosts[3:]}
host_cores = g.get_free_cores(p, 3, 3, 0)
#测试没有碎片核的情况
#获取核
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores['full']) / count
for i in range(count):
cid = random_sha1()
used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]}
# not using a port
c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
assert c is not None
containers.append(c)
host.occupy_cores(cores, 0)
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 1
assert len(part_cores) == 0
assert len(host.containers.all()) == 1
assert host.count == 1
assert len(containers) == 3
assert len(v.containers.all()) == 3
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores.all()) == 3
assert len(c.part_cores.all()) == 0
all_core_labels = sorted(['0', '1', '2', '3', ])
used_full_core_labels = [core.label for core in c.full_cores.all()]
used_part_core_labels = [core.label for core in c.part_cores.all()]
free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels)
#释放核
for c in containers:
c.delete()
assert len(v.containers.all()) == 0
assert g.get_max_containers(p, 3, 0) == 3
host_cores = g.get_free_cores(p, 3, 3, 0)
assert len(host_cores) == 3
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(part_cores) == 0
assert len(host.containers.all()) == 0
assert host.count == 0
#测试有碎片的情况
#获取核
host_cores = g.get_free_cores(p, 3, 3, 4)
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores['full']) / count
for i in range(count):
cid = random_sha1()
used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container],
'part': cores['part']}
# not using a port
c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
assert c is not None
containers.append(c)
host.occupy_cores(cores, 4)
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 0
assert len(part_cores) == 1
assert part_cores[0].used == 4
assert len(host.containers.all()) == 1
assert host.count == 1
assert len(containers) == 3
assert len(v.containers.all()) == 3
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores.all()) == 3
assert len(c.part_cores.all()) == 1
all_core_labels = sorted(['0', '1', '2', '3', ])
used_full_core_labels = [core.label for core in c.full_cores.all()]
used_part_core_labels = [core.label for core in c.part_cores.all()]
free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels)
#释放核
for c in containers:
c.delete(4)
assert len(v.containers.all()) == 0
assert g.get_max_containers(p, 3, 0) == 3
host_cores = g.get_free_cores(p, 3, 3, 0)
assert len(host_cores) == 3
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(host.containers.all()) == 0
assert host.count == 0
#获取
host_cores = g.get_free_cores(p, 6, 1, 5)
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores['full']) / count
for i in range(count):
cid = random_sha1()
used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container],
'part': cores['part'][i:i+1]}
# not using a port
c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env')
assert c is not None
containers.append(c)
host.occupy_cores(used_cores, 5)
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 1
assert len(part_cores) == 0
assert len(host.containers.all()) == 2
assert host.count == 2
assert len(containers) == 6
assert len(v.containers.all()) == 6
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores.all()) == 1
assert len(c.part_cores.all()) == 1
##释放核
for c in containers:
c.delete(5)
assert len(v.containers.all()) == 0
assert g.get_max_containers(p, 3, 0) == 3
host_cores = g.get_free_cores(p, 3, 3, 0)
assert len(host_cores) == 3
for host in g.private_hosts.all():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(part_cores) == 0
assert len(host.containers.all()) == 0
assert host.count == 0
#!/usr/bin/env python
# encoding: utf-8
from eru.app import create_app_with_celery
from eru.models import db, Group, Pod, Host, App
from tests.utils import random_ipv4, random_string, random_uuid, random_sha1
host_number = int(raw_input("how many hosts: "))
app, _ = create_app_with_celery
app.config['TESTING'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:@localhost:3306/erutest'
with app.app_context():
db.create_all()
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '')
v = a.add_version(random_sha1())
g = Group.create('group', 'group')
p = Pod.create('pod', 'pod', 10, -1)
p.assigned_to_group(g)
hosts = [Host.create(p, random_ipv4(), random_string(), random_uuid(), 24, 4096) for i in range(host_number)]
for host in hosts:
host.assigned_to_group(g)
def list_app_env(name):
app = App.get_by_name(name)
if not app:
raise EruAbortException(consts.HTTP_BAD_REQUEST)
return {'r': 0, 'msg': 'ok', 'data': app.list_resource_config()}
def list_all_apps():
return App.list_all(g.start, g.limit)
def test_container(test_db):
a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git')
assert a is not None
assert a.id == a.user_id
v = a.add_version(random_sha1())
assert v is not None
assert v.app.id == a.id
assert v.name == a.name
assert len(v.containers.all()) == 0
assert len(v.tasks.all()) == 0
p = Pod.create('pod', 'pod', 10, -1)
hosts = [
Host.create(p, random_ipv4(), random_string(prefix='host'),
random_uuid(), 4, 4096) for i in range(6)
]
for host in hosts[3:]:
host.set_public()
host_ids1 = {h.id for h in hosts[:3]}
host_ids2 = {h.id for h in hosts[3:]}
host_cores = centralized_schedule(p, 3, 3, 0)
#测试没有碎片核的情况
#获取核
containers = []
for (host, count), cores in host_cores.iteritems():
host.occupy_cores(cores, 0)
cores_per_container = len(cores['full']) / count
for i in range(count):
cid = random_sha1()
used_cores = {
'full':
cores['full'][i * cores_per_container:(i + 1) *
cores_per_container]
}
c = Container.create(cid,
host,
v,
random_string(),
'entrypoint',
used_cores,
'env',
nshare=0)
assert c is not None
containers.append(c)
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 1
assert len(part_cores) == 0
assert len(host.containers.all()) == 1
assert host.count == 1
assert len(containers) == 3
assert len(v.containers.all()) == 3
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores) == 3
assert len(c.part_cores) == 0
all_core_labels = sorted([
'0',
'1',
'2',
'3',
])
used_full_core_labels = [core.label for core in c.full_cores]
used_part_core_labels = [core.label for core in c.part_cores]
free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
assert all_core_labels == sorted(used_full_core_labels +
used_part_core_labels +
free_core_labels)
#释放核
for c in containers:
c.delete()
assert len(v.containers.all()) == 0
assert get_max_container_count(p, 3, 0) == 3
host_cores = centralized_schedule(p, 3, 3, 0)
assert len(host_cores) == 3
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(part_cores) == 0
assert len(host.containers.all()) == 0
assert host.count == 4
#测试有碎片的情况
#获取核
host_cores = centralized_schedule(p, 3, 3, 4)
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores['full']) / count
host.occupy_cores(cores, 4)
for i in range(count):
cid = random_sha1()
used_cores = {
'full':
cores['full'][i * cores_per_container:(i + 1) *
cores_per_container],
'part':
cores['part']
}
# not using a port
c = Container.create(cid,
host,
v,
random_string(),
'entrypoint',
used_cores,
'env',
nshare=4)
assert c is not None
containers.append(c)
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 0
assert len(part_cores) == 1
assert part_cores[0].remain == 6
assert len(host.containers.all()) == 1
assert host.count == D('0.6')
assert len(containers) == 3
assert len(v.containers.all()) == 3
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores) == 3
assert len(c.part_cores) == 1
all_core_labels = sorted([
'0',
'1',
'2',
'3',
])
used_full_core_labels = [core.label for core in c.full_cores]
used_part_core_labels = [core.label for core in c.part_cores]
free_core_labels = [core.label for core in c.host.get_free_cores()[0]]
assert all_core_labels == sorted(used_full_core_labels +
used_part_core_labels +
free_core_labels)
#释放核
for c in containers:
c.delete()
assert len(v.containers.all()) == 0
assert get_max_container_count(p, 3, 0) == 3
host_cores = centralized_schedule(p, 3, 3, 0)
assert len(host_cores) == 3
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(host.containers.all()) == 0
assert host.count == 4
#获取
host_cores = centralized_schedule(p, 6, 1, 5)
containers = []
for (host, count), cores in host_cores.iteritems():
cores_per_container = len(cores['full']) / count
for i in range(count):
cid = random_sha1()
used_cores = {
'full':
cores['full'][i * cores_per_container:(i + 1) *
cores_per_container],
'part':
cores['part'][i:i + 1],
}
host.occupy_cores(used_cores, 5)
# not using a port
c = Container.create(cid,
host,
v,
random_string(),
'entrypoint',
used_cores,
'env',
nshare=5)
assert c is not None
containers.append(c)
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 1
assert len(part_cores) == 0
assert len(host.containers.all()) == 2
assert host.count == D('1')
assert len(containers) == 6
assert len(v.containers.all()) == 6
for c in containers:
assert c.host_id in host_ids1
assert c.host_id not in host_ids2
assert c.app.id == a.id
assert c.version.id == v.id
assert c.is_alive
assert len(c.full_cores) == 1
assert len(c.part_cores) == 1
##释放核
for c in containers:
c.delete()
assert len(v.containers.all()) == 0
assert get_max_container_count(p, 3, 0) == 3
host_cores = centralized_schedule(p, 3, 3, 0)
assert len(host_cores) == 3
for host in p.get_private_hosts():
full_cores, part_cores = host.get_free_cores()
assert len(full_cores) == 4
assert len(part_cores) == 0
assert len(host.containers.all()) == 0
assert host.count == 4