def test_send_all_samples_sends_each_samples_with_project_release_date(self): # Given self.submitter.send_sample = MagicMock(return_value='UPDATED') test_date = datetime.now().isoformat() project_attributes = { 'releaseDate': test_date } project_attributes = make_ingest_entity('projects', random_id(), random_uuid(), project_attributes) self.submission.map_ingest_entity(project_attributes) biomaterials = self.map_random_biomaterials(self.submission, 5) calls = [] call_count = 0 expected_result = {'UPDATED': []} for entity in biomaterials: call_count += 1 calls.append(call(entity, test_date)) expected_result['UPDATED'].append(entity) # When result = self.submitter.send_all_samples(self.submission) # Then self.submitter.send_sample.assert_has_calls(calls, any_order=True) self.assertEqual(call_count, self.submitter.send_sample.call_count) self.assertDictEqual(expected_result, result)
def test_container_release_cores(test_db): a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git') v = a.add_version(random_sha1()) p = Pod.create('pod', 'pod', 10, -1) host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0) for core in host.cores: assert core.host_id == host.id assert core.remain == 10 containers = [] cores = sorted(host.cores, key=operator.attrgetter('label')) for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)): used_cores = {'full': fcores, 'part': pcores} host.occupy_cores(used_cores, 5) c = Container.create(random_sha1(), host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=5) containers.append(c) cores = sorted(host.cores, key=operator.attrgetter('label')) for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)): for core in fcores: assert core.remain == 0 for core in pcores: assert core.remain == 5 for c in containers: c.delete() cores = sorted(host.cores, key=operator.attrgetter('label')) for fcores, pcores in zip(chunked(cores[:100], 10), chunked(cores[100:], 10)): for core in fcores: assert core.remain == 10 for core in pcores: assert core.remain == 10
def _create_data(core_share, max_share_core, host_count): group = Group.create('group', 'group') pod = Pod.create('pod', 'pod', core_share, max_share_core) for _ in range(host_count): host = Host.create(pod, random_ipv4(), random_string(), random_uuid(), 16, 4096) host.assigned_to_group(group) return group, pod
def setUp(self) -> None: # Given self.test_type = random_id() self.test_id = random_id() self.test_uuid = random_uuid() self.test_case = make_ingest_entity(self.test_type, self.test_id, self.test_uuid) self.submission = HcaSubmission() # When self.test_entity = self.submission.map_ingest_entity(self.test_case)
def map_random_biomaterials(submission: HcaSubmission, entity_count: int) -> List[Entity]: biomaterials = [] for i in range(0, entity_count): biomaterials.append( submission.map_ingest_entity( make_ingest_entity('biomaterials', random_id(), random_uuid()) ) ) return biomaterials
def test_duplication_entity_with_differing_content_is_ignored(self): # Given ignored_uuid = random_uuid() duplicate_test_case = make_ingest_entity(self.test_type, self.test_id, ignored_uuid) # When duplicate_entity = self.submission.map_ingest_entity(duplicate_test_case) # Then self.assertEqual(self.test_entity, duplicate_entity) self.assertEqual(self.test_uuid, self.test_entity.attributes['uuid']['uuid']) self.assertNotEqual(ignored_uuid, self.test_entity.attributes['uuid']['uuid'])
def test_get_max_container_count_single_host(test_db): pod = Pod.create('pod', 'pod', 10, -1) Host.create(pod, random_ipv4(), random_string(), random_uuid(), 64, 4096) assert get_max_container_count(pod, ncore=1, nshare=0) == 64 assert get_max_container_count(pod, ncore=2, nshare=0) == 32 assert get_max_container_count(pod, ncore=3, nshare=0) == 21 assert get_max_container_count(pod, ncore=4, nshare=0) == 16 assert get_max_container_count(pod, ncore=5, nshare=0) == 12 assert get_max_container_count(pod, ncore=1, nshare=5) == 42 assert get_max_container_count(pod, ncore=2, nshare=5) == 25
def test_submit_sample_return_error(self): # Given error_msg = f'TestCaseError{random_id(3)}' self.biosamples.send_sample = MagicMock(side_effect=Exception(error_msg)) test_case = self.submission.map_ingest_entity( make_ingest_entity('biomaterials', random_id(), random_uuid())) # When result = self.submitter.send_sample(test_case) # Then self.assertEqual('ERRORED', result) self.assertDictEqual(test_case.get_errors(), {'content.biomaterial_core.biosamples_accession': [f'BioSamples Error: {error_msg}']})
def test_submit_sample_converts_and_sends_sample(self): # Given test_case = self.submission.map_ingest_entity( make_ingest_entity('biomaterials', random_id(), random_uuid()) ) # When result = self.submitter.send_sample(test_case) # Then self.converter.convert.assert_called_once_with(test_case.attributes, release_date=None, accession=None) self.biosamples.send_sample.assert_called_once_with(self.empty_sample) self.assertEqual('UPDATED', result)
def test_container_transform(test_db): a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '') assert a is not None v = a.add_version(random_sha1()) v2 = a.add_version(random_sha1()) assert v is not None assert v.app.id == a.id assert v.name == a.name assert len(v.containers.all()) == 0 assert len(v.tasks.all()) == 0 g = Group.create('group', 'group') p = Pod.create('pod', 'pod') assert p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6)] for host in hosts[:3]: host.assigned_to_group(g) assert g.get_max_containers(p, 3, 0) == 3 host_cores = g.get_free_cores(p, 3, 3, 0) assert len(host_cores) == 3 containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores) / count for i in range(count): cid = random_sha1() used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]} c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env') assert c is not None containers.append(c) host.occupy_cores(cores, 0) for host in g.private_hosts.all(): assert len(host.get_free_cores()[0]) == 1 assert len(host.containers.all()) == 1 assert host.count == 1 assert len(containers) == 3 assert len(v.containers.all()) == 3 cids = [c.container_id for c in containers] for c in containers: host = c.host cid = c.container_id c.transform(v2, random_sha1(), random_string()) assert c.container_id != cid new_cids = [c.container_id for c in containers] assert new_cids != cids
def test_network(test_db): n = Network.create('net', '10.1.0.0/16') assert n is not None assert len(n.ips.all()) == 0 assert n.hostmask_string == '16' assert n.pool_size == 65436 assert n.used_count == 0 assert n.used_gate_count == 0 assert n.gate_pool_size == 100 ip = n.acquire_ip() assert ip is not None assert ip.network_id == n.id assert ip.vethname == '' assert not ip.container_id assert ip.hostmask == n.hostmask_string assert ip.vlan_seq_id == n.id assert ip.address.startswith('10.1') assert len(n.ips.all()) == 1 assert n.pool_size == 65435 assert n.used_count == 1 ip.release() assert len(n.ips.all()) == 0 assert n.pool_size == 65436 assert n.used_count == 0 p = Pod.create('pod', 'pod', 10, -1) host = Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) gate = n.acquire_gateway_ip(host) assert gate is not None assert gate.network_id == n.id assert gate.vlan_address.startswith('10.1.0.') assert gate.vlan_seq_id == n.id assert gate.name == 'vlan.%02d.br' % n.id g = VLanGateway.get_by_host_and_network(host.id, n.id) assert g is not None assert g.id == gate.id assert len(host.list_vlans()) == 1 assert n.used_gate_count == 1 assert n.gate_pool_size == 99 gate.release() assert n.used_gate_count == 0 assert n.gate_pool_size == 100 assert VLanGateway.get_by_host_and_network(host.id, n.id) is None assert len(host.list_vlans()) == 0
def test_get_max_container_count_single_host(test_db): group = Group.create('group', 'group') pod = Pod.create('pod', 'pod', 10, -1) host = Host.create(pod, random_ipv4(), random_string(), random_uuid(), 64, 4096) host.assigned_to_group(group) assert get_max_container_count(group, pod, ncore=1, nshare=0) == 64 assert get_max_container_count(group, pod, ncore=2, nshare=0) == 32 assert get_max_container_count(group, pod, ncore=3, nshare=0) == 21 assert get_max_container_count(group, pod, ncore=4, nshare=0) == 16 assert get_max_container_count(group, pod, ncore=5, nshare=0) == 12 assert get_max_container_count(group, pod, ncore=1, nshare=5) == 42 assert get_max_container_count(group, pod, ncore=2, nshare=5) == 25
def test_host(test_db): g = Group.create('group', 'group') p = Pod.create('pod', 'pod') assert p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6)] for host in hosts: assert host is not None assert len(host.cores.all()) == 4 assert len(host.get_free_cores()) == 4 assert len(g.private_hosts.all()) == 0 assert g.get_max_containers(p, 1) == 0 assert g.get_free_cores(p, 1, 1) == {} for host in hosts[:3]: host.assigned_to_group(g) host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} assert len(g.private_hosts.all()) == 3 assert g.get_max_containers(p, 1) == 12 host_cores = g.get_free_cores(p, 12, 1) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 4 assert len(cores) == 4 assert g.get_max_containers(p, 3) == 3 host_cores = g.get_free_cores(p, 3, 3) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 1 assert len(cores) == 3 assert g.get_max_containers(p, 2) == 6 host_cores = g.get_free_cores(p, 4, 2) assert len(host_cores) == 2 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 2 assert len(cores) == 4
def test_occupy_and_release_cores(test_db): g = Group.create('group', 'group') p = Pod.create('pod', 'pod', 10, -1) host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0) assert p.assigned_to_group(g) assert host.assigned_to_group(g) for core in host.cores: assert core.host_id == host.id assert core.remain == 10 cores = { 'full': host.cores[:100], 'part': host.cores[100:], } host.occupy_cores(cores, 5) for core in host.cores[:100]: assert core.remain == 0 for core in host.cores[100:]: assert core.remain == 5 host.release_cores(cores, 5) for core in host.cores: assert core.remain == 10 host.occupy_cores(cores, 8) for core in host.cores[:100]: assert core.remain == 0 for core in host.cores[100:]: assert core.remain == 2 host.release_cores(cores, 8) for core in host.cores: assert core.remain == 10 cores = { 'full': host.cores[:50], 'part': host.cores[50:100], } host.occupy_cores(cores, 8) for core in host.cores[:50]: assert core.remain == 0 for core in host.cores[50:100]: assert core.remain == 2 host.release_cores(cores, 8) for core in host.cores: assert core.remain == 10
def test_occupy_and_release_cores(test_db): p = Pod.create('pod', 'pod', 10, -1) host = Host.create(p, random_ipv4(), random_string(), random_uuid(), 200, 0) for core in host.cores: assert core.host_id == host.id assert core.remain == 10 cores = { 'full': host.cores[:100], 'part': host.cores[100:], } host.occupy_cores(cores, 5) for core in host.cores[:100]: assert core.remain == 0 for core in host.cores[100:]: assert core.remain == 5 host.release_cores(cores, 5) for core in host.cores: assert core.remain == 10 host.occupy_cores(cores, 8) for core in host.cores[:100]: assert core.remain == 0 for core in host.cores[100:]: assert core.remain == 2 host.release_cores(cores, 8) for core in host.cores: assert core.remain == 10 cores = { 'full': host.cores[:50], 'part': host.cores[50:100], } host.occupy_cores(cores, 8) for core in host.cores[:50]: assert core.remain == 0 for core in host.cores[50:100]: assert core.remain == 2 host.release_cores(cores, 8) for core in host.cores: assert core.remain == 10
def map_random_entities( submission: HcaSubmission, entity_count: int, entity_type: str = None ): entity_list = [] for i in range(0, entity_count): entity_list.append( submission.map_ingest_entity( make_ingest_entity( entity_type if entity_type else random_id(), random_id(), random_uuid() ) ) ) return entity_list
def create_test_suite(): appyaml = { 'appname': 'app', 'entrypoints': { 'web': { 'cmd': 'python app.py', 'ports': ['5000/tcp'], }, 'daemon': { 'cmd': 'python daemon.py', }, 'service': { 'cmd': 'python service.py' }, }, 'build': 'pip install -r ./requirements.txt', } app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git') version = app.add_version(random_sha1()) appconfig = version.appconfig appconfig.update(**appyaml) appconfig.save() pod = Pod.create('pod', 'pod') hosts = [ Host.create(pod, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(4) ] containers = [] for (host, count), cores in centralized_schedule(pod, 4, 4, 0).iteritems(): cores_per_container = len(cores) / count for i in range(count): cid = random_sha1() used_cores = { 'full': cores['full'][i * cores_per_container:(i + 1) * cores_per_container] } c = Container.create(cid, host, version, random_string(), 'web', used_cores, 'env') containers.append(c) host.occupy_cores(cores, 0) return app, version, pod, hosts, containers
def create_test_suite(): appyaml = { 'appname': 'app', 'entrypoints': { 'web': { 'cmd': 'python app.py', 'ports': ['5000/tcp'], }, 'daemon': { 'cmd': 'python daemon.py', }, 'service': { 'cmd': 'python service.py' }, }, 'build': 'pip install -r ./requirements.txt', } app = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', 'token') version = app.add_version(random_sha1()) appconfig = version.appconfig appconfig.update(**appyaml) appconfig.save() group = Group.create('group', 'group') pod = Pod.create('pod', 'pod') pod.assigned_to_group(group) hosts = [Host.create(pod, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(4)] for host in hosts: host.assigned_to_group(group) containers = [] for (host, count), cores in group.get_free_cores(pod, 4, 4, 0).iteritems(): cores_per_container = len(cores) / count for i in range(count): cid = random_sha1() used_cores = cores['full'][i*cores_per_container:(i+1)*cores_per_container] c = Container.create(cid, host, version, random_string(), 'entrypoint', used_cores, 'env') containers.append(c) host.occupy_cores(cores, 0) return app, version, group, pod, hosts, containers
def test_container(test_db): a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '') assert a is not None assert a.id == a.user_id v = a.add_version(random_sha1()) assert v is not None assert v.app.id == a.id assert v.name == a.name assert len(v.containers.all()) == 0 assert len(v.tasks.all()) == 0 g = Group.create('group', 'group') p = Pod.create('pod', 'pod', 10, -1) assert p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6)] for host in hosts[:3]: host.assigned_to_group(g) host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} host_cores = g.get_free_cores(p, 3, 3, 0) #测试没有碎片核的情况 #获取核 containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores['full']) / count for i in range(count): cid = random_sha1() used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container]} # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env') assert c is not None containers.append(c) host.occupy_cores(cores, 0) for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 1 assert len(part_cores) == 0 assert len(host.containers.all()) == 1 assert host.count == 1 assert len(containers) == 3 assert len(v.containers.all()) == 3 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores.all()) == 3 assert len(c.part_cores.all()) == 0 all_core_labels = sorted(['0', '1', '2', '3', ]) used_full_core_labels = [core.label for core in c.full_cores.all()] used_part_core_labels = [core.label for core in c.part_cores.all()] free_core_labels = [core.label for core in c.host.get_free_cores()[0]] assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels) #释放核 for c in containers: c.delete() assert len(v.containers.all()) == 0 assert g.get_max_containers(p, 3, 0) == 3 host_cores = g.get_free_cores(p, 3, 3, 0) assert len(host_cores) == 3 for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(host.containers.all()) == 0 assert host.count == 0 #测试有碎片的情况 #获取核 host_cores = g.get_free_cores(p, 3, 3, 4) containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores['full']) / count for i in range(count): cid = random_sha1() used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container], 'part': cores['part']} # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env') assert c is not None containers.append(c) host.occupy_cores(cores, 4) for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 0 assert len(part_cores) == 1 assert part_cores[0].used == 4 assert len(host.containers.all()) == 1 assert host.count == 1 assert len(containers) == 3 assert len(v.containers.all()) == 3 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores.all()) == 3 assert len(c.part_cores.all()) == 1 all_core_labels = sorted(['0', '1', '2', '3', ]) used_full_core_labels = [core.label for core in c.full_cores.all()] used_part_core_labels = [core.label for core in c.part_cores.all()] free_core_labels = [core.label for core in c.host.get_free_cores()[0]] assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels) #释放核 for c in containers: c.delete(4) assert len(v.containers.all()) == 0 assert g.get_max_containers(p, 3, 0) == 3 host_cores = g.get_free_cores(p, 3, 3, 0) assert len(host_cores) == 3 for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(host.containers.all()) == 0 assert host.count == 0 #获取 host_cores = g.get_free_cores(p, 6, 1, 5) containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores['full']) / count for i in range(count): cid = random_sha1() used_cores = {'full': cores['full'][i*cores_per_container:(i+1)*cores_per_container], 'part': cores['part'][i:i+1]} # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env') assert c is not None containers.append(c) host.occupy_cores(used_cores, 5) for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 1 assert len(part_cores) == 0 assert len(host.containers.all()) == 2 assert host.count == 2 assert len(containers) == 6 assert len(v.containers.all()) == 6 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores.all()) == 1 assert len(c.part_cores.all()) == 1 ##释放核 for c in containers: c.delete(5) assert len(v.containers.all()) == 0 assert g.get_max_containers(p, 3, 0) == 3 host_cores = g.get_free_cores(p, 3, 3, 0) assert len(host_cores) == 3 for host in g.private_hosts.all(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(host.containers.all()) == 0 assert host.count == 0
#!/usr/bin/env python # encoding: utf-8 from eru.app import create_app_with_celery from eru.models import db, Group, Pod, Host, App from tests.utils import random_ipv4, random_string, random_uuid, random_sha1 host_number = int(raw_input("how many hosts: ")) app, _ = create_app_with_celery app.config['TESTING'] = True app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:@localhost:3306/erutest' with app.app_context(): db.create_all() a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '') v = a.add_version(random_sha1()) g = Group.create('group', 'group') p = Pod.create('pod', 'pod', 10, -1) p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(), random_uuid(), 24, 4096) for i in range(host_number)] for host in hosts: host.assigned_to_group(g)
def test_submit_sample_return_creation(self): # Given self.biosamples.send_sample = MagicMock(return_value={'accession': 'SAMEA1234567'}) test_case = self.submission.map_ingest_entity(make_ingest_entity('biomaterials', random_id(), random_uuid())) # When result = self.submitter.send_sample(test_case) # Then self.assertEqual('CREATED', result)
def test_host(test_db): p = Pod.create('pod', 'pod', 10, -1) hosts = [ Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6) ] for host in hosts: host.set_public() assert host is not None assert len(host.cores) == 4 full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(p.get_private_hosts()) == 0 assert get_max_container_count(p, 1, 0) == 0 assert centralized_schedule(p, 1, 1, 0) == {} for host in hosts[:3]: host.set_private() host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} assert get_max_container_count(p, 1, 0) == 12 host_cores = centralized_schedule(p, 12, 1, 0) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 4 assert len(cores['full']) == 4 assert get_max_container_count(p, 3, 0) == 3 host_cores = centralized_schedule(p, 3, 3, 0) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 1 assert len(cores['full']) == 3 assert get_max_container_count(p, 2, 0) == 6 host_cores = centralized_schedule(p, 4, 2, 0) assert len(host_cores) == 2 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 2 assert len(cores['full']) == 4 assert get_max_container_count(p, 1, 1) == 9 host_cores = centralized_schedule(p, 3, 1, 1) assert len(host_cores) == 1 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 3 assert len(cores['full']) == 3 assert len(cores['part']) == 3 assert get_max_container_count(p, 2, 3) == 3 host_cores = centralized_schedule(p, 3, 2, 3) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 1 assert len(cores['full']) == 2 assert len(cores['part']) == 1
def _create_data(core_share, max_share_core, host_count): pod = Pod.create('pod', 'pod', core_share, max_share_core) for _ in range(host_count): Host.create(pod, random_ipv4(), random_string(), random_uuid(), 16, 4096) return pod
def test_container(test_db): a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git') assert a is not None assert a.id == a.user_id v = a.add_version(random_sha1()) assert v is not None assert v.app.id == a.id assert v.name == a.name assert len(v.containers.all()) == 0 assert len(v.tasks.all()) == 0 p = Pod.create('pod', 'pod', 10, -1) hosts = [ Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6) ] for host in hosts[3:]: host.set_public() host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} host_cores = centralized_schedule(p, 3, 3, 0) #测试没有碎片核的情况 #获取核 containers = [] for (host, count), cores in host_cores.iteritems(): host.occupy_cores(cores, 0) cores_per_container = len(cores['full']) / count for i in range(count): cid = random_sha1() used_cores = { 'full': cores['full'][i * cores_per_container:(i + 1) * cores_per_container] } c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=0) assert c is not None containers.append(c) for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 1 assert len(part_cores) == 0 assert len(host.containers.all()) == 1 assert host.count == 1 assert len(containers) == 3 assert len(v.containers.all()) == 3 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores) == 3 assert len(c.part_cores) == 0 all_core_labels = sorted([ '0', '1', '2', '3', ]) used_full_core_labels = [core.label for core in c.full_cores] used_part_core_labels = [core.label for core in c.part_cores] free_core_labels = [core.label for core in c.host.get_free_cores()[0]] assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels) #释放核 for c in containers: c.delete() assert len(v.containers.all()) == 0 assert get_max_container_count(p, 3, 0) == 3 host_cores = centralized_schedule(p, 3, 3, 0) assert len(host_cores) == 3 for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(host.containers.all()) == 0 assert host.count == 4 #测试有碎片的情况 #获取核 host_cores = centralized_schedule(p, 3, 3, 4) containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores['full']) / count host.occupy_cores(cores, 4) for i in range(count): cid = random_sha1() used_cores = { 'full': cores['full'][i * cores_per_container:(i + 1) * cores_per_container], 'part': cores['part'] } # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=4) assert c is not None containers.append(c) for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 0 assert len(part_cores) == 1 assert part_cores[0].remain == 6 assert len(host.containers.all()) == 1 assert host.count == D('0.6') assert len(containers) == 3 assert len(v.containers.all()) == 3 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores) == 3 assert len(c.part_cores) == 1 all_core_labels = sorted([ '0', '1', '2', '3', ]) used_full_core_labels = [core.label for core in c.full_cores] used_part_core_labels = [core.label for core in c.part_cores] free_core_labels = [core.label for core in c.host.get_free_cores()[0]] assert all_core_labels == sorted(used_full_core_labels + used_part_core_labels + free_core_labels) #释放核 for c in containers: c.delete() assert len(v.containers.all()) == 0 assert get_max_container_count(p, 3, 0) == 3 host_cores = centralized_schedule(p, 3, 3, 0) assert len(host_cores) == 3 for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(host.containers.all()) == 0 assert host.count == 4 #获取 host_cores = centralized_schedule(p, 6, 1, 5) containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores['full']) / count for i in range(count): cid = random_sha1() used_cores = { 'full': cores['full'][i * cores_per_container:(i + 1) * cores_per_container], 'part': cores['part'][i:i + 1], } host.occupy_cores(used_cores, 5) # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env', nshare=5) assert c is not None containers.append(c) for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 1 assert len(part_cores) == 0 assert len(host.containers.all()) == 2 assert host.count == D('1') assert len(containers) == 6 assert len(v.containers.all()) == 6 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.full_cores) == 1 assert len(c.part_cores) == 1 ##释放核 for c in containers: c.delete() assert len(v.containers.all()) == 0 assert get_max_container_count(p, 3, 0) == 3 host_cores = centralized_schedule(p, 3, 3, 0) assert len(host_cores) == 3 for host in p.get_private_hosts(): full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(host.containers.all()) == 0 assert host.count == 4
def test_host(test_db): g = Group.create('group', 'group') p = Pod.create('pod', 'pod', 10, -1) assert p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6)] for host in hosts: assert host is not None assert len(host.cores) == 4 full_cores, part_cores = host.get_free_cores() assert len(full_cores) == 4 assert len(part_cores) == 0 assert len(g.private_hosts.all()) == 0 assert get_max_container_count(g, p, 1, 0) == 0 assert centralized_schedule(g, p, 1, 1, 0) == {} for host in hosts[:3]: host.assigned_to_group(g) host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} assert len(g.private_hosts.all()) == 3 assert get_max_container_count(g, p, 1, 0) == 12 host_cores = centralized_schedule(g, p, 12, 1, 0) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 4 assert len(cores['full']) == 4 assert get_max_container_count(g, p, 3, 0) == 3 host_cores = centralized_schedule(g, p, 3, 3, 0) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 1 assert len(cores['full']) == 3 assert get_max_container_count(g, p, 2, 0) == 6 host_cores = centralized_schedule(g, p, 4, 2, 0) assert len(host_cores) == 2 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 2 assert len(cores['full']) == 4 assert get_max_container_count(g, p, 1, 1) == 9 host_cores = centralized_schedule(g, p, 3, 1, 1) assert len(host_cores) == 1 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 3 assert len(cores['full']) == 3 assert len(cores['part']) == 3 assert get_max_container_count(g, p, 2, 3) == 3 host_cores = centralized_schedule(g, p, 3, 2, 3) assert len(host_cores) == 3 for (host, count), cores in host_cores.iteritems(): assert host.id in host_ids1 assert host.id not in host_ids2 assert count == 1 assert len(cores['full']) == 2 assert len(cores['part']) == 1
def test_container(test_db): a = App.get_or_create('app', 'http://git.hunantv.com/group/app.git', '') assert a is not None v = a.add_version(random_sha1()) assert v is not None assert v.app.id == a.id assert v.name == a.name assert len(v.containers.all()) == 0 assert len(v.tasks.all()) == 0 g = Group.create('group', 'group') p = Pod.create('pod', 'pod') assert p.assigned_to_group(g) hosts = [Host.create(p, random_ipv4(), random_string(prefix='host'), random_uuid(), 4, 4096) for i in range(6)] for host in hosts[:3]: host.assigned_to_group(g) host_ids1 = {h.id for h in hosts[:3]} host_ids2 = {h.id for h in hosts[3:]} assert g.get_max_containers(p, 3) == 3 host_cores = g.get_free_cores(p, 3, 3) assert len(host_cores) == 3 containers = [] for (host, count), cores in host_cores.iteritems(): cores_per_container = len(cores) / count for i in range(count): cid = random_sha1() used_cores = cores[i*cores_per_container:(i+1)*cores_per_container] # not using a port c = Container.create(cid, host, v, random_string(), 'entrypoint', used_cores, 'env') assert c is not None containers.append(c) host.occupy_cores(cores) for host in g.private_hosts.all(): assert len(host.get_free_cores()) == 1 assert len(host.containers.all()) == 1 assert host.count == 1 assert len(containers) == 3 assert len(v.containers.all()) == 3 for c in containers: assert c.host_id in host_ids1 assert c.host_id not in host_ids2 assert c.app.id == a.id assert c.version.id == v.id assert c.is_alive assert len(c.cores.all()) == 3 all_core_labels = sorted(['0', '1', '2', '3', ]) used_core_labels = [core.label for core in c.cores.all()] free_core_labels = [core.label for core in c.host.get_free_cores()] assert all_core_labels == sorted(used_core_labels + free_core_labels) for c in containers: c.delete() assert len(v.containers.all()) == 0 assert g.get_max_containers(p, 3) == 3 host_cores = g.get_free_cores(p, 3, 3) assert len(host_cores) == 3 for host in g.private_hosts.all(): assert len(host.get_free_cores()) == 4 assert len(host.containers.all()) == 0 assert host.count == 0