def init_db():
db = db_utils.init_db('sqlite:///:memory:')
assert type(db) is neat.db.Database
assert isinstance(db.hosts, Table)
assert isinstance(db.vms, Table)
assert isinstance(db.vm_resource_usage, Table)
assert isinstance(db.host_states, Table)
assert db.hosts.c.keys() == \
['id', 'hostname', 'cpu_mhz', 'cpu_cores', 'ram']
assert db.host_resource_usage.c.keys() == \
['id', 'host_id', 'timestamp', 'cpu_mhz']
assert list(db.host_resource_usage.foreign_keys)[0].target_fullname \
== 'hosts.id'
assert db.vms.c.keys() == \
['id', 'uuid']
assert db.vm_resource_usage.c.keys() == \
['id', 'vm_id', 'timestamp', 'cpu_mhz']
assert list(db.vm_resource_usage.foreign_keys)[0].target_fullname \
== 'vms.id'
assert db.vm_migrations.c.keys() == \
['id', 'vm_id', 'host_id', 'timestamp']
keys = set([list(db.vm_migrations.foreign_keys)[0].target_fullname,
list(db.vm_migrations.foreign_keys)[1].target_fullname])
assert keys == set(['vms.id', 'hosts.id'])
assert db.host_states.c.keys() == \
['id', 'host_id', 'timestamp', 'state']
assert list(db.host_states.foreign_keys)[0].target_fullname \
== 'hosts.id'
assert db.host_overload.c.keys() == \
['id', 'host_id', 'timestamp', 'overload']
assert list(db.host_overload.foreign_keys)[0].target_fullname \
== 'hosts.id'
def insert_select():
db = db_utils.init_db('sqlite:///:memory:')
db.vms.insert().execute(uuid='test')
assert db.vms.select().execute().first()['uuid'] == 'test'
db.vm_resource_usage.insert().execute(vm_id=1, cpu_mhz=1000)
assert db.vm_resource_usage.select(). \
execute().first()['cpu_mhz'] == 1000
def test_switch_hosts_on(self):
db = db_utils.init_db('sqlite:///:memory:')
with MockTransaction:
expect(subprocess).call(any_of(['ether-wake -i eth0 mac1',
'etherwake -i eth0 mac1']),
shell=True).once()
expect(subprocess).call(any_of(['ether-wake -i eth0 mac2',
'etherwake -i eth0 mac2']),
shell=True).once()
expect(manager).host_mac('h1').and_return('mac1').once()
expect(db).insert_host_states({
'h1': 1,
'h2': 1}).once()
manager.switch_hosts_on(db, 'eth0', {'h2': 'mac2'}, ['h1', 'h2'])
with MockTransaction:
expect(subprocess).call(any_of(['ether-wake -i eth0 mac1',
'etherwake -i eth0 mac1']),
shell=True).once()
expect(subprocess).call(any_of(['ether-wake -i eth0 mac2',
'etherwake -i eth0 mac2']),
shell=True).once()
expect(manager).host_mac('h1').and_return('mac1').once()
expect(manager).host_mac('h2').and_return('mac2').once()
expect(db).insert_host_states({
'h1': 1,
'h2': 1}).once()
manager.switch_hosts_on(db, 'eth0', {}, ['h1', 'h2'])
def append_vm_data_remotely(
vms=dict_(
keys=str_(of='abc123-', min_length=36, max_length=36),
values=tuple_(int_(min=1, max=3000),
list_(of=int_(min=1, max=3000),
min_length=0, max_length=10)),
min_length=0, max_length=5
)
):
db = db_utils.init_db('sqlite:///:memory:')
initial_data = []
data_to_submit = {}
final_data = {}
for uuid, data in vms.items():
vm_id = db.select_vm_id(uuid)
data_to_submit[uuid] = data[0]
final_data[uuid] = list(data[1])
final_data[uuid].append(data[0])
for cpu_mhz in data[1]:
initial_data.append({'vm_id': vm_id,
'cpu_mhz': cpu_mhz})
if initial_data:
db.vm_resource_usage.insert().execute(initial_data)
collector.append_vm_data_remotely(db, data_to_submit)
for uuid, data in final_data.items():
assert db.select_cpu_mhz_for_vm(uuid, 11) == data
def select_host_ids():
db = db_utils.init_db('sqlite:///:memory:')
assert db.select_host_ids() == {}
hosts = {}
hosts['host1'] = db.update_host('host1', 1, 1, 1)
hosts['host2'] = db.update_host('host2', 1, 1, 1)
assert db.select_host_ids() == hosts
def append_host_data_remotely(
hostname=str_(of='abc123', min_length=5, max_length=10),
cpu_mhz=int_(min=0, max=3000)
):
db = db_utils.init_db('sqlite:///:memory:')
db.update_host(hostname, 1, 1, 1)
collector.append_host_data_remotely(db, hostname, cpu_mhz)
assert db.select_cpu_mhz_for_host(hostname, 1) == [cpu_mhz]
def insert_host_cpu_mhz(
hostname=str_(of='abc123', min_length=5, max_length=10),
cpu_mhz=list_(of=int_(min=0, max=3000), min_length=1, max_length=10)
):
db = db_utils.init_db('sqlite:///:memory:')
db.update_host(hostname, 1, 1, 1)
for value in cpu_mhz:
db.insert_host_cpu_mhz(hostname, value)
assert db.select_cpu_mhz_for_host(hostname, len(cpu_mhz)) == cpu_mhz
def insert_select():
db = db_utils.init_db('sqlite:///:memory:')
db.vms.insert().execute(uuid='x' * 36).inserted_primary_key[0]
vm_id = db.vms.insert().execute(uuid='vm' * 18).inserted_primary_key[0]
host_id = db.update_host('host', 1, 1, 1)
db.insert_vm_migration('vm' * 18, 'host')
result = db.vm_migrations.select().execute().first()
assert result[1] == vm_id
assert result[2] == host_id
def select_vm_id(
uuid1=str_(of='abc123-', min_length=36, max_length=36),
uuid2=str_(of='abc123-', min_length=36, max_length=36)
):
db = db_utils.init_db('sqlite:///:memory:')
result = db.vms.insert().execute(uuid=uuid1)
vm_id = result.inserted_primary_key[0]
assert db.select_vm_id(uuid1) == vm_id
assert db.select_vm_id(uuid2) == vm_id + 1
def select_host_characteristics():
db = db_utils.init_db('sqlite:///:memory:')
assert db.select_host_characteristics() == ({}, {}, {})
db.update_host('host1', 3000, 4, 4000)
db.update_host('host2', 3500, 8, 8000)
assert db.select_host_characteristics() == \
({'host1': 3000, 'host2': 3500},
{'host1': 4, 'host2': 8},
{'host1': 4000, 'host2': 8000})
def select_cpu_mhz_for_host(
hostname=str_(of='abc123', min_length=5, max_length=10),
cpu_mhz=list_(of=int_(min=0, max=3000), min_length=0, max_length=10),
n=int_(min=1, max=10)
):
db = db_utils.init_db('sqlite:///:memory:')
host_id = db.update_host(hostname, 1, 1, 1)
for mhz in cpu_mhz:
db.host_resource_usage.insert().execute(
host_id=host_id,
cpu_mhz=mhz)
assert db.select_cpu_mhz_for_host(hostname, n) == cpu_mhz[-n:]
def select_cpu_mhz_for_vm(
uuid=str_(of='abc123-', min_length=36, max_length=36),
cpu_mhz=list_(of=int_(min=0, max=3000), min_length=0, max_length=10),
n=int_(min=1, max=10)
):
db = db_utils.init_db('sqlite:///:memory:')
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
for mhz in cpu_mhz:
db.vm_resource_usage.insert().execute(
vm_id=vm_id,
cpu_mhz=mhz)
assert db.select_cpu_mhz_for_vm(uuid, n) == cpu_mhz[-n:]
def execute(uuid=str_(of="abc123-", min_length=36, max_length=36)):
with MockTransaction:
db = db_utils.init_db("sqlite:///:memory:")
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
time = datetime.datetime.today()
for i in range(10):
db.vm_resource_usage.insert().execute(vm_id=1, cpu_mhz=i, timestamp=time.replace(second=i))
state = {"db": db, "time_delta": datetime.timedelta(seconds=5)}
expect(cleaner).today().and_return(time.replace(second=10)).once()
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(10)
cleaner.execute({}, state)
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(5, 10)
def select_host_id():
db = db_utils.init_db('sqlite:///:memory:')
host1_id = db.hosts.insert().execute(
hostname='host1',
cpu_mhz=1,
cpu_cores=1,
ram=1).inserted_primary_key[0]
host2_id = db.hosts.insert().execute(
hostname='host2',
cpu_mhz=1,
cpu_cores=1,
ram=1).inserted_primary_key[0]
assert db.select_host_id('host1') == host1_id
assert db.select_host_id('host2') == host2_id
def cleanup_host_resource_usage(
hostname=str_(of='abc123', min_length=5, max_length=10)
):
db = db_utils.init_db('sqlite:///:memory:')
host_id = db.update_host(hostname, 1, 1, 1)
time = datetime.datetime.today()
for i in range(10):
db.host_resource_usage.insert().execute(
host_id=1,
cpu_mhz=i,
timestamp=time.replace(second=i))
assert db.select_cpu_mhz_for_host(hostname, 100) == range(10)
db.cleanup_host_resource_usage(time.replace(second=5))
assert db.select_cpu_mhz_for_host(hostname, 100) == range(5, 10)
def cleanup_vm_resource_usage(
uuid=str_(of='abc123-', min_length=36, max_length=36)
):
db = db_utils.init_db('sqlite:///:memory:')
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
time = datetime.datetime.today()
for i in range(10):
db.vm_resource_usage.insert().execute(
vm_id=1,
cpu_mhz=i,
timestamp=time.replace(second=i))
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(10)
db.cleanup_vm_resource_usage(time.replace(second=5))
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(5, 10)
def select_last_cpu_mhz_for_vms(
vms=dict_(
keys=str_(of='abc123-', min_length=36, max_length=36),
values=list_(of=int_(min=1, max=3000),
min_length=0, max_length=10),
min_length=0, max_length=3
)
):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for uuid, data in vms.items():
for value in data:
db.insert_vm_cpu_mhz({uuid: value})
if data:
res[uuid] = data[-1]
assert db.select_last_cpu_mhz_for_vms() == res
def select_active_hosts(
hosts=dict_(
keys=str_(of='abc123', min_length=1, max_length=5),
values=list_(of=int_(min=0, max=1),
min_length=0, max_length=10),
min_length=0, max_length=3
)
):
db = db_utils.init_db('sqlite:///:memory:')
res = []
for host, data in hosts.items():
db.update_host(host, 1, 1, 1)
for state in data:
db.insert_host_states({host: state})
if data and data[-1] == 1 or not data:
res.append(host)
assert set(db.select_active_hosts()) == set(res)
def test_switch_hosts_off(self):
db = db_utils.init_db('sqlite:///:memory:')
with MockTransaction:
expect(subprocess).call('ssh h1 "sleep"', shell=True).once()
expect(subprocess).call('ssh h2 "sleep"', shell=True).once()
expect(db).insert_host_states({
'h1': 0,
'h2': 0}).once()
manager.switch_hosts_off(db, 'sleep', ['h1', 'h2'])
with MockTransaction:
expect(subprocess).call.never()
expect(db).insert_host_states({
'h1': 0,
'h2': 0}).once()
manager.switch_hosts_off(db, '', ['h1', 'h2'])
def test_insert_host_states(self):
db = db_utils.init_db('sqlite:///:memory:')
hosts = {}
hosts['host1'] = db.update_host('host1', 1, 1, 1)
hosts['host2'] = db.update_host('host2', 1, 1, 1)
db.insert_host_states({'host1': 0, 'host2': 1})
db.insert_host_states({'host1': 0, 'host2': 0})
db.insert_host_states({'host1': 1, 'host2': 1})
result = db.host_states.select().execute().fetchall()
host1 = [x[3] for x in sorted(filter(
lambda x: x[1] == hosts['host1'],
result), key=lambda x: x[0])]
self.assertEqual(host1, [0, 0, 1])
host2 = [x[3] for x in sorted(filter(
lambda x: x[1] == hosts['host2'],
result), key=lambda x: x[0])]
self.assertEqual(host2, [1, 0, 1])
def fetch_remote_data(x=dict_(keys=str_(of='abc123-',
min_length=36,
max_length=36),
values=list_(of=int_(min=0, max=3000),
min_length=0,
max_length=10),
min_length=0,
max_length=3),
data_length=int_(min=1, max=10)):
db = db_utils.init_db('sqlite:///:memory:')
if x:
for uuid, data in x.items():
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
for mhz in data:
db.vm_resource_usage.insert().execute(vm_id=vm_id,
cpu_mhz=mhz)
x[uuid] = data[-data_length:]
assert collector.fetch_remote_data(db, data_length, x.keys()) == x
def select_host_states(hosts=dict_(keys=str_(of='abc123',
min_length=1,
max_length=5),
values=list_(of=int_(min=0, max=1),
min_length=0,
max_length=10),
min_length=0,
max_length=3)):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for host, data in hosts.items():
db.update_host(host, 1, 1, 1)
for state in data:
db.insert_host_states({host: state})
if data:
res[host] = data[-1]
else:
res[host] = 1
assert db.select_host_states() == res
def update_host():
db = db_utils.init_db('sqlite:///:memory:')
db.update_host('host1', 3000, 4, 4000)
hosts = db.hosts.select().execute().fetchall()
assert len(hosts) == 1
host = hosts[0]
assert host['hostname'] == 'host1'
assert host['cpu_mhz'] == 3000
assert host['cpu_cores'] == 4
assert host['ram'] == 4000
db.update_host('host1', 3500, 8, 8000L)
hosts = db.hosts.select().execute().fetchall()
assert len(hosts) == 1
host = hosts[0]
assert host['hostname'] == 'host1'
assert host['cpu_mhz'] == 3500
assert host['cpu_cores'] == 8
assert host['ram'] == 8000L
def select_last_cpu_mhz_for_hosts(
hosts=dict_(
keys=str_(of='abc123', min_length=5, max_length=10),
values=list_(of=int_(min=1, max=3000),
min_length=0, max_length=10),
min_length=0, max_length=3
)
):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for hostname, data in hosts.items():
db.update_host(hostname, 1, 1, 1)
for value in data:
db.insert_host_cpu_mhz(hostname, value)
if data:
res[hostname] = data[-1]
else:
res[hostname] = 0
assert db.select_last_cpu_mhz_for_hosts() == res
def select_host_states(
hosts=dict_(
keys=str_(of='abc123', min_length=1, max_length=5),
values=list_(of=int_(min=0, max=1),
min_length=0, max_length=10),
min_length=0, max_length=3
)
):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for host, data in hosts.items():
db.update_host(host, 1, 1, 1)
for state in data:
db.insert_host_states({host: state})
if data:
res[host] = data[-1]
else:
res[host] = 1
assert db.select_host_states() == res
def select_last_cpu_mhz_for_hosts(hosts=dict_(keys=str_(of='abc123',
min_length=5,
max_length=10),
values=list_(of=int_(
min=1, max=3000),
min_length=0,
max_length=10),
min_length=0,
max_length=3)):
db = db_utils.init_db('sqlite:///:memory:')
res = {}
for hostname, data in hosts.items():
db.update_host(hostname, 1, 1, 1)
for value in data:
db.insert_host_cpu_mhz(hostname, value)
if data:
res[hostname] = data[-1]
else:
res[hostname] = 0
assert db.select_last_cpu_mhz_for_hosts() == res
def fetch_remote_data(
x=dict_(
keys=str_(of='abc123-', min_length=36, max_length=36),
values=list_(of=int_(min=0, max=3000),
min_length=0, max_length=10),
min_length=0, max_length=3
),
data_length=int_(min=1, max=10)
):
db = db_utils.init_db('sqlite:///:memory:')
if x:
for uuid, data in x.items():
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
for mhz in data:
db.vm_resource_usage.insert().execute(
vm_id=vm_id,
cpu_mhz=mhz)
x[uuid] = data[-data_length:]
assert collector.fetch_remote_data(db, data_length, x.keys()) == x
def test_insert_host_states(self):
db = db_utils.init_db('sqlite:///:memory:')
hosts = {}
hosts['host1'] = db.update_host('host1', 1, 1, 1)
hosts['host2'] = db.update_host('host2', 1, 1, 1)
db.insert_host_states({'host1': 0, 'host2': 1})
db.insert_host_states({'host1': 0, 'host2': 0})
db.insert_host_states({'host1': 1, 'host2': 1})
result = db.host_states.select().execute().fetchall()
host1 = [
x[3]
for x in sorted(filter(lambda x: x[1] == hosts['host1'], result),
key=lambda x: x[0])
]
self.assertEqual(host1, [0, 0, 1])
host2 = [
x[3]
for x in sorted(filter(lambda x: x[1] == hosts['host2'], result),
key=lambda x: x[0])
]
self.assertEqual(host2, [1, 0, 1])
def test_switch_hosts_on(self):
db = db_utils.init_db('sqlite:///:memory:')
with MockTransaction:
expect(subprocess).call('ether-wake -i eth0 mac1', shell=True).once()
expect(subprocess).call('ether-wake -i eth0 mac2', shell=True).once()
expect(manager).host_mac('h1').and_return('mac1').once()
expect(db).insert_host_states({
'h1': 1,
'h2': 1}).once()
manager.switch_hosts_on(db, 'eth0', {'h2': 'mac2'}, ['h1', 'h2'])
with MockTransaction:
expect(subprocess).call('ether-wake -i eth0 mac1', shell=True).once()
expect(subprocess).call('ether-wake -i eth0 mac2', shell=True).once()
expect(manager).host_mac('h1').and_return('mac1').once()
expect(manager).host_mac('h2').and_return('mac2').once()
expect(db).insert_host_states({
'h1': 1,
'h2': 1}).once()
manager.switch_hosts_on(db, 'eth0', {}, ['h1', 'h2'])
def execute(
uuid=str_(of='abc123-', min_length=36, max_length=36)
):
with MockTransaction:
db = db_utils.init_db('sqlite:///:memory:')
result = db.vms.insert().execute(uuid=uuid)
vm_id = result.inserted_primary_key[0]
time = datetime.datetime.today()
for i in range(10):
db.vm_resource_usage.insert().execute(
vm_id=1,
cpu_mhz=i,
timestamp=time.replace(second=i))
state = {
'db': db,
'time_delta': datetime.timedelta(seconds=5)}
expect(cleaner).today(). \
and_return(time.replace(second=10)).once()
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(10)
cleaner.execute({}, state)
assert db.select_cpu_mhz_for_vm(uuid, 100) == range(5, 10)
def log_host_overload():
db = db_utils.init_db('sqlite:///:memory:')
with MockTransaction:
expect(db).insert_host_overload('host', 1).once()
assert collector.log_host_overload(db, 0.9, 'host', -1, 3000, 2800)
with MockTransaction:
expect(db).insert_host_overload('host', 0).once()
assert not collector.log_host_overload(db, 0.9, 'host', -1, 3000, 2600)
with MockTransaction:
expect(db).insert_host_overload('host', 1).once()
assert collector.log_host_overload(db, 0.9, 'host', 0, 3000, 2800)
with MockTransaction:
expect(db).insert_host_overload('host', 0).once()
assert not collector.log_host_overload(db, 0.9, 'host', 1, 3000, 2600)
with MockTransaction:
expect(db).insert_host_overload.never()
assert collector.log_host_overload(db, 0.9, 'host', 1, 3000, 2800)
with MockTransaction:
expect(db).insert_host_overload.never()
assert not collector.log_host_overload(db, 0.9, 'host', 0, 3000, 2600)
def select_inactive_hosts(hosts=dict_(keys=str_(of='abc123',
min_length=1,
max_length=5),
values=list_(of=int_(min=0, max=1),
min_length=0,
max_length=10),
min_length=0,
max_length=3)):
hosts = {
'1ab': [0],
'3222': [0, 0, 1, 1, 1, 1, 0, 0],
'b222b': [0, 0, 1, 1, 1, 0, 1]
}
db = db_utils.init_db('sqlite:///:memory:')
res = []
for host, data in hosts.items():
db.update_host(host, 1, 1, 1)
for state in data:
db.insert_host_states({host: state})
if data and data[-1] == 0:
res.append(host)
assert set(db.select_inactive_hosts()) == set(res)