def hello(vm_ip, **args):
"""
First function which must be called by VMs ctx module. It registers VM with status 'running ctx',
also serves a special role when creating farms (tracking head, and worker nodes)
@parameter{vm_ip,string}
@parameter{args}
"""
vm = VM.get_by_ip(vm_ip)
log.debug(vm.user_id, "Hello from vm %d ip: %s" % (vm.id, vm_ip))
vm.ctx_api_version = args.get('version', None)
vm.state = vm_states['running ctx']
if vm.ssh_username and vm.ssh_key:
Command.execute('add_ssh_key', vm.user_id, vm.id, user=vm.ssh_username, ssh_key=vm.ssh_key)
if vm.is_head():
Command.register_head(vm)
elif vm.is_farm():
Command.register_node(vm)
try:
vm.save(update_fields=['state', 'ctx_api_version'])
except Exception, e:
log.error(vm.user_id, "Cannot update database for vm %d: %s" % (vm.id, e.message))
return response('ctx_error', "Cannot update database: %s" % e.message)
def destroy(vms):
"""
@parameter{vms}
@response result
"""
results = []
for vm in vms:
vm = VM.objects.get(pk=vm.id)
log.debug(vm.user.id,
"Killing VM id: %s, state: %s" % (vm.id, vm.state))
# Check for VM state
if vm.state in (vm_states['closing'], vm_states['saving']):
results.append({'status': 'vm_already_closing', 'data': ''})
continue
if vm.state in (vm_states['erased'], vm_states['closed']):
results.append({'status': 'vm_wrong_state', 'data': ''})
continue
vm.save_vm = 0
try:
vm.save()
transaction.commit()
vm.lv_destroy()
except Exception, e:
log.exception(vm.user.id, 'error destroying VM: %s' % str(e))
results.append({'status': 'vm_destroy', 'data': ''})
message.error(vm.user_id, 'vm_destroy', {
'id': vm.id,
'name': vm.name
})
continue
results.append({'status': 'ok', 'data': ''})
def get_unused_ipnetwork(self, mask):
"""
@returns Unused subnetwork represented by IPNetwork object
"""
networks = []
for network in self.usernetwork_set.all():
networks.append(network.to_ipnetwork())
networks = sorted(networks)
log.debug(1, 'Networks: %s' % str(networks))
if self.mask > mask:
raise CMException('network_to_large')
if len(networks) == 0:
return IPNetwork(self.address + '/' + str(mask))
if IPNetwork(self.address + '/' + str(mask)).network < networks[0].network:
return IPNetwork(self.address + '/' + str(mask))
# Find matching hole in existing networks
for i in xrange(len(networks) - 1):
n = IPNetwork(str(networks[i].next().ip) + "/" + str(mask))
if networks[i] < n and n < networks[i + 1]:
return n
# If previous fails, try to fit network at end of pool
n = IPNetwork(str(networks[-1].next().network) + "/" + str(mask))
log.debug(1, 'Trying: %s' % str(n))
if networks[-1].network < n.network and n.network < self.to_ipnetwork().next().network:
return n
else:
# or raise exception, if this is not possible
raise CMException("network_unavailable")
def create(caller_id, name, address, directory, capacity):
"""
Registers new Storage.
@cmview_admin_cm
@param_post{name,string} libvirt's pool name
@param_post{address,string} storage ip address or hostname
@param_post{directory,string} directory on storage
@param_post{capacity,int} maximum storage capacity [MB]
@raises{storage_already_exist,CMException}
@raises{storage_create,CMException}
"""
#error if already exists a storage with the same name
if Storage.objects.filter(name__exact=name).exists():
raise CMException('storage_already_exist')
try:
st = Storage()
st.name = name
st.address = address
st.dir = directory
st.capacity = capacity
st.state = storage_states['ok']
except Exception, e:
log.debug(caller_id, 'Cannot register storage - missing element: %s' % str(e))
raise CMException('storage_create')
def get_command(remote_ip, **kw):
"""
@param_post{remote_ip,string}
@param_post{kw,dict} keyword params
@returns{Command} next command from the que to the asking VM
"""
vm = VM.get_by_ip(remote_ip)
log.debug(0, "Get first command for %s" % vm.id)
command = vm.command_set.filter(state=command_states['pending']).order_by('id')
if len(command) == 0:
return response('ctx_no_command')
command = command[0]
log.debug(0, "First command is %s" % command.id)
command.state = command_states['executing']
command.save()
d = command.dict()
r = response('ok', d)
if int(kw.get('version', 0)) < VERSION:
f = file(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'actions.py'), 'r')
r['actions_file'] = f.read()
f.close()
return r
def hello(vm_ip, **args):
"""
First function which must be called by VMs ctx module. It registers VM with status 'running ctx',
also serves a special role when creating farms (tracking head, and worker nodes)
@parameter{vm_ip,string}
@parameter{args}
"""
vm = VM.get_by_ip(vm_ip)
log.debug(vm.user_id, "Hello from vm %d ip: %s" % (vm.id, vm_ip))
vm.ctx_api_version = args.get('version', None)
vm.state = vm_states['running ctx']
if vm.ssh_username and vm.ssh_key:
Command.execute('add_ssh_key',
vm.user_id,
vm.id,
user=vm.ssh_username,
ssh_key=vm.ssh_key)
if vm.is_head():
Command.register_head(vm)
elif vm.is_farm():
Command.register_node(vm)
try:
vm.save(update_fields=['state', 'ctx_api_version'])
except Exception, e:
log.error(
vm.user_id,
"Cannot update database for vm %d: %s" % (vm.id, e.message))
return response('ctx_error',
"Cannot update database: %s" % e.message)
def assign(self, lease):
if lease.vm == None:
raise CMException('lease_not_attached')
self.lease = lease
self.save()
log.debug(
0, "Attaching ip with comand: %s" % str([
'ssh', '-i', '/var/lib/cc1/.ssh/id_rsa',
'%s@%s' % (lease.vm.node.username, lease.vm.node.address),
'sudo /usr/sbin/cc1_node_public_ip attach %d %s %s' %
(lease.vm.id, lease.vm_address, self.address)
]))
p = subprocess.Popen([
'ssh', '-i', '/var/lib/cc1/.ssh/id_rsa',
'%s@%s' % (lease.vm.node.username, lease.vm.node.address),
'sudo /usr/sbin/cc1_node_public_ip attach %d %s %s' %
(lease.vm.id, lease.vm_address, self.address)
],
stdout=subprocess.PIPE)
p.wait()
log.debug(self.user.id, p.stdout.read())
if p.returncode != 0:
log.error(self.user.id, "SSH error: %d" % p.returncode)
raise CMException('public_ip_failed')
def request(caller_id, mask, name):
"""
Tries to allocate network with specified mask for caller.
@cmview_user
@param_post{mask}
@param_post{name}
@response{dict} UserNetwork.dict property of the newly created UserNetwork
"""
new_net = None
user = User.get(caller_id)
for available_network in AvailableNetwork.objects.all():
log.debug(
user.id,
"Trying to allocate in %s" % str(available_network.to_ipnetwork()))
if available_network.state == available_network_states['ok']:
try:
net = available_network.get_unused_ipnetwork(mask)
new_net = UserNetwork()
new_net.address = str(net.network)
new_net.mask = mask
new_net.name = name
new_net.available_network = available_network
new_net.user = user
new_net.save()
new_net.allocate()
return new_net.dict
except:
continue
if new_net == None:
raise CMException('available_network_not_found')
def get_unused_ipnetwork(self, mask):
"""
@returns Unused subnetwork represented by IPNetwork object
"""
networks = []
for network in self.usernetwork_set.all():
networks.append(network.to_ipnetwork())
networks = sorted(networks)
log.debug(1, "Networks: %s" % str(networks))
if self.mask > mask:
raise CMException("network_to_large")
if len(networks) == 0:
return IPNetwork(self.address + "/" + str(mask))
if IPNetwork(self.address + "/" + str(mask)).network < networks[0].network:
return IPNetwork(self.address + "/" + str(mask))
# Find matching hole in existing networks
for i in xrange(len(networks) - 1):
n = IPNetwork(str(networks[i].next().ip) + "/" + str(mask))
if networks[i] < n and n < networks[i + 1]:
return n
# If previous fails, try to fit network at end of pool
n = IPNetwork(str(networks[-1].next().network) + "/" + str(mask))
log.debug(1, "Trying: %s" % str(n))
if networks[-1].network < n.network and n.network < self.to_ipnetwork().next().network:
return n
else:
# or raise exception, if this is not possible
raise CMException("network_unavailable")
def request(caller_id, mask, name):
"""
Tries to allocate network with specified mask for caller.
@cmview_user
@param_post{mask}
@param_post{name}
@response{dict} UserNetwork.dict property of the newly created UserNetwork
"""
new_net = None
user = User.get(caller_id)
for available_network in AvailableNetwork.objects.all():
log.debug(user.id, "Trying to allocate in %s" % str(available_network.to_ipnetwork()))
if available_network.state == available_network_states['ok']:
try:
net = available_network.get_unused_ipnetwork(mask)
new_net = UserNetwork()
new_net.address = str(net.network)
new_net.mask = mask
new_net.name = name
new_net.available_network = available_network
new_net.user = user
new_net.save()
new_net.allocate()
return new_net.dict
except:
continue
if new_net == None:
raise CMException('available_network_not_found')
def register_node(vm):
"""
Called from CLM when registering worker nodes of the farm
@parameter{vm,vm} VM database mapper
"""
log.debug(vm.user_id, "machine %d: registered as worker node" % vm.id)
try:
hosts = vm.farm.hosts()
log.debug(
vm.user_id, "vm: %d, host list to inject into WNs: %s" %
(vm.id, str(hosts)))
Command.execute('add_ssh_key',
vm.user_id,
vm.id,
user=vm.ssh_username,
ssh_key=vm.ssh_key)
Command.execute('update_hosts',
vm.user_id,
vm.id,
hosts_list=hosts,
user=vm.ssh_username)
Command.execute('set_hostname',
vm.user_id,
vm.id,
hostname=vm.name.replace(vm.farm.name, 'farm'))
except Exception:
log.exception(vm.user_id,
'configuring farm failed for machine %d' % vm.id)
raise Exception('configuring farm failed')
log.info(vm.user_id, 'WN %d registered' % vm.id)
def create(caller_id, name, address, directory, capacity):
"""
Registers new Storage.
@cmview_admin_cm
@param_post{name,string} libvirt's pool name
@param_post{address,string} storage ip address or hostname
@param_post{directory,string} directory on storage
@param_post{capacity,int} maximum storage capacity [MB]
@raises{storage_already_exist,CMException}
@raises{storage_create,CMException}
"""
#error if already exists a storage with the same name
if Storage.objects.filter(name__exact=name).exists():
raise CMException('storage_already_exist')
try:
st = Storage()
st.name = name
st.address = address
st.dir = directory
st.capacity = capacity
st.state = storage_states['ok']
except Exception, e:
log.debug(caller_id,
'Cannot register storage - missing element: %s' % str(e))
raise CMException('storage_create')
def destroy(vms):
"""
@parameter{vms}
@response result
"""
results = []
for vm in vms:
vm = VM.objects.get(pk=vm.id)
log.debug(vm.user.id, "Killing VM id: %s, state: %s" % (vm.id, vm.state))
# Check for VM state
if vm.state in (vm_states['closing'], vm_states['saving']):
results.append({'status': 'vm_already_closing', 'data': ''})
continue
if vm.state in (vm_states['erased'], vm_states['closed']):
results.append({'status': 'vm_wrong_state', 'data': ''})
continue
vm.save_vm = 0
try:
vm.save()
transaction.commit()
vm.lv_destroy()
except Exception, e:
log.exception(vm.user.id, 'error destroying VM: %s' % str(e))
results.append({'status': 'vm_destroy', 'data': ''})
message.error(vm.user_id, 'vm_destroy', {'id': vm.id, 'name': vm.name})
continue
results.append({'status': 'ok', 'data': ''})
def assign(self, lease):
if lease.vm == None:
raise CMException('lease_not_attached')
self.lease = lease
self.save()
log.debug(0, "Attaching ip with comand: %s" % str(['ssh',
'-i',
'/var/lib/cc1/.ssh/id_rsa',
'%s@%s' % (lease.vm.node.username, lease.vm.node.address),
'sudo /usr/sbin/cc1_node_public_ip attach %d %s %s' % (lease.vm.id, lease.vm_address, self.address)]))
p = subprocess.Popen(['ssh',
'-i',
'/var/lib/cc1/.ssh/id_rsa',
'%s@%s' % (lease.vm.node.username, lease.vm.node.address),
'sudo /usr/sbin/cc1_node_public_ip attach %d %s %s' % (lease.vm.id, lease.vm_address, self.address)],
stdout=subprocess.PIPE)
p.wait()
log.debug(self.user.id, p.stdout.read())
if p.returncode != 0:
log.error(self.user.id, "SSH error: %d" % p.returncode)
raise CMException('public_ip_failed')
def execute(command_list):
try:
log.debug(0, "Execute command: %s" % str(command_list))
log_file = file('/var/log/cc1/cm_thread.log', 'a')
r = subprocess.call(command_list, stdout=log_file, stderr=log_file)
log_file.close()
except Exception, e:
log.error(0, "Execute command %s failed: %s" % (str(command_list), e))
def wait_pool_is_unused(self, conn, pool_name):
for i in range(60):
try:
conn.storagePoolLookupByName(pool_name)
log.debug(self.user_id, "Waiting for pool...")
time.sleep(10 * random.random())
except Exception:
log.debug(self.user_id, "Pool doesn't exists")
break
def wait_pool_is_unused(self, conn, pool_name):
for i in range(60):
try:
conn.storagePoolLookupByName(pool_name)
log.debug(self.user_id, "Waiting for pool...")
time.sleep(10 * random.random())
except Exception:
log.debug(self.user_id, "Pool doesn't exists")
break
def execute(name, user_id, vm_id, **kwargs):
"""
Method executes command @prm{name} on the specified VM.
User with id @prm{user_id} must be the owner of that VM.
@parameter{name,string} name of the function to execute
@parameter{user_id,long} id of the declared VM owner
@parameter{vm_id,int} id of the VM on which command needs to be executed
@parameter{kwargs,dict} keyword args for the called function
@raises{ctx_timeout,CMException}
@raises{ctx_execute_command,CMException}
"""
vm = VM.get(user_id, vm_id)
try:
cmd = Command.add_command(name, user_id, vm_id, **kwargs)
transaction.commit()
log.debug(user_id, "Command state %s for machine %s" % (cmd.state, vm_id))
dom = vm.lv_domain()
dom.sendKey(0, 500, [113], 1, 0)
retry = 3
retry_factor = 1.2
retry_time = 1
try:
while retry > 0:
log.debug(user_id, "Check if command %s is finished for machine %s" % (cmd.id, vm_id))
Command.objects.update()
cmd = Command.objects.get(id=cmd.id)
log.debug(user_id, "Checked command status: %s, %s, %s" % (cmd.state, command_states['finished'], bool(cmd.state == command_states['finished'])))
if cmd.state == command_states['finished']:
log.debug(user_id, "Response %s from machine %s" % (cmd.response, vm_id))
break
elif cmd.state == command_states['failed']:
raise CMException('ctx_' + name)
retry -= 1
retry_time *= retry_factor
sleep(retry_time)
except:
raise
finally:
cmd.delete()
if retry == 0:
log.debug(user_id, "Command %s for machine %s - TIMEOUT" % (name, vm_id))
raise CMException('ctx_timeout')
return cmd.response or ''
except CMException:
raise
except Exception:
log.exception(user_id, 'Execute command')
raise CMException('ctx_execute_command')
def execute_with_output(command_list):
try:
log.debug(0, "Execute command (with output): %s" % str(command_list))
p = subprocess.Popen(command_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
r = p.communicate()
except Exception, e:
log.error(
0, "Execute command (with output) %s failed: %s" %
(str(command_list), e))
def attach_node(self):
template = loader.get_template('networking/routed.xml')
context = Context({
'lease': self,
'vm': self.vm
})
network_xml = template.render(context)
log.debug(self.user_network.user.id, "Rendered network template:\n%s" % network_xml)
conn = libvirt.open(self.vm.node.conn_string)
net = conn.networkDefineXML(network_xml)
net.create()
def copy_to_storage(self, vm, image):
"""
Copy vm image from node images pool to selected image on storage. Image
entity should be createt before calling this function
"""
log.debug(vm.user.id, "Preparing temporary storage pool")
if not os.path.exists(os.path.dirname(image.path)):
os.makedirs(os.path.dirname(image.path))
log.debug(vm.user.id, str(['ssh', vm.node.ssh_string, 'cp /images/%d %s' % (vm.id, image.path)]))
if subprocess.call(['ssh', vm.node.ssh_string, 'cp /images/%d %s' % (vm.id, image.path)]):
raise CMException('image_copy_to_storage')
def release_resources(self):
"""
Method releases node's resources.
"""
log.debug(self.user.id, "Detaching vnc for vm %d" % self.id)
if self.vnc_enabled == vnc_states['attached']:
try:
self.detach_vnc()
except Exception, e:
log.debug(self.user.id, str(e))
self.set_state('failed')
self.node.lock()
def attach_node(self):
template = loader.get_template('networking/routed.xml')
context = Context({
'lease': self,
'vm': self.vm
})
network_xml = template.render(context)
log.debug(self.user_network.user.id, "Rendered network template:\n%s" % network_xml)
conn = libvirt.open(self.vm.node.conn_string)
net = conn.networkDefineXML(network_xml)
net.create()
def release_resources(self):
"""
Method releases node's resources.
"""
log.debug(self.user.id, "Detaching vnc for vm %d" % self.id)
if self.vnc_enabled == vnc_states['attached']:
try:
self.detach_vnc()
except Exception, e:
log.debug(self.user.id, str(e))
self.set_state('failed')
self.node.lock()
def attach(self, vm):
"""
Attaches this StorageImage to specified VM. It searches for first free
device and if there's any, tries to attach this to it via Libvirt.
Further it updates DB information.
@parameter{vm,VM} instance of the existing VM
@raises{storage_image_attach,CMException} no free device found or cannot
attach StorageImage
"""
domain = vm.lv_domain()
log.debug(self.user.id, self.disk_controller)
# Get all block devices and find first, unused sdX
# attached_devices = [d.disk_dev for d in Session.query(StorageImage).filter(StorageImage.vm_id == vm.id).all()]
attached_devices = [
d.disk_dev for d in StorageImage.objects.filter(vm_id__exact=vm.id)
]
free_dev = None
# find the first free numbers to be given to disk volume (sda is now integer)
for i in range(2, 12):
if not i in attached_devices:
free_dev = i
break
if free_dev == None:
raise CMException('storage_image_attach')
try:
device_desc = """<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='%(path)s'/>
<target dev='%(dev)s' bus='%(bus)s'/>
<alias name='%(bus)s-%(dev)s'/>
</disk>""" % {
'path': self.path,
# disk_dev name will be in format sd+letter corresponding to the number (e.g: 2->sdb)
'dev': 'sd%s' % chr(free_dev + 98),
'bus': self.disk_controller_name
}
log.debug(self.user.id, device_desc)
domain.attachDevice(device_desc)
except:
log.exception(self.user.id, 'storage attach')
raise CMException('storage_image_attach')
# Update database information
self.disk_dev = free_dev
self.vm = vm
def lock(caller_id, storage_id):
"""
Locks specified Storage.
@cmview_admin_cm
@param_post{storage_id,int}
"""
try:
st = Storage.objects.get(pk=storage_id)
st.state = storage_states['locked']
st.save()
except Exception, e:
log.debug(caller_id, 'Cannot lock storage: %s' % str(e))
raise CMException('storage_lock')
def add(new_user_id):
"""
Adds existing CLM User to CM Users.
@cmview_guest
@param_post{new_user_id,int} id of the existing CLM User
@response{None}
"""
user = User.create(new_user_id)
try:
user.save()
except Exception, e:
log.debug(0, "Adding to DB: %s" % str(e))
raise CMException('user_create')
def add(new_user_id):
"""
Adds existing CLM User to CM Users.
@cmview_guest
@param_post{new_user_id,int} id of the existing CLM User
@response{None}
"""
user = User.create(new_user_id)
try:
user.save()
except Exception, e:
log.debug(0, "Adding to DB: %s" % str(e))
raise CMException('user_create')
def lock(caller_id, storage_id):
"""
Locks specified Storage.
@cmview_admin_cm
@param_post{storage_id,int}
"""
try:
st = Storage.objects.get(pk=storage_id)
st.state = storage_states['locked']
st.save()
except Exception, e:
log.debug(caller_id, 'Cannot lock storage: %s' % str(e))
raise CMException('storage_lock')
def attach(self, vm):
"""
Attaches this StorageImage to specified VM. It searches for first free
device and if there's any, tries to attach this to it via Libvirt.
Further it updates DB information.
@parameter{vm,VM} instance of the existing VM
@raises{storage_image_attach,CMException} no free device found or cannot
attach StorageImage
"""
domain = vm.lv_domain()
log.debug(self.user.id, self.disk_controller)
# Get all block devices and find first, unused sdX
# attached_devices = [d.disk_dev for d in Session.query(StorageImage).filter(StorageImage.vm_id == vm.id).all()]
attached_devices = [d.disk_dev for d in StorageImage.objects.filter(vm_id__exact=vm.id)]
free_dev = None
# find the first free numbers to be given to disk volume (sda is now integer)
for i in range(2, 12):
if not i in attached_devices:
free_dev = i
break
if free_dev == None:
raise CMException('storage_image_attach')
try:
device_desc = """<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='%(path)s'/>
<target dev='%(dev)s' bus='%(bus)s'/>
<alias name='%(bus)s-%(dev)s'/>
</disk>""" % {
'path': self.path,
# disk_dev name will be in format sd+letter corresponding to the number (e.g: 2->sdb)
'dev': 'sd%s' % chr(free_dev + 98),
'bus': self.disk_controller_name
}
log.debug(self.user.id, device_desc)
domain.attachDevice(device_desc)
except:
log.exception(self.user.id, 'storage attach')
raise CMException('storage_image_attach')
# Update database information
self.disk_dev = free_dev
self.vm = vm
def add(new_user_id):
"""
Function adds new user to DB and creates its home directory.
@cmview_guest
@dictkey{new_user_id,int}
@response{None}
"""
user = User.create(new_user_id)
try:
user.save()
except Exception, e:
log.debug(0, "Adding to DB: %s" % str(e))
raise CMException('user_create')
def destroy(farms):
"""
Destroyes farms' VMs (Head and Worker Nodes of each farm) without saving them.
@parameter{farms,list} list of farms to destroy
@response{list(dict)} list of statuses returned by destroyed VMs
@raises{farm_wrong_state,CMException}
@raises{farm_destroy,CMException}
"""
from cm.models.vm import VM
vm_resp = []
for farm in farms:
# those are states in which farm can not be destroyed
if farm.state in (farm_states['init'], farm_states['closing'], farm_states['closed']):
raise CMException('farm_wrong_state')
for farm in farms:
# stop all threads
if farm.state == farm_states['init_head']:
for vm in farm.vms.all():
if vm.is_head():
continue
vm.release_resources()
vm.state = vm_states['closed']
vm.stop_time = datetime.now()
vm.save()
log.debug(vm.user.id, "vm state %s" % vm.state)
r = VM.destroy([farm.head])
else:
for vm in farm.vms.all():
if vm.state == vm_states['init']:
raise CMException('farm_wrong_state')
log.debug(farm.user_id, "killing wn: %s" % farm.vms)
r = VM.destroy(farm.vms.all())
if True in [x['status'] != 'ok' for x in r]:
farm.state = farm_states['failed']
try:
farm.save()
except Exception:
raise CMException('farm_destroy')
vm_resp.append(r)
farm.state = farm_states['closed']
try:
farm.save()
except Exception:
raise CMException('farm_destroy')
log.debug(farm.user_id, "session commited")
for vm in farm.vms.all():
log.debug(vm.user.id, "vm state %s" % vm.state)
return vm_resp
def destroy(farms):
"""
Destroyes farms' VMs (Head and Worker Nodes of each farm) without saving them.
@parameter{farms,list} list of farms to destroy
@response{list(dict)} list of statuses returned by destroyed VMs
@raises{farm_wrong_state,CMException}
@raises{farm_destroy,CMException}
"""
from cm.models.vm import VM
vm_resp = []
for farm in farms:
# those are states in which farm can not be destroyed
if farm.state in (farm_states["init"], farm_states["closing"], farm_states["closed"]):
raise CMException("farm_wrong_state")
for farm in farms:
# stop all threads
if farm.state == farm_states["init_head"]:
for vm in farm.vms.all():
if vm.is_head():
continue
vm.release_resources()
vm.state = vm_states["closed"]
vm.stop_time = datetime.now()
vm.save()
log.debug(vm.user.id, "vm state %s" % vm.state)
r = VM.destroy([farm.head])
else:
for vm in farm.vms.all():
if vm.state == vm_states["init"]:
raise CMException("farm_wrong_state")
log.debug(farm.user_id, "killing wn: %s" % farm.vms)
r = VM.destroy(farm.vms.all())
if True in [x["status"] != "ok" for x in r]:
farm.state = farm_states["failed"]
try:
farm.save()
except Exception:
raise CMException("farm_destroy")
vm_resp.append(r)
farm.state = farm_states["closed"]
try:
farm.save()
except Exception:
raise CMException("farm_destroy")
log.debug(farm.user_id, "session commited")
for vm in farm.vms.all():
log.debug(vm.user.id, "vm state %s" % vm.state)
return vm_resp
def add_command(name, user_id, vm_id, **kwargs):
"""
@parameter{name,string} Command to add for machine @prm{vm_id}
@parameter{user_id,int}
@parameter{vm_id,int}
@parameter{kwargs,dict} key word args for the called function
"""
cmd = Command()
cmd.vm_id = vm_id
cmd.name = name
cmd.args = json.dumps(kwargs)
cmd.state = command_states['pending']
cmd.response = None
log.debug(user_id, "Add command %s for machine %s" % (name, vm_id))
cmd.save()
return cmd
def add_command(name, user_id, vm_id, **kwargs):
"""
@parameter{name,string} Command to add for machine @prm{vm_id}
@parameter{user_id,int}
@parameter{vm_id,int}
@parameter{kwargs,dict} key word args for the called function
"""
cmd = Command()
cmd.vm_id = vm_id
cmd.name = name
cmd.args = json.dumps(kwargs)
cmd.state = command_states['pending']
cmd.response = None
log.debug(user_id, "Add command %s for machine %s" % (name, vm_id))
cmd.save()
return cmd
def libvirt_template(self):
"""
@returns{string} Libvirt XML template
"""
try:
lv_template = loader.get_template("%s.xml" % self.node.driver)
c = Context({'vm': self,
'uuid': uuid.uuid1(),
'memory': self.template.memory * 1024,
'cpu': self.template.cpu,
'image_path': self.path
})
# and render it
domain_template = lv_template.render(c)
except Exception, e:
log.debug(self.user.id, str(e))
def ctx_log(*arg, **kw):
"""
Decorator for functions requiring only \b guest's privilidges.
src.cm.utils.decorators.genericlog() is called with parameters:
- \c is_user=False
- \c is_superuser=False
@par Decorated function's declaration
@code
@guest_log[(log=<False|True>)]
function (*arg, **kw)
@endcode
@par Decorated function's call
@code
function (*arg, **kw)
@endcode
"""
def logwrapper(fun):
@wraps(fun)
def wrapper(request, *args, **kwargs):
data = request.GET.dict()
data['remote_ip'] = request.META.get('REMOTE_ADDR')
gen_exception = False
log_enabled = kw.get('log', False)
name = '%s.%s' % (fun.__module__.replace('cm.views.',
''), fun.__name__)
if log_enabled:
log.debug(0, '=' * 100)
log.debug(0, 'Function: %s' % name)
log.debug(0, 'Args:\n%s' % json.dumps(data, indent=4))
with transaction.commit_manually():
try:
# Execute function
resp = fun(**data)
transaction.commit()
except CMException, e:
transaction.rollback()
log.exception(0, 'CMException %s' % e)
resp = e.response
except Exception, e:
transaction.rollback()
gen_exception = True
resp = response('cm_error', str(e))
if resp['status'] != 'ok' and not log_enabled:
log.debug(0, '=' * 100)
log.debug(0, 'Function: %s' % name)
log.debug(0, 'ARGS: %s' % str(data))
if resp['status'] != 'ok' or log_enabled:
if gen_exception:
log.exception(0, 'General exception')
log.debug(0, 'Response: %s' % resp or 'None')
return HttpResponse(json.dumps(resp, default=json_convert))
def attach(self, vm):
"""
Attaches this StorageImage to specified VM. It searches for first free
device and if there's any, tries to attach this to it via Libvirt.
Further it updates DB information.
@parameter{vm,VM} instance of the existing VM
@raises{storage_image_attach,CMException} no free device found or cannot
attach StorageImage
"""
domain = vm.lv_domain()
log.debug(self.user.id, self.disk_controller)
disk_controller_name = disk_controllers_reversed[self.disk_controller]
# Get all block devices and find first, unused sdX
# attached_devices = [d.disk_dev for d in Session.query(StorageImage).filter(StorageImage.vm_id == vm.id).all()]
attached_devices = [
d.disk_dev for d in IsoImage.objects.filter(vm_id__exact=vm.id)
]
free_dev = 'sdz'
if free_dev == attached_devices:
raise CMException('iso_image_attach')
try:
device_desc = """<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='%(path)s'/>
<target dev='%(dev)s' bus='%(bus)s'/>
<alias name='%(bus)s-%(dev)s'/>
</disk>""" % {
'path': self.path,
'dev': 'sd%s' % free_dev,
'bus': disk_controller_name
}
log.debug(self.user.id, device_desc)
domain.attachDevice(device_desc)
except:
log.exception(self.user.id, 'iso attach')
raise CMException('iso_image_attach')
# Update database information
self.disk_dev = free_dev
self.vm = vm
def lock(caller_id, storage_id):
"""
Locks storage with id \c storage_id.
@cmview_admin_cm
@parameter{caller_id,int}
@parameter{storage_id,int}
@response{None}
"""
try:
st = Storage.objects.get(pk=storage_id)
st.state = storage_states['locked']
st.save()
except Exception, e:
log.debug(caller_id, 'Cannot lock storage: %s' % str(e))
raise CMException('storage_lock')
def ctx_log(*arg, **kw):
"""
Decorator for functions requiring only \b guest's privilidges.
src.cm.utils.decorators.genericlog() is called with parameters:
- \c is_user=False
- \c is_superuser=False
@par Decorated function's declaration
@code
@guest_log[(log=<False|True>)]
function (*arg, **kw)
@endcode
@par Decorated function's call
@code
function (*arg, **kw)
@endcode
"""
def logwrapper(fun):
@wraps(fun)
def wrapper(request, *args, **kwargs):
data = request.GET.dict()
data['remote_ip'] = request.META.get('REMOTE_ADDR')
gen_exception = False
log_enabled = kw.get('log', False)
name = '%s.%s' % (fun.__module__.replace('cm.views.', ''), fun.__name__)
if log_enabled:
log.debug(0, '=' * 100)
log.debug(0, 'Function: %s' % name)
log.debug(0, 'Args:\n%s' % json.dumps(data, indent=4))
with transaction.commit_manually():
try:
# Execute function
resp = fun(**data)
transaction.commit()
except CMException, e:
transaction.rollback()
log.exception(0, 'CMException %s' % e)
resp = e.response
except Exception, e:
transaction.rollback()
gen_exception = True
resp = response('cm_error', str(e))
if resp['status'] != 'ok' and not log_enabled:
log.debug(0, '=' * 100)
log.debug(0, 'Function: %s' % name)
log.debug(0, 'ARGS: %s' % str(data))
if resp['status'] != 'ok' or log_enabled:
if gen_exception:
log.exception(0, 'General exception')
log.debug(0, 'Response: %s' % resp or 'None')
return HttpResponse(json.dumps(resp, default=json_convert))
def libvirt_template(self):
"""
@returns{string} Libvirt XML template
"""
try:
lv_template = loader.get_template("%s.xml" % self.node.driver)
c = Context({
'vm': self,
'uuid': uuid.uuid1(),
'memory': self.template.memory * 1024,
'cpu': self.template.cpu,
'image_path': self.path
})
# and render it
domain_template = lv_template.render(c)
except Exception, e:
log.debug(self.user.id, str(e))
def reset(self):
"""
Restarts VM.
-# Connects to Libvirt.
-# Sets VM's state as *restart*.
-# Restarts it.
-# Sets VM's state back as *running*.
"""
log.debug(self.vm.user_id, "VM Reboot")
self.vm.set_state('restart')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot set vm state')
return
try:
domain = self.vm.lv_domain()
domain.reset(0)
self.vm.libvirt_id = domain.ID()
except:
self.vm.node.lock()
message.error(self.vm.user_id, 'vm_restart', {
'id': self.vm.id,
'name': self.vm.name
})
self.vm.set_state('failed')
log.exception(self.vm.user_id, 'Cannot restart machine')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot update libvirt ID')
return
return
# Update state state
self.vm.set_state('running')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot set vm state')
return
def attach(self, vm):
"""
Attaches this StorageImage to specified VM. It searches for first free
device and if there's any, tries to attach this to it via Libvirt.
Further it updates DB information.
@parameter{vm,VM} instance of the existing VM
@raises{storage_image_attach,CMException} no free device found or cannot
attach StorageImage
"""
domain = vm.lv_domain()
log.debug(self.user.id, self.disk_controller)
disk_controller_name = disk_controllers_reversed[self.disk_controller]
# Get all block devices and find first, unused sdX
# attached_devices = [d.disk_dev for d in Session.query(StorageImage).filter(StorageImage.vm_id == vm.id).all()]
attached_devices = [d.disk_dev for d in IsoImage.objects.filter(vm_id__exact=vm.id)]
free_dev = 'sdz'
if free_dev == attached_devices:
raise CMException('iso_image_attach')
try:
device_desc = """<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='%(path)s'/>
<target dev='%(dev)s' bus='%(bus)s'/>
<alias name='%(bus)s-%(dev)s'/>
</disk>""" % {
'path': self.path,
'dev': 'sd%s' % free_dev,
'bus': disk_controller_name
}
log.debug(self.user.id, device_desc)
domain.attachDevice(device_desc)
except:
log.exception(self.user.id, 'iso attach')
raise CMException('iso_image_attach')
# Update database information
self.disk_dev = free_dev
self.vm = vm
def register_head(vm):
"""
Head registration process:
- Creates ssh keys and sets their values for WN;
- Inserts VMs into the database;
- Then starts VMThreads which create actual machines.
Called when registering farms head.
@parameter{vm,VM} instance of the VM to be registered as head
"""
log.debug(vm.user_id, "machine %d: registered as head" % vm.id)
log.debug(vm.user_id, "creating lock for machine %d in farm %d" % (vm.id, vm.farm_id))
# skip if farm is already configured - reboot head
if vm.is_head() == True and vm.farm.state == farm_states['running']:
return
vms = []
if vm.farm.state == farm_states['init_head']:
vm.farm.state = farm_states['running']
vm.farm.save()
log.info(vm.user_id, 'generating ssh keys on head %d' % vm.id)
try:
r = Command.execute('generate_key', vm.user_id, vm.id)
r = json.loads(r)
log.info(vm.user_id, 'generated key: %s for machine %d' % (r, vm.id))
for wn in vm.farm.vms.all():
wn.ssh_username = '******'
wn.ssh_key = r
wn.save()
if not wn.is_head():
vms.append(wn)
ssh_username = '******'
ssh_key = r
log.debug(vm.user_id, 'appended %d vms to farm [id:%d]' % (vm.farm.vms.count() - 1, vm.id)) # excluding head
Command.add_command('add_ssh_key', vm.user_id, vm.id, user=ssh_username, ssh_key=ssh_key)
Command.add_command('update_hosts', vm.user_id, vm.id, hosts_list=vm.farm.hosts(), user=ssh_username)
Command.execute('set_hostname', vm.user_id, vm.id, hostname=vm.name.replace(vm.farm.name, 'farm'))
except Exception:
log.exception(vm.user_id, '')
vm.farm.state = farm_states['unconfigured']
message.error(vm.id, 'farm_create', {'id': vm.farm.id, 'name': vm.farm.name})
log.info(vm.user_id, 'Head %d registered' % vm.id)
shared = {"counter": len(vms), "lock": threading.Lock()}
for vm in vms:
thread = VMThread(vm, 'create', shared)
thread.start()
log.debug(vm.user_id, 'vm thread created [vm id:%d]' % vm.id)
def network_template(self):
"""
@returns{string} Libvirt network XML template
"""
# Try to configure django
try:
django_settings.configure()
except:
pass
try:
# Open template file
template = open("%s/%s-network.xml" % (settings.TEMPLATE_DIR, settings.NETWORK_TYPE)).read()
# Create django template
lv_template = loader.get_template_from_string(template)
c = Context({'vm': self})
lv_template = lv_template.render(c)
except Exception, e:
log.debug(self.user.id, str(e))
def copy_to_node(self, vm):
"""
Copy vm image from storage to node
@raises{vm_create,CMException}
"""
r = subprocess.call(['ssh', vm.node.ssh_string, 'chmod a+rw %s' % (vm.system_image.path)])
log.debug(vm.user.id, "Copy image by ssh")
log.debug(vm.user.id, str(['ssh', vm.node.ssh_string, 'cp %s /images/%d' % (vm.system_image.path, vm.id)]))
if subprocess.call(['ssh', vm.node.ssh_string, 'cp %s /images/%d' % (vm.system_image.path, vm.id)]):
message.error(vm.user_id, 'vm_create', {'id': vm.id, 'name': vm.name})
raise CMException('vm_create')
if subprocess.call(['ssh', vm.node.ssh_string, 'chmod a+rw /images/%d' % vm.id]):
message.error(vm.user_id, 'vm_create', {'id': vm.id, 'name': vm.name})
raise CMException('vm_create')
return
def reset(self):
"""
Restarts VM.
-# Connects to Libvirt.
-# Sets VM's state as *restart*.
-# Restarts it.
-# Sets VM's state back as *running*.
"""
log.debug(self.vm.user_id, "VM Reboot")
self.vm.set_state('restart')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot set vm state')
return
try:
domain = self.vm.lv_domain()
domain.reset(0)
self.vm.libvirt_id = domain.ID()
except:
self.vm.node.lock()
message.error(self.vm.user_id, 'vm_restart', {'id': self.vm.id, 'name': self.vm.name})
self.vm.set_state('failed')
log.exception(self.vm.user_id, 'Cannot restart machine')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot update libvirt ID')
return
return
# Update state state
self.vm.set_state('running')
try:
self.vm.save()
except:
log.exception(self.vm.user_id, 'Cannot set vm state')
return
class VMThread(threading.Thread):
def __init__(self, vm, action, shared=None):
threading.Thread.__init__(self)
self.vm = vm
self.action = action
self.shared = shared
def terminate(self):
log.info(0, "Terminate vm %d" % (self.vm.id))
return self._Thread__stop()
def create(self):
"""
Starts VM's thread.
-# Gets VM's record from database (basing on vm_id) (if exists).
-# Copies image chosen for this VM.
-# Connects to Libvirt and generate template for it.
-# Creates Libvirt domain.
-# Sets VM's state as *running*
-# If VM is element of farm, it sets proper farm state.
"""
try:
log.info(
self.vm.user_id, "Copy image from %s to %s" %
(self.vm.system_image.path, self.vm.path))
self.vm.system_image.copy_to_node(self.vm)
except Exception, e:
log.exception(self.vm.user_id,
'Libvirt error for %d: %s' % (self.vm.id, e))
self.vm.set_state('failed')
message.error(self.vm.user_id, 'vm_create', {
'id': self.vm.id,
'name': self.vm.name
})
self.vm.node.lock()
self.vm.save(update_fields=['state'])
return
#TODO: network part
log.debug(self.vm.user_id, "Attaching network")
try:
for lease in self.vm.lease_set.all():
lease.attach_node()
except Exception, e:
log.exception(self.vm.user_id, "Cannot create network")
self.vm.set_state('failed')
self.vm.save(update_fields=['state'])
message.error(self.vm.user_id, 'vm_create', {
'id': self.vm.id,
'name': self.vm.name
})
self.vm.node.lock()
self.vm.node.save()
return
def register_node(vm):
"""
Called from CLM when registering worker nodes of the farm
@parameter{vm,vm} VM database mapper
"""
log.debug(vm.user_id, "machine %d: registered as worker node" % vm.id)
try:
hosts = vm.farm.hosts()
log.debug(vm.user_id, "vm: %d, host list to inject into WNs: %s" % (vm.id, str(hosts)))
Command.execute('add_ssh_key', vm.user_id, vm.id, user=vm.ssh_username, ssh_key=vm.ssh_key)
Command.execute('update_hosts', vm.user_id, vm.id, hosts_list=hosts, user=vm.ssh_username)
Command.execute('set_hostname', vm.user_id, vm.id, hostname=vm.name.replace(vm.farm.name, 'farm'))
except Exception:
log.exception(vm.user_id, 'configuring farm failed for machine %d' % vm.id)
raise Exception('configuring farm failed')
log.info(vm.user_id, 'WN %d registered' % vm.id)
def copy_to_storage(self, vm, image):
"""
Copy vm image from node images pool to selected image on storage. Image
entity should be createt before calling this function
"""
log.debug(vm.user.id, "Preparing temporary storage pool")
if not os.path.exists(os.path.dirname(image.path)):
os.makedirs(os.path.dirname(image.path))
log.debug(
vm.user.id,
str([
'ssh', vm.node.ssh_string,
'cp /images/%d %s' % (vm.id, image.path)
]))
if subprocess.call([
'ssh', vm.node.ssh_string,
'cp /images/%d %s' % (vm.id, image.path)
]):
raise CMException('image_copy_to_storage')
def format(self):
if not os.path.exists(os.path.dirname(self.image.path)):
os.makedirs(os.path.dirname(self.image.path))
format_cmd = disk_format_commands[disk_filesystems_reversed[self.filesystem]].split()
if format_cmd:
tmp_dir = '/var/lib/cc1/images-tmp/'
tmp_path = os.path.join(tmp_dir, os.path.split(self.image.path)[1])
if not os.path.exists(os.path.dirname(tmp_dir)):
os.makedirs(os.path.dirname(tmp_dir))
else:
tmp_path = str(self.image.path)
log.debug(self.image.user.id, 'stage [1/6] truncate partition file')
if self.exec_cmd(['truncate', '-s', '%dM' % self.image.size, '%s' % tmp_path]):
return 'failed'
self.set_progress(random.randint(0, 15))
if format_cmd:
format_cmd.append('%s' % tmp_path)
log.debug(self.image.user.id, 'stage [2/6] creating partition filesystem')
if self.exec_cmd(format_cmd):
return 'failed'
self.set_progress(random.randint(15, 50))
log.debug(self.image.user.id, 'stage [3/6] creating disk')
if self.exec_cmd(['/usr/bin/ddrescue', '-S', '-o', '1048576', '%s' % tmp_path, str(self.image.path)]):
return 'failed'
self.set_progress(random.randint(50, 80))
log.debug(self.image.user.id, 'stage [4/6] creating new partition table')
if self.exec_cmd(['/sbin/parted', '-s', str(self.image.path), 'mklabel', 'msdos']):
return 'failed'
self.set_progress(random.randint(80, 90))
log.debug(self.image.user.id, 'stage [5/6] adding partition')
if self.exec_cmd(['/sbin/parted', '-s', str(self.image.path), 'mkpart', 'primary', '1048576b', '100%']):
return 'failed'
self.set_progress(random.randint(90, 100))
log.info(self.image.user.id, 'disk succesfully formatted')
def run(self):
if os.path.exists(self.image.path):
self.image.state = image_states['failed']
self.image.save(update_fields=['state'])
log.error(self.image.user.id, "Destination image %d for user %d exists! Aborting creation" % (self.image.id, self.image.user.id))
return
self.image.progress = 0
if self.format() == 'failed':
self.image.state = image_states['failed']
self.image.save(update_fields=['state'])
else:
self.image.progress = 100
self.image.state = image_states['ok']
self.image.save(update_fields=['state', 'progress'])
log.debug(self.image.user.id, 'stage [6/6] cleaning..')
try:
os.remove('%s' % os.path.join('/var/lib/cc1/images-tmp/', os.path.split(self.image.path)[1]))
except Exception, e:
log.error(self.image.user.id, 'error remove file: %s' % str(e))
def network_template(self):
"""
@returns{string} Libvirt network XML template
"""
# Try to configure django
try:
django_settings.configure()
except:
pass
try:
# Open template file
template = open(
"%s/%s-network.xml" %
(settings.TEMPLATE_DIR, settings.NETWORK_TYPE)).read()
# Create django template
lv_template = loader.get_template_from_string(template)
c = Context({'vm': self})
lv_template = lv_template.render(c)
except Exception, e:
log.debug(self.user.id, str(e))
def delete(self):
"""
Method releases resources taken by deleted ex VM.
"""
VM.objects.update()
self = VM.objects.get(pk=self.id)
if self.save_vm > 0:
log.debug(self.user.id, 'Saving image')
self.save_image()
log.debug(self.user.id, 'Removing image')
self.remove()
log.debug(self.user.id, 'Releasing resources')
self.release_resources()
# Update vm state
self.set_state('closed')
"""
#TODO:
if self.is_head():
self.farm.state = farm_states['closed']
"""
self.stop_time = datetime.now()
try:
self.save(update_fields=['state', 'stop_time'])
except Exception, e:
log.exception(self.user.id, "Cannot commit changes: %s" % e)
def wrapper(request, *args, **kwargs):
data = request.GET.dict()
data['remote_ip'] = request.META.get('REMOTE_ADDR')
gen_exception = False
log_enabled = kw.get('log', False)
name = '%s.%s' % (fun.__module__.replace('cm.views.',
''), fun.__name__)
if log_enabled:
log.debug(0, '=' * 100)
log.debug(0, 'Function: %s' % name)
log.debug(0, 'Args:\n%s' % json.dumps(data, indent=4))
with transaction.commit_manually():
try:
# Execute function
resp = fun(**data)
transaction.commit()
except CMException, e:
transaction.rollback()
log.exception(0, 'CMException %s' % e)
resp = e.response
except Exception, e:
transaction.rollback()
gen_exception = True
resp = response('cm_error', str(e))
def delete(self):
"""
Method releases resources taken by deleted ex VM.
"""
VM.objects.update()
self = VM.objects.get(pk=self.id)
if self.save_vm > 0:
log.debug(self.user.id, 'Saving image')
self.save_image()
log.debug(self.user.id, 'Removing image')
self.remove()
log.debug(self.user.id, 'Releasing resources')
self.release_resources()
# Update vm state
self.set_state('closed')
"""
#TODO:
if self.is_head():
self.farm.state = farm_states['closed']
"""
self.stop_time = datetime.now()
try:
self.save(update_fields=['state', 'stop_time'])
except Exception, e:
log.exception(self.user.id, "Cannot commit changes: %s" % e)
def finish_command(remote_ip, command_id, status, returns=None, **kw):
"""
REST stub for finish_command
@param_post{remote_ip,string}
@param_post{command_id,string} hash string identyfing command
@param_post{status,string}
@param_post{returns,dict} dictionary containing VM returned values
@param_post{kw,dict} keyword params
"""
vm = VM.get_by_ip(remote_ip)
if returns:
returns = json.dumps(returns)
log.debug(0, "Select command %s %s" % (command_id, status))
try:
command = vm.command_set.get(id=command_id)
except Command.DoesNotExist:
return
log.debug(0, "Finish command %s" % command)
if command is None:
for c in Command.objects.all():
log.debug(0, 'Finish - Available cmds id:%s, state:%s, name:%s, vmid:%s' % (c.id, c.state, c.name, c.vm_id))
return
log.debug(vm.user_id, "command state %s" % command.state)
command.response = returns
command.state = command_states[status]
log.debug(vm.user_id, "Finish command %s" % command.id)
command.save()
r = response('ok')
if int(kw.get('version', 0)) < VERSION:
f = file(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'actions.py'), 'r')
r['actions_file'] = f.read()
f.close()
return r
