def process_expiring_vms_task():
with app.app_context():
proxmox = connect_proxmox()
db = connect_db()
connect_starrs()
pools = get_pools(proxmox, db)
expired_vms = []
for pool in pools:
user = User(pool)
expiring_vms = []
vms = user.vms
for vm in vms:
vm = VM(vm['vmid'])
days = (vm.expire - datetime.date.today()).days
if days in [10, 7, 3, 1, 0, -1, -2, -3, -4, -5, -6]:
expiring_vms.append([vm.id, vm.name, days])
if days <= 0:
expired_vms.append([vm.id, vm.name, days])
vm.stop()
elif days <= -7:
logging.info(
'Deleting {} ({}) as it has been at least a week since expiration.'
.format(vm.name, vm.id))
send_stop_ssh_tunnel(vm.id)
delete_vm_task(vm.id)
if expiring_vms:
send_vm_expire_email(pool, expiring_vms)
if expired_vms:
send_rtp_vm_delete_email(expired_vms)
def create_vm_task(user, name, cores, memory, disk, iso):
with app.app_context():
job = get_current_job()
proxmox = connect_proxmox()
db = connect_db()
starrs = connect_starrs()
logging.info('[{}] Creating VM.'.format(name))
set_job_status(job, 'creating VM')
vmid = create_vm(proxmox, user, name, cores, memory, disk, iso)
logging.info(
'[{}] Waiting until Proxmox is done provisioning.'.format(name))
set_job_status(job, 'waiting for Proxmox')
timeout = 20
retry = 0
while retry < timeout:
if not VM(vmid).is_provisioned():
retry += 1
time.sleep(3)
continue
break
if retry == timeout:
logging.info('[{}] Failed to provision, deleting.'.format(name))
set_job_status(job, 'failed to provision')
delete_vm_task(vmid)
return
logging.info('[{}] Registering in STARRS.'.format(name))
set_job_status(job, 'registering in STARRS')
vm = VM(vmid)
ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
register_starrs(starrs, name, app.config['STARRS_USER'], vm.get_mac(),
ip)
set_job_status(job, 'setting VM expiration')
get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
logging.info('[{}] VM successfully provisioned.'.format(name))
set_job_status(job, 'complete')
def delete_vm_task(vmid):
with app.app_context():
db = connect_db()
starrs = connect_starrs()
vm = VM(vmid)
vm.delete()
delete_starrs(starrs, vm.name)
delete_vm_expire(db, vmid)
def iso_eject(vmid):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
vm.eject_iso()
return '', 200
else:
return '', 403
def iso_mount(vmid, iso):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
iso = '{}:iso/{}'.format(app.config['PROXMOX_ISO_STORAGE'], iso)
vm = VM(vmid)
vm.mount_iso(iso)
return '', 200
else:
return '', 403
def vm_disk(vmid, disk, size):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
usage_check = user.check_usage(0, 0, size)
if usage_check:
return usage_check
vm.resize_disk(disk, size)
return '', 200
else:
return '', 403
def set_boot_order(vmid):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
boot_order = []
for key in sorted(request.form):
boot_order.append(request.form[key])
vm = VM(vmid)
vm.set_boot_order(boot_order)
return '', 200
else:
return '', 403
def vm_console(vmid):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
stop_ssh_tunnel(vm.id, ssh_tunnels)
port = str(5900 + int(vmid))
token = add_vnc_target(port)
node = '{}.csh.rit.edu'.format(vm.node)
logging.info('creating SSH tunnel to %s for VM %s', node, vm.id)
tunnel = start_ssh_tunnel(node, port)
ssh_tunnels.append(tunnel)
vm.start_vnc(port)
return token, 200
else:
return '', 403
def vm_cpu(vmid, cores):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
cur_cores = vm.cpu
if cores >= cur_cores:
if vm.qmpstatus == 'running' or vm.qmpstatus == 'paused':
usage_check = user.check_usage(cores - cur_cores, 0, 0)
else:
usage_check = user.check_usage(cores, 0, 0)
if usage_check:
return usage_check
vm.set_cpu(cores)
return '', 200
else:
return '', 403
def vm_mem(vmid, mem):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
cur_mem = vm.mem // 1024
if mem >= cur_mem:
if vm.qmpstatus == 'running' or vm.qmpstatus == 'paused':
usage_check = user.check_usage(0, mem - cur_mem, 0)
else:
usage_check = user.check_usage(0, mem, 0)
if usage_check:
return usage_check
vm.set_mem(mem * 1024)
return '', 200
else:
return '', 403
def process_expiring_vms_task():
with app.app_context():
proxmox = connect_proxmox()
db = connect_db()
starrs = connect_starrs()
pools = get_pools(proxmox, db)
for pool in pools:
user = User(pool)
expiring_vms = []
vms = user.vms
for vm in vms:
vm = VM(vm['vmid'])
days = (vm.expire - datetime.date.today()).days
if days in [10, 7, 3, 1, 0]:
name = vm.name
expiring_vms.append([vm.name, days])
if days == 0:
vm.stop()
if expiring_vms:
send_vm_expire_email('com6056', expiring_vms)
def vm_renew(vmid):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
renew_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
for interface in vm.interfaces:
if interface[2] != 'No IP':
renew_ip(starrs, interface[2])
return '', 200
else:
return '', 403
def vm_console(vmid):
user = User(session['userinfo']['preferred_username'])
proxmox = connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
port = str(5900 + int(vmid))
token = add_vnc_target(port)
node = "{}.csh.rit.edu".format(vm.node)
tunnel = next(
(tunnel
for tunnel in ssh_tunnels if tunnel.local_bind_port == int(port)),
None)
if tunnel:
if tunnel.ssh_host != node:
print(
"Tunnel already exists for VM {} to the wrong Proxmox node."
.format(vmid))
tunnel.stop()
ssh_tunnels.remove(tunnel)
print("Creating SSH tunnel to {} for VM {}.".format(
node, vmid))
tunnel = start_ssh_tunnel(node, port)
ssh_tunnels.append(tunnel)
vm.start_vnc(port)
else:
print("Tunnel already exists to {} for VM {}.".format(
node, vmid))
else:
print("Creating SSH tunnel to {} for VM {}.".format(node, vmid))
tunnel = start_ssh_tunnel(node, port)
ssh_tunnels.append(tunnel)
vm.start_vnc(port)
return token, 200
else:
return '', 403
def delete_vm_task(vmid):
with app.app_context():
db = connect_db()
starrs = connect_starrs()
vm = VM(vmid)
# do this before deleting the VM since it is hard to reconcile later
retry = 0
while retry < 3:
try:
delete_starrs(starrs, vm.name)
break
except:
retry += 1
time.sleep(3)
continue
if vm.status != 'stopped':
vm.stop()
retry = 0
while retry < 10:
time.sleep(3)
if vm.status == 'stopped':
break
retry += 1
vm.delete()
delete_vm_expire(db, vmid)
def vm_details(vmid):
user = User(session['userinfo']['preferred_username'])
proxmox = connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
usage_check = user.check_usage(vm.cpu, vm.mem, 0)
return render_template('vm_details.html',
user=user,
vm=vm,
usage=user.usage,
limits=user.limits,
usage_check=usage_check)
else:
abort(403)
def usage(self):
usage = {}
usage['cpu'] = 0
usage['mem'] = 0
usage['disk'] = 0
if self.rtp:
return usage
vms = self.vms
for vm in vms:
if 'status' in vm:
vm = VM(vm['vmid'])
if vm.status in ('running', 'paused'):
usage['cpu'] += int(vm.cpu)
usage['mem'] += int(vm.mem) / 1024
for disk in vm.disks:
usage['disk'] += int(disk[1])
return usage
def usage(self):
usage = dict()
usage['cpu'] = 0
usage['mem'] = 0
usage['disk'] = 0
if self.rtp:
return usage
vms = self.vms
for vm in vms:
if 'status' in vm:
vm = VM(vm['vmid'])
if vm.status == 'running' or vm.status == 'paused':
usage['cpu'] += int(vm.cpu * vm.config.get('sockets', 1))
usage['mem'] += (int(vm.mem) / 1024)
for disk in vm.disks:
usage['disk'] += int(disk[1])
return usage
def delete_vm_task(vmid):
with app.app_context():
db = connect_db()
starrs = connect_starrs()
vm = VM(vmid)
if vm.status != 'stopped':
vm.stop()
retry = 0
while retry < 10:
time.sleep(3)
if vm.status == 'stopped':
break
retry += 1
vm.delete()
delete_starrs(starrs, vm.name)
delete_vm_expire(db, vmid)
def setup_template_task(template_id, name, user, ssh_key, cores, memory):
with app.app_context():
job = get_current_job()
proxmox = connect_proxmox()
starrs = connect_starrs()
db = connect_db()
print("[{}] Retrieving template info for template {}.".format(
name, template_id))
template = get_template(db, template_id)
print("[{}] Cloning template {}.".format(name, template_id))
job.meta['status'] = 'cloning template'
job.save_meta()
vmid, mac = clone_vm(proxmox, template_id, name, user)
print("[{}] Registering in STARRS.".format(name))
job.meta['status'] = 'registering in STARRS'
job.save_meta()
ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
register_starrs(starrs, name, app.config['STARRS_USER'], mac, ip)
get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
print("[{}] Giving Proxmox some time to finish cloning.".format(name))
job.meta['status'] = 'waiting for Proxmox'
time.sleep(15)
print("[{}] Setting CPU and memory.".format(name))
job.meta['status'] = 'setting CPU and memory'
job.save_meta()
vm = VM(vmid)
vm.set_cpu(cores)
vm.set_mem(memory)
print("[{}] Applying cloud-init config.".format(name))
job.meta['status'] = 'applying cloud-init'
vm.set_ci_user(user)
vm.set_ci_ssh_key(ssh_key)
vm.set_ci_network()
print(
"[{}] Waiting for STARRS to propogate before starting VM.".format(
name))
job.meta['status'] = 'waiting for STARRS'
job.save_meta()
time.sleep(90)
print("[{}] Starting VM.".format(name))
job.meta['status'] = 'starting VM'
job.save_meta()
vm.start()
print("[{}] Template successfully provisioned.".format(name))
job.meta['status'] = 'completed'
job.save_meta()
def setup_template_task(template_id, name, user, ssh_key, cores, memory):
with app.app_context():
job = get_current_job()
proxmox = connect_proxmox()
starrs = connect_starrs()
db = connect_db()
logging.info('[{}] Retrieving template info for template {}.'.format(
name, template_id))
get_template(db, template_id)
logging.info('[{}] Cloning template {}.'.format(name, template_id))
set_job_status(job, 'cloning template')
vmid = clone_vm(proxmox, template_id, name, user)
logging.info(
'[{}] Waiting until Proxmox is done provisioning.'.format(name))
set_job_status(job, 'waiting for Proxmox')
timeout = 25
retry = 0
while retry < timeout:
if not VM(vmid).is_provisioned():
retry += 1
time.sleep(12)
continue
break
if retry == timeout:
logging.info('[{}] Failed to provision, deleting.'.format(name))
set_job_status(job, 'failed to provision')
delete_vm_task(vmid)
return
logging.info('[{}] Registering in STARRS.'.format(name))
set_job_status(job, 'registering in STARRS')
vm = VM(vmid)
ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
register_starrs(starrs, name, app.config['STARRS_USER'], vm.get_mac(),
ip)
get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
logging.info('[{}] Setting CPU and memory.'.format(name))
set_job_status(job, 'setting CPU and memory')
vm.set_cpu(cores)
vm.set_mem(memory)
logging.info('[{}] Applying cloud-init config.'.format(name))
set_job_status(job, 'applying cloud-init')
vm.set_ci_user(user)
vm.set_ci_ssh_key(ssh_key)
vm.set_ci_network()
logging.info(
'[{}] Waiting for STARRS to propogate before starting VM.'.format(
name))
set_job_status(job, 'waiting for STARRS')
job.save_meta()
time.sleep(90)
logging.info('[{}] Starting VM.'.format(name))
set_job_status(job, 'starting VM')
job.save_meta()
vm.start()
logging.info('[{}] Template successfully provisioned.'.format(name))
set_job_status(job, 'completed')
job.save_meta()
def vm_power(vmid, action):
user = User(session['userinfo']['preferred_username'])
connect_proxmox()
if user.rtp or int(vmid) in user.allowed_vms:
vm = VM(vmid)
if action == 'start':
vmconfig = vm.config
usage_check = user.check_usage(vmconfig['cores'],
vmconfig['memory'], 0)
if usage_check:
return usage_check
vm.start()
elif action == 'stop':
vm.stop()
send_stop_ssh_tunnel(vmid)
elif action == 'shutdown':
vm.shutdown()
send_stop_ssh_tunnel(vmid)
elif action == 'reset':
vm.reset()
elif action == 'suspend':
vm.suspend()
send_stop_ssh_tunnel(vmid)
elif action == 'resume':
vm.resume()
return '', 200
else:
return '', 403
def setup_template_task(template_id, name, user, password, cores, memory):
with app.app_context():
job = get_current_job()
proxmox = connect_proxmox()
starrs = connect_starrs()
db = connect_db()
print("[{}] Retrieving template info for template {}.".format(
name, template_id))
template = get_template(db, template_id)
print("[{}] Cloning template {}.".format(name, template_id))
job.meta['status'] = 'cloning template'
job.save_meta()
vmid, mac = clone_vm(proxmox, template_id, name, user)
print("[{}] Registering in STARRS.".format(name))
job.meta['status'] = 'registering in STARRS'
job.save_meta()
ip = get_next_ip(starrs, app.config['STARRS_IP_RANGE'])
register_starrs(starrs, name, app.config['STARRS_USER'], mac, ip)
get_vm_expire(db, vmid, app.config['VM_EXPIRE_MONTHS'])
print("[{}] Setting CPU and memory.".format(name))
job.meta['status'] = 'setting CPU and memory'
job.save_meta()
vm = VM(vmid)
vm.set_cpu(cores)
vm.set_mem(memory)
print(
"[{}] Waiting for STARRS to propogate before starting VM.".format(
name))
job.meta['status'] = 'waiting for STARRS'
job.save_meta()
time.sleep(90)
print("[{}] Starting VM.".format(name))
job.meta['status'] = 'starting VM'
job.save_meta()
vm.start()
print("[{}] Waiting for VM to start before SSHing.".format(name))
job.meta['status'] = 'waiting for VM to start'
job.save_meta()
time.sleep(20)
print("[{}] Creating SSH session.".format(name))
job.meta['status'] = 'creating SSH session'
job.save_meta()
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
retry = 0
while retry < 30:
try:
client.connect(ip,
username=template['username'],
password=template['password'])
break
except:
retry += 1
time.sleep(3)
print("[{}] Running user creation commands.".format(name))
job.meta['status'] = 'running user creation commands'
job.save_meta()
stdin, stdout, stderr = client.exec_command("useradd {}".format(user))
exit_status = stdout.channel.recv_exit_status()
root_password = gen_password(32)
stdin, stdout, stderr = client.exec_command(
"echo '{}' | passwd root --stdin".format(root_password))
exit_status = stdout.channel.recv_exit_status()
stdin, stdout, stderr = client.exec_command(
"echo '{}' | passwd '{}' --stdin".format(password, user))
exit_status = stdout.channel.recv_exit_status()
stdin, stdout, stderr = client.exec_command(
"passwd -e '{}'".format(user))
exit_status = stdout.channel.recv_exit_status()
stdin, stdout, stderr = client.exec_command(
"echo '{} ALL=(ALL:ALL) ALL' | sudo EDITOR='tee -a' visudo".format(
user))
exit_status = stdout.channel.recv_exit_status()
client.close()
print("[{}] Template successfully provisioned.".format(name))
job.meta['status'] = 'completed'
job.save_meta()
