def get_segregated_requests(request_list):
install_requests = []
clone_requests = []
disk_requests = []
edit_requests = []
install_object_store_requests = []
install_container_requests=[]
for req in request_list:
if req['request_type'] == VM_TASK_CREATE:
install_requests.append(req)
elif req['request_type'] == VM_TASK_CLONE:
clone_requests.append(req)
elif req['request_type'] == VM_TASK_ATTACH_DISK:
disk_requests.append(req)
elif req['request_type'] == VM_TASK_EDIT_CONFIG:
edit_requests.append(req)
elif req['request_type'] == Object_Store_TASK_CREATE:
install_object_store_requests.append(req)
elif req['request_type'] == CONTAINER_TASK_CREATE:
install_container_requests.append(req)
if not config.getboolean("GENERAL_CONF","docker_enabled"):
install_container_requests = None
return (install_requests, clone_requests, disk_requests, edit_requests, install_object_store_requests,install_container_requests)
def push_email(to_address,
email_subject,
email_message,
reply_to_address,
cc_addresses=[]):
if config.getboolean("MAIL_CONF", "mail_active"):
logger.debug("Sending mail to %s with subject %s" %
(to_address, email_subject))
rtn = mail.send(to=to_address,
subject=email_subject,
message=email_message,
reply_to=reply_to_address,
cc=cc_addresses)
logger.error("ERROR:: " + str(mail.error))
logger.info("EMAIL STATUS:: " + str(rtn))
def is_nic_setup():
return config.getboolean("GENERAL_CONF", "nic_setup")
def is_object_store_enabled():
return config.getboolean("GENERAL_CONF", "object_store_enabled")
def is_docker_enabled():
return config.getboolean("GENERAL_CONF", "docker_enabled")
def is_vm_enabled():
return config.getboolean("GENERAL_CONF", "vm_enabled")
def push_email(to_address, email_subject, email_message, reply_to_address, cc_addresses=[]):
if config.getboolean("MAIL_CONF","mail_active"):
logger.debug("Sending mail to %s with subject %s" %(to_address, email_subject))
rtn = mail.send(to=to_address, subject=email_subject, message = email_message, reply_to=reply_to_address, cc=cc_addresses)
logger.error("ERROR:: " + str(mail.error))
logger.info("EMAIL STATUS:: " + str(rtn))
def inference(data, models, resize):
threshold = config['param'].getfloat('threshold')
threshold_edge = config['param'].getfloat('threshold_edge')
threshold_mark = config['param'].getfloat('threshold_mark')
ensemble_policy = config['post']['ensemble']
# sub-rountine to convert output tensor to numpy
def convert(t):
assert isinstance(t, (torch.FloatTensor, torch.cuda.FloatTensor))
if len(t) == 0:
return None
# pixel wise ensemble output of models
t = torch.mean(t, 0, True)
# to numpy array
t = t.cpu()
t = t.numpy()[0]
if ensemble_policy == 'vote':
t = np.where(t >= 0.5, 1., 0.) # majority vote
# channel first [C, H, W] -> channel last [H, W, C]
t = np.transpose(t, (1, 2, 0))
# Remove single-dimensional channel from the shape of an array
t = np.squeeze(t)
t = align_size(t, size, resize)
return t
# get input data
uid = data['uid']
size = data['size']
inputs = data['image']
# prepare input variables
inputs = inputs.unsqueeze(0)
if torch.cuda.is_available():
inputs = inputs.cuda()
if not resize:
inputs = pad_tensor(inputs, size)
else:
inputs = Variable(inputs)
y_s = y_c = y_m = torch.FloatTensor()
if torch.cuda.is_available():
y_s, y_c, y_m = y_s.cuda(), y_c.cuda(), y_m.cuda()
for model in models:
model_name = type(model).__name__.lower()
with_contour = config.getboolean(model_name,
'branch_contour',
fallback=False)
with_marker = config.getboolean(model_name,
'branch_marker',
fallback=False)
# predict model output
c = m = Variable()
if torch.cuda.is_available():
c, m = c.cuda(), m.cuda()
s = model(inputs)
# convert predict to numpy array
if with_contour and with_marker:
s, c, m = s
elif with_contour:
s, c = s
# concat outputs
if ensemble_policy == 'avg':
y_s = torch.cat([y_s, s.data], 0)
y_c = torch.cat([y_c, c.data], 0)
y_m = torch.cat([y_m, m.data], 0)
elif ensemble_policy == 'vote':
y_s = torch.cat([y_s, (s.data > threshold).float()], 0)
y_c = torch.cat([y_c, (c.data > threshold_edge).float()], 0)
y_m = torch.cat([y_m, (m.data > threshold_mark).float()], 0)
else:
raise NotImplementedError("Ensemble policy not implemented")
return uid, convert(y_s), convert(y_c), convert(y_m)
def valid(loader, model, epoch, writer, n_step):
iou = AverageMeter() # semantic IoU
iou_c = AverageMeter() # contour IoU
iou_m = AverageMeter() # marker IoU
losses = AverageMeter()
only_contour = config['contour'].getboolean('exclusive')
weight_map = config['param'].getboolean('weight_map')
model_name = config['param']['model']
with_contour = config.getboolean(model_name, 'branch_contour')
with_marker = config.getboolean(model_name, 'branch_marker')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Sets the model in evaluation mode.
model.eval()
for i, data in enumerate(loader):
# get the inputs
inputs = data['image'].to(device)
labels = data['label'].to(device)
labels_c = data['label_c'].to(device)
labels_m = data['label_m'].to(device)
# get loss weight
weights = None
if weight_map and 'weight' in data:
weights = data['weight'].to(device)
# forward step
outputs = model(inputs)
if with_contour and with_marker:
outputs, outputs_c, outputs_m = outputs
elif with_contour:
outputs, outputs_c = outputs
# compute loss
if only_contour:
loss = contour_criterion(outputs, labels_c)
else:
# weight_criterion equals to segment_criterion if weights is none
loss = focal_criterion(outputs, labels, weights)
if with_contour:
loss += focal_criterion(outputs_c, labels_c, weights)
if with_marker:
loss += focal_criterion(outputs_m, labels_m, weights)
# measure accuracy and record loss (Non-instance level IoU)
losses.update(loss.item(), inputs.size(0))
if only_contour:
batch_iou = iou_mean(outputs, labels_c)
else:
batch_iou = iou_mean(outputs, labels)
iou.update(batch_iou, inputs.size(0))
if with_contour:
batch_iou_c = iou_mean(outputs_c, labels_c)
iou_c.update(batch_iou_c, inputs.size(0))
if with_marker:
batch_iou_m = iou_mean(outputs_m, labels_m)
iou_m.update(batch_iou_m, inputs.size(0))
# end of loop, dump epoch summary
writer.add_scalar('CV/epoch_loss', losses.avg, epoch)
writer.add_scalar('CV/epoch_iou', iou.avg, epoch)
writer.add_scalar('CV/epoch_iou_c', iou_c.avg, epoch)
writer.add_scalar('CV/epoch_iou_m', iou_m.avg, epoch)
print(
'Epoch: [{0}]\t\tcross-validation\t'
'Loss: N/A (avg: {loss.avg:.4f})\t'
'IoU: {iou.avg:.3f} (Coutour: {iou_c.avg:.3f}, Marker: {iou_m.avg:.3f})\t'
.format(
epoch, loss=losses, iou=iou, iou_c=iou_c, iou_m=iou_m
)
)
return iou.avg # return epoch average iou
def is_docker_enabled():
return config.getboolean("GENERAL_CONF","docker_enabled")
def is_vm_enabled():
return config.getboolean("GENERAL_CONF","vm_enabled")
def inference(data, models, resize):
threshold = config['param'].getfloat('threshold')
threshold_edge = config['param'].getfloat('threshold_edge')
threshold_mark = config['param'].getfloat('threshold_mark')
tta = config['valid'].getboolean('test_time_augment')
ensemble_policy = config['valid']['ensemble']
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# sub-rountine to convert output tensor to numpy
def convert(t):
assert isinstance(t, (torch.FloatTensor, torch.cuda.FloatTensor))
if len(t) == 0:
return None
# pixel wise ensemble output of models
t = torch.mean(t, 0, True)
# to numpy array
t = t.to('cpu').numpy()[0]
if ensemble_policy == 'vote':
t = np.where(t >= 0.5, 1., 0.) # majority vote
# channel first [C, H, W] -> channel last [H, W, C]
t = np.transpose(t, (1, 2, 0))
# Remove single-dimensional channel from the shape of an array
t = np.squeeze(t)
t = align_size(t, size, resize)
return t
# get input data
uid = data['uid']
size = data['size']
inputs = data['image']
# prepare input variables
inputs = inputs.unsqueeze(0)
inputs = inputs.to(device)
if tta:
txf_funcs = [lambda x: x,
lambda x: flip(x, 2), # up down flip
lambda x: flip(x, 3), # left right flip
lambda x: flip(flip(x, 3), 2),
]
else:
txf_funcs = [lambda x: x]
y_s = y_c = y_m = torch.FloatTensor().to(device)
for model in models:
model_name = type(model).__name__.lower()
with_contour = config.getboolean(model_name, 'branch_contour')
with_marker = config.getboolean(model_name, 'branch_marker')
# predict model output
for txf in txf_funcs:
# apply test time transform
x = inputs
x = txf(x)
# padding
if not resize:
x = pad_tensor(x, size)
# inference model
s = model(x)
# handle multi-head
c = m = torch.FloatTensor().to(device)
if with_contour and with_marker:
s, c, m = s
elif with_contour:
s, c = s
# crop padding
if not resize:
w, h = size
s = s[:, :, :h, :w]
c = c[:, :, :h, :w] if len(c) > 0 else c
m = m[:, :, :h, :w] if len(m) > 0 else m
# reverse flip
s = txf(s)
c = txf(c)
m = txf(m)
# concat outputs
if ensemble_policy == 'avg':
y_s = torch.cat([y_s, s], 0)
if len(c) > 0:
y_c = torch.cat([y_c, c], 0)
if len(m) > 0:
y_m = torch.cat([y_m, m], 0)
elif ensemble_policy == 'vote':
y_s = torch.cat([y_s, (s > threshold).float()], 0)
if len(c) > 0:
y_c = torch.cat([y_c, (c > threshold_edge).float()], 0)
if len(m) > 0:
y_m = torch.cat([y_m, (m > threshold_mark).float()], 0)
else:
raise NotImplementedError("Ensemble policy not implemented")
return uid, convert(y_s), convert(y_c), convert(y_m)
from cont_handler import Container, addip
from gluon import current
from helper import get_context_path, get_docker_daemon_address, \
get_nginx_server_address, log_exception , config
from images import getImageProfile
from log_handler import logger
import docker
import os
import random
import remote_vm_task as remote_machine
if config.getboolean("GENERAL_CONF", "docker_enabled"):
cert_path = os.path.join(get_context_path(), 'modules/certs/')
tls_config = docker.tls.TLSConfig(client_cert=(cert_path + 'cert.pem',
cert_path + 'key.pem'),
verify=cert_path + 'ca.pem')
docker_machine = get_docker_daemon_address()
#client = docker.Client(base_url='https://:10.237.20.236:3376',version='auto', tls=tls_config)
client = docker.Client(base_url='https://' + docker_machine[0] + ':' +
docker_machine[1],
version='auto',
tls=tls_config)
Container.setclient(client)
def proxieddomain(name):
#alias = '.baadalgateway.cse.iitd.ac.in'
alias = '.docker.iitd.ac.in'
address = name + alias
return address[1:]
Field('name', 'string', length = 255, notnull = True, unique = True),
Field('value', 'string', length = 255, notnull = True))
db.define_table('organisation',
Field('name', 'string', length = 255, notnull = True, unique = True),
Field('details', 'string', length = 255),
Field('public_ip', 'string',length = 15),
Field('admin_mailid', 'string', length = 50),
format = '%(details)s')
from gluon.tools import Auth
auth = Auth(db)
from gluon.tools import Mail
mail = Mail()
if config.getboolean("MAIL_CONF","mail_active"):
mail.settings.server = config.get("MAIL_CONF","mail_server")
mail.settings.sender = config.get("MAIL_CONF","mail_sender")
mail.settings.login = config.get("MAIL_CONF","mail_login")
mail.settings.tls = literal_eval(config.get("MAIL_CONF","mail_server_tls"))
#added to make auth and db objects available in modules
from gluon import current # @Reimport
current.auth = auth
current.db = db
current.auth_type = config.get("AUTH_CONF","auth_type")
## configure custom auth tables
auth.settings.table_user_name = 'user'
auth.settings.table_group_name = 'user_group'
auth.settings.table_membership_name = 'user_membership'
def is_object_store_enabled():
return config.getboolean("GENERAL_CONF","object_store_enabled")
def train(loader, model, optimizer, epoch, writer):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
iou = AverageMeter() # semantic IoU
iou_c = AverageMeter() # contour IoU
iou_m = AverageMeter() # marker IoU
print_freq = config['train'].getfloat('print_freq')
only_contour = config['contour'].getboolean('exclusive')
weight_map = config['param'].getboolean('weight_map')
model_name = config['param']['model']
with_contour = config.getboolean(model_name, 'branch_contour')
with_marker = config.getboolean(model_name, 'branch_marker')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Sets the module in training mode.
model.train()
end = time.time()
n_step = len(loader)
for i, data in enumerate(loader):
# measure data loading time
data_time.update(time.time() - end)
# split sample data
inputs = data['image'].to(device)
labels = data['label'].to(device)
labels_c = data['label_c'].to(device)
labels_m = data['label_m'].to(device)
# get loss weight
weights = None
if weight_map and 'weight' in data:
weights = data['weight'].to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward step
outputs = model(inputs)
if with_contour and with_marker:
outputs, outputs_c, outputs_m = outputs
elif with_contour:
outputs, outputs_c = outputs
# compute loss
if only_contour:
loss = contour_criterion(outputs, labels_c)
else:
# weight_criterion equals to segment_criterion if weights is none
loss = focal_criterion(outputs, labels, weights)
if with_contour:
loss += focal_criterion(outputs_c, labels_c, weights)
if with_marker:
loss += focal_criterion(outputs_m, labels_m, weights)
# compute gradient and do backward step
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# measure accuracy and record loss
# NOT instance-level IoU in training phase, for better speed & instance separation handled in post-processing
losses.update(loss.item(), inputs.size(0))
if only_contour:
batch_iou = iou_mean(outputs, labels_c)
else:
batch_iou = iou_mean(outputs, labels)
iou.update(batch_iou, inputs.size(0))
if with_contour:
batch_iou_c = iou_mean(outputs_c, labels_c)
iou_c.update(batch_iou_c, inputs.size(0))
if with_marker:
batch_iou_m = iou_mean(outputs_m, labels_m)
iou_m.update(batch_iou_m, inputs.size(0))
# log to summary
#step = i + epoch * n_step
#writer.add_scalar('training/loss', loss.item(), step)
#writer.add_scalar('training/batch_elapse', batch_time.val, step)
#writer.add_scalar('training/batch_iou', iou.val, step)
#writer.add_scalar('training/batch_iou_c', iou_c.val, step)
#writer.add_scalar('training/batch_iou_m', iou_m.val, step)
if (i + 1) % print_freq == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Time: {batch_time.avg:.2f} (io: {data_time.avg:.2f})\t'
'Loss: {loss.val:.4f} (avg: {loss.avg:.4f})\t'
'IoU: {iou.avg:.3f} (Coutour: {iou_c.avg:.3f}, Marker: {iou_m.avg:.3f})\t'
.format(
epoch, i, n_step, batch_time=batch_time,
data_time=data_time, loss=losses, iou=iou, iou_c=iou_c, iou_m=iou_m
)
)
# end of loop, dump epoch summary
writer.add_scalar('training/epoch_loss', losses.avg, epoch)
writer.add_scalar('training/epoch_iou', iou.avg, epoch)
writer.add_scalar('training/epoch_iou_c', iou_c.avg, epoch)
writer.add_scalar('training/epoch_iou_m', iou_m.avg, epoch)
return iou.avg # return epoch average iou
def is_nic_setup():
return config.getboolean("GENERAL_CONF","nic_setup")
Field('name', 'string', length = 255, notnull = True, unique = True),
Field('value', 'string', length = 255, notnull = True))
db.define_table('organisation',
Field('name', 'string', length = 255, notnull = True, unique = True),
Field('details', 'string', length = 255),
Field('public_ip', 'string',length = 15),
Field('admin_mailid', 'string', length = 50),
format = '%(details)s')
from gluon.tools import Auth
auth = Auth(db)
from gluon.tools import Mail
mail = Mail()
if config.getboolean("MAIL_CONF","mail_active"):
mail.settings.server = config.get("MAIL_CONF","mail_server")
mail.settings.sender = config.get("MAIL_CONF","mail_sender")
mail.settings.login = config.get("MAIL_CONF","mail_login")
mail.settings.tls = literal_eval(config.get("MAIL_CONF","mail_server_tls"))
#added to make auth and db objects available in modules
from gluon import current # @Reimport
current.auth = auth
current.db = db
current.auth_type = config.get("AUTH_CONF","auth_type")
## configure custom auth tables
auth.settings.table_user_name = 'user'
auth.settings.table_group_name = 'user_group'
auth.settings.table_membership_name = 'user_membership'
# -*- coding: utf-8 -*-
###################################################################################
# Added to enable code completion in IDE's.
if 0:
import gluon
global auth; auth = gluon.tools.Auth()
from gluon import T,request,response,URL,H2
from applications.baadal.models import * # @UnusedWildImport
###################################################################################
from helper import get_constant, config
from auth_user import is_auth_type_db
from maintenance import BAADAL_STATUS_UP, BAADAL_STATUS_DOWN, BAADAL_STATUS_UP_IN_PROGRESS, BAADAL_STATUS_DOWN_IN_PROGRESS
docker_enabled = config.getboolean("GENERAL_CONF","docker_enabled")
response.title = request.application
response.google_analytics_id = None
response.top_menu = [
(T('About'), False, URL('default','index')),
(T('FAQ'), False, URL('default','faq')),
(T('Team Baadal'), False, URL('default','team')),
(T('Contact'), False, URL('default','contact'))
]
if auth.is_logged_in():
response.user_menu = [
(H2('USER MENU'),False, dict(_href='#', _id='menu_user')),
(T('Home'), False, URL('default','index')),
(T('Request VM'), False, URL('user','request_vm')),
(T('Request Object Store'), False, URL('user','request_object_store')),
(T('Pending Requests'), False, URL('user','list_my_requests')),
(T('My VMs'), False, URL('user','list_my_vm')),
