def temp_list(self, args):
templates_service = self.c.system_service().templates_service()
try:
if args.prefix is not None:
search_prefix = 'name=' + args.prefix + '*'
else:
search_prefix = '*'
templates = templates_service.list(search=search_prefix)
rows = []
for template in templates:
cores = template.cpu.topology.cores * template.cpu.topology.sockets * template.cpu.topology.threads
memory = template.memory / 1024 / 1024
description = ""
if template.description is not None:
description = template.description
rows.append([
template.id, template.name,
str(memory) + ' M', cores,
str(description)
])
output = Output(["ID", "Name", "Mem", "CPU", "Description"], rows)
output.print_table()
self.c.close()
except sdk.Error as err:
print "Failed to list templates, %s" % str(err)
def vm_show(self, args):
vms_service = self.c.system_service().vms_service()
try:
rows = []
vm = vms_service.list(search=args.vm_name)[0]
vm_service = vms_service.vm_service(vm.id)
# Print basic information
rows.append(["Name", vm.name])
rows.append(["ID", vm.id])
rows.append(["Status", str(vm.status).upper()])
rows.append(["Memory", str(vm.memory / 1024 / 1024) + 'M'])
rows.append([
"CPU", vm.cpu.topology.cores * vm.cpu.topology.sockets *
vm.cpu.topology.threads
])
# Print Disk information
disk_attachments_service = vm_service.disk_attachments_service()
disk_attachments = disk_attachments_service.list()
for disk_attachment in disk_attachments:
disk = self.c.follow_link(disk_attachment.disk)
rows.append([
"Disks",
[
disk.name, disk.id,
str(disk.provisioned_size / 1024 / 1024 / 1024) + 'G'
]
])
# We use nics_service() instead of reported_devices_service()
# because we need to get NIC id and name
nics_service = vm_service.nics_service().list()
for nic in nics_service:
for device in nic.reported_devices:
if device.ips is not None:
for ip in device.ips:
if ip.version.value == 'v4':
rows.append([
"Active Nics",
[
nic.name, nic.mac.address, nic.id,
ip.address
]
])
if device.ips is None:
rows.append([
"Inactive Nics",
[nic.name, nic.mac.address, nic.id]
])
output = Output(["Item", "Value"], rows)
output.print_table()
except sdk.Error as err:
print str(err)
except IndexError:
print "Error: No such VM %s" % args.vm_name
def forward(self, inputs):
item_x, context_x, user_id = split_input(inputs, requires_grad=False,
dtype=[torch.long, torch.float, torch.long], device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_last_hidden, norm_attn) = self.encode(item_x, context_x, user_id)
global_preference = self.concat_to_item_tfm(item_enc_hidden)
# calculate p(v | sequence) based on the global preference
# (batch x num_item_units) times (|V| x num_item_units) -> (batch x |V|)
item_logits = torch.einsum('bd,vd->bv', global_preference, self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
if self.lambda_score == 1:
_, recommended_list = torch.topk(item_probs, self.recommendation_len, sorted=True)
else:
# DIVERSITY WITH Maximal Marginal Relevance
init_preds = item_probs.argmax(dim=-1)
recommended_list = init_preds.detach().view(batch_size, 1)
for _ in range(self.recommendation_len - 1):
# calculate diversity using the embeddings of the given ids
selected = self.compute_distance(item_probs, recommended_list)
recommended_list = torch.cat([recommended_list, selected.view(batch_size, 1)], dim=-1)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
return Output(preds=recommended_list, logits=item_logits,
attention=norm_attn, context_logits=None, context_preds=None)
def net_list(self, args):
networks_service = self.c.system_service().networks_service()
try:
networks = networks_service.list()
rows = []
for network in networks:
comment = ""
description = ""
if network.description is not None:
description = network.description
if network.comment is not None:
comment = network.comment
rows.append([network.id, network.name, comment, description])
output = Output(["ID", "Name", "Comment", "Description"], rows)
output.print_table()
self.c.close()
except sdk.Error as err:
print "Failed to list networks, %s" % str(err)
def sd_list(self, args):
try:
sds_service = self.c.system_service().storage_domains_service()
rows = []
for sd in sds_service.list():
if sd.available is not None and sd.used is not None:
rows.append([
sd.id, sd.name, sd.storage.type,
str((sd.available + sd.used) / 1024 / 1024 / 1024) +
'G',
str(sd.available / 1024 / 1024 / 1024) + 'G'
])
else:
rows.append(
[sd.id, sd.name, sd.storage.type, "N/A", "N/A"])
output = Output(["ID", "Name", "Type", "Total", "Free"], rows)
output.print_table()
except sdk.Error as err:
print "Failed to list disks, %s" % str(err)
def forward(self, inputs):
item_x, _, _ = inputs
lengths = (item_x != 0).sum(dim=-1)
total_length = item_x.shape[1]
batch_size = item_x.shape[0]
mask = item_x.ne(0)
# Look up embeddings
item_x_embedded = self.item_embedding(item_x)
item_x_embedded = dropout(item_x_embedded,
p=self.dropout_rate,
training=self.training)
item_x = pack_padded_sequence(item_x_embedded,
lengths,
batch_first=True,
enforce_sorted=False)
# Do forward passes GRU stuff
item_enc_all, item_enc_hidden = self.item_encoder_GRU(item_x)
item_enc_all = pad_packed_sequence(item_enc_all,
total_length=total_length,
batch_first=True)
item_enc_all = item_enc_all[0] # 0 contains the padded sequences
item_enc_all = dropout(item_enc_all,
p=self.dropout_rate,
training=self.training)
item_enc_hidden = dropout(item_enc_hidden,
p=self.dropout_rate,
training=self.training)
# Attention
item_enc_hidden, attn, norm_attn = self.attention(
item_enc_hidden.reshape(batch_size, -1).unsqueeze(1),
item_enc_all,
item_enc_all,
mask=mask.unsqueeze(1))
global_preference = self.hidden_to_item_tfm(item_enc_hidden).reshape(
-1, self.num_item_units)
item_logits = torch.einsum('bd,vd->bv', global_preference,
self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
_, recommended_list = torch.topk(item_probs,
self.recommendation_len,
sorted=True)
else:
recommended_list = torch.zeros(1, device=self.device)
recommended_list = recommended_list.requires_grad_(False)
return Output(preds=recommended_list,
logits=item_logits,
attention=norm_attn.view(batch_size, -1),
context_logits=None,
context_preds=None)
def forward(self, inputs):
item_x, context_x, user_id = split_input(
inputs,
requires_grad=False,
dtype=[torch.long, torch.float, torch.long],
device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_hidden,
norm_attn) = self.encode(item_x, context_x, user_id)
context_predict_input = torch.cat((item_enc_hidden, ctxt_enc_hidden),
dim=-1)
context_logits = self.context_MLP(context_predict_input)
context_probs = context_logits.softmax(dim=-1)
context_preds = context_logits.argmax(dim=-1)
context_y_embedded = torch.einsum('bc, cd -> bd', context_probs,
self.context_embedding)
context_y_embedded = dropout(context_y_embedded,
p=self.dropout_rate,
training=self.training)
global_preference = self.bilinear_mapping(item_enc_hidden,
context_y_embedded)
# calculate p(v | sequence) based on the global preference
item_logits = torch.einsum('bd,vd->bv', global_preference,
self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
recommended_list = []
init_preds = item_probs.argmax(dim=-1)
recommended_list.append(init_preds.detach())
satisfactions = context_probs
for _ in range(self.recommendation_len - 1):
# calculate diversity using the embeddings of the given ids
# selected = self.compute_distance(item_probs, recommended_list)
selected, satisfactions = self.compute_highest_marginal_utility(
item_probs, satisfactions, recommended_list)
recommended_list.append(selected)
recommended_list = torch.stack(recommended_list, 1)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
context_preds = context_preds.requires_grad_(False)
return Output(preds=recommended_list,
logits=item_logits,
attention=norm_attn,
context_logits=context_logits.view(batch_size, -1),
context_preds=context_preds)
def vm_list(self, args):
# Call ovirtsdk to get vms_service
vms_service = self.c.system_service().vms_service()
# We check whether prefix has been given or not
if args.prefix is not None:
search_prefix = 'name=' + args.prefix + '*'
else:
search_prefix = 'name=' + environ.get('TITAMU_VM_PREFIX') + '*'
try:
# Get vm list by using vms_service
vms = vms_service.list(
search=search_prefix,
case_sensitive=False,
)
rows = []
for vm in vms:
addr = ""
comment = ""
if vm.comment is not None:
comment = vm.comment
vm_service = vms_service.vm_service(vm.id)
# We use reported_devices_service method to get all the
# NICs that have IP address assigned
for device in vm_service.reported_devices_service().list():
if device.ips is not None and vm.status.value == 'up':
for ip in device.ips:
# cchen: ip.version is a enum so we have to use "value" to
# get its real value we are assuming that only the IP which
# starts with '10' is something that we are interested in
if ip.version.value == 'v4' and ip.address.startswith(
'10'):
addr += ip.address + ' '
rows.append(
[vm.id, vm.name,
vm.status.value.upper(), addr, comment])
output = Output(["ID", "Name", "Status", "Networks", "Comment"],
rows)
output.print_table()
self.c.close()
except sdk.Error as err:
print "Failed to list VMs, %s" % str(err)
def disk_list(self, args):
try:
vms_service = self.c.system_service().vms_service()
vm = vms_service.list(search=args.vm_name)[0]
vm_service = vms_service.vm_service(vm.id)
disk_attachments_service = vm_service.disk_attachments_service()
disk_attachments = disk_attachments_service.list()
rows = []
for disk_attachment in disk_attachments:
disk = self.c.follow_link(disk_attachment.disk)
rows.append([
disk.id, disk.name,
str(disk.provisioned_size / 1024 / 1024 / 1024) + "G",
str(disk.status).upper()
])
output = Output(["ID", "Name", "Size", "Status"], rows)
output.print_table()
self.c.close()
except sdk.Error as err:
print "Failed to list disks, %s" % str(err)
def forward(self, inputs):
item_x, context_x, user_id = split_input(
inputs,
requires_grad=False,
dtype=[torch.long, torch.float, torch.long],
device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_hidden,
norm_attn) = self.encode(item_x, context_x, user_id)
# user_embeddings = self.user_embedding(user_id) # (batch x num_user_units)
context_predict_input = torch.cat((item_enc_hidden, ctxt_enc_hidden),
dim=-1)
context_logits = self.context_MLP(context_predict_input)
context_probs = context_logits.softmax(dim=-1)
context_preds = context_logits.argmax(dim=-1)
context_y_embedded = torch.einsum('bc, cd -> bd', context_probs,
self.context_embedding)
context_y_embedded = dropout(context_y_embedded,
p=self.dropout_rate,
training=self.training)
global_preference = self.bilinear_mapping(item_enc_hidden,
context_y_embedded)
# calculate p(v | sequence) based on the global preference
item_logits = torch.einsum('bd,vd->bv', global_preference,
self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
_, recommended_list = torch.topk(
item_probs,
self.recommendation_len,
sorted=True,
)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
context_preds = context_preds.requires_grad_(False)
return Output(preds=recommended_list,
logits=item_logits,
attention=norm_attn,
context_logits=context_logits,
context_preds=context_preds)
def make_storage_template():
cft = CloudFormationTemplate(description="Refinery Platform storage")
# Parameters
cft.parameters.add(
Parameter('StaticBucketName', 'String', {
'Description': 'Name of S3 bucket for Django static files',
}))
cft.parameters.add(
Parameter(
'MediaBucketName',
'String',
{
'Description':
'Name of S3 bucket for Django media files',
# make names DNS-compliant without periods (".") for
# compatibility with virtual-hosted-style access and S3
# Transfer Acceleration
'AllowedPattern':
'[a-z0-9\-]+',
'ConstraintDescription':
'must only contain lower case letters, numbers, and '
'hyphens',
}))
cft.parameters.add(
Parameter(
'IdentityPoolName', 'String', {
'Default': 'Refinery Platform',
'Description': 'Name of Cognito identity pool for S3 uploads',
}))
cft.parameters.add(
Parameter(
'DeveloperProviderName', 'String', {
'Default':
'login.refinery',
'Description':
'"domain" by which Cognito will refer to users',
'AllowedPattern':
'[a-z\-\.]+',
'ConstraintDescription':
'must only contain lower case letters, periods, '
'underscores, and hyphens'
}))
# Resources
cft.resources.add(
Resource(
'StaticStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('StaticBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['GET'],
'AllowedHeaders': ['Authorization'],
'MaxAge': 3000,
}]
},
}),
DeletionPolicy('Retain'),
))
cft.resources.add(
Resource(
'MediaStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('MediaBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['POST', 'PUT', 'DELETE'],
'AllowedHeaders': ['*'],
'ExposedHeaders': ['ETag'],
'MaxAge': 3000,
}]
}
}),
DeletionPolicy('Retain'),
))
# Cognito Identity Pool for Developer Authenticated Identities Authflow
# http://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html
cft.resources.add(
Resource(
'IdentityPool', 'AWS::Cognito::IdentityPool',
Properties({
'IdentityPoolName': ref('IdentityPoolName'),
'AllowUnauthenticatedIdentities': False,
'DeveloperProviderName': ref('DeveloperProviderName'),
})))
cft.resources.add(
Resource(
'IdentityPoolAuthenticatedRole',
'AWS::Cognito::IdentityPoolRoleAttachment',
Properties({
'IdentityPoolId': ref('IdentityPool'),
'Roles': {
'authenticated': get_att('CognitoS3UploadRole', 'Arn'),
}
})))
upload_role_trust_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": {
"Federated": "cognito-identity.amazonaws.com"
},
"Action": "sts:AssumeRoleWithWebIdentity",
"Condition": {
"StringEquals": {
"cognito-identity.amazonaws.com:aud": ref('IdentityPool')
},
"ForAnyValue:StringLike": {
"cognito-identity.amazonaws.com:amr": "authenticated"
}
}
}]
}
upload_access_policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["cognito-identity:*"],
"Resource": "*"
}, {
"Action": ["s3:PutObject", "s3:AbortMultipartUpload"],
"Effect": "Allow",
"Resource": {
"Fn::Sub":
"arn:aws:s3:::${MediaStorageBucket}/uploads/"
"${!cognito-identity.amazonaws.com:sub}/*"
}
}]
}
cft.resources.add(
Resource(
'CognitoS3UploadRole', 'AWS::IAM::Role',
Properties({
'AssumeRolePolicyDocument':
upload_role_trust_policy,
'Policies': [{
'PolicyName': 'AuthenticatedS3UploadPolicy',
'PolicyDocument': upload_access_policy,
}]
})))
# Outputs
cft.outputs.add(
Output('IdentityPoolId', ref('IdentityPool'),
{'Fn::Sub': '${AWS::StackName}IdentityPoolId'},
'Cognito identity pool ID'))
return cft
def make_storage_template():
cft = CloudFormationTemplate(description="Refinery Platform storage")
# Parameters
cft.parameters.add(
Parameter('StaticBucketName', 'String', {
'Description': 'Name of S3 bucket for Django static files',
}))
cft.parameters.add(
Parameter(
'MediaBucketName',
'String',
{
'Description':
'Name of S3 bucket for Django media files',
# make names DNS-compliant without periods (".") for compatibility
# with virtual-hosted-style access and S3 Transfer Acceleration
'AllowedPattern':
'[a-z0-9\-]+',
'ConstraintDescription':
'must only contain lower case letters, numbers, and hyphens',
}))
# Resources
cft.resources.add(
Resource(
'StaticStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('StaticBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['GET'],
'AllowedHeaders': ['Authorization'],
'MaxAge': 3000,
}]
},
}),
DeletionPolicy('Retain'),
))
cft.resources.add(
Resource(
'MediaStorageBucket',
'AWS::S3::Bucket',
Properties({
'BucketName': ref('MediaBucketName'),
'AccessControl': 'PublicRead',
'CorsConfiguration': {
'CorsRules': [{
'AllowedOrigins': ['*'],
'AllowedMethods': ['POST', 'PUT', 'DELETE'],
'AllowedHeaders': ['*'],
'ExposedHeaders': ['ETag'],
'MaxAge': 3000,
}]
}
}),
DeletionPolicy('Retain'),
))
cft.outputs.add(
Output('MediaBucketName', ref('MediaStorageBucket'),
{'Fn::Sub': '${AWS::StackName}Media'},
'Name of S3 bucket for Django media files'))
return cft
#!/usr/bin/env python
import argparse
from utils import Output, is_valid_user, arguments
from classes import Model
parser = argparse.ArgumentParser(description='Performs a data modeling pipeline',
usage='python model.py user password [OPTIONS]')
input_username, input_password, db_name, db_host = arguments(parser)
output = Output()
def model_data():
model = Model(input_username, input_password, db_name, db_host)
# Create course content similarity DataFrame
output.write('Create a course content similarity DataFrame')
output.warning('This process can take a long time')
output.start_spinner('Creating a course content similarity DataFrame')
try:
model.create_course_content_similarity_df()
output.spinner_success()
except Exception as err:
output.spinner_fail(str(err))
exit(1)
# Save courses content similarities to database
