def enlistImages(self):
banner(f"Fetching image list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Image list {self.cloudname}.")
image_list = ec2_instance.describe_images()
opDict= []
opDict = [{'ImageId':i.get('ImageId'), 'ImageType': i.get('ImageType'), 'Description': i.get('Description'), 'Name': i.get('Name'), 'State':i.get('State'),'Public': i.get('Public'),'CreationDate':i.get('CreationDate')} for i in image_list['Images']]
StopWatch.stop(f"Image list {self.cloudname}.")
t = format(StopWatch.get(f"Image list {self.cloudname}."), '.2f')
banner(f"Image list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} images fetched. Time taken {t} \nPrinting first 5 sample images:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Image_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Image list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def createDatasetGroup(self):
from cloudmesh.common.StopWatch import StopWatch
from StatusIndicator import StatusIndicator
import time
def create_uuid(project_name):
import random
id = random.randrange(100000)
return project_name + '_' + str(id)
self.DATASET_FREQUENCY = "D"
self.TIMESTAMP_FORMAT = "yyyy-MM-dd hh:mm:ss"
proj = 'timeseries'
self.project = create_uuid(proj)
print(self.project)
self.datasetName = self.project + '_ds'
self.datasetGroupName = self.project + '_dsg'
self.s3DataPath = "s3://" + self.bucket_name + "/" + self.key
StopWatch.start('to_dsg')
create_dataset_group_response = self.forecast_srv.create_dataset_group(
DatasetGroupName=self.datasetGroupName,
Domain="CUSTOM",
)
datasetGroupArn = create_dataset_group_response['DatasetGroupArn']
self.datasetGroupArn = datasetGroupArn
StopWatch.stop('to_dsg')
print(StopWatch.get('to_dsg'))
return self.datasetGroupArn
def load(self, name):
"""
Load cached model
:param name:
:return:
"""
cm = CmDatabase()
# USER env variable is required by StopWatch
if os.getenv('USER'):
# Do nothing
VERBOSE("USER env variable is already defined")
else:
os.environ['USER'] = '******'
test = cm.find(cloud="local", kind="cache", query={"name": {'$regex': f"{name}"}})
cached_file = test[0]['cached_file']
Console.info(f"Loading serialized model: {cached_file}")
StopWatch.start(f"Load pickle {name}")
deserialized_model = self._load_pickle(cached_file)
StopWatch.stop(f"Load pickle {name}")
time_taken = StopWatch.get(f"Load pickle {name}")
# TODO: figure out how useful the duration is and return to client if required
deserialized_model_dict = {
"model_name": name,
"model_object": deserialized_model,
"duration": time_taken # duration of deserialization function
}
return deserialized_model
def postcmd(self, stop, line):
StopWatch.stop("command")
if "timer" not in self.variable:
self.variable["timer"] = "off"
if self.variable["timer"].lower() in ['on', 'true']:
print("Timer: {:.4f}s ({})".format(StopWatch.get("command"),
line.strip()))
return stop
def _create(self, **arguments):
arguments = dotdict(arguments)
r = []
StopWatch.start(f"create vm {arguments.name}")
cm = {
'kind': "vm",
'name': arguments.name,
'group': arguments.group,
'cloud': self.cloudname(),
'status': 'booting'
}
entry = {}
entry.update(cm=cm, name=arguments.name)
result = CmDatabase.UPDATE(entry, progress=False)[0]
data = {}
dryrun = False
if "dryrun" in arguments:
dryrun = arguments.dryrun
data = {"dryrun": True}
else:
arguments.timeout = 360
data = self.p.create(**arguments)
# print('entry')
# pprint(entry)
# print('data')
pprint(data)
entry.update(data)
StopWatch.stop(f"create vm {arguments.name}")
t = format(StopWatch.get(f"create vm {arguments.name}"), '.2f')
cm['creation_time'] = t
entry.update({'cm': cm})
if arguments.metadata:
entry.update({"metadata": arguments.metadata})
else:
entry.update({
"metadata":
str({
"cm": cm,
"image": arguments.image,
"size": arguments.size
})
})
cm['status'] = 'available'
self.p.set_server_metadata(arguments.name, cm)
result = CmDatabase.UPDATE(entry, progress=False)[0]
return result
def demo_stopwatch(self):
count = 1
StopWatch.start("demo")
while (count < 5):
ping = Shell.ping(host='localhost', count=count)
print(ping)
count += count
StopWatch.stop("demo")
print("\n Total time elapsed:", StopWatch.get("demo", digits=5))
def stopwatch_demo():
"""
This program measure the time elapse of sorting
"""
StopWatch.start("Sort Test")
np.sort(np.random.normal(size=1000))
StopWatch.stop("Sort Test")
print('Sorting took:', StopWatch.get("Sort Test"), 's')
return
def postcmd(self, stop, line):
StopWatch.stop("command")
try:
variable = Variables()
if "timer" not in variable:
variable["timer"] = "False"
if str(variable["timer"].lower()) in ['on', 'true']:
command_time = StopWatch.get("command")
load_time = StopWatch.get("load")
line_strip = line.strip()
print(f"# Timer: {command_time:.4f}s "
f"Load: {load_time:.4f}s "
f"{line_strip}")
variable.close()
except Exception as e:
Error.traceback(error=e)
return stop
def getfibonacci():
banner("Fetching 100th number of the Fibonacci series")
StopWatch.start("Fibo")
a, b = 0, 1
for i in range(100):
# print(a)
a, b = a + b, a
print(b)
StopWatch.stop("Fibo")
print(StopWatch.get("Fibo",5))
def sampleStopWatch(self):
""" function to use stop watch """
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('test')
sleep(1)
StopWatch.stop('test')
print(StopWatch.get('test'))
StopWatch.benchmark()
def f(test):
msg = "This is a test {test}".format(**locals())
print(" jj ", locals())
from cloudmesh.common.debug import VERBOSE
d = {'test': 'Gergor'}
VERBOSE(d, "a", "RED", 100)
from cloudmesh.common.console import Console
msg = 'my message'
Console.ok(msg) # prins a green message
Console.error(msg) # prins a red message proceeded with ERROR
Console.msg(msg) # prins a regular black message
from cloudmesh.common.variables import Variables
variables = Variables()
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
m = {'key': 'value'}
VERBOSE(m)
a = {'p': "ac"}
print(a['p'])
from cloudmesh.common.Shell import Shell
result = Shell.execute('pwd')
print(result)
result = Shell.execute('ls', ['-l', '-a'])
print(result)
result = Shell.execute('ls', '-l -a')
print(result)
result = Shell.ls('-aux')
print(result)
result = Shell.ls('-a')
print(result)
result = Shell.pwd()
print(result)
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('test')
sleep(1)
StopWatch.stop('test')
print(StopWatch.get('test'))
def postcmd(self, stop, line):
StopWatch.stop("command")
try:
variable = Variables()
if "timer" not in variable:
variable["timer"] = "off"
if variable["timer"].lower() in ['on', 'true']:
print("Timer: {:.4f}s ({})".format(StopWatch.get("command"),
line.strip()))
variable.close()
except Exception as e:
Error.traceback(error=e)
return stop
def enlistFlavors(self):
print("\n")
banner(f"Fetching list of flavors for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'pricing',
aws_access_key_id=str(self.ACCESS_KEY),
aws_secret_access_key=str(self.SECRET_KEY),
region_name='us-east-1'
)
StopWatch.start(f"Flavor list {self.cloudname}.")
print("CALLING FUNC.")
#flavor_list = ec2_instance.describe_services(ServiceCode='AmazonEC2')
flavor_list = ec2_instance.get_products(ServiceCode='AmazonEC2')
print("CALLED FUNC.")
opDict= []
for i in flavor_list['PriceList']:
i = json.loads(i)
opDict.append(
{
'productFamily':i['product']['productFamily'],
'memory':i['product']['attributes']['memory'],
'instanceType':i['product']['attributes']['instanceType'],
'tenancy':i['product']['attributes']['tenancy'],
'pubdate':i['publicationDate']
}
)
StopWatch.stop(f"Flavor list {self.cloudname}.")
t = format(StopWatch.get(f"Flavor list {self.cloudname}."), '.2f')
banner(f"Flavor list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} flavors fetched. Time taken {t} \nPrinting first 5 sample flavors:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Flavor_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Flavor list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def test_stopwatch_loop(self):
HEADING()
cumulate = False
dt = 0.1
n = 10
for i in range(0, n):
StopWatch.start("stopwatch loop")
time.sleep(dt)
StopWatch.stop("stopwatch loop")
StopWatch.status("stopwatch loop", True)
t = StopWatch.get("stopwatch loop")
print(t)
assert t >= dt
t = StopWatch.sum("stopwatch loop", digits=4)
print(t)
assert t >= n * dt
CC = CloudCommon({
"ElementOne": "One",
"ElementTwo": "Two",
"ElementThree": {
"SubElementOne": "Three",
"SubElementTwo": "Two"
}
})
# Shows usage of Flatdict (E.Cloudmesh.Common.3)
flat = FlatDict(CC.getItems(), sep='.')
# each banner completes part of 1.a
banner("Normal Dictionary")
print(CC.getItems())
banner("Flat Dictionary")
print(flat)
CC.PrintItemsInDotDict()
banner("Usage of VERBOSE to show info about my dict")
variables = Variables()
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
VERBOSE(CC.getItems())
banner("Execute a shell command for pip")
res = Shell.pip()
print(res)
banner("Let's sleep for three seconds and check our stopwatch")
time.sleep(3)
StopWatch.stop('script')
# Shows StopWatch functionality (E.Cloudmesh.Common.5)
print("Time to Complete: " + str(StopWatch.get('script')) + " seconds")
# fa19-516-140
#This program demonistrate the use of Stopwatch
#functions which been stored in cloudmesh.common.StopWatch
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
#starting Stopwatch timer counting
StopWatch.start("mywatch")
#pausing the program for 15 minutes
sleep(15)
#stoping Stopwatch timer counting
StopWatch.stop("mywatch")
#printing the elapsed time between starting and stoping the stopwatch
# (should be quale to the program execution time, 15 second on our demonstration )
print("Time Elapsed =", StopWatch.get("mywatch"), "(seconds)")
'''
E.Cloudmesh.Common.5
Develop a program that demonstrates the use of cloudmesh.common.StopWatch.
'''
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('test')
sleep(10)
StopWatch.stop('test')
print("Total Time is:", StopWatch.get('test'))
from cloudmesh.common.StopWatch import StopWatch
import time
#Develop a program that demonstartes the use of cloudmesh.common.StopWatch.
StopWatch.start("time check")
time.sleep(5)
StopWatch.stop("time check")
print(StopWatch.get("time check"))
def exercise5(self):
HEADING()
time.sleep(1)
StopWatch.stop("test")
print(StopWatch.get("test"))
def home():
StopWatch.start("test")
sleep(1)
StopWatch.stop("test")
msg = {"msg": "It's working!", "runtime": StopWatch.get("test")}
return jsonify(msg)
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('time')
sleep(1)
a = 1 + 1
print(a)
sleep(1)
b = 2 + 2
print(b)
sleep(1)
print("A+B =", a + b)
sleep(1)
StopWatch.stop('time')
print("running time:", StopWatch.get('time'))
# E.Cloudmesh.Common.5
## Develop a program that demonstartes the use of cloudmesh.common.StopWatch.
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
import sys
StopWatch.start("1")
for _ in range(0, 10):
sleep(0.3)
sys.stdout.write('.')
print('done')
StopWatch.stop("1")
print(StopWatch.get("1"))
StopWatch.benchmark()
def do_kubernetes(self, args, arguments):
"""
::
Usage:
kubernetes name NAME
kubernetes size SIZE
kubernetes image IMAGE
kubernetes flavor FLAVOR
kubernetes cloud CLOUD
kubernetes cluster info
kubernetes cluster deploy
kubernetes cluster benchmark
Arguments:
NAME name of the cluster
SIZE size of the clusteri
IMAGE image of the cluster
FLAVOR flavor of the vm
CLOUD cloud on which the cluster will be created
This command does some useful things.
"""
default = Default()
if arguments.name and arguments.NAME:
print("Set name to {}.".format(arguments.NAME))
default["kubernetes", "name"] = arguments.NAME
elif arguments.size and arguments.SIZE:
print(" Set size to {}.".format(arguments.SIZE))
default["kubernetes", "size"] = arguments.SIZE
elif arguments.image and arguments.IMAGE:
print(" set image to {}.".format(arguments.IMAGE))
default["kubernetes", "image"] = arguments.IMAGE
elif arguments.flavor and arguments.FLAVOR:
print(" set flavor to {}.".format(arguments.FLAVOR))
default["kubernetes", "flavor"] = arguments.FLAVOR
elif arguments.cloud and arguments.CLOUD:
print(" set cloud to {}.".format(arguments.CLOUD))
default["kubernetes", "cloud"] = arguments.CLOUD
elif arguments.cluster and arguments.info:
print("Cluster details:")
print("\tCloud\t: {}".format(default["kubernetes", "cloud"]))
print("\tName\t: {}".format(default["kubernetes", "name"]))
print("\tSize\t: {}".format(default["kubernetes", "size"]))
print("\tImage\t: {}".format(default["kubernetes", "image"]))
print("\tFlavor\t: {}".format(default["kubernetes", "flavor"]))
print("\nIf any of the above details are None/False,")
print(
"please set them using the appropriate command, before deploying the cluster."
)
elif arguments.cluster and arguments.deploy:
if default["kubernetes", "name"] is not None and default[
"kubernetes", "size"] is not None and default[
"kubernetes", "image"] is not None and default[
"kubernetes", "flavor"] is not None and default[
"kubernetes", "cloud"] is not None:
stopwatch = StopWatch()
stopwatch.start('cluster creation for kubernetes')
print("Creating cluster {}...".format(default["kubernetes",
"name"]))
# Define a cluster
command = "cm cluster define --secgroup mesos-secgroup --name {} --count {} --image {} --flavor {} -C {}".format(
default["kubernetes", "name"],
default["kubernetes", "size"], default["kubernetes",
"image"],
default["kubernetes", "flavor"], default["kubernetes",
"cloud"])
os.system(command)
# Use defined cluster
command = "cm default cloud={}".format(default["kubernetes",
"cloud"])
os.system(command)
# Use defined cluster
command = "cm cluster use {}".format(default["kubernetes",
"name"])
os.system(command)
# Create defined cluster
command = "cm cluster allocate"
os.system(command)
# Make hosts file
self.make_hosts()
stopwatch.stop('cluster creation for kubernetes')
print(
'Time Taken for creating clusters required for kubernetes:'
+ str(stopwatch.get('cluster creation for kubernetes')) +
' seconds')
stopwatch = StopWatch()
stopwatch.start('prereq for kubernetes')
# Run ansible script for setting up the various installables
print("Running the setup needed for Kubernetes")
command = 'ansible-playbook ~/cloudmesh.kubernetes/ansiblescript/installations.yml -i ~/cloudmesh.kubernetes/ansiblescript/inventory.txt -e "cloud={}"'.format(
default["kubernetes", "cloud"])
os.system(command)
stopwatch.stop('prereq for kubernetes')
print(
'Time Taken for installing various pre requites for kubernetes:'
+ str(stopwatch.get('prereq for kubernetes')) + ' seconds')
stopwatch = StopWatch()
stopwatch.start('Kubernetes installables')
# Run ansible script for setting up kubernetes cluster
print("Running the Kubernetes setup")
command = 'ansible-playbook ~/cloudmesh.kubernetes/ansiblescript/kubernetes.yml -i ~/cloudmesh.kubernetes/ansiblescript/inventory.txt -e "cloud={}"'.format(
default["kubernetes", "cloud"])
os.system(command)
stopwatch.stop('Kubernetes installables')
print(
'Time Taken for installing kubernetes related packages:' +
str(stopwatch.get('Kubernetes installables')) + ' seconds')
stopwatch = StopWatch()
stopwatch.start('Kubernetes cluster creation')
# Run ansible script for installing kubernetes on master node
print("Installing Kubernetes on master node")
command = 'ansible-playbook ~/cloudmesh.kubernetes/ansiblescript/master.yml -i ~/cloudmesh.kubernetes/ansiblescript/inventory.txt -e "cloud={}"'.format(
default["kubernetes", "cloud"])
os.system(command)
# Run ansible script for joining slaves to master node to make a kubernetes cluster
print("Joining the slaves")
command = 'ansible-playbook ~/cloudmesh.kubernetes/ansiblescript/slaves.yml -i ~/cloudmesh.kubernetes/ansiblescript/inventory.txt -e "cloud={}"'.format(
default["kubernetes", "cloud"])
os.system(command)
stopwatch.stop('Kubernetes cluster creation')
print('Time Taken for deploying Kubernetes cluster:' +
str(stopwatch.get('Kubernetes cluster creation')) +
' seconds')
print("Ansible tasks have been successfully completed.")
print(
"Cluster {} created and Kubernetes is running on cluster.".
format(default["kubernetes", "name"]))
default["kubernetes", "deploy"] = True
else:
print("Please set all the required variables.")
elif arguments.cluster and arguments.benchmark:
if default["kubernetes", "name"] is not None and default[
"kubernetes", "size"] is not None and default[
"kubernetes", "image"] is not None and default[
"kubernetes", "flavor"] is not None and default[
"kubernetes",
"cloud"] is not None and default["kubernetes",
"deploy"]:
stopwatch = StopWatch()
stopwatch.start('Kubernetes benchmark')
# Running the spam detection application
print(
"Running the spam detection application on kubernetes cluster"
)
command = 'ansible-playbook ~/cloudmesh.kubernetes/ansiblescript/runningapplicationonkubernetes.yml -i ~/cloudmesh.kubernetes/ansiblescript/inventory.txt -e "cloud={}"'.format(
default["kubernetes", "cloud"])
os.system(command)
stopwatch.stop('Kubernetes benchmark')
print(
'Time Taken for running the Spam Detection application:' +
str(stopwatch.get('Kubernetes benchmark')) + ' seconds')
print(
"Cluster {} created and Kubernetes is running on cluster.".
format(default["kubernetes", "name"]))
default["kubernetes", "deploy"] = False
else:
print(
"Please set all the required variables and deploy a kubernetes cluster before running benchmarks on it."
)
default.close()
return ""
# sp20-516-231 E.Cloudmesh.Common.5
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
print('cubing every number from 0 to 4,999,999...')
StopWatch.start('cubes')
#sleep(1)
for i in range(5000000):
x = i**3
StopWatch.stop('cubes')
print(StopWatch.get('cubes'))
banner("Cloud Programming")
mesg = "console test"
# test Console
Console.ok(mesg)
Console.msg(mesg)
## Print the Heading, with ###
HEADING("Heading")
##VERBOSE
variables = Variables()
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
m = dict(key1="value1", key2="value2", key3="value3")
VERBOSE(m)
## one more timer sleep
sleep(1)
StopWatch.stop("test")
print(StopWatch.get("test"))
## Clear timer variable
StopWatch.clear()
print(StopWatch.get("test"))
# fal19-516-251 E.Cloudmesh.Common.5
# develop a program that demonstrates the use of cloudmesh.common.StopWatch
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
if __name__ == "__main__":
StopWatch.start("abc")
a = 0
for i in range(1, 100000000):
a = a + 1
StopWatch.stop("abc")
print(StopWatch.get("abc"))
# E.Cloudmesh.Common.5
# Develop a program that demostrate a use of cloudmesh.common.StopWatch
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start("My stop watch")
sleep(2)
StopWatch.stop("My stop watch")
print(StopWatch.get("My stop watch"))
# sp20-516-233 E.Cloudmesh.Common.5
# Develop a program that demonstrates the use ofcloudmesh.common.StopWatch
from cloudmesh.common.StopWatch import StopWatch
if __name__ == "__main__":
# compare how fast the different implementations are
numbers = [100, 200, 300, 400, 500, 600, 700, 800, 900,
1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900,
2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900]
print("start of test1")
StopWatch.start("test1")
total_squares = 0
for n in numbers:
total_squares = total_squares + (n * n)
StopWatch.stop("test1")
print(StopWatch.get("test1"))
print("start of test2")
StopWatch.start("test2")
total_squares2 = 0
for i in range(0, len(numbers)):
total_squares2 += (n * n)
StopWatch.stop("test2")
print(StopWatch.get("test2"))
print("Benchmark")
StopWatch.benchmark()
def decorated_func(*args, **kwargs):
StopWatch.start(func.__name__)
result = func(*args, **kwargs)
StopWatch.stop(func.__name__)
print(StopWatch.get(func.__name__))
return result
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('test1')
y = []
for i in range(1000000):
y.append(i + 1)
print(len(y))
StopWatch.stop('test1')
print(StopWatch.get('test1', digits=4))
StopWatch.start('test2')
x = [i + 1 for i in range(1000000)]
print(len(x))
StopWatch.stop('test2')
print(StopWatch.get('test2', digits=4))
StopWatch.benchmark()
