def get_google_pricing(refresh=False):
# connect to cm db and check for Google info
cm = CmDatabase()
googleinfo = cm.collection('gcp-frugal')
if googleinfo.estimated_document_count() > 0 and not refresh:
Console.msg(f"Using local db gcp flavors...")
return googleinfo
else:
Console.msg(f"Pulling gcp flavor price information...")
googleinfo = requests.get(
'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json?v=1570117883807'
).json()['gcp_price_list']
google_list = []
for machine, locations in googleinfo.items():
if type(
locations
) is dict and 'cores' in locations and 'memory' in locations:
cores = locations['cores']
if cores == 'shared':
continue
memory = locations['memory']
for location in locations:
# 'cores' is end of regions, so stop if found
if location == 'cores':
break
else:
if type(locations[location]) is str:
print(locations[location])
google_list.append(
np.array([
'gcp', machine, location,
float(cores),
float(memory),
float(locations[location])
]))
googleinforeturn = np.stack(google_list, axis=0)
googleinfo = np.stack(googleinforeturn, axis=0)
googleinfo = helpers.format_mat(googleinfo)
# convert to list of dicts
googleinfo = googleinfo.to_dict('records')
# write back to cm db
for entry in googleinfo:
entry["cm"] = {
"kind": 'frugal',
"driver": 'gcp',
"cloud": 'gcp',
"name": str(entry['machine-name'] + '-' + entry['location']),
"updated": str(datetime.utcnow()),
}
Console.msg(f"Writing back to db ...")
cm.update(googleinfo, progress=True)
return cm.collection('gcp-frugal')
class DatabaseUpdate:
"""
Prints the dict of the result but does not add it to the DB
Example:
@DatabaseUpdate("test-collection")
def foo(x):
return {"test": "hello"}
"""
def __init__(self, **kwargs):
self.database = CmDatabase()
def __call__(self, f):
def wrapper(*args, **kwargs):
current = f(*args, **kwargs)
if type(current) == dict:
current = [current]
result = self.database.update(current)
return result
return wrapper
class DatabaseAddOld:
"""
Save the method's output to a MongoDB collection
if the output is a dict or list of dicts.
Example:
@DatabaseUpdate("test-collection")
def foo(x):
return {"test": "hello"}
"""
def __init__(self, collection="cloudmesh", replace=False):
self.database = CmDatabase()
self.replace = replace
self.collection = collection
self.name = Name()
def __call__(self, f):
def wrapper(*args, **kwargs):
result = f(*args, **kwargs)
result["cmid"] = str(self.name)
result["cmcounter"] = str(self.name.counter)
result["created"] = result["modified"] = str(datetime.utcnow())
self.name.incr()
if result is not None:
result["created"] = result["modified"] = str(datetime.utcnow())
r = self.database.update(result, collection=self.collection,
replace=self.replace)
return result
return wrapper
def _cancel(self, specification):
cm = CmDatabase()
entries = cm.find(cloud=self.name, kind='storage')
id = specification['cm']['name']
if id == 'None':
for entry in entries:
if entry['status'] == 'waiting':
entry['status'] = "cancelled"
else:
for entry in entries:
if entry['cm']['id'] == id and entry['status'] == 'waiting':
entry['status'] = "cancelled"
break
cm.update(entries)
specification['status'] = 'completed'
return specification
def get_azure_pricing(refresh=False):
cm = CmDatabase()
azureinfo = cm.collection('azure-frugal')
# check to see if azure-frugal already exists in db
if azureinfo.estimated_document_count() > 0 and not refresh:
Console.msg(f"Using local db azure flavors...")
return azureinfo
# get local db azure flavors (empty if it does not exist yet)
azureinfo = cm.collection('azure-flavor')
# create provider to get region
try:
azureprovider = azureprv.Provider(
name='azure',
configuration="~/.cloudmesh/cloudmesh.yaml")
except:
Console.msg("No azure credentials")
return
region = azureprovider.LOCATION
priceregion = location_conv_dict[region]
if azureinfo.estimated_document_count() == 0 or refresh:
# use provider to fetch info
Console.msg(f"Pulling azure flavors...")
azureinfo = azureprovider.flavors()
else:
# use local, just turn it into a list for matching iteration use
azureinfo = list(azureinfo.find())
# get pricing and limit to what is needed
Console.msg(f"Pulling azure flavor price information...")
pricing = requests.get(
'https://azure.microsoft.com/api/v3/pricing/virtual-machines/calculator/?culture=en-us&discount=mosp&v=20191002-1500-96990').json()
offers = pricing['offers']
modoffers = {}
for key, val in offers.items():
newkey = key.replace('_', '').replace('-', '')
modoffers[newkey] = val
azure_list = []
for flavor in azureinfo:
key = flavor['name'].lower()
if key[0] is 'b':
# basic
search = 'linux' + key[6:].replace('_', '').replace('-',
'').replace(
'promo', '') + 'basic'
elif key[0] is 's':
# standard
search = 'linux' + key[9:].replace('_', '').replace('-',
'').replace(
'promo', '') + 'standard'
elif key[0] is 'l':
# low_priority
search = 'linux' + key[13:].replace('_', '').replace('-',
'').replace(
'promo', '') + 'standard'
else:
print('no matches on key')
continue
try:
found = modoffers[search]
except:
# print('machine match failure on ' + search)
continue
# now to fit it into the frame
azure_list.append(np.array(
['azure', flavor['name'], region, found['cores'], found['ram'],
found['prices']['perhour'][priceregion]['value']]))
azureinfo = np.stack(azure_list, axis=0)
azureinfo = helpers.format_mat(azureinfo)
# convert to list of dicts
azureinfo = azureinfo.to_dict('records')
# write back to cm db
for entry in azureinfo:
entry["cm"] = {
"kind": 'frugal',
"driver": 'azure',
"cloud": 'azure',
"name": str(entry['machine-name'] + '-' + entry['location']),
"updated": str(datetime.utcnow()),
}
Console.msg(f"Writing back to db ...")
cm.update(azureinfo)
return cm.collection('azure-frugal')
class DatabaseUpdate:
"""
The data base decorator utomatically replaces an entry in the database with
the dictionary returned by a function.
It is added to a MongoDB collection. The location is determined from the
values in the dictionary.
The name of the collection is determined from cloud and kind:
cloud-kind
In addition each entry in the collection has a name that must be unique in
that collection.
IN most examples it is pest to separate the updload from the actual return
class. This way we essentially provide two functions one that provide the
dict and another that is responsible for the upload to the database.
Example:
cloudmesh.example.foo contains:
class Provider(object)
def entries(self):
return {
"cloud": "foo",
"kind"": "entries",
"name": "test01"
"test": "hello"}
cloudmesh.example.bar contains:
class Provider(object)
def entries(self):
return {
"cloud": "bar",
"kind"": "entries",
"name": "test01"
"test": "hello"}
cloudmesh.example.provider.foo:
from cloudmesh.example.foo import Provider as FooProvider
from cloudmesh.example.foo import Provider as BarProvider
class Provider(object)
def __init__(self, provider):
if provider == "foo":
provider = FooProvider()
elif provider == "bar":
provider = BarProvider()
@DatabaseUpdate
def entries(self):
provider.entries()
Separating the database and the dictionary creation allows the developer to
implement different providers but only use one class with the same methods
to interact for all providers with the database.
In the combined provider a find function to for example search for entries
by name across collections could be implemented.
"""
# noinspection PyUnusedLocal
def __init__(self, **kwargs):
self.database = CmDatabase()
def __call__(self, f):
def wrapper(*args, **kwargs):
current = f(*args, **kwargs)
if type(current) == dict:
current = [current]
result = self.database.update(current)
return result
return wrapper
class TestMongo:
def setup(self):
self.database = CmDatabase()
self.name = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=1)
def test_00_status(self):
HEADING()
#print(self.name)
#print(self.name.counter)
#print(self.name.id(counter=100))
self.database.clear()
r = self.database.find()
pprint(r)
assert len(r) == 0
def test_01_status(self):
HEADING()
r = self.database.status()
# pprint(r)
assert "Connection refused" not in r
d = {}
for field in ['uptime', 'pid', 'version', 'host']:
d[field] = r[field]
print(Printer.attribute(d))
assert d is not None
def test_02_update(self):
HEADING()
entries = [{"name": "Gregor"}, {"name": "Laszewski"}]
for entry in entries:
entry["cmid"] = str(self.name)
entry["cmcounter"] = self.name.counter
self.name.incr()
self.database.update(entries)
r = self.database.find()
pprint(r)
assert len(r) == 2
def test_03_update(self):
HEADING()
r = self.database.find(name="Gregor")
pprint(r)
assert r[0]['name'] == "Gregor"
def test_04_update(self):
HEADING()
entries = [{
"cmcounter": 1,
"name": "gregor"
}, {
"cmcounter": 2,
"name": "laszewski"
}]
pprint(entries)
for entry in entries:
counter = entry["cmcounter"]
print("Counter:", counter)
entry["cmid"] = self.name.id(counter=counter)
self.database.update(entries, replace=False)
r = self.database.find()
pprint(r)
def test_05_update(self):
HEADING()
r = self.database.find(name="gregor")
pprint(r)
assert r[0]["name"] == "gregor"
def test_06_find_by_counter(self):
HEADING()
r = self.database.find_by_counter(1)
pprint(r)
assert r[0]["name"] == "gregor"
r = self.database.find_by_counter(2)
pprint(r)
assert r[0]["name"] == "laszewski"
def test_07_decorator_update(self):
HEADING()
@DatabaseUpdate(collection="cloudmesh")
def entry():
name = Name()
print(name)
print("OOOO", str(name), name.counter)
d = {
"cmid": str(name),
"cmcounter": name.counter,
"name": "albert"
}
name.incr()
pprint(d)
return d
a = entry()
r = self.database.find_by_counter(3)
pprint(r)
def test_08_decorator_add(self):
HEADING()
@DatabaseAdd(collection="cloudmesh")
def entry():
d = {"name": "zweistein"}
return d
a = entry()
r = self.database.find()
pprint(r)
assert len(r) == 4
def test_09_overwrite(self):
HEADING()
r = self.database.find(name="gregor")[0]
pprint(r)
r["color"] = "red"
self.database.update([r], replace=True)
r = self.database.find(color="red")
pprint(r)
assert len(r) == 1
def test_10_fancy(self):
HEADING()
counter = 1
n = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=counter)
print(n)
entries = [{
"cmcounter": counter,
"cmid": str(n),
"name": "gregor",
"phone": "android"
}]
self.database.update(entries, replace=True)
r = self.database.find()
pprint(r)
assert len(r) == 4
def get_aws_pricing(refresh=False):
# connect to the db
cm = CmDatabase()
# check to see if AWS frugal entries already exist
awsinfo = cm.collection('aws-frugal')
if awsinfo.estimated_document_count() > 0 and not refresh:
# frugal information alredy exists, so return it
Console.msg(f"Using local db aws flavors...")
return awsinfo
# helper function to parse aws info
def aws_parse(input):
if ('location' not in x['attributes'] or 'vcpu' not in x[
'attributes'] or 'price' not in x or x['attributes'][
'memory'] == 'NA'):
return
return np.array(['aws', x['sku'], x['attributes']['location'],
float(x['attributes']['vcpu']),
float(x['attributes']['memory'][:-4].replace(',', '')),
float(x['price']['pricePerUnit']['USD'])])
# check to see if general flavor entries exist
awsinfo = cm.collection('aws-flavor')
aws_list = []
if awsinfo.estimated_document_count() > 0 and not refresh:
# can get info directly from db, just need to reshape
Console.msg(f"Calculating with local db aws flavors...")
for x in awsinfo.find():
tempray = aws_parse(x)
if tempray is not None:
aws_list.append(aws_parse(x))
else:
Console.msg(f"refreshing aws flavors in now ...")
try:
awsprovider = awsprv.Provider(
name='aws',
configuration="~/.cloudmesh/cloudmesh.yaml")
except:
Console.msg("No aws credentials")
return
awsinfo = awsprovider.flavors()
for x in awsinfo:
tempray = aws_parse(x)
if tempray is not None:
aws_list.append(aws_parse(x))
awsinfo = np.stack(aws_list, axis=0)
awsinfo = helpers.format_mat(awsinfo)
# convert to list of dicts
awsinfo = awsinfo.to_dict('records')
# write back to cm db
for entry in awsinfo:
entry["cm"] = {
"kind": 'frugal',
"driver": 'aws',
"cloud": 'aws',
"name": str(entry['machine-name'] + '-' + entry['location']),
"updated": str(datetime.utcnow()),
}
Console.msg(f"Writing back to db ...")
cm.update(awsinfo, progress=True)
return cm.collection('aws-frugal')
def benchmark(self):
#get current cloud and create provider
var_list = Variables(filename="~/.cloudmesh/var-data")
cloud = var_list['cloud']
name = var_list['vm']
newProvider = Provider(name=cloud)
#get vm
cm = CmDatabase()
try:
vm = cm.find_name(name, "vm")[0]
except IndexError:
Console.error(f"could not find vm {name}")
# get file path of the benchmark
filepath = path.dirname(path.dirname(
path.abspath(__file__))) + '/api/benchmark.py'
filepath = filepath.replace('\\', '/')
# prepare command to run the file
vmcom = VmCommand()
try:
Console.msg('waiting for vm to be reachable...')
Console.msg('wait')
newProvider.wait(vm=vm)
except:
Console.msg('could not reach vm for benchmark')
return
try:
Console.msg(f'moving benchmark file to vm...')
Console.msg(f'put ' + filepath + ' /home/ubuntu')
vmcom.do_vm('put ' + filepath + ' /home/ubuntu')
except:
Console.msg(
f'could not ssh into vm, make sure one is running and reachable'
)
return
try:
Console.msg(f'executing the benchmark...')
Console.msg(
'ssh --command=\"chmod +x benchmark.py;./benchmark.py;rm benchmark.py;exit\"'
)
benchtime = newProvider.ssh(
vm=vm,
command=
"chmod +x benchmark.py;./benchmark.py;rm benchmark.py;exit")
except:
Console.msg(
f'could not ssh into vm, make sure one is running and reachable'
)
return
print("successfully benchmarked")
benchtime = float(benchtime.strip())
#add the benchmark, cloud, vm, and time to db
benchdict = {}
benchdict['cloud'] = cloud
benchdict['name'] = name
benchdict['ImageId'] = vm['ImageId']
benchdict['flavor'] = vm['InstanceType']
benchdict['region'] = vm['Placement']['AvailabilityZone']
benchdict['BenchmarkTime'] = benchtime
benchdict['updated'] = str(datetime.utcnow())
benchdict["cm"] = {
"kind": 'frugal-benchmark',
"driver": cloud,
"cloud": cloud,
"name": name,
"updated": str(datetime.utcnow()),
}
cm.update(benchdict, progress=True)
return ""
