def get_google_pricing(refresh=False): # connect to cm db and check for Google info cm = CmDatabase() googleinfo = cm.collection('gcp-frugal') if googleinfo.estimated_document_count() > 0 and not refresh: Console.msg(f"Using local db gcp flavors...") return googleinfo else: Console.msg(f"Pulling gcp flavor price information...") googleinfo = requests.get( 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json?v=1570117883807' ).json()['gcp_price_list'] google_list = [] for machine, locations in googleinfo.items(): if type( locations ) is dict and 'cores' in locations and 'memory' in locations: cores = locations['cores'] if cores == 'shared': continue memory = locations['memory'] for location in locations: # 'cores' is end of regions, so stop if found if location == 'cores': break else: if type(locations[location]) is str: print(locations[location]) google_list.append( np.array([ 'gcp', machine, location, float(cores), float(memory), float(locations[location]) ])) googleinforeturn = np.stack(google_list, axis=0) googleinfo = np.stack(googleinforeturn, axis=0) googleinfo = helpers.format_mat(googleinfo) # convert to list of dicts googleinfo = googleinfo.to_dict('records') # write back to cm db for entry in googleinfo: entry["cm"] = { "kind": 'frugal', "driver": 'gcp', "cloud": 'gcp', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(googleinfo, progress=True) return cm.collection('gcp-frugal')
class DatabaseUpdate: """ Prints the dict of the result but does not add it to the DB Example: @DatabaseUpdate("test-collection") def foo(x): return {"test": "hello"} """ def __init__(self, **kwargs): self.database = CmDatabase() def __call__(self, f): def wrapper(*args, **kwargs): current = f(*args, **kwargs) if type(current) == dict: current = [current] result = self.database.update(current) return result return wrapper
class DatabaseAddOld: """ Save the method's output to a MongoDB collection if the output is a dict or list of dicts. Example: @DatabaseUpdate("test-collection") def foo(x): return {"test": "hello"} """ def __init__(self, collection="cloudmesh", replace=False): self.database = CmDatabase() self.replace = replace self.collection = collection self.name = Name() def __call__(self, f): def wrapper(*args, **kwargs): result = f(*args, **kwargs) result["cmid"] = str(self.name) result["cmcounter"] = str(self.name.counter) result["created"] = result["modified"] = str(datetime.utcnow()) self.name.incr() if result is not None: result["created"] = result["modified"] = str(datetime.utcnow()) r = self.database.update(result, collection=self.collection, replace=self.replace) return result return wrapper
def _cancel(self, specification): cm = CmDatabase() entries = cm.find(cloud=self.name, kind='storage') id = specification['cm']['name'] if id == 'None': for entry in entries: if entry['status'] == 'waiting': entry['status'] = "cancelled" else: for entry in entries: if entry['cm']['id'] == id and entry['status'] == 'waiting': entry['status'] = "cancelled" break cm.update(entries) specification['status'] = 'completed' return specification
def get_azure_pricing(refresh=False): cm = CmDatabase() azureinfo = cm.collection('azure-frugal') # check to see if azure-frugal already exists in db if azureinfo.estimated_document_count() > 0 and not refresh: Console.msg(f"Using local db azure flavors...") return azureinfo # get local db azure flavors (empty if it does not exist yet) azureinfo = cm.collection('azure-flavor') # create provider to get region try: azureprovider = azureprv.Provider( name='azure', configuration="~/.cloudmesh/cloudmesh.yaml") except: Console.msg("No azure credentials") return region = azureprovider.LOCATION priceregion = location_conv_dict[region] if azureinfo.estimated_document_count() == 0 or refresh: # use provider to fetch info Console.msg(f"Pulling azure flavors...") azureinfo = azureprovider.flavors() else: # use local, just turn it into a list for matching iteration use azureinfo = list(azureinfo.find()) # get pricing and limit to what is needed Console.msg(f"Pulling azure flavor price information...") pricing = requests.get( 'https://azure.microsoft.com/api/v3/pricing/virtual-machines/calculator/?culture=en-us&discount=mosp&v=20191002-1500-96990').json() offers = pricing['offers'] modoffers = {} for key, val in offers.items(): newkey = key.replace('_', '').replace('-', '') modoffers[newkey] = val azure_list = [] for flavor in azureinfo: key = flavor['name'].lower() if key[0] is 'b': # basic search = 'linux' + key[6:].replace('_', '').replace('-', '').replace( 'promo', '') + 'basic' elif key[0] is 's': # standard search = 'linux' + key[9:].replace('_', '').replace('-', '').replace( 'promo', '') + 'standard' elif key[0] is 'l': # low_priority search = 'linux' + key[13:].replace('_', '').replace('-', '').replace( 'promo', '') + 'standard' else: print('no matches on key') continue try: found = modoffers[search] except: # print('machine match failure on ' + search) continue # now to fit it into the frame azure_list.append(np.array( ['azure', flavor['name'], region, found['cores'], found['ram'], found['prices']['perhour'][priceregion]['value']])) azureinfo = np.stack(azure_list, axis=0) azureinfo = helpers.format_mat(azureinfo) # convert to list of dicts azureinfo = azureinfo.to_dict('records') # write back to cm db for entry in azureinfo: entry["cm"] = { "kind": 'frugal', "driver": 'azure', "cloud": 'azure', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(azureinfo) return cm.collection('azure-frugal')
class DatabaseUpdate: """ The data base decorator utomatically replaces an entry in the database with the dictionary returned by a function. It is added to a MongoDB collection. The location is determined from the values in the dictionary. The name of the collection is determined from cloud and kind: cloud-kind In addition each entry in the collection has a name that must be unique in that collection. IN most examples it is pest to separate the updload from the actual return class. This way we essentially provide two functions one that provide the dict and another that is responsible for the upload to the database. Example: cloudmesh.example.foo contains: class Provider(object) def entries(self): return { "cloud": "foo", "kind"": "entries", "name": "test01" "test": "hello"} cloudmesh.example.bar contains: class Provider(object) def entries(self): return { "cloud": "bar", "kind"": "entries", "name": "test01" "test": "hello"} cloudmesh.example.provider.foo: from cloudmesh.example.foo import Provider as FooProvider from cloudmesh.example.foo import Provider as BarProvider class Provider(object) def __init__(self, provider): if provider == "foo": provider = FooProvider() elif provider == "bar": provider = BarProvider() @DatabaseUpdate def entries(self): provider.entries() Separating the database and the dictionary creation allows the developer to implement different providers but only use one class with the same methods to interact for all providers with the database. In the combined provider a find function to for example search for entries by name across collections could be implemented. """ # noinspection PyUnusedLocal def __init__(self, **kwargs): self.database = CmDatabase() def __call__(self, f): def wrapper(*args, **kwargs): current = f(*args, **kwargs) if type(current) == dict: current = [current] result = self.database.update(current) return result return wrapper
class TestMongo: def setup(self): self.database = CmDatabase() self.name = Name(experiment="exp", group="grp", user="******", kind="vm", counter=1) def test_00_status(self): HEADING() #print(self.name) #print(self.name.counter) #print(self.name.id(counter=100)) self.database.clear() r = self.database.find() pprint(r) assert len(r) == 0 def test_01_status(self): HEADING() r = self.database.status() # pprint(r) assert "Connection refused" not in r d = {} for field in ['uptime', 'pid', 'version', 'host']: d[field] = r[field] print(Printer.attribute(d)) assert d is not None def test_02_update(self): HEADING() entries = [{"name": "Gregor"}, {"name": "Laszewski"}] for entry in entries: entry["cmid"] = str(self.name) entry["cmcounter"] = self.name.counter self.name.incr() self.database.update(entries) r = self.database.find() pprint(r) assert len(r) == 2 def test_03_update(self): HEADING() r = self.database.find(name="Gregor") pprint(r) assert r[0]['name'] == "Gregor" def test_04_update(self): HEADING() entries = [{ "cmcounter": 1, "name": "gregor" }, { "cmcounter": 2, "name": "laszewski" }] pprint(entries) for entry in entries: counter = entry["cmcounter"] print("Counter:", counter) entry["cmid"] = self.name.id(counter=counter) self.database.update(entries, replace=False) r = self.database.find() pprint(r) def test_05_update(self): HEADING() r = self.database.find(name="gregor") pprint(r) assert r[0]["name"] == "gregor" def test_06_find_by_counter(self): HEADING() r = self.database.find_by_counter(1) pprint(r) assert r[0]["name"] == "gregor" r = self.database.find_by_counter(2) pprint(r) assert r[0]["name"] == "laszewski" def test_07_decorator_update(self): HEADING() @DatabaseUpdate(collection="cloudmesh") def entry(): name = Name() print(name) print("OOOO", str(name), name.counter) d = { "cmid": str(name), "cmcounter": name.counter, "name": "albert" } name.incr() pprint(d) return d a = entry() r = self.database.find_by_counter(3) pprint(r) def test_08_decorator_add(self): HEADING() @DatabaseAdd(collection="cloudmesh") def entry(): d = {"name": "zweistein"} return d a = entry() r = self.database.find() pprint(r) assert len(r) == 4 def test_09_overwrite(self): HEADING() r = self.database.find(name="gregor")[0] pprint(r) r["color"] = "red" self.database.update([r], replace=True) r = self.database.find(color="red") pprint(r) assert len(r) == 1 def test_10_fancy(self): HEADING() counter = 1 n = Name(experiment="exp", group="grp", user="******", kind="vm", counter=counter) print(n) entries = [{ "cmcounter": counter, "cmid": str(n), "name": "gregor", "phone": "android" }] self.database.update(entries, replace=True) r = self.database.find() pprint(r) assert len(r) == 4
def get_aws_pricing(refresh=False): # connect to the db cm = CmDatabase() # check to see if AWS frugal entries already exist awsinfo = cm.collection('aws-frugal') if awsinfo.estimated_document_count() > 0 and not refresh: # frugal information alredy exists, so return it Console.msg(f"Using local db aws flavors...") return awsinfo # helper function to parse aws info def aws_parse(input): if ('location' not in x['attributes'] or 'vcpu' not in x[ 'attributes'] or 'price' not in x or x['attributes'][ 'memory'] == 'NA'): return return np.array(['aws', x['sku'], x['attributes']['location'], float(x['attributes']['vcpu']), float(x['attributes']['memory'][:-4].replace(',', '')), float(x['price']['pricePerUnit']['USD'])]) # check to see if general flavor entries exist awsinfo = cm.collection('aws-flavor') aws_list = [] if awsinfo.estimated_document_count() > 0 and not refresh: # can get info directly from db, just need to reshape Console.msg(f"Calculating with local db aws flavors...") for x in awsinfo.find(): tempray = aws_parse(x) if tempray is not None: aws_list.append(aws_parse(x)) else: Console.msg(f"refreshing aws flavors in now ...") try: awsprovider = awsprv.Provider( name='aws', configuration="~/.cloudmesh/cloudmesh.yaml") except: Console.msg("No aws credentials") return awsinfo = awsprovider.flavors() for x in awsinfo: tempray = aws_parse(x) if tempray is not None: aws_list.append(aws_parse(x)) awsinfo = np.stack(aws_list, axis=0) awsinfo = helpers.format_mat(awsinfo) # convert to list of dicts awsinfo = awsinfo.to_dict('records') # write back to cm db for entry in awsinfo: entry["cm"] = { "kind": 'frugal', "driver": 'aws', "cloud": 'aws', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(awsinfo, progress=True) return cm.collection('aws-frugal')
def benchmark(self): #get current cloud and create provider var_list = Variables(filename="~/.cloudmesh/var-data") cloud = var_list['cloud'] name = var_list['vm'] newProvider = Provider(name=cloud) #get vm cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") # get file path of the benchmark filepath = path.dirname(path.dirname( path.abspath(__file__))) + '/api/benchmark.py' filepath = filepath.replace('\\', '/') # prepare command to run the file vmcom = VmCommand() try: Console.msg('waiting for vm to be reachable...') Console.msg('wait') newProvider.wait(vm=vm) except: Console.msg('could not reach vm for benchmark') return try: Console.msg(f'moving benchmark file to vm...') Console.msg(f'put ' + filepath + ' /home/ubuntu') vmcom.do_vm('put ' + filepath + ' /home/ubuntu') except: Console.msg( f'could not ssh into vm, make sure one is running and reachable' ) return try: Console.msg(f'executing the benchmark...') Console.msg( 'ssh --command=\"chmod +x benchmark.py;./benchmark.py;rm benchmark.py;exit\"' ) benchtime = newProvider.ssh( vm=vm, command= "chmod +x benchmark.py;./benchmark.py;rm benchmark.py;exit") except: Console.msg( f'could not ssh into vm, make sure one is running and reachable' ) return print("successfully benchmarked") benchtime = float(benchtime.strip()) #add the benchmark, cloud, vm, and time to db benchdict = {} benchdict['cloud'] = cloud benchdict['name'] = name benchdict['ImageId'] = vm['ImageId'] benchdict['flavor'] = vm['InstanceType'] benchdict['region'] = vm['Placement']['AvailabilityZone'] benchdict['BenchmarkTime'] = benchtime benchdict['updated'] = str(datetime.utcnow()) benchdict["cm"] = { "kind": 'frugal-benchmark', "driver": cloud, "cloud": cloud, "name": name, "updated": str(datetime.utcnow()), } cm.update(benchdict, progress=True) return ""