def enlistInstances(self):
print("\n")
banner(f"Fetching instance list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Instance list {self.cloudname}.")
vm_instance_list = ec2_instance.describe_instances()
opDict= []
opDict= [{'ImageId':i.get('ImageId'), 'InstanceId':i.get('InstanceId'), 'InstanceType':i.get('InstanceType'), 'KeyName':i.get('KeyName'), 'LaunchTime':i.get('LaunchTime'), 'VpcId':i.get('VpcId'), 'Zone':i['Placement']['AvailabilityZone'], 'State':i['State']['Name']} for i in vm_instance_list['Reservations'][0]['Instances']]
StopWatch.stop(f"Instance list {self.cloudname}.")
t = format(StopWatch.get(f"Instance list {self.cloudname}."), '.2f')
banner(f"Instance list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} instances fetched. Time taken {t} \nPrinting first 5 sample instances:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Instance_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Instance list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def enlistImages(self):
banner(f"Fetching image list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Image list {self.cloudname}.")
image_list = ec2_instance.describe_images()
opDict= []
opDict = [{'ImageId':i.get('ImageId'), 'ImageType': i.get('ImageType'), 'Description': i.get('Description'), 'Name': i.get('Name'), 'State':i.get('State'),'Public': i.get('Public'),'CreationDate':i.get('CreationDate')} for i in image_list['Images']]
StopWatch.stop(f"Image list {self.cloudname}.")
t = format(StopWatch.get(f"Image list {self.cloudname}."), '.2f')
banner(f"Image list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} images fetched. Time taken {t} \nPrinting first 5 sample images:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Image_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Image list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def benchmark(cls, sysinfo=True):
"""
prints out all timers in a convenient benchmark tabble
:return:
:rtype:
"""
#
# PRINT PLATFORM
#
if sysinfo:
data_platform = systeminfo()
print(
Printer.attribute(data_platform,
["Machine Arribute", "Time/s"]))
#
# PRINT TIMERS
#
timers = StopWatch.keys()
data_timers = {}
for timer in timers:
data_timers[timer] = {
'time': round(StopWatch.get(timer), 2),
'timer': timer
}
for attribute in [
"node", "system", "machine", "mac_version", "win_version"
]:
data_timers[timer][attribute] = data_platform[attribute]
# print(Printer.attribute(data_timers, header=["Command", "Time/s"]))
print(
Printer.write(data_timers,
order=[
"timer", "time", "node", "system", "mac_version",
"win_version"
]))
print()
print(
Printer.write(data_timers,
order=[
"timer", "time", "node", "system", "mac_version",
"win_version"
],
output="csv"))
def test_02_list_os(self):
HEADING()
ostypes = self.p.list_os()
print(Printer.write(
ostypes,
order=["id", "64_bit", "description", "family_descr", "family_id"],
header=["id", "64_bit", "description", "family_descr", "family_id"]))
def info(cls, cluster, format='json', all=False):
if all:
result = Shell.ssh(cluster, 'sinfo --format=\"%all\"')
else:
result = Shell.ssh(cluster, 'sinfo --format=\"%P|%a|%l|%D|%t|%N\"')
# ignore leading lines till header is found
l = result.splitlines()
for i, res in enumerate(l):
if 'PARTITION|AVAIL|' in res:
result = "\n".join(l[i:])
break
parser = TableParser(strip=False)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=[
'cluster', 'partition', 'avail',
'timelimit', 'nodes', 'state',
'nodelist', 'updated'
],
output=format))
def Print(self, data, output=None):
"""
print registry
:param data:
:param output:
:return:
"""
if output == "table":
order = self.output[Registry.kind]['order'] # not pretty
header = self.output[Registry.kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def boot(self, order='price', refresh=False, cloud=None):
clouds = ['aws', 'azure', 'gcp']
if cloud in clouds:
clouds = [cloud]
Console.msg(f"Checking to see which providers are bootable ...")
reachdict = {}
for cloud in clouds:
try:
tempProv = Provider(
name=cloud, configuration="~/.cloudmesh/cloudmesh.yaml")
Console.msg(cloud + " reachable ...")
reachdict[cloud] = tempProv
except:
Console.msg(cloud + " not available ...")
flavorframe = self.list(order, 10000000, refresh, printit=False)
keysya = list(reachdict.keys())
flavorframe = flavorframe[flavorframe['provider'].isin(keysya)]
Console.msg(f"Showing top 5 options, booting first option now...")
converted = flavorframe.head(5).to_dict('records')
print(Printer.write(converted))
cheapest = converted[0]
var_list = Variables(filename="~/.cloudmesh/var-data")
var_list['cloud'] = cheapest['provider']
Console.msg(f'new cloud is ' + var_list['cloud'] +
', booting up the vm with flavor ' +
cheapest['machine-name'])
vmcom = VmCommand()
vmcom.do_vm('boot --flavor=' + cheapest['machine-name'])
return ""
def Print(self, data, output=None):
"""
print output in a structured format
:param data: input data to be printed out
:param output: type of structured output
:return: structured output
"""
if output == "table":
order = self.output[RegistryPickle.kind]['order'] # not pretty
header = self.output[RegistryPickle.kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def _ssh(self, command, timeout=1):
"""
An internal ssh command that allows to ssh into a remote host while having a small timeout value
Args:
command (str): the command to be executed
timeout (int): the number of seconds for a timeout
Returns:
the result of the executed command as a string.
It ignores errors and includes them into the result string.
"""
_command = f"ssh -o ConnectTimeout={timeout} -o StrictHostKeyChecking=no {command}"
# print (_command)
if os_is_windows:
import subprocess
hostname = subprocess.run(['hostname'],
capture_output=True,
text=True).stdout.strip()
results = Host.ssh(hosts=hostname, command=_command)
print(Printer.write(results))
for entry in results:
print(str(entry["stdout"]))
r = str(entry["stdout"])
else:
r = Shell.run(_command).strip()
return r
def Print(self, data, kind=None, output="table"):
"""
Print out the result dictionary as table(by default) or json.
:param data: dic returned from volume functions
:param kind: kind of provider
:param output: "table" or "json"
:return:
"""
if kind is None and len(data) > 0:
kind = data[0]["cm"]["kind"]
if output == "table":
order = self.provider.output[kind]['order']
header = self.provider.output[kind]['header']
if 'humanize' in self.provider.output[kind]:
humanize = self.provider.output[kind]['humanize']
else:
humanize = None
print(Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output,
humanize=humanize)
)
else:
print(Printer.write(data, output=output))
def run(self,
script=None,
hosts=None,
username=None,
processors=4,
verbose=False):
results = []
if type(hosts) != list:
hosts = Parameter.expand(hosts)
for command in script.splitlines():
print(hosts, "->", command)
if command.startswith("#") or command.strip() == "":
pass
# print (command)
elif len(hosts) == 1 and hosts[0] == self.hostname:
os.system(command)
elif len(hosts) == 1 and hosts[0] != self.hostname:
host = hosts[0]
os.system(f"ssh {host} {command}")
else:
result = Host.ssh(hosts=hosts,
command=command,
username=username,
key="~/.ssh/id_rsa.pub",
processors=processors,
executor=os.system)
results.append(result)
if verbose:
pprint(results)
for result in results:
print(Printer.write(result, order=['host', 'stdout']))
return results
def Print(self, data, output=None, kind=None):
if output == "table":
if kind == "secrule":
# this is just a temporary fix, both in sec.py and here the
# secgruops and secrules should be separated
result = []
for group in data:
# for rule in group['security_group_rules']:
# rule['name'] = group['name']
result.append(group)
data = result
order = self.output[kind]['order'] # not pretty
header = self.output[kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def Print(self, data, output=None, kind=None):
# TODO: Joaquin
if output == "table":
if kind == "secrule":
result = []
for group in data:
for rule in group['security_group_rules']:
rule['name'] = group['name']
result.append(rule)
data = result
order = self.output[kind]['order'] # not pretty
header = self.output[kind]['header'] # not pretty
humanize = self.output[kind]['humanize'] # not pretty
print(Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output,
humanize=humanize)
)
else:
print(Printer.write(data, output=output))
def test_004_table(self):
HEADING("Printer.write of a table object")
output = Printer.write(self.d,
order=None,
header=None,
output="table",
sort_keys=True)
print(output)
def print_keys(keys):
print(
Printer.write(
keys,
sort_keys=["name"],
order=["name", "type", "fingerprint", "comment"],
header=["Name", "Type", "Fingerprint", "Comment"],
output=arguments.output))
def enlistFlavors(self):
print("\n")
banner(f"Fetching list of flavors for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'pricing',
aws_access_key_id=str(self.ACCESS_KEY),
aws_secret_access_key=str(self.SECRET_KEY),
region_name='us-east-1'
)
StopWatch.start(f"Flavor list {self.cloudname}.")
print("CALLING FUNC.")
#flavor_list = ec2_instance.describe_services(ServiceCode='AmazonEC2')
flavor_list = ec2_instance.get_products(ServiceCode='AmazonEC2')
print("CALLED FUNC.")
opDict= []
for i in flavor_list['PriceList']:
i = json.loads(i)
opDict.append(
{
'productFamily':i['product']['productFamily'],
'memory':i['product']['attributes']['memory'],
'instanceType':i['product']['attributes']['instanceType'],
'tenancy':i['product']['attributes']['tenancy'],
'pubdate':i['publicationDate']
}
)
StopWatch.stop(f"Flavor list {self.cloudname}.")
t = format(StopWatch.get(f"Flavor list {self.cloudname}."), '.2f')
banner(f"Flavor list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} flavors fetched. Time taken {t} \nPrinting first 5 sample flavors:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Flavor_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Flavor list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def test_005_csv(self):
HEADING("Printer.write of a csv object")
output = Printer.write(self.d,
order=None,
header=None,
output="csv",
sort_keys=True)
print(output)
def test_ssh_processors(self):
print()
for processors in range(1, len(hosts)):
print("Processors:", processors)
results = self.ssh(processors=processors)
print(Printer.write(results))
for result in results:
assert result["success"]
def _print(results):
arguments.output = arguments.output or 'table'
if arguments.output == 'table':
print(
Printer.write(results, order=['host', 'success',
'stdout']))
else:
pprint(results)
def test_001_yaml(self):
HEADING("Printer.write of a yaml object")
output = Printer.write(self.d,
order=None,
header=None,
output="yaml",
sort_keys=True)
print(output)
assert ":" in output
def test_002_json(self):
HEADING("Printer.write of a json object")
output = Printer.write(self.d,
order=None,
header=None,
output="json",
sort_keys=True)
print(output)
assert "{" in output
def test_003_dict(self):
HEADING("Printer.write of a dict object")
output = Printer.write(self.d,
order=None,
header=None,
output="dict",
sort_keys=True)
pprint(output)
assert type(output) == dict
def test_8_list_secgroups(self):
HEADING()
secgroups = self.p.list_secgroups()
for secgroup in secgroups:
print(secgroup["name"])
rules = self.p.list_secgroup_rules(secgroup["name"])
print(Printer.write(rules,
sort_keys=["ip_protocol", "from_port", "to_port", "ip_range"],
order=["ip_protocol", "from_port", "to_port", "ip_range"],
header=["ip_protocol", "from_port", "to_port", "ip_range"])
)
def _print(results):
arguments.output = arguments.output or 'table'
if arguments.output in ['table', 'yaml']:
print(
Printer.write(
results,
order=['host', 'success', 'stdout', 'stderr'],
output=arguments.output))
else:
pprint(results)
def Print(kind, list):
if kind == "group":
output = ""
else:
output = groups.output
print(Printer.write(list,
sort_keys=output[kind]['sort_keys'],
order=output[kind]['order'],
header=output[kind]['header'],
output=arguments.output))
def test_members(self):
HEADING()
members = g.members(name="test")
pprint(members)
print(Printer.write(members))
for member in members:
assert member['name'] in services
assert len(members) == len(services)
def test_ping_processor(self):
print()
for processors in range(1, len(hosts)):
print("Processors:", processors)
results = self.ping(processors=processors)
print(
Printer.write(
results, order=['host', 'success', 'max', 'min',
'stddev']))
for result in results:
assert result['success']
def _LIST_PRINT(l, output, order=None):
if output in ["yaml", "dict", "json"]:
l_converted = _convert(l)
else:
l_converted = l
result = Printer.write(l_converted, order=order, output=output)
if output in ["table", "yaml", "json", "csv"]:
print(result)
else:
pprint(result)
def do_plugin(self, args, arguments):
"""
::
Usage:
plugin install PLUGIN [-s]
plugin uninstall PLUGIN
plugin list
plugin ? [--format=FORMAT]
Arguments:
PLUGIN the name of the plugin
Description:
plugin available
lists the available plugins
plugin list
lists the plugin
plugin install
installs the given plugin
plugin uninstall
uninstalls the given plugin
"""
if arguments['--format'] is None:
arguments['--format'] = 'table'
# print (arguments)
if arguments.install:
plugins = PluginManager()
plugins.load()
if arguments["-s"]:
plugins.source_install(arguments.PLUGIN)
else:
plugins.pip_install(arguments.PLUGIN)
elif arguments.uninstall:
plugins = PluginManager()
plugins.load()
plugins.uninstall(arguments.PLUGIN)
elif '?' in arguments:
plugins = PluginManager()
plugins.load()
print(
Printer.write(plugins.data['plugins'],
output=arguments["--format"],
order=["name", "status", "description"],
sort_keys="name"))
def print(self, order=None, header=None, output="table"):
"""
prints the inventory in the output format
:param order:
:type order:
:param header:
:type header:
:return:
:rtype:
"""
order = order or self.order
header = header or self.header
print(Printer.write(self.data, order=order, header=header))
