def Print(self, data, output=None):
"""
print registry
:param data:
:param output:
:return:
"""
if output == "table":
order = self.output[Registry.kind]['order'] # not pretty
header = self.output[Registry.kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def Print(self, data, output=None, kind=None):
if output == "table":
if kind == "secrule":
# this is just a temporary fix, both in sec.py and here the
# secgruops and secrules should be separated
result = []
for group in data:
# for rule in group['security_group_rules']:
# rule['name'] = group['name']
result.append(group)
data = result
order = self.output[kind]['order'] # not pretty
header = self.output[kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def enlistImages(self):
banner(f"Fetching image list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Image list {self.cloudname}.")
image_list = ec2_instance.describe_images()
opDict= []
opDict = [{'ImageId':i.get('ImageId'), 'ImageType': i.get('ImageType'), 'Description': i.get('Description'), 'Name': i.get('Name'), 'State':i.get('State'),'Public': i.get('Public'),'CreationDate':i.get('CreationDate')} for i in image_list['Images']]
StopWatch.stop(f"Image list {self.cloudname}.")
t = format(StopWatch.get(f"Image list {self.cloudname}."), '.2f')
banner(f"Image list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} images fetched. Time taken {t} \nPrinting first 5 sample images:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Image_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Image list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def Print(self, data, output=None, kind=None):
# TODO: Joaquin
if output == "table":
if kind == "secrule":
result = []
for group in data:
for rule in group['security_group_rules']:
rule['name'] = group['name']
result.append(rule)
data = result
order = self.output[kind]['order'] # not pretty
header = self.output[kind]['header'] # not pretty
humanize = self.output[kind]['humanize'] # not pretty
print(Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output,
humanize=humanize)
)
else:
print(Printer.write(data, output=output))
def enlistInstances(self):
print("\n")
banner(f"Fetching instance list for {self.cloudname} cloud service.")
if self.cloudname == "aws":
try:
ec2_instance = boto3.client(
'ec2',
aws_access_key_id=self.ACCESS_KEY,
aws_secret_access_key=self.SECRET_KEY,
region_name=self.REGION_ID
)
StopWatch.start(f"Instance list {self.cloudname}.")
vm_instance_list = ec2_instance.describe_instances()
opDict= []
opDict= [{'ImageId':i.get('ImageId'), 'InstanceId':i.get('InstanceId'), 'InstanceType':i.get('InstanceType'), 'KeyName':i.get('KeyName'), 'LaunchTime':i.get('LaunchTime'), 'VpcId':i.get('VpcId'), 'Zone':i['Placement']['AvailabilityZone'], 'State':i['State']['Name']} for i in vm_instance_list['Reservations'][0]['Instances']]
StopWatch.stop(f"Instance list {self.cloudname}.")
t = format(StopWatch.get(f"Instance list {self.cloudname}."), '.2f')
banner(f"Instance list fetched for {self.cloudname} cloud service.\nTotal {len(opDict)} instances fetched. Time taken {t} \nPrinting first 5 sample instances:")
print(Printer.write(opDict[:6], output='table'))
#Saving complete list to a file
opFile = f"{self.cloudname}_Instance_list.txt"
with open(opFile,'w') as fo:
print(Printer.write(opDict, output='table'), file=fo)
except Exception as e:
Console.error(f"Instance list of {self.cloudname} can\'t be fetched. Error:\n{e}")
else:
Console.error(f"Provider {self.cloudname} not supported")
raise ValueError(f"provider {self.cloudname} not supported")
def Print(self, data, kind=None, output="table"):
"""
Print out the result dictionary as table(by default) or json.
:param data: dic returned from volume functions
:param kind: kind of provider
:param output: "table" or "json"
:return:
"""
if kind is None and len(data) > 0:
kind = data[0]["cm"]["kind"]
if output == "table":
order = self.provider.output[kind]['order']
header = self.provider.output[kind]['header']
if 'humanize' in self.provider.output[kind]:
humanize = self.provider.output[kind]['humanize']
else:
humanize = None
print(Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output,
humanize=humanize)
)
else:
print(Printer.write(data, output=output))
def Print(self, data, output=None):
"""
print output in a structured format
:param data: input data to be printed out
:param output: type of structured output
:return: structured output
"""
if output == "table":
order = self.output[RegistryPickle.kind]['order'] # not pretty
header = self.output[RegistryPickle.kind]['header'] # not pretty
# humanize = self.output[kind]['humanize'] # not pretty
print(
Printer.flatwrite(
data,
sort_keys=["name"],
order=order,
header=header,
output=output,
# humanize=humanize
))
else:
print(Printer.write(data, output=output))
def Print(self, data, output, kind):
if output == "table":
order = self.output[kind]['order']
header = self.output[kind]['header']
print(Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output, )
)
else:
print(Printer.write(data, output=output))
def pretty_print(self, data, data_type, output=None):
if output == "table":
order = self.output[data_type]['order'] # not pretty
header = self.output[data_type]['header'] # not pretty
sort_keys = self.output[data_type]['sort_keys']
print(
Printer.flatwrite(
data,
sort_keys=sort_keys,
order=order,
header=header,
output=output,
))
else:
print(Printer.write(data, output=output))
def benchmark(cls, sysinfo=True):
"""
prints out all timers in a convenient benchmark tabble
:return:
:rtype:
"""
#
# PRINT PLATFORM
#
if sysinfo:
data_platform = systeminfo()
print(
Printer.attribute(data_platform,
["Machine Arribute", "Time/s"]))
#
# PRINT TIMERS
#
timers = StopWatch.keys()
data_timers = {}
for timer in timers:
data_timers[timer] = {
'time': round(StopWatch.get(timer), 2),
'timer': timer
}
for attribute in [
"node", "system", "machine", "mac_version", "win_version"
]:
data_timers[timer][attribute] = data_platform[attribute]
# print(Printer.attribute(data_timers, header=["Command", "Time/s"]))
print(
Printer.write(data_timers,
order=[
"timer", "time", "node", "system", "mac_version",
"win_version"
]))
print()
print(
Printer.write(data_timers,
order=[
"timer", "time", "node", "system", "mac_version",
"win_version"
],
output="csv"))
def _ssh(self, command, timeout=1):
"""
An internal ssh command that allows to ssh into a remote host while having a small timeout value
Args:
command (str): the command to be executed
timeout (int): the number of seconds for a timeout
Returns:
the result of the executed command as a string.
It ignores errors and includes them into the result string.
"""
_command = f"ssh -o ConnectTimeout={timeout} -o StrictHostKeyChecking=no {command}"
# print (_command)
if os_is_windows:
import subprocess
hostname = subprocess.run(['hostname'],
capture_output=True,
text=True).stdout.strip()
results = Host.ssh(hosts=hostname, command=_command)
print(Printer.write(results))
for entry in results:
print(str(entry["stdout"]))
r = str(entry["stdout"])
else:
r = Shell.run(_command).strip()
return r
def keypair_refresh(self):
"""
List all the available key pair
associated with account
:returns: None
:rtype: NoneType
"""
driver = self._get_driver()
db_client = Evemongo_client()
db_client.delete(KEYPAIR)
key_pair_objs = driver.list_key_pairs()
n =1
e = {}
for kp in key_pair_objs:
data = {}
data['name'] = kp.name
data['fingerprint'] = kp.fingerprint
e[n] = data
n = n + 1
db_client.post(KEYPAIR, data)
Console.ok(str(Printer.dict_table(e, order=['name', 'fingerprint'])))
return
def info(cls, cluster, format='json', all=False):
if all:
result = Shell.ssh(cluster, 'sinfo --format=\"%all\"')
else:
result = Shell.ssh(cluster, 'sinfo --format=\"%P|%a|%l|%D|%t|%N\"')
# ignore leading lines till header is found
l = result.splitlines()
for i, res in enumerate(l):
if 'PARTITION|AVAIL|' in res:
result = "\n".join(l[i:])
break
parser = TableParser(strip=False)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=[
'cluster', 'partition', 'avail',
'timelimit', 'nodes', 'state',
'nodelist', 'updated'
],
output=format))
def test_04_list_vm(self):
HEADING()
vms = self.p.list()
pprint(vms)
print(Printer.flatwrite(vms,
sort_keys=("name"),
order=["name",
"state",
"extra.task_state",
"extra.vm_state",
"extra.userId",
"extra.key_name",
"private_ips",
"public_ips"],
header=["Name",
"State",
"Task state",
"VM state",
"User Id",
"SSHKey",
"Private ips",
"Public ips"])
)
def location_refresh(self, print_location):
"""
List all
available locations
:returns: location objects
:rtype: NoneType
"""
driver = self._get_driver()
locations = driver.list_locations()
db_client = Evemongo_client()
db_client.delete(LOCATION)
n = 1
e = {}
#<EC2NodeLocation: id=0, name=us-west-1a, country=USA, availability_zone=<ExEC2AvailabilityZone: name=us-west-1a,
# zone_state=available, region_name=us-west-1> driver=Amazon EC2>,
for location in locations:
data = {}
data['id'] = location.id
data['name'] = location.name
data['country'] = location.country
data['availability_zone'] = location.availability_zone.name
data['zone_state'] = location.availability_zone.zone_state
data['region_name'] = location.availability_zone.region_name
#data['provider'] = location.driver.name
e[n] = data
n = n + 1
db_client.post(LOCATION, data)
if print_location == True:
Console.ok(str(Printer.dict_table(e, order=['id','name','country', 'availability_zone', 'zone_state','region_name','provider'])))
return locations
def network_list(self, kwargs=None):
"""List of docker networks
:returns: None
:rtype: NoneType
"""
try:
scode, networks = Rest.get('Network')
except docker.errors.APIError as e:
Console.error(e.explanation)
return
if len(networks) == 0:
Console.info("No network exist")
return
n = 1
e = {}
data = []
for network in networks:
d = {}
d['Ip'] = network['Ip']
d['Id'] = network['Id']
d['Name'] = network['Name']
d['Containers'] = network['Containers']
e[n] = d
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['Ip', 'Id', 'Name', 'Containers'])))
def key_selector(keys):
'''
This is a helper method for ssh key selection
THIS IS JUST A SAFETY MEASURE, PLEASE DON'T MIND IT
:param keys:
:return:
'''
tmp_keys = keys[:]
# indices = range(1,len(tmp_keys)+1)
for key_idx, key in enumerate(keys):
key['idx'] = key_idx + 1
print(
Printer.flatwrite(
tmp_keys,
sort_keys=["idx"],
order=['idx', 'KeyName', 'KeyFingerprint'],
header=['Index', 'Key Name', "Key Fingerprint"],
output="table",
humanize=None))
# Console.msg("Please select one of the AWS key indices from the table above: ")
picked = 0
while picked < 1 or picked > len(keys):
try:
picked = int(
input(
"Please select one of the AWS key indices from the table above: "
))
except ValueError:
pass
return keys[picked - 1]
def volume_delete(self, volume_name):
"""
Deletes the volumes
with specified volume_name
:returns: None
:rtype: NoneType
"""
driver = self._get_driver()
volume_objs = self.volume_refresh(False)
e = {}
isDeleted = False
for vol in volume_objs:
if vol.name == volume_name :
isDeleted = driver.destroy_volume(vol)
#print(vol)
data = {}
data['id'] = vol.id
data['name'] = vol.name
data['size'] = vol.size
data['driver'] = vol.driver.name
e[1] = data
break
Console.ok(str(Printer.dict_table(e, order=['id','name', 'size', 'driver'])))
print("Is deleted - ", isDeleted)
return
def volume_refresh(self, print_objs):
"""
List all the successfuly created
volumes
:returns: volumes object
:rtype: NoneType
"""
driver = self._get_driver()
volumes = driver.list_volumes()
db_client = Evemongo_client()
db_client.delete(VOLUME)
e = {}
n = 1
for vol in volumes:
#print(vol)
data = {}
data['id'] = vol.id
data['name'] = vol.name
data['size'] = vol.size
data['driver'] = vol.driver.name
e[n] = data
n = n + 1
db_client.post(VOLUME, data)
if print_objs == True :
Console.ok(str(Printer.dict_table(e, order=['id','name', 'size','driver'])))
return volumes
def keypair_create(self, key_pair):
"""
Creates the key pair
required for node object
:returns: None
:rtype: NoneType
"""
driver = self._get_driver()
key_pair_obj = driver.create_key_pair(key_pair)
#print("keypair is created !",name)
n = 0 ;
e = {}
# parse flavors
data = {}
data['name'] = str(key_pair_obj.name)
data['fingerprint'] = str(key_pair_obj.fingerprint)
e[n] = data
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['name', 'fingerprint', 'driver'])))
#Store the created keypair in db
db_client = Evemongo_client()
db_client.post(KEYPAIR, data)
return
def node_refresh(self):
"""Refresh of swarm nodes
:returns: None
:rtype: NoneType
"""
filter = {}
filter['Swarmmode'] = 'Manager'
scode, hosts = Rest.get('Host', filter)
filter = {}
n = 1
e = {}
data = []
for host in hosts:
os.environ["DOCKER_HOST"] = host['Ip'] + ":" + str(host['Port'])
filter = {}
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
self.client = docker.from_env()
try:
nodes = self.client.nodes.list()
except docker.errors.APIError as e:
Console.error(e.explanation)
return
if len(nodes) == 0:
Console.info("No nodes exist for manager" +
os.environ["DOCKER_HOST"].split(':'))
continue
n = 1
e = {}
data = []
for node in nodes:
d = {}
node_dict = node.__dict__['attrs']
d['Id'] = node_dict['ID']
data.append(node_dict)
d['Role'] = node_dict['Spec']['Role']
d['Status'] = node_dict['Status']['State']
if d['Role'] == 'manager':
d['Ip'] = node_dict['ManagerStatus']['Addr'].split(':')[0]
d['Manager Ip'] = ''
else:
d['Ip'] = node_dict['Status']['Addr']
d['Manager Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
d['Host Name'] = node_dict['Description']['Hostname']
e[n] = d
n = n + 1
Console.ok(
str(
Printer.dict_table(e,
order=[
'Ip', 'Host Name', 'Id', 'Role',
'Status', 'Manager Ip'
])))
Rest.delete('Node')
Rest.post('Node', data)
def run(self,
script=None,
hosts=None,
username=None,
processors=4,
verbose=False):
results = []
if type(hosts) != list:
hosts = Parameter.expand(hosts)
for command in script.splitlines():
print(hosts, "->", command)
if command.startswith("#") or command.strip() == "":
pass
# print (command)
elif len(hosts) == 1 and hosts[0] == self.hostname:
os.system(command)
elif len(hosts) == 1 and hosts[0] != self.hostname:
host = hosts[0]
os.system(f"ssh {host} {command}")
else:
result = Host.ssh(hosts=hosts,
command=command,
username=username,
key="~/.ssh/id_rsa.pub",
processors=processors,
executor=os.system)
results.append(result)
if verbose:
pprint(results)
for result in results:
print(Printer.write(result, order=['host', 'stdout']))
return results
def container_list(self, kwargs=None):
"""List of docker containers
:returns: None
:rtype: NoneType
"""
try:
scode, containers = Rest.get('Container')
except docker.errors.APIError as e:
Console.error(e.explanation)
return
if len(containers) == 0:
print("No containers exist")
return
n = 1
e = {}
for container in containers:
d = {}
d['Ip'] = container['Ip']
d['Id'] = container['Id']
d['Name'] = container['Name']
d['Image'] = container['Config']['Image']
d['Status'] = container['State']['Status']
d['StartedAt'] = container['State']['StartedAt']
e[n] = d
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['Ip', 'Id', 'Name', 'Image', 'Status', 'StartedAt'])))
def host_list(self):
"""List of docker containers
:returns: None
:rtype: NoneType
"""
try:
scode, hosts = Rest.get('Host')
except Exception as e:
Console.error(e.message)
return
if len(hosts) == 0:
print("No hosts exist")
return
n = 1
e = {}
for host in hosts:
d = {}
d['Ip'] = str(host['Ip'])
d['Name'] = str(host['Name'])
d['Port'] = str(host['Port'])
d['Swarmmode'] = str(host['Swarmmode'])
e[n] = d
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['Ip', 'Name', 'Port', 'Swarmmode'])))
def print_images(self):
images = self.p.images()
print(
Printer.flatwrite(images,
sort_keys=["name"],
order=["name", "provider", "version"],
header=["Name", "Provider", "Version"]))
def test_02_list_os(self):
HEADING()
ostypes = self.p.list_os()
print(Printer.write(
ostypes,
order=["id", "64_bit", "description", "family_descr", "family_id"],
header=["id", "64_bit", "description", "family_descr", "family_id"]))
def test_list_vm(self):
HEADING()
vms = self.p.list()
# pprint(vms)
# TODO: bug th eprint function is not implemented
# print (self.p.Print(vms, kind="vm"))
# '''
print(Printer.flatwrite(vms,
sort_keys=["name"],
order=["name",
"state",
"extra.properties.hardwareProfile.vmSize",
"extra.properties.storageProfile.imageReference.sku",
"extra.properties.storageProfile.osDisk.osType",
"extra.properties.storageProfile.osDisk.diskSizeGB",
"extra.properties.osProfile.adminUsername",
"private_ips",
"public_ips"],
header=["Name",
"State",
"vmSize",
"Image",
"OS Type",
"Disk (GB)",
"Admin User",
"Private ips",
"Public ips"])
)
def images_list(self, kwargs=None):
"""List of docker images
:returns: None
:rtype: NoneType
"""
try:
scode, images = Rest.get('Image')
except docker.errors.APIError as e:
Console.error(e.explanation)
return
if len(images) == 0:
Console.info("No images exist")
return
n = 1
e = {}
for image in images:
d = {}
d['Ip'] = image['Ip']
d['Id'] = image['Id']
if image['RepoTags'] == None:
d['Repository'] = image['RepoDigests'][0]
else:
d['Repository'] = image['RepoTags'][0]
# d['Size'] = image['Size']
d['Size(GB)'] = round(image['Size'] / float(1 << 30), 2) # Converting the size to GB
e[n] = d
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['Ip', 'Id', 'Repository', 'Size(GB)'])))
def print_table(result, status=None, source=None, target=None):
op_result = []
for idx, i in enumerate(result):
op_dict = dict()
op_dict['idx'] = idx + 1
op_dict['source'] = source
op_dict['name'] = i['fileName']
op_dict['size'] = i['contentLength']
op_dict['lastmodified'] = i['lastModificationDate']
op_dict['type'] = 'File'
op_dict['status'] = status
op_dict['target'] = target
op_result.append(op_dict)
# pprint(op_result)
table = Printer.flatwrite(op_result,
sort_keys=["idx"],
order=[
"idx", "source", "target", "name",
"size", "type", "lastmodified", "status"
],
header=[
"S.No.", "Source CSP", "Target CSP",
"Name", "Size", "Type", "Creation",
"Status"
])
print(table)
return op_result
def boot(self, order='price', refresh=False, cloud=None):
clouds = ['aws', 'azure', 'gcp']
if cloud in clouds:
clouds = [cloud]
Console.msg(f"Checking to see which providers are bootable ...")
reachdict = {}
for cloud in clouds:
try:
tempProv = Provider(
name=cloud, configuration="~/.cloudmesh/cloudmesh.yaml")
Console.msg(cloud + " reachable ...")
reachdict[cloud] = tempProv
except:
Console.msg(cloud + " not available ...")
flavorframe = self.list(order, 10000000, refresh, printit=False)
keysya = list(reachdict.keys())
flavorframe = flavorframe[flavorframe['provider'].isin(keysya)]
Console.msg(f"Showing top 5 options, booting first option now...")
converted = flavorframe.head(5).to_dict('records')
print(Printer.write(converted))
cheapest = converted[0]
var_list = Variables(filename="~/.cloudmesh/var-data")
var_list['cloud'] = cheapest['provider']
Console.msg(f'new cloud is ' + var_list['cloud'] +
', booting up the vm with flavor ' +
cheapest['machine-name'])
vmcom = VmCommand()
vmcom.do_vm('boot --flavor=' + cheapest['machine-name'])
return ""
