def host_delete(self, addr):
"""Deletes docker host
:param str addr: Address for docker
:returns: None
:rtype: NoneType
"""
try:
filter = {}
filter['Ip'] = addr.split(':')[0]
r = Rest.delete('Host', filter)
# Delete Host should delete all Containers and Networks for the host
r = Rest.delete('Container', filter)
r = Rest.delete('Network', filter)
Console.ok('Host ' + addr + ' is deleted')
except Exception as e:
Console.error(e.message)
return
def run(self):
Console.ok("starting server")
# if self.server is not None:
# self.server_command = "--server={server}".format(**self.__dict__)
# command = ("connexion run {spec} {server_command} --debug".format(
# **self.__dict__))
# VERBOSE(command, label="OpenAPI Server", verbose=1)
# r = Shell.live(command)
sys.path.append(self.directory)
app = connexion.App(__name__, specification_dir=self.directory)
# app.app["config"]["DEBUG"] = True
# ### app.add_cls(self.directory)
app.add_api(self.path)
app.run(host=self.host,
port=self.port,
debug=self.debug,
server=self.server)
def create_lifecycle_config_files(self, config_path, config_file_data):
'''Creates lifecycle config files'''
try:
# Get file path
service_config_file = os.path.expanduser(config_path)
# Create config file in .cloudmesh directory
f = open(service_config_file, 'w') # Opens in write-only mode
f.write(json.dumps(config_file_data))
f.close()
# Debug
Console.ok(
"Successfully created temp config file in .cloudmesh directory"
)
except Exception as error:
Console.error(error, prefix=True, traceflag=True)
return
def volume_list(self, print_objs):
"""
List all the successfuly stored
volumes
:returns: None
:rtype: NoneType
"""
#Fetch the list of images from db
db_client = Evemongo_client()
volumes = db_client.get(VOLUME)
e = {}
n = 1
for vol in volumes:
e[n] = vol
n = n + 1
if print_objs == True :
Console.ok(str(Printer.dict_table(e, order=['id','name', 'size','driver'])))
return volumes
def test(self, port):
Console.msg("Running Test on Local...")
# The user may specify multiple ports. Selecting only the first port.
port = Parameter.expand(port)
port = port[0]
Console.msg("PORT=" + port)
self.start_local(port=port, ip=None, dbpath=None)
Console.msg("Local instance started. Listening for requests...")
command = "mongo 127.0.0.1:" + port + " --eval \"printjson(db.serverStatus())\""
returncode = os.system(command)
if (returncode == 0):
Console.ok("Success!")
else:
Console.error(
"Test Error! Check physical connections. Port may be already in use."
)
banner("MongoDB Test Completed")
return 1
def container_delete(self, containerName=None, kwargs=None):
"""Deleting docker container
:param str containerName: Name of docker container
:returns: None
:rtype: NoneType
"""
try:
container = self.client.containers.get(containerName)
container.remove(**kwargs)
filter = {}
filter['Id'] = container.__dict__['attrs']['Id']
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
Rest.delete('Container', filter)
Console.ok('Container ' + container.name + ' is deleted')
except docker.errors.APIError as e:
Console.error(e.explanation)
return
def remove(service, name):
removed_item = None
try:
# Update the google cloud section of cloudmesh.yaml config file.
config = Config()
config_service = config["cloudmesh"][service]
if name in config_service:
removed_item = config_service.pop(name, None)
config.save()
Console.ok(f"Removed {name} from {service} service.")
else:
Console.warning(
f"{name} is not registered for cloudmesh.{service}")
except Exception as se:
Console.error(f"Error removing {service}-{name} :: {se}")
return removed_item
def check_python(cls):
"""
checks if the python version is supported
:return: True if it is supported
"""
python_version = sys.version_info[:3]
v_string = [str(i) for i in python_version]
if python_version[0] == 2:
Console.error(f"You are running an unsupported version "
f"of python: {python_version}")
Console.error("Please update to python version 3.7")
sys.exit(1)
elif python_version[0] == 3:
python_version_s = '.'.join(v_string)
if (python_version[0] == 3) and (python_version[1] >= 7) and \
(python_version[2] >= 0):
Console.ok(f"You are running a supported version of python: "
f"{python_version_s}")
else:
Console.error(
"WARNING: You are running an unsupported version of "
"python: {:}".format(python_version_s))
Console.error(" We recommend you update your python")
# pip_version = pip.__version__
python_version, pip_version = cls.get_python()
if int(pip_version.split(".")[0]) >= 18:
Console.ok(
f"You are running a supported version of pip: {pip_version}")
else:
Console.error("WARNING: You are running an old version of pip: " +
str(pip_version))
Console.error(" We recommend you update your pip with \n")
Console.error(" pip install -U pip\n")
def check_venv(self):
# banner(f"checking python venv")
if self.is_venv():
Console.ok("OK. you are running in a venv")
print(" VIRTUAL_ENV=", os.environ.get("VIRTUAL_ENV"), sep="")
else:
Console.error("You are not running in a venv")
if "ENV3" not in os.environ.get("VIRTUAL_ENV"):
Console.warning("your venv is not called ENV3. That may be ok")
if platform.system() == "Windows":
venv = os.environ.get("VIRTUAL_ENV")
where = path_expand(venv)
activate_path = f"{where}\\Scripts\\activate.bat"
# check if the dir exists at where
if os.path.isdir(where):
Console.ok("OK. ENV3 directory exists")
else:
Console.error("ENV3 directory does not exists")
# check if activate exists in ~\ENV3\Scripts\activate.bat
if os.path.isfile(activate_path):
Console.ok(f"OK. Activate exists in {activate_path}")
else:
Console.error(f"Could not find {activate_path}")
def set_auth(self):
"""
add admin account into the MongoDB admin database
"""
mode = self.data['MODE']
if mode == 'docker':
Console.error("* Docker is not yet supported")
raise NotImplementedError
if platform.lower(
) == 'win32': # don't remove this otherwise init won't work in windows, eval should start with double quote in windows
self.data['MONGO'] = f"{self.mongo_home}\\bin\mongo"
script = """ "{MONGO}" --eval "db.getSiblingDB('admin').createUser({{ user:'******',pwd:'{MONGO_PASSWORD}',roles:[{{role:'root',db:'admin'}}]}}) ; db.shutdownServer()" """.format(
**self.data)
# print(script)
try:
# result = Shell.run2(script)
p = subprocess.Popen(script,
shell=True,
stdout=subprocess.PIPE,
stderr=STDOUT)
result = p.stdout.read().decode('utf-8')
except Exception as e:
print(e)
return
else:
script = """mongo --eval 'db.getSiblingDB("admin").createUser({{user:"******",pwd:"{MONGO_PASSWORD}",roles:[{{role:"root",db:"admin"}}]}})'""".format(
**self.data)
result = Script.run(script)
if "Successfully added user" in result:
Console.ok("Administrative user created.")
elif "already exists" in result:
Console.error('admin user already exists.')
else:
Console.error("Problem creating the administrative user. Check "
"the yaml file and make sure the password and "
"username are not TBD.")
def boot(self, **kwargs):
arg = dotdict(kwargs)
arg.cwd = kwargs.get("cwd", None)
# get the dir based on name
print("ARG")
pprint(arg)
vms = self.to_dict(self.nodes())
print("VMS", vms)
arg = self._get_specification(**kwargs)
pprint(arg)
if arg.name in vms:
Console.error("vm {name} already booted".format(**arg),
traceflag=False)
return None
else:
self.create(**kwargs)
Console.ok("{name} created".format(**arg))
Console.ok("{directory}/{name} booting ...".format(**arg))
result = Shell.execute("vagrant", ["up", arg.name],
cwd=arg.directory)
Console.ok("{name} ok.".format(**arg))
return result
def container_status_change(self, status=None, containerName=None, kwargs=None):
"""Change status of docker container
:param str status: Docker container status to be changed to
:param str containerName: Name of Docker container
:returns: None
:rtype: NoneType
"""
if status is None:
Console.info("No status specified")
return
try:
container = self.client.containers.get(containerName)
# need to check this ..
if status is "start":
container.start(**kwargs)
elif status is "pause":
container.pause(**kwargs)
elif status is "unpause":
container.unpause(**kwargs)
elif status is "stop":
container.stop(**kwargs)
else:
Console.error('Invalid Commmand')
return
container = self.client.containers.get(containerName)
filter = {}
container_dict = container.__dict__['attrs']
filter['Id'] = container_dict['Id']
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
container_dict['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
Rest.post('Container', container_dict, filter)
Console.ok('Container ' + container.name + ' status changed to ' + status)
except docker.errors.APIError as e:
Console.error(e.explanation)
return
def join(self, addr, type, kwargs=None):
"""Creates docker Swarm
:param str addr: Address of Swarm Manager
:returns: None
:rtype: NoneType
"""
man_list = []
man_list.append(addr)
savehost = os.environ["DOCKER_HOST"]
os.environ["DOCKER_HOST"] = addr.split(':')[0] + ":4243"
self.client = docker.from_env()
if type not in ['Manager', 'Worker']:
Console.error('Valid values are Manager or Worker')
return
token = self.client.swarm.attrs['JoinTokens']
os.environ["DOCKER_HOST"] = savehost
self.client = docker.from_env()
rcode = self.client.swarm.join(remote_addrs=man_list,
join_token=token[type],
listen_addr="0.0.0.0:2377",
**kwargs)
filter = {}
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
scode, hosts = Rest.get('Host', filter)
d = {}
for host in hosts:
d['Ip'] = host['Ip']
d['Name'] = host['Name']
d['Port'] = host['Port']
d['Swarmmode'] = type
d['SwarmmanagerIp'] = addr.split(':')[0]
d['Swarmhost'] = True
Rest.post('Host', d, filter)
self.node_refresh()
Console.ok("Node Joined Swarm")
def service_delete(self, name):
"""List of docker images
:param str name: name for service
:returns: None
:rtype: NoneType
"""
try:
service = self.client.services.get(name)
service.remove()
filter = {}
filter['ID'] = service.id
filter['Name'] = service.name
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
Rest.delete('Service', filter)
except docker.errors.APIError as e:
Console.error(e.explanation)
return
Console.ok("Service " + name + " is deleted")
def images_list(self, kwargs=None):
"""List of docker images
:returns: None
:rtype: NoneType
"""
try:
scode, images = Rest.get('Image')
except docker.errors.APIError as e:
Console.error(e.explanation)
return
if len(images) == 0:
Console.info("No images exist")
return
n = 1
e = {}
for image in images:
d = {}
d['Ip'] = image['Ip']
d['Id'] = image['Id']
if image['RepoTags'] == None:
d['Repository'] = image['RepoDigests'][0]
else:
d['Repository'] = image['RepoTags'][0]
# d['Size'] = image['Size']
d['Size(GB)'] = round(image['Size'] / float(1 << 30),
2) # Converting the size to GB
e[n] = d
n = n + 1
Console.ok(
str(
Printer.dict_table(
e, order=['Ip', 'Id', 'Repository', 'Size(GB)'])))
def do_banner(self, args, arguments):
"""
::
Usage:
banner [-c CHAR] [-n WIDTH] [-i INDENT] [-r COLOR] TEXT...
Arguments:
TEXT... The text message from which to create the banner
CHAR The character for the frame.
WIDTH Width of the banner
INDENT indentation of the banner
COLOR the color
Options:
-c CHAR The character for the frame. [default: #]
-n WIDTH The width of the banner. [default: 70]
-i INDENT The width of the banner. [default: 0]
-r COLOR The color of the banner. [default: NORMAL]
Prints a banner form a one line text message.
"""
Console.ok("banner")
n = int(arguments['-n'])
c = arguments['-c']
i = int(arguments['-i'])
color = arguments['-r'].upper()
line = ' '.join(arguments['TEXT'])
Console.init()
hline = i * " " + str((n - i) * c)
Console.cprint(color=color, prefix="", message=hline)
Console.cprint(color=color,
prefix="",
message=i * " " + c + " " + line)
Console.cprint(color=color, prefix="", message=hline)
return ""
def image_refresh(self):
"""List of amazon images
get store it in db
:returns: None
:rtype: NoneType
"""
# get driver
driver = self._get_driver()
# get image list and print
images = driver.list_images()
db_client = Evemongo_client()
if len(images) == 0:
print("Error in fetching new list ...Showing existing images")
self.image_list()
else:
#r = db_client.delete(IMAGE)
#print(images)
#print("storing in db")
#db_client.perform_delete(IMAGE)
n = 0 ;
e = {}
for image in images:
# parse flavors
data = {}
data['id'] = str(image .id)
data['name'] = str(image.name)
data['driver'] = str(image.driver)
# store it in mongodb
db_client.post(IMAGE, data)
e[n] = data
n = n + 1
Console.ok(str(Printer.dict_table(e, order=['id', 'name', 'driver'])))
#print(images)
return
def network_delete(self, name):
"""List of docker images
:param str name: name for network
:returns: None
:rtype: NoneType
"""
try:
network = self.client.networks.get(name)
network.remove()
filter = {}
filter['ID'] = network.id
filter['Name'] = network.name
filter['Ip'] = os.environ["DOCKER_HOST"].split(':')[0]
Rest.delete('Network', filter)
except docker.errors.APIError as e:
Console.error(e.explanation)
return
Console.ok("Network " + name + " is deleted")
def generate(self,
key="india",
host="india.futuresystems.org",
username=None,
force=False,
verbose=False):
"""
adds a host to the config file with given parameters. #TODO: make sure this is better documented
:param key: the key
:param host: the host
:param username: the username
:param force: not used
:param verbose: prints debug messages
:return:
"""
data = {
"host": host,
"key": key,
"username": username
}
if verbose and key in self.names():
Console.error("{key} already in ~/.ssh/config".format(**data), traceflag=False)
return ""
else:
entry = dedent("""
Host {key}
Hostname {host}
User {username}
""".format(**data))
try:
with open(self.filename, "a") as config_ssh:
config_ssh.write(entry)
config_ssh.close()
self.load()
if verbose:
Console.ok("Added india to ~/.ssh/config")
except Exception as e:
if verbose:
Console.error(e.message)
def info(self):
self.filename = path_expand(self.filename)
banner("Configuration")
Console.ok('Data File: {:}'.format(self.filename))
Console.ok('Object Attributes: {:}'.format(', '.join(self.order)))
Console.ok('Objects: {:}'.format(len(self.data)))
print(70 * "#")
def list(self,
source=None,
source_obj=None,
target=None,
target_obj=None,
recursive=True):
"""
To enlist content of "target object"
:param source: source CSP - awss3/azure/local, None for list method
:param source_obj: It can be file or folder, None for list method
:param target: target CSP - awss3/azure/local
:param target_obj: It can be file or folder
:param recursive: enlist directories/sub-directories
:return: dictionary enlisting objects
"""
print("CALLING AZURE BLOB STORAGE PROVIDER'S LIST METHOD")
# print("=============> ", target_obj)
if target_obj:
target_obj = target_obj.replace("\\", "/")
else:
target_obj = '/'
# print(target_obj, recursive)
result = self.storage_provider.list(source=target_obj,
recursive=recursive)
if result is None:
return Console.error(f"List of object(s) couldn't be fetched from "
f"{target} CSP for object {target_obj}. "
f"Please check.")
else:
Console.ok(f"\nList of objects from {target} CSP for object "
f"{target_obj}:\n")
# pprint(result)
return self.print_table(result,
status="Available",
source=source,
target=target)
def add_secgroup_rule(
self,
name=None, # group name
port=None,
protocol=None,
ip_range=None):
"""
Add rule to named security group
:param name: Name of the security group to which rfule needs to be added
:param port: The start and end port range for the TCP and UDP protocols
:param protocol:
:param ip_range:
:return:
"""
try:
portmin, portmax = port.split(":")
except ValueError:
portmin = -1
portmax = -1
try:
data = self.ec2_client.authorize_security_group_ingress(
GroupName=name,
IpPermissions=[
{
'IpProtocol': protocol,
'FromPort': int(portmin),
'ToPort': int(portmax),
'IpRanges': [{
'CidrIp': ip_range
}]
},
])
Console.ok(f'Ingress Successfully Set as {data}')
except ClientError as e:
Console.info("Rule couldn't be added to security group")
def _install_dnsmasq(cls):
"""
Uses apt-get to install package dnsmasq
:return:
"""
if cls.dryrun:
Console.info('Installing dnsmasq...')
else:
banner("""
Installing dnsmasq. Please wait for installation to complete.
""")
StopWatch.start('install dnsmasq')
# cls._system(f'sudo apt-get install -y dnsmasq')
# Use os.system to display the installation feedback
os.system('sudo apt-get install -y dnsmasq')
StopWatch.stop('install dnsmasq')
StopWatch.status('install dnsmasq', True)
Console.ok("Finished installing dnsmasq")
def check_command(self, command, test=None, show=True):
# banner(f"testing command: {command}")
try:
if sys.platform in ["win32"]:
result = Shell.run2(command).strip()
else:
result = Shell.run(command).strip()
if test not in result:
if show:
Console.error(f"{test} not found in {result}")
else:
Console.error(f"{command} not found")
else:
if show:
Console.ok(f"OK. {test} found in {result}")
else:
Console.ok(f"OK. {command} found")
except:
Console.error(f"command '{command}' not successful")
def keypair_get(self, key_pair):
"""
Get the keypair object
associated with name
:returns: keypar object
:rtype: NoneType
"""
driver = self._get_driver()
e = {}
key_pair_obj= {}
try:
key_pair_obj = driver.get_key_pair(key_pair)
except Exception:
Console.error("Key does not exist")
else:
data = {}
data['name'] = key_pair_obj.name
data['fingerprint'] = key_pair_obj.fingerprint
e[1] = data
Console.ok(str(Printer.dict_table(e, order=['name', 'fingerprint'])))
return key_pair_obj
def list(self, source=None):
bucket = self.google_client.bucket(self.bucket)
keys = []
match = []
for blob in bucket.list_blobs():
datetimeObj = blob.time_created
timestampStr = datetimeObj.strftime("%d-%b-%Y %H:%M:%S")
keys.append("Name: " + blob.name + " , Created: " + timestampStr)
#pprint(blob.name)
if (blob.name).startswith(source):
match.append("Name: " + blob.name + " , Created: " +
timestampStr)
if match is not None:
pprint(f"Listing from source - {source}")
pprint(match)
else:
Console.ok(
f"{source} cannot be found in Google bucket{self.bucket}")
pprint(f"Listing contents from bucket - {self.bucket}")
pprint(keys)
return keys
def delete(self, storage_provider, storage_bucket_name):
'''Deletes the lifecycle configuration defined for a bucket.
:param storage_provider: Name of the cloud service provider
:param storage_bucket_name: Name of the storage bucket
:exception: Exception
:returns: Result of operation as string
'''
try:
# Invoke service
result = self.s3_client.delete_bucket_lifecycle(
Bucket=storage_bucket_name)
# Debug
Console.ok(json.dumps(result, indent=4, sort_keys=True))
except ClientError as error:
Console.error(error, prefix=True, traceflag=True)
return False
return result
def add(entry=None,
base="cloudmesh.cloud",
path="~/.cloudmesh/cloudmesh.yaml"):
try:
_entry = dedent(entry)
data = yaml.safe_load(_entry)
name, entry = Entry.extract(data, base)
if Entry.verify(entry):
Console.ok("Verification passed")
config = Config() # todo: add the path
config[base][name] = entry
config.save()
else:
Console.error("entry format is wrong")
return ""
except yaml.YAMLError:
Console.error(f"parsing YAML entry: {entry}")
sys.exit()
def update(provider, kind, service, name, attributes):
try:
sample = Register.get_sample(provider, kind, service, name,
attributes)
if sample is None:
Console.error("The sample is not fully filled out.")
return ""
# Add the entry into cloudmesh.yaml file.
config = Config()
config_path = config.location.config()
Entry.add(entry=sample,
base=f"cloudmesh.{service}",
path=config_path)
Console.ok(f"Registered {service} service for {kind}"
f" provider with name {name}.")
except Exception as se:
Console.error(f"Error updating {service}-{name} :: {se}")
return ""
def get(self, storage_provider, storage_bucket_name):
'''Loads the lifecycle configuration defined for a bucket.
:param storage_provider: Name of the cloud service provider
:param storage_bucket_name: Name of the storage bucket
:exception: Exception
:returns: Result of operation as string
'''
try:
# Invoke GCP gsutil via CMS command shell
result = Shell.execute(self.SERVICE_CLI, [
self.SERVICE_COMMAND_LIFECYCLE, self.SERVICE_SUBCOMMAND_GET,
self.STORAGE_BUCKET_PROTOCOL + storage_bucket_name
])
# Debug
Console.ok(result)
except Exception as error:
Console.error(error, prefix=True, traceflag=True)
return result
