def json_to_yaml(cls, name, filename="~/.cloudmesh/security/google.json"):
"""
Given a json file downloaded from google, copies the content into the
cloudmesh yaml file, while overwriting or creating a new compute provider
:param cls:
:param name:
:param filename: Service Account Key file downloaded from google cloud.
:return: None
"""
path = path_expand(filename)
# Open and load the JSON file.
with open(path, "r") as file:
d = json.load(file)
# Get the project id and client email.
project_id = d["project_id"]
client_email = d["client_email"]
# Format the sample with json file details.
format_sample = cls.sample.format_map(locals())
# Convert the yaml sample to JSON.
google_yaml = yaml.load(format_sample, Loader=yaml.SafeLoader)
# Extract the google compute section
google_config = google_yaml["cloudmesh"]["cloud"]
# Update the google cloud section of cloudmesh.yaml config file.
config = Config()
config["cloudmesh"]["cloud"][name] = google_config
config.save()
banner("Result")
pprint(config["cloudmesh"]["cloud"][name])
def edit(key, caps=True, show=False):
global gui_enabled
if not gui_enabled:
Console.Warning(
"Cloudmesh Gui not supported, can not find tkinter")
return ""
config = Config()
entry = dict(FlatDict(config[key], sep='.'))
layout = [[gui.Text(f'Cloudmesh Configuration Editor: {key}')]]
length = 1
for _key, _value in entry.items():
length = max(length, len(_key))
length = length + 3
for _key, _value in entry.items():
if caps:
label = _key.capitalize()
else:
label = _key
secrets = Config.secrets()
if _key in secrets and not show:
field = [
gui.Text(label, size=(length, 1)),
gui.InputText(key=f"{key}.{_key}",
password_char="*",
default_text=_value)
]
else:
field = [
gui.Text(label, size=(length, 1)),
gui.InputText(key=f"{key}.{_key}", default_text=_value)
]
layout.append(field)
layout.append([gui.Submit(), gui.Cancel()])
window = gui.Window('Cloudmesh Configuration Editor',
layout,
background_color="white")
event, values = window.Read()
window.Close()
if event == "Submit":
for _key, _value in values.items():
config[_key] = _value
if show:
Console.ok(f"{_key}={_value}")
config.save()
else:
print(event)
def activate():
global gui_enabled
if not gui_enabled:
Console.Warning(
"Cloudmesh Gui not supported, can not find tkinter")
return ""
config = Config()
clouds = list(config["cloudmesh.cloud"].keys())
gui.SetOptions(text_justification='right')
layout = [[
gui.Text('Cloudmesh Cloud Activation', font=('Helvetica', 16))
], [gui.Text('Compute Services')]]
layout.append([gui.Text('_' * 100, size=(65, 1))])
for cloud in clouds:
tbd = "TBD" in str(config[f"cloudmesh.cloud.{cloud}.credentials"])
active = config[f"cloudmesh.cloud.{cloud}.cm.active"]
if tbd:
color = 'red'
else:
color = "green"
choice = [
gui.Checkbox(cloud,
key=cloud,
text_color=color,
default=active)
]
layout.append(choice)
layout.append([gui.Text('_' * 100, size=(65, 1))])
layout.append([gui.Submit(), gui.Cancel()])
window = gui.Window('Cloudmesh Configuration',
layout,
font=("Helvetica", 12))
event, values = window.Read()
if event == "Submit":
for cloud in values:
active = values[cloud] or False
config[f"cloudmesh.cloud.{cloud}.cm.active"] = str(active)
if active:
Console.ok(f"Cloud {cloud} is active")
config.save()
else:
print(event)
class AzRegister(object):
def __init__(self, cloud='azure'):
self.config = Config()
self.credentials = self.config[f'cloudmesh.cloud.{cloud}.credentials']
def set_credentials(self, creds):
self.credentials['AZURE_TENANT_ID'] = creds['Access key ID']
self.credentials['AZURE_SUBSCRIPTION_ID'] = creds['Secret access key']
self.credentials['AZURE_APPLICATION_ID'] = creds['Secret access key']
self.credentials['AZURE_SECRET_KEY'] = creds['Secret access key']
self.config.save()
def azString2Dict(self, azString):
azDict = dict()
for i in azString.strip().splitlines():
x = i.split(":")
if len(x) == 2:
azDict[x[0].strip(' ,"')] = x[1].strip(' ,"')
return azDict
def register(self, cloud='azure'):
# Opens web browser and prompts user to login
subprocess.Popen('az login')
# once user has logged in, collects account information, such as subscription id
accountInfo = subprocess.getoutput('az account show')
print(accountInfo)
azoutput = self.azString2Dict(accountInfo)
AZURE_SUBSCRIPTION_ID = azoutput['id']
AZURE_TENANT_ID = azoutput['tenantId']
# WARNING: FOLLOWING CODE WILL RENDER OLD SECRET KEY INVALID
azAppKeyStr = subprocess.getoutput(
'az ad sp create-for-rbac --name http://cloudmesh')
azAppKeyDict = self.azString2Dict(azAppKeyStr)
AZURE_APPLICATION_ID = azAppKeyDict['appId']
AZURE_SECRET_KEY = azAppKeyDict['password']
creds = {
'AZURE_SUBSCRIPTION_ID': AZURE_SUBSCRIPTION_ID,
'AZURE_TENANT_ID': AZURE_TENANT_ID,
'AZURE_APPLICATION_ID': AZURE_APPLICATION_ID,
'AZURE_SECRET_KEY': AZURE_SECRET_KEY
}
self.set_credentials(creds)
Console.info(
"Azure Tenant, Subscription, Application, and Secret Key have been added to the cloudmesh.yaml file."
)
def json_to_yaml(name, filename="~/.cloudmesh/google.json"):
"""
given a json file downloaded from google, copies the content into the cloudmesh yaml file, while overwriting or creating a new storage provider
:param filename:
:return:
"""
# creates cloud,esh.storgae.{name}
path = path_expand(filename)
with open(path, "r") as file:
d = json.load(file)
config = Config()
#
# BUG START FROM THE sample
#
element = {
"cm": {
"name": name,
"active": 'true',
"heading": "GCP",
"host": "https://console.cloud.google.com/storage",
"kind": "google",
"version": "TBD",
"service": "storage"
},
"default": {
"directory": "cloudmesh_gcp",
"Location_type": "Region",
"Location": "us - east1",
"Default_storage_class": "Standard",
"Access_control": "Uniform",
"Encryption": "Google-managed",
"Link_URL":
"https://console.cloud.google.com/storage/browser/cloudmesh_gcp",
"Link_for_gsutil": "gs://cloudmesh_gcp"
},
"credentials": d
}
config["cloudmesh"]["storage"][name] = element
config.save()
pprint(config["cloudmesh"]["storage"][name])
def remove(service, name):
removed_item = None
try:
# Update the google cloud section of cloudmesh.yaml config file.
config = Config()
config_service = config["cloudmesh"][service]
if name in config_service:
removed_item = config_service.pop(name, None)
config.save()
Console.ok(f"Removed {name} from {service} service.")
else:
Console.warning(
f"{name} is not registered for cloudmesh.{service}")
except Exception as se:
Console.error(f"Error removing {service}-{name} :: {se}")
return removed_item
def add(entry=None,
base="cloudmesh.cloud",
path="~/.cloudmesh/cloudmesh.yaml"):
try:
_entry = dedent(entry)
data = yaml.safe_load(_entry)
name, entry = Entry.extract(data, base)
if Entry.verify(entry):
Console.ok("Verification passed")
config = Config() # todo: add the path
config[base][name] = entry
config.save()
else:
Console.error("entry format is wrong")
return ""
except yaml.YAMLError:
Console.error(f"parsing YAML entry: {entry}")
sys.exit()
class AWSRegister(object):
def __init__(self, cloud='aws'):
self.config = Config()
self.credentials = self.config[f'cloudmesh.cloud.{cloud}.credentials']
def set_credentials(self, creds):
self.credentials['EC2_ACCESS_ID'] = creds['Access key ID'][0]
self.credentials['EC2_SECRET_KEY'] = creds['Secret access key'][0]
self.config.save()
def register(self, cloud='aws'):
if platform == "linux" or platform == "linux2":
# check if chrome installed
chrome_ver1 = subprocess.getoutput(
'google-chrome-stable --version')
chrome_ver2 = subprocess.getoutput('google-chrome --version')
if 'not found' in chrome_ver1.lower(
) or 'not found' in chrome_ver2.lower():
Console.error("google chrome is not installed")
return
# self.driver = webdriver.Chrome()
try:
self.driver = webdriver.Chrome()
# register = AWSRegister(self.driver)
except WebDriverException as e:
Console.error(e)
Console.error(
"Chrome geckodriver not installed. Follow these steps for installation: \n"
"1) Download the driver from the following link: \n\t "
"https://sites.google.com/a/chromium.org/chromedriver/downloads \n"
"2) Copy the `chromedriver` to '/usr/bin' \n"
"3) Set the permission using:\n\t"
"'sudo chmod +x /usr/bin/chromedriver'")
return
credentials_file_name = self.create_user()
credentials_csv_path = Path.home().joinpath('Downloads').joinpath(
credentials_file_name).resolve()
cloudmesh_folder = Path.home().joinpath('.cloudmesh').resolve()
os.rename(
credentials_csv_path,
cloudmesh_folder.joinpath(credentials_file_name).resolve())
Console.info("{filename} moved to ~/.cloudmesh folder".format(
filename=credentials_file_name))
creds = pandas.read_csv("{cm}/{filename}".format(
cm=cloudmesh_folder, filename=credentials_file_name))
self.set_credentials(creds)
self.config.save()
Console.info(
"AWS 'Access Key ID' and 'Secret Access Key' in the cloudmesh.yaml updated"
)
elif platform == "darwin":
chrome = Path(
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome")
if not chrome.is_file():
Console.error("Google chrome is not installed")
return
try:
self.driver = webdriver.Chrome()
except WebDriverException as e:
Console.error(e)
Console.error(
"Chrome geckodriver not installed. Follow these steps for installation: \n"
"1) Download the driver from the following link: \n\t "
"https://sites.google.com/a/chromium.org/chromedriver/downloads \n"
"2) Copy the `chromedriver` to '/usr/local/bin' \n"
"3) Set the permission using:\n\t"
"'chmod +x /usr/local/bin/chromedriver'")
return
credentials_file_name = self.create_user()
credentials_csv_path = Path.home().joinpath('Downloads').joinpath(
credentials_file_name).resolve()
# check if the DOwanloaded file exists
# Path("~/.cloudmesh/{credentials_file_name}).resolve()
cloudmesh_folder = Path.home().joinpath('.cloudmesh').resolve()
os.rename(
credentials_csv_path,
cloudmesh_folder.joinpath(credentials_file_name).resolve())
Console.info(
f"{credentials_file_name} moved to ~/.cloudmesh folder")
creds = pandas.read_csv("{cm}/{filename}".format(
cm=cloudmesh_folder, filename=credentials_file_name))
self.set_credentials(creds)
Console.info(
"AWS 'Access Key ID' and 'Secret Access Key' in the cloudmesh.yaml updated"
)
elif platform == "win32":
chrome = Path(
"C:\Program Files (x86)\Google\Chrome\Application\Chrome.exe")
if not chrome.is_file():
Console.error("Google chrome is not installed")
return
try:
self.driver = webdriver.Chrome()
except WebDriverException as e:
Console.error(e)
Console.error(
"Chrome geckodriver not installed. Follow these steps for installation: \n"
"1) Download the driver from the following link: \n\t "
"https://sites.google.com/a/chromium.org/chromedriver/downloads \n"
"2) Copy the `chromedriver` to path, for instance you can add "
"it to the followtin path: "
"\n\t %USERPROFILE%\\AppData\\Local\\Microsoft\\WindowsApps"
)
return
credentials_file_name = self.create_user()
credentials_csv_path = Path.home().joinpath('Downloads').joinpath(
credentials_file_name).resolve()
cloudmesh_folder = Path.home().joinpath('.cloudmesh').resolve()
os.rename(
credentials_csv_path,
cloudmesh_folder.joinpath(credentials_file_name).resolve())
Console.info(
f"{credentials_file_name} moved to ~/.cloudmesh folder")
creds = pandas.read_csv("{cm}/{filename}".format(
cm=cloudmesh_folder, filename=credentials_file_name))
self.set_credentials(creds)
Console.info(
"AWS 'Access Key ID' and 'Secret Access Key' in the cloudmesh.yaml updated"
)
def slow_typer(self, element, text):
for character in text:
element.send_keys(character)
sleep(random.random() * 0.05)
def check_captcha(self):
if "Type the characters seen in the image below" in self.driver.page_source:
text = input(
"Captcha encountered. Please enter the captcha and press Submit then press Enter to continue"
)
while (text != ""):
text = input(
"Captcha encountered. Please enter the captcha and press Submit then press Enter to continue"
)
def create_user(self):
'''
Creates the user or if the user exists creates another access key
:return: the name of the file containing the access key
'''
email = input("Enter your email: ")
passw = getpass.getpass("Enter your password: "******"https://console.aws.amazon.com/iam/home#/users")
assert "Amazon Web Services" in self.driver.title, "Unexpected login page, aborting"
sleep(1.5)
self.driver.find_element_by_id("resolving_input").send_keys(email)
self.driver.find_element_by_id("next_button").click()
sleep(1.5)
self.check_captcha()
self.driver.find_element_by_id("password").send_keys(passw)
self.driver.find_element_by_id("signin_button").click()
sleep(1.5)
self.check_captcha()
# adding cloudmesh user
if self.user_exists():
self.create_accesskey_on_current_account()
return "accessKeys.csv"
self.driver.find_element_by_link_text("Add user").click()
sleep(1.5)
self.driver.find_element_by_id("awsui-textfield-17").send_keys(
"cloudmesh")
self.driver.find_element_by_name("accessKey").click()
sleep(1.5)
self.driver.find_element_by_class_name("wizard-next-button").click()
sleep(1.5)
self.driver.find_element_by_class_name("awsui-util-pt-s").click()
sleep(1.5)
self.driver.find_element_by_xpath(
"//awsui-textfield[@ng-model='createGroupModal.groupName']/input"
).send_keys("cloudmesh")
sleep(1.5)
self.driver.find_element_by_xpath(
'//*/policies-table//table-search//search/div/input').send_keys(
"AmazonEC2FullAccess")
sleep(1.5)
self.driver.find_element_by_xpath(
'//div[@data-item-id="arn:aws:iam::aws:policy/AmazonEC2FullAccess"]//awsui-checkbox//div'
).click()
sleep(1.5)
self.driver.find_element_by_xpath(
"//awsui-button[@click-tracker='CreateGroup']").click()
sleep(1.5)
Console.info("'cloudmesh' group created")
self.driver.find_element_by_class_name("wizard-next-button").click()
sleep(1.5)
self.driver.find_element_by_class_name("wizard-next-button").click()
sleep(1.5)
Console.info("'cloudmesh' user created")
self.driver.find_element_by_class_name("wizard-next-button").click()
sleep(1.5)
self.driver.find_element_by_xpath(
"//awsui-button[@text='Download .csv']").click()
Console.info("credentials.csv downloaded")
sleep(2)
self.driver.find_element_by_xpath(
"//awsui-button[@text='Close']").click()
sleep(4)
return "credentials.csv"
def user_exists(self):
sleep(1.5)
return len(self.driver.find_elements_by_link_text("cloudmesh")) > 0
def create_accesskey_on_current_account(self):
self.driver.find_element_by_xpath(
'//a[@href="#/users/cloudmesh"]').click()
sleep(1.5)
self.driver.find_element_by_link_text("Security credentials").click()
sleep(1.5)
while len(
self.driver.find_elements_by_xpath(
'//span[text()="Make inactive"]')) == 2:
input(
"Two access keys already exist for the user, remove one manually before creating another one and press Enter"
)
self.driver.find_element_by_xpath(
'//button[.//span[text()="Create access key"]]').click()
sleep(1.5)
self.driver.find_element_by_xpath(
'//*[@id="modal-container"]//awsui-button[contains(@text,"Download .csv file")]'
).click()
sleep(1.5)
Console.info("accessKeys.csv downloaded")
else:
color = "green"
choice = [gui.Checkbox(cloud,
text_color=color,
default=active)]
layout.append(choice)
layout.append([gui.Text('_' * 100, size=(65, 1))])
layout.append([gui.Submit(), gui.Cancel()])
window = gui.Window('Cloudmesh Configuration', layout, font=("Helvetica", 12))
event, values = window.Read()
selected = []
for i in range(0, len(clouds)):
cloud = clouds[i]
if values[i]:
selected.append(cloud)
Console.ok(f"Activate Cloud {cloud}")
for cloud in clouds:
active = False
if cloud in selected:
active = True
config[f"cloudmesh.cloud.{cloud}.cm.active"] = str(active)
config.save()
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list --cloud=CLOUDS [--output=OUTPUT]
key list --source=ssh [--dir=DIR] [--output=OUTPUT]
key list --source=git [--output=OUTPUT] [--username=USERNAME]
key list [--output=OUTPUT]
key init
key add NAME --filename=FILENAME [--output=OUTPUT]
key add [NAME] [--source=FILENAME]
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
key delete NAMES [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [VMS] [--dryrun]
key group upload [NAMES] [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun]
key group add [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun]
key group add --file=FILENAME
key group delete [--group=GROUPNAMES] [NAMES] [--dryrun]
key group list [--group=GROUPNAMES] [--output=OUTPUT]
key group export --group=GROUNAMES --filename=FILENAME
key gen (rsa | ssh) [--filename=FILENAME] [--nopass] [--set_path]
key verify (ssh | pem) --filename=FILENAME [--pub]
Arguments:
VMS Parameterized list of virtual machines
CLOUDS The clouds
NAME The name of the key.
SOURCE db, ssh, all
KEYNAME The desired full path name to the key file
OUTPUT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key is located
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--filename=FILENAME the name and full path to the file
--nopass Flag indicating if the key has no password
--output=OUTPUT the format of the output [default: table]
--pub Indicates that the public key is passed in
--set_path Sets the security key paths to KEYNAME
--source=SOURCE the source for the keys
--username=USERNAME the source for the keys [default: none]
Description:
Please note that some values are read from the cloudmesh.yaml
file. One such value is cloudmesh.profile.user
Manages public keys is an essential component of accessing
virtual machine sin the cloud. There are a number of sources
where you can find public keys. This includes teh ~/.ssh
directory and for example github.
Keys will be uploaded into cloudmesh database with the add
command under the given NAME. If the name is not specified the name
cloudmesh.profile.user is assumed.
key add NAME --source=ssh
adds the default key in ~/.ssh/id_rsa.pub
key add NAME --source=FILENAME
adds the key specified by the filename with the given name
key add NAME --git --username=username
adds a named github key from a user with the given github
username.
key set
adds the ~/.ssh/id_rsa.pub key with the name specified in
cloudmesh.profile.user.
It also sets the variable key to that user.
Once the keys are uploaded to github, they can be listed
To list these keys the following list functions are provided.
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--output=OUTPUT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list NAMES
lists all keys in the named virtual machines.
List command can use the [--output=OUTPUT] option
list the keys loaded to cloudmesh in the given format:
json, yaml, table. table is default. The NAME can be
specified and if omitted the name cloudmesh.profile.user
is assumed.
To get keys from the cloudmesh database the following commands
are available:
key delete NAMES
deletes the Named keys. This may also have an impact on groups
key rename NAME NEW
renames the key from NAME to NEW in the cloudmesh database.
Group management of keys is an important concept in cloudmesh,
allowing multiple users to be added to virtual machines while
managing the keys associated with them. The keys must be uploaded
to cloudmesh database with a name so they can be used in a
group. The --dryrun option executes the command without
uploading the information to the clouds. If no group name is
specified the group name default is assumed. If no cloudnamesh
are specified, all active clouds are assumed. active clouds
can be set in the cloudmesh.yaml file.
key group delete [GROUPNAMES] [NAMES] [--dryrun]
deletes the named keys from the named groups.
key group list [GROUPNAMES] [--output=OUTPUT]
list the key names and details in the group.
key group upload [GROUPNAMES] [CLOUDS] [--dryrun]
uploads the named groups to the specified clouds.
In some cases you may want to store the public keys in files. For
this reason we support the following commands.
key group add --group=GROUPNAME --file=FILENAME
the command adds the keys to the given group. The keys are
written in the files in yaml format.
key group export --group=GROUNAMES --filename=FILENAME
the command exports the keys to the given group. The keys are
written in the files in yaml format.
The yaml format is as follows:
cloudmesh:
keys:
NAMEOFKEY:
name: NAMEOFKEY
key: ssh-rsa AAAA..... comment
group:
- GROUPNAME
...
If a key is included in multiple groups they will be added
to the grouplist of the key
"""
def print_keys(keys):
print(
Printer.write(
keys,
sort_keys=["name"],
order=["name", "type", "fingerprint", "comment"],
header=["Name", "Type", "Fingerprint", "Comment"],
output=arguments.output))
map_parameters(arguments, 'cloud', 'dir', 'dryrun', 'filename', 'name',
'nopass', 'output', 'pub', 'pwd', 'set_path', 'source')
variables = Variables()
if arguments.list and arguments.source == "git":
config = Config()
username = config["cloudmesh.profile.github"]
keys = SSHkey().get_from_git(username)
print_keys(keys)
return ""
elif arguments.list and arguments.source == "ssh":
# this is much simpler
sshkey = SSHkey()
print_keys([sshkey])
return ""
elif arguments.list and arguments.cloud:
clouds = Parameter.expand(arguments.cloud)
if len(clouds) == 0:
variables = Variables()
cloudname = variables['cloud']
clouds = [cloudname]
keys = []
for cloud in clouds:
print(f"cloud {cloud}")
provider = Provider(name=cloud)
keys = provider.keys()
provider.Print(keys, output=arguments.output, kind="key")
return ""
elif arguments.list:
cloud = "local"
db = CmDatabase()
keys = db.find(collection=f"{cloud}-key")
print_keys(keys)
return ""
elif arguments.add:
"""
key add [NAME] [--source=FILENAME] # NOT IMPLEMENTED YET
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
"""
key = Key()
if arguments["--source"] == "ssh":
name = arguments.NAME or "ssh"
key.add(name, "ssh")
elif arguments["--source"] == "git":
name = arguments.NAME or "git"
key.add("git", "git")
else:
config = Config()
name = config["cloudmesh.profile.user"]
kind = "ssh"
key.add(name, kind)
elif arguments.init:
"""
key init
"""
config = Config()
username = config["cloudmesh.profile.user"]
if username == "TBD":
Console.error(
"Please set cloudmesh.profile.user in ~/.cloudmesh.yaml")
u = os.environ["USER"].lower().replace(" ", "")
Console.msg(
f"To change it you can use the command. Define a NAME such as '{u}' e.g."
)
Console.msg("")
Console.msg(f" cms config set cloudmesh.profile.user={u}")
Console.msg("")
return ""
key = Key()
key.add(username, "ssh")
variables['key'] = username
elif arguments.upload:
"""
key upload [NAMES] [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [VMS] [--dryrun]
"""
names = Parameter.expand(arguments.NAMES)
# this may have a bug if NAMES is ommitted
#
# Step 0. Set keyname to variable
#
if names is None or len(names) == 0:
config = Config()
username = config["cloudmesh.profile.user"]
names = [username]
if len(names) == 1:
name = names[0]
variables = Variables()
if "key" in variables:
old = variables["key"]
if old != name:
Console.msg(
f"Changing defualt key from {old} to {name}")
variables["key"] = name
#
# Step 1. keys = find keys to upload
#
cloud = "local"
db = CmDatabase()
db_keys = db.find(collection=f"{cloud}-key")
keys = []
for key in db_keys:
if key["name"] in names:
keys.append(key)
if len(keys) == 0:
Console.error(
f"No keys with the names {names} found in cloudmesh. \n"
" Use the command 'key add' to add the key.")
#
# Step 2. iterate over the clouds to upload
#
clouds, names = Arguments.get_cloud_and_names(
"list", arguments, variables)
for cloud in clouds:
print(f"cloud {cloud}")
provider = Provider(name=cloud)
for key in db_keys:
name = key['name']
if name in names:
try:
r = provider.key_upload(key)
Console.ok(f"upload key '{name} successful'. ")
except ValueError as e:
Console.error(
f"key '{name} already exists in {cloud}.")
return ""
elif arguments.delete and arguments.cloud and arguments.NAMES:
# key delete NAMES --cloud=CLOUDS [--dryrun]
names = Parameter.expand(arguments.NAMES)
clouds = Parameter.expand(arguments.cloud)
for cloud in clouds:
provider = Provider(name=cloud)
for name in names:
if arguments.dryrun:
Console.ok(f"Dryrun: delete {name} in {cloud}")
else:
images = provider.key_delete(name)
return ""
elif arguments.gen:
"""
key gen (rsa | ssh) [--filename=FILENAME] [--nopass] [--set_path]
Generate an RSA key pair with pem or ssh encoding for the public
key. The private key is always encoded as a PEM file.
"""
config = Config()
# Check if password will be requested
ap = not arguments.nopass
if not ap:
Console.warning("Private key will NOT have a password")
cnt = yn_choice(message="Continue, despite risk?", default="N")
if not cnt:
sys.exit()
# Discern the name of the public and private keys
rk_path = None
uk_path = None
if arguments.filename:
if arguments.filename[-4:] == ".pub":
rk_path = path_expand(arguments.name[-4:])
uk_path = path_expand(arguments.name)
elif arguments.filename[-5:] == ".priv":
rk_path = path_expand(arguments.name)
uk_path = path_expand(arguments.name[-5:])
else:
rk_path = path_expand(arguments.filename)
uk_path = rk_path + ".pub"
else:
rk_path = path_expand(config['cloudmesh.security.privatekey'])
uk_path = path_expand(config['cloudmesh.security.publickey'])
# Set the path if requested
if arguments.set_path and arguments.filename:
config['cloudmesh.security.privatekey'] = rk_path
config['cloudmesh.security.publickey'] = uk_path
config.save()
Console.msg(f"\nPrivate key: {rk_path}")
Console.msg(f"Public key: {uk_path}\n")
# Generate the Private and Public keys
kh = KeyHandler()
r = kh.new_rsa_key()
u = kh.get_pub_key(priv=r)
# Serialize and write the private key to the path
sr = kh.serialize_key(key=r,
key_type="PRIV",
encoding="PEM",
format="PKCS8",
ask_pass=ap)
kh.write_key(key=sr, path=rk_path)
# Determine the public key format and encoding
enc = None
forma = None
if arguments.ssh:
enc = "SSH"
forma = "SSH"
elif arguments.rsa:
enc = "PEM"
forma = "SubjectInfo"
# Serialize and write the public key to the path
su = kh.serialize_key(key=u,
key_type="PUB",
encoding=enc,
format=forma,
ask_pass=False)
kh.write_key(key=su, path=uk_path)
Console.ok("Success")
elif arguments.verify:
"""
key verify (ssh | pem) --filename=FILENAME --pub
Verifies the encoding (pem or ssh) of the key (private or public)
"""
kh = KeyHandler()
fp = arguments.filename
kt = None
enc = None
# Discern key type
if arguments.pub:
kt = "public"
# Discern public key encoding
if arguments.ssh:
enc, e = "OpenSSH", "SSH"
elif arguments.pem: #PEM encoding
enc = e = "PEM"
# Load the public key, if no error occurs formatting is correct
u = kh.load_key(path=fp,
key_type="PUB",
encoding=e,
ask_pass=False)
else:
kt, enc = "private", "PEM"
# Load the private key to verify the formatting and password of
# the key file. If no error occurs the format and pwd are correct
r = kh.load_key(path=fp,
key_type="PRIV",
encoding=enc,
ask_pass=True)
m = f"Success the {kt} key {fp} has proper {enc} format"
Console.ok(m)
elif arguments.delete and arguments.NAMES:
# key delete NAMES [--dryrun]
names = Parameter.expand(arguments.NAMES)
cloud = "local"
db = CmDatabase()
db_keys = db.find(collection=f"{cloud}-key")
error = []
for key in db_keys:
name = key['name']
if name in names:
if arguments.dryrun:
Console.ok(f"Dryrun: delete {name}")
else:
db.delete(collection="local-key", name=name)
Console.ok(f"delete {name}")
return ""
elif arguments.group:
raise NotImplementedError
return ""
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config cat [less]
config check
config secinit
config encrypt
config decrypt
config edit [ATTRIBUTE]
config set ATTRIBUTE=VALUE
config get ATTRIBUTE [--output=OUTPUT]
config value ATTRIBUTE
config ssh keygen
config ssh verify
config ssh check
config ssh pem
config cloud verify NAME [KIND]
config cloud edit [NAME] [KIND]
config cloud list NAME [KIND] [--secrets]
Arguments:
SOURCE the file to encrypted or decrypted.
an .enc is added to the filename or removed form it
dependent of if you encrypt or decrypt
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the container and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
Options:
--name=KEYNAME The name of a key
--output=OUTPUT The output format [default: yaml]
--secrets Print the secrets. Use carefully.
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys must be generated with
ssh-keygen -t rsa -m pem
openssl rsa -in ~/.ssh/id_rsa -out ~/.ssh/id_rsa.pem
or
cms config ssh keygen
Key validity can be checked with
cms config check
The key password can be verified with
cms config verify
ssh-add
cms config encrypt ~/.cloudmesh/cloudmesh.yaml
cms config decrypt ~/.cloudmesh/cloudmesh.yaml
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
In case the ATTRIBUTE is the name of a cloud defined under
cloudmesh.cloud, the value will be written into the credentials
attributes for that cloud this way you can safe a lot of
typing. An example is
cms config set aws.AWS_TEST=Gregor
which would write the AWS_TEST attribute in the credentials
of the cloud aws. This can naturally be used to set for
example username and password.
"""
# d = Config() #~/.cloudmesh/cloudmesh.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc
map_parameters(arguments, "keep", "secrets", "output")
source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml")
destination = source + ".enc"
if arguments.cloud and arguments.edit and arguments.NAME is None:
path = path_expand("~/.cloudmesh/cloudmesh.yaml")
print(path)
Shell.edit(path)
return ""
cloud = arguments.NAME
kind = arguments.KIND
if kind is None:
kind = "cloud"
configuration = Config()
if arguments.cloud and arguments.verify:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "verify"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml")
print(yaml.dump(result))
flat = flatten(service, sep=".")
for attribute in flat:
if "TBD" in str(flat[attribute]):
Console.error(
f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD")
elif arguments.cloud and arguments.list:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "list"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml")
lines = yaml.dump(result).split("\n")
secrets = not arguments.secrets
result = Config.cat_lines(lines, mask_secrets=secrets)
print(result)
elif arguments.cloud and arguments.edit:
#
# there is a duplicated code in config.py for this
#
action = "edit"
banner(
f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml")
credentials = configuration[f"cloudmesh.{kind}.{cloud}.credentials"]
print(yaml.dump(credentials))
for attribute in credentials:
if "TBD" in credentials[str(attribute)]:
value = credentials[attribute]
result = input(f"Please enter {attribute}[{value}]: ")
credentials[attribute] = result
# configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials
print(yaml.dump(
configuration[f"cloudmesh.{kind}.{cloud}.credentials"]))
elif arguments["edit"] and arguments["ATTRIBUTE"]:
attribute = arguments.ATTRIBUTE
config = Config()
config.edit(attribute)
config.save()
return ""
elif arguments.cat:
content = Config.cat()
import shutil
columns, rows = shutil.get_terminal_size(fallback=(80, 24))
lines = content.split("\n")
counter = 1
for line in lines:
if arguments.less:
if counter % (rows - 2) == 0:
x = input().split("\n")[0].strip()
if x != '' and x in 'qQxX':
return ""
print(line)
counter += 1
return ""
elif arguments.check and not arguments.ssh:
Config.check()
elif arguments.encrypt:
config = Config()
config.encrypt()
elif arguments.decrypt:
config = Config()
config.decrypt()
elif arguments.ssh and arguments.verify:
e = EncryptFile(source, destination)
e.pem_verify()
elif arguments.ssh and arguments.check:
e = EncryptFile(source, destination)
key = path_expand("~/.ssh/id_rsa")
r = e.check_key(key)
if r:
Console.ok(f"Key {key} is valid")
# does not work as it does not change it to pem format
# e.check_passphrase()
elif arguments.ssh and arguments.pem:
e = EncryptFile(source, destination)
r = e.pem_create()
elif arguments.set:
config = Config()
clouds = config["cloudmesh.cloud"].keys()
line = arguments["ATTRIBUTE=VALUE"]
attribute, value = line.split("=", 1)
cloud, field = attribute.split(".", 1)
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
config[attribute] = value
config.save()
elif arguments.value:
config = Config()
attribute = arguments.ATTRIBUTE
if not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
raise Console.error("the variable is a dict")
else:
print(f"{value}")
except Exception as e:
print(e)
return ""
elif arguments.secinit:
config = Config()
secpath = path_expand(config['cloudmesh.security.secpath'])
gcm_path = f"{secpath}/gcm" # Location of nonces and keys for encryption
if not os.path.isdir(gcm_path):
Shell.mkdir(gcm_path) # Use Shell that makes all dirs as needed
elif arguments.get:
print()
config = Config()
clouds = config["cloudmesh.cloud"].keys()
attribute = arguments.ATTRIBUTE
try:
cloud, field = attribute.split(".", 1)
field = f".{field}"
except:
cloud = attribute
field = ""
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
print(Printer.write(value, output=arguments.output))
else:
print(f"{attribute}={value}")
except Exception as e:
print(e)
return ""
elif arguments.ssh and arguments.keygen:
e = EncryptFile(source, destination)
e.ssh_keygen()
return ""
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list --cloud=CLOUDS [--output=OUTPUT]
key list --source=ssh [--dir=DIR] [--output=OUTPUT]
key list --source=git [--output=OUTPUT] [--username=USERNAME]
key list [--output=OUTPUT]
key init
key add NAME --filename=FILENAME [--output=OUTPUT]
key add [NAME] [--source=FILENAME]
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
key delete NAMES [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [VMS] [--dryrun]
key group upload [NAMES] [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun]
key group add [NAMES] [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun]
key group delete [--group=GROUPNAMES] [NAMES] [--dryrun]
key group list [--group=GROUPNAMES] [--output=OUTPUT]
key group export --group=GROUNAMES --filename=FILENAME
key gen (ssh | pem) [--filename=FILENAME] [--nopass] [--set_path] [--force]
key reformat (ssh | pem) [--filename=FILENAME] [--format=FORMAT]
[--nopass] [--pub]
key verify (ssh | pem) [--filename=FILENAME] [--pub] [--check_pass]
Arguments:
VMS Parameterized list of virtual machines
CLOUDS The clouds
NAME The name of the key.
SOURCE db, ssh, all
OUTPUT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key is located
FORMAT Desired key format (SubjectInfo, SSH, OpenSSL, PKCS8)
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--check_pass Flag where program query user for password
--filename=FILENAME the name and full path to the file
--nopass Flag indicating if the key has no password
--output=OUTPUT the format of the output [default: table]
--pub Indicates that the public key is passed in
--set_path Sets the cloudmesh encryption key path to
the full path of the generated keys
--source=SOURCE the source for the keys
--username=USERNAME the source for the keys [default: none]
Description:
Please note that some values are read from the cloudmesh.yaml
file. One such value is cloudmesh.profile.user
Management of public keys is an essential component of accessing
virtual machines in the cloud. There are a number of sources
where you can find public keys. This includes the ~/.ssh
directory and for example github. If you do not already have a
public-private key pair they can be generated using cloudmesh
key gen ssh
This will create the public-private keypair of ~/.ssh/id_rsa
and ~/.ssh/id_rsa.pub in OpenSSH format
key gen pem
This will create the public-private keypair of ~/.ssh/id_rsa
and ~/.ssh/id_rsa.pub in PEM format
key gen (ssh | pem) --filename=~/.cloudmesh/foobar
This will generate the public-private key pair of
~/.cloudmesh/foobar and ~/.cloudmesh/foobar.pub
key gen (ssh | pem) --filename=~/.cloudmesh/foobar --set_path
This will generate the keys as stated above, but it will
also set cloudmesh to use these keys for encryption.
Keys can also be verified for their formatting and passwords.
By default cloudmesh checks ~/.ssh/id_rsa and ~/.ssh/id_rsa.pub
If the key is password protected the formatting can only be
verified if the password is provided (--check_pass argument)
key verify pem
Verifies that ~/.ssh/id_rsa has PEM format
key verify ssh --pub
Verifies that ~/.ssh/id_rsa.pub has OpenSSH format
key verify pem --filename=~/.cloudmesh/foobar
Verifies if the private key located at ~/.cloudmesh/foobar
is password protected
key verify pem --filenam=~/.cloudmesh/foobar --check_pass
Request the password to the file, then checks if the
key is in proper PEM format
You may find the need to keep the values of your keys but
different encodings or formats. These aspects of your key can
also be changed using cloudmesh.
key reformat pem
Will reformat the ~/.id_rsa.pub key from PEM to OpenSSH
key reformat ssh
Will reformat the ~/.id_rsa.pub key from OpenSSH to PEM
key reformat --filename=~/.id_rsa --format=PKCS8
Will reformat the private key to PKCS8 format
Keys will be uploaded into cloudmesh database with the add
command under the given NAME. If the name is not specified the name
cloudmesh.profile.user is assumed.
key add NAME --source=ssh
adds the default key in ~/.ssh/id_rsa.pub
key add NAME --source=FILENAME
adds the key specified by the filename with the given name
key add NAME --git --username=username
adds a named github key from a user with the given github
username.
key set
adds the ~/.ssh/id_rsa.pub key with the name specified in
cloudmesh.profile.user.
It also sets the variable key to that user.
Once the keys are uploaded to github, they can be listed
To list these keys the following list functions are provided.
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--output=OUTPUT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list NAMES
lists all keys in the named virtual machines.
List command can use the [--output=OUTPUT] option
list the keys loaded to cloudmesh in the given format:
json, yaml, table. table is default. The NAME can be
specified and if omitted the name cloudmesh.profile.user
is assumed.
To get keys from the cloudmesh database the following commands
are available:
key delete NAMES
deletes the Named keys. This may also have an impact on groups
key rename NAME NEW
renames the key from NAME to NEW in the cloudmesh database.
Group management of keys is an important concept in cloudmesh,
allowing multiple users to be added to virtual machines while
managing the keys associated with them. The keys must be uploaded
to cloudmesh database with a name so they can be used in a
group. The --dryrun option executes the command without
uploading the information to the clouds. If no group name is
specified the group name default is assumed. If no cloudnamesh
are specified, all active clouds are assumed. active clouds
can be set in the cloudmesh.yaml file.
key group delete [GROUPNAMES] [NAMES] [--dryrun]
deletes the named keys from the named groups.
key group list [GROUPNAMES] [--output=OUTPUT]
list the key names and details in the group.
key group upload [GROUPNAMES] [CLOUDS] [--dryrun]
uploads the named groups to the specified clouds.
In some cases you may want to store the public keys in files. For
this reason we support the following commands.
key group add --group=GROUPNAME --file=FILENAME
the command adds the keys to the given group. The keys are
written in the files in yaml format.
key group export --group=GROUNAMES --filename=FILENAME
the command exports the keys to the given group. The keys are
written in the files in yaml format.
The yaml format is as follows:
cloudmesh:
keys:
NAMEOFKEY:
name: NAMEOFKEY
key: ssh-rsa AAAA..... comment
group:
- GROUPNAME
...
If a key is included in multiple groups they will be added
to the grouplist of the key
"""
def print_keys(keys):
print(
Printer.write(
keys,
sort_keys=["name"],
order=["name", "type", "fingerprint", "comment"],
header=["Name", "Type", "Fingerprint", "Comment"],
output=arguments.output))
map_parameters(arguments, 'check_pass', 'cloud', 'dir', 'dryrun',
'filename', 'force', 'format', 'name', 'nopass',
'output', 'pub', 'pwd', 'set_path', 'source')
variables = Variables()
if arguments.list and arguments.source == "git":
config = Config()
username = config["cloudmesh.profile.github"]
keys = SSHkey().get_from_git(username)
print_keys(keys)
return ""
elif arguments.list and arguments.source == "ssh":
# this is much simpler
sshkey = SSHkey()
print_keys([sshkey])
return ""
elif arguments.list and arguments.cloud:
clouds = Parameter.expand(arguments.cloud)
if len(clouds) == 0:
variables = Variables()
cloudname = variables['cloud']
clouds = [cloudname]
keys = []
for cloud in clouds:
print(f"cloud {cloud}")
provider = Provider(name=cloud)
keys = provider.keys()
provider.Print(keys, output=arguments.output, kind="key")
return ""
elif arguments.list:
cloud = "local"
db = CmDatabase()
keys = db.find(collection=f"{cloud}-key")
print_keys(keys)
return ""
elif arguments.add:
"""
key add [NAME] [--source=FILENAME] # NOT IMPLEMENTED YET
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
"""
key = Key()
if arguments["--source"] == "ssh":
name = arguments.NAME or "ssh"
key.add(name, "ssh")
elif arguments["--source"] == "git":
name = arguments.NAME or "git"
key.add("git", "git")
else:
config = Config()
name = config["cloudmesh.profile.user"]
kind = "ssh"
key.add(name, kind)
elif arguments.init:
"""
key init
"""
config = Config()
username = config["cloudmesh.profile.user"]
if username == "TBD":
Console.error(
"Please set cloudmesh.profile.user in ~/.cloudmesh.yaml")
u = os.environ["USER"].lower().replace(" ", "")
Console.msg(
f"To change it you can use the command. Define a NAME such as '{u}' e.g."
)
Console.msg("")
Console.msg(f" cms config set cloudmesh.profile.user={u}")
Console.msg("")
return ""
key = Key()
key.add(username, "ssh")
variables['key'] = username
elif arguments.upload:
"""
key upload [NAMES] [--cloud=CLOUDS] [--dryrun]
key upload [NAMES] [VMS] [--dryrun]
"""
names = Parameter.expand(arguments.NAMES)
# this may have a bug if NAMES is ommitted
#
# Step 0. Set keyname to variable
#
if names is None or len(names) == 0:
config = Config()
username = config["cloudmesh.profile.user"]
names = [username]
if len(names) == 1:
name = names[0]
variables = Variables()
if "key" in variables:
old = variables["key"]
if old != name:
Console.msg(
f"Changing default key from {old} to {name}")
variables["key"] = name
#
# Step 1. keys = find keys to upload
#
cloud = "local"
db = CmDatabase()
db_keys = db.find(collection=f"{cloud}-key")
keys = []
for key in db_keys:
if key["name"] in names:
keys.append(key)
if len(keys) == 0:
Console.error(
f"No keys with the names {names} found in cloudmesh. \n"
" Use the command 'key add' to add the key.")
#
# Step 2. iterate over the clouds to upload
#
clouds, vmnames = Arguments.get_cloud_and_names(
"list", arguments, variables)
for cloud in clouds:
print(f"cloud {cloud}")
provider = Provider(name=cloud)
for key in db_keys:
name = key['name']
if name in names:
try:
r = provider.key_upload(key)
Console.ok(f"upload key '{name} successful'. ")
except ValueError as e:
Console.error(
f"key '{name} already exists in {cloud}.")
return ""
elif arguments.delete and arguments.cloud and arguments.NAMES:
# key delete NAMES --cloud=CLOUDS [--dryrun]
names = Parameter.expand(arguments.NAMES)
clouds = Parameter.expand(arguments.cloud)
for cloud in clouds:
provider = Provider(name=cloud)
for name in names:
if arguments.dryrun:
Console.ok(f"Dryrun: delete {name} in {cloud}")
else:
images = provider.key_delete(name)
return ""
elif arguments.gen:
"""
key gen (ssh | pem) [--filename=FILENAME] [--nopass] [--set_path]
[--force]
Generate an RSA key pair with pem or ssh encoding for the public
key. The private key is always encoded as a PEM file.
"""
config = Config()
# Check if password will be requested
ap = not arguments.nopass
if not ap:
Console.warning("Private key will NOT have a password")
cnt = yn_choice(message="Continue, despite risk?", default="N")
if not cnt:
sys.exit()
# Discern the name of the public and private keys
rk_path = None
uk_path = None
if arguments.filename:
fp = path_expand(arguments.filename)
fname, fext = os.path.splitext(fp)
if fext == ".pub" or fext == ".ssh":
rk_path = fname
uk_path = fp
elif fext == ".priv" or fext == ".pem":
rk_path = fp
uk_path = fname + ".pub"
else:
rk_path = fp
uk_path = rk_path + ".pub"
else:
rk_path = path_expand("~/.ssh/id_rsa")
uk_path = rk_path + ".pub"
# Check if the file exist, if so confirm overwrite
def check_exists(path):
if os.path.exists(path):
Console.info(f"{path} already exists")
ovwr_r = yn_choice(message=f"overwrite {path}?",
default="N")
if not ovwr_r:
Console.info(f"Not overwriting {path}. Quitting")
sys.exit()
if not arguments.force:
check_exists(rk_path)
check_exists(uk_path)
# Set the path if requested
if arguments.set_path:
config['cloudmesh.security.privatekey'] = rk_path
config['cloudmesh.security.publickey'] = uk_path
config.save()
Console.msg(f"\nPrivate key: {rk_path}")
Console.msg(f"Public key: {uk_path}\n")
# Generate the Private and Public keys
kh = KeyHandler()
r = kh.new_rsa_key()
u = kh.get_pub_key(priv=r)
# Serialize and write the private key to the path
sr = kh.serialize_key(key=r,
key_type="PRIV",
encoding="PEM",
format="PKCS8",
ask_pass=ap)
# Force write the key (since we check file existence above)
kh.write_key(key=sr, path=rk_path, force=True)
# Determine the public key format and encoding
enc = None
forma = None
if arguments.ssh:
enc = "SSH"
forma = "SSH"
elif arguments.pem:
enc = "PEM"
forma = "SubjectInfo"
# Serialize and write the public key to the path
su = kh.serialize_key(key=u,
key_type="PUB",
encoding=enc,
format=forma,
ask_pass=False)
# Force write the key (since we check file existence above)
kh.write_key(key=su, path=uk_path, force=True)
Console.ok("Success")
elif arguments.verify:
"""
key verify (ssh | pem) [--filename=FILENAME] [--pub] [--check_pass]
Verifies the encoding (pem or ssh) of the key (private or public)
"""
# Initialize variables
kh = KeyHandler()
# Determine filepath
fp = None
if arguments.filename is None:
config = Config()
kp = path_expand("~/.ssh/id_rsa")
if arguments.pub:
fp = kp + ".pub"
else:
fp = kp
else:
fp = arguments.filename
# Discern key type
kt = enc = None
ap = True
if arguments.pub:
# Load the public key, if no error occurs formatting is correct
kt, kta, ap = "public", "PUB", False
# Discern public key encoding
if arguments.ssh:
enc, e = "OpenSSH", "SSH"
elif arguments.pem: # PEM encoding
enc = e = "PEM"
else:
# Load the private key to verify the format and password of the
# key file. If no error occurs the format and pwd are correct
kt, kta = "private", "PRIV"
enc = e = "PEM"
ap = False
if arguments.check_pass:
ap = True
try:
k = kh.load_key(path=fp, key_type=kta, encoding=e, ask_pass=ap)
m = f"Success the {kt} key {fp} has proper {enc} format"
Console.ok(m)
except ValueError as e:
# The formatting was incorrect
m = f"Failure, {kt} key {fp} does not have proper {enc} format"
Console.error(m)
raise e
except TypeError as e:
# Success, we didn't ask the user for the key password and
# we received an error for not entering the password, thus
# the key is password protectd
if not arguments.check_pass:
Console.ok("The key is password protected")
else:
# Error Message handled in kh.load_key()
raise e
elif arguments.reformat:
"""
key reformat (ssh | pem) [--filename=FILENAME] [--format=FORMAT]
[--nopass] [--pub]
Restructures a key's format, encoding, and password
"""
# Initialize variables
kh = KeyHandler()
# Determine key type
fname, fext = os.path.splitext(arguments.filename)
kt = "PRIV"
if arguments.pub or fext == ".pub":
kt = "PUB"
# Determine new encoding
use_pem = True
if arguments.ssh:
use_pem = False
kh.reformat_key(path=arguments.filename,
key_type=kt,
use_pem=use_pem,
new_format=arguments.format,
ask_pass=not arguments.nopass)
elif arguments.delete and arguments.NAMES:
# key delete NAMES [--dryrun]
names = Parameter.expand(arguments.NAMES)
cloud = "local"
db = CmDatabase()
db_keys = db.find(collection=f"{cloud}-key")
error = []
for key in db_keys:
name = key['name']
if name in names:
if arguments.dryrun:
Console.ok(f"Dryrun: delete {name}")
else:
db.delete(collection="local-key", name=name)
Console.ok(f"delete {name}")
return ""
elif arguments.group:
raise NotImplementedError
return ""
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config cat [less]
config check
config edit [ATTRIBUTE]
config set ATTRIBUTE=VALUE
config get ATTRIBUTE [--output=OUTPUT]
config value ATTRIBUTE
config cloud verify NAME [KIND]
config cloud edit [NAME] [KIND]
config cloud list NAME [KIND] [--secrets]
config security add (--secret=REGEXP | --exception=REGEXP )
config security rmv (--secret=REGEXP | --exception=REGEXP )
config security list
Arguments:
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the container and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
REGEXP python regular expression
Options:
--name=KEYNAME The name of a key
--nopass Indicates if private key is password protected
--output=OUTPUT The output format [default: yaml]
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys can be generated with
cms key gen (ssh | pem)
Key validity and password can be verified with
cms key verify (ssh | pem)
key verify (ssh | pem) [--filename=FILENAME] [--pub]
ssh-add
Setting configuration
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
In case the ATTRIBUTE is the name of a cloud defined under
cloudmesh.cloud, the value will be written into the credentials
attributes for that cloud this way you can safe a lot of
typing. An example is
cms config set aws.AWS_TEST=Gregor
which would write the AWS_TEST attribute in the credentials
of the cloud aws. This can naturally be used to set for
example username and password.
"""
# d = Config() #~/.cloudmesh/cloudmesh.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc
map_parameters(arguments, "exception", "keep", "nopass", "output",
"secrets")
source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml")
destination = source + ".enc"
if arguments.cloud and arguments.edit and arguments.NAME is None:
path = path_expand("~/.cloudmesh/cloudmesh.yaml")
print(path)
Shell.edit(path)
return ""
cloud = arguments.NAME
kind = arguments.KIND
if kind is None:
kind = "cloud"
configuration = Config()
if arguments.cloud and arguments.verify:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "verify"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
print(yaml.dump(result))
flat = flatten(service, sep=".")
for attribute in flat:
if "TBD" in str(flat[attribute]):
Console.error(
f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD"
)
elif arguments.cloud and arguments.list:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "list"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
lines = yaml.dump(result).splitlines()
secrets = not arguments.secrets
result = Config.cat_lines(lines, mask_secrets=secrets)
print(result)
elif arguments.cloud and arguments.edit:
#
# there is a duplicated code in config.py for this
#
action = "edit"
banner(
f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml"
)
credentials = configuration[
f"cloudmesh.{kind}.{cloud}.credentials"]
print(yaml.dump(credentials))
for attribute in credentials:
if "TBD" in credentials[str(attribute)]:
value = credentials[attribute]
result = input(f"Please enter {attribute}[{value}]: ")
credentials[attribute] = result
# configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials
print(
yaml.dump(
configuration[f"cloudmesh.{kind}.{cloud}.credentials"]))
elif arguments["edit"] and arguments["ATTRIBUTE"]:
attribute = arguments.ATTRIBUTE
config = Config()
config.edit(attribute)
config.save()
return ""
elif arguments.cat:
content = Config.cat()
import shutil
columns, rows = shutil.get_terminal_size(fallback=(80, 24))
lines = content.splitlines()
counter = 1
for line in lines:
if arguments.less:
if counter % (rows - 2) == 0:
x = input()
if x != '' and 'q' in x.lower():
return ""
print(line)
counter += 1
return ""
elif arguments.check:
Config.check()
elif arguments.set:
config = Config()
clouds = config["cloudmesh.cloud"].keys()
line = arguments["ATTRIBUTE=VALUE"]
attribute, value = line.split("=", 1)
cloud, field = attribute.split(".", 1)
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
config[attribute] = value
config.save()
elif arguments.value:
config = Config()
attribute = arguments.ATTRIBUTE
if not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
raise Console.error("the variable is a dict")
else:
print(f"{value}")
except Exception as e:
print(e)
return ""
elif arguments.get:
print()
config = Config()
clouds = config["cloudmesh.cloud"].keys()
attribute = arguments.ATTRIBUTE
try:
cloud, field = attribute.split(".", 1)
field = f".{field}"
except:
cloud = attribute
field = ""
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
print(Printer.write(value, output=arguments.output))
else:
print(f"{attribute}={value}")
except Exception as e:
print(e)
return ""
return ""
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config cat [less]
config check
config secinit
config security add (--secret=REGEXP | --exception=REGEXP )
config security rmv (--secret=REGEXP | --exception=REGEXP )
config security list
config encrypt
config decrypt [--nopass]
config edit [ATTRIBUTE]
config set ATTRIBUTE=VALUE
config get ATTRIBUTE [--output=OUTPUT]
config value ATTRIBUTE
config cloud verify NAME [KIND]
config cloud edit [NAME] [KIND]
config cloud list NAME [KIND] [--secrets]
Arguments:
SOURCE the file to encrypted or decrypted.
an .enc is added to the filename or removed form it
dependent of if you encrypt or decrypt
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the container and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
REGEXP python regular expression
Options:
--secret=REGEXP ensures all attributes within cloudmesh.yaml
whose dot path matches REGEXP are not encrypted
(even if listed in secrets)
--exception=REGEXP ensures attributes within cloudmesh.yaml whose
dot path matches REGEXP are encrypted
--name=KEYNAME The name of a key
--nopass Indicates if private key is password protected
--output=OUTPUT The output format [default: yaml]
--secrets Print the secrets. Use carefully.
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys can be generated with
cms key gen (ssh | pem)
Key validity and password can be verified with
cms key verify (ssh | pem)
key verify (ssh | pem) [--filename=FILENAME] [--pub]
ssh-add
cms config encrypt
Encrypts the config data at-rest. This means that the data
is encrypted when not in use. This command is reliant upon
the cloudmesh.security.secrets attribute and the
cloudmesh.security.exceptions attribute within the
cloudmesh.yaml file. Note, that the encrypted data is not
encrypted upon request/query to the attribute. This means
you must decrypt the config when needed in use and
re-encrypt when not using the file, or delivering the file.
1. cloudmesh.security.secrets:
This attribute will hold a list of python regular
expressions that detail which attributes will be
encrypted by the command.
ex) .*: will encrypt all attributes
ex) .*mdbpwd.*: will encrypt all paths with mdbpwd
2. cloudmesh.security.exceptions:
This attribute will hold a list of python regular
expressions that detail which attributes will not
be encrypted by the command.
ex) .*pubkey.*: ensures no pubkey path is encrypted
security add --secret=REGEXP
Adds valid REGEXP to the cloudmesh.security.secrets section
security rmv --secret=REGEXP
Removes REGEXP from the cloudmesh.security.secrets section
security add --exception=REGEXP
Adds valid REGEXP to cloudmesh.security.exceptions section
security rmv --exception=REGEXP
Removes REGEXP from cloudmesh.security.exceptions section
security list
Prints a list of all the attribute dot-paths that are
referenced by cms config encryption and decryption commands
cms config decrypt
Decrypts the config data that was held in rest. This
command decrypts and attributes that were encrypted
using the sister `cms config encrypt` command.
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
In case the ATTRIBUTE is the name of a cloud defined under
cloudmesh.cloud, the value will be written into the credentials
attributes for that cloud this way you can safe a lot of
typing. An example is
cms config set aws.AWS_TEST=Gregor
which would write the AWS_TEST attribute in the credentials
of the cloud aws. This can naturally be used to set for
example username and password.
"""
# d = Config() #~/.cloudmesh/cloudmesh.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc
map_parameters(arguments, "exception", "keep", "nopass", "output",
"secret", "secrets")
source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml")
destination = source + ".enc"
if arguments.cloud and arguments.edit and arguments.NAME is None:
path = path_expand("~/.cloudmesh/cloudmesh.yaml")
print(path)
Shell.edit(path)
return ""
cloud = arguments.NAME
kind = arguments.KIND
if kind is None:
kind = "cloud"
configuration = Config()
if arguments.cloud and arguments.verify:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "verify"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
print(yaml.dump(result))
flat = flatten(service, sep=".")
for attribute in flat:
if "TBD" in str(flat[attribute]):
Console.error(
f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD"
)
elif arguments.cloud and arguments.list:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "list"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
lines = yaml.dump(result).split("\n")
secrets = not arguments.secrets
result = Config.cat_lines(lines, mask_secrets=secrets)
print(result)
elif arguments.cloud and arguments.edit:
#
# there is a duplicated code in config.py for this
#
action = "edit"
banner(
f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml"
)
credentials = configuration[
f"cloudmesh.{kind}.{cloud}.credentials"]
print(yaml.dump(credentials))
for attribute in credentials:
if "TBD" in credentials[str(attribute)]:
value = credentials[attribute]
result = input(f"Please enter {attribute}[{value}]: ")
credentials[attribute] = result
# configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials
print(
yaml.dump(
configuration[f"cloudmesh.{kind}.{cloud}.credentials"]))
elif arguments["edit"] and arguments["ATTRIBUTE"]:
attribute = arguments.ATTRIBUTE
config = Config()
config.edit(attribute)
config.save()
return ""
elif arguments.cat:
content = Config.cat()
import shutil
columns, rows = shutil.get_terminal_size(fallback=(80, 24))
lines = content.split("\n")
counter = 1
for line in lines:
if arguments.less:
if counter % (rows - 2) == 0:
x = input().split("\n")[0].strip()
if x != '' and x in 'qQxX':
return ""
print(line)
counter += 1
return ""
elif arguments.check:
Config.check()
elif arguments.encrypt:
config = Config()
config.encrypt()
elif arguments.decrypt:
config = Config()
config.decrypt(ask_pass=not arguments.nopass)
elif arguments.set:
config = Config()
clouds = config["cloudmesh.cloud"].keys()
line = arguments["ATTRIBUTE=VALUE"]
attribute, value = line.split("=", 1)
cloud, field = attribute.split(".", 1)
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
config[attribute] = value
config.save()
elif arguments.value:
config = Config()
attribute = arguments.ATTRIBUTE
if not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
raise Console.error("the variable is a dict")
else:
print(f"{value}")
except Exception as e:
print(e)
return ""
elif arguments.secinit:
config = Config()
secpath = path_expand(config['cloudmesh.security.secpath'])
if not os.path.isdir(secpath):
Shell.mkdir(secpath) # Use Shell that makes all dirs as needed
elif arguments.security and arguments.list:
config = Config()
secrets = config.get_list_secrets()
for s in secrets:
Console.msg(s)
elif arguments.security:
# Get the regular expression from command line
regexp = None
if arguments.secret:
regexp = arguments.secret
elif arguments.exception:
regexp = arguments.exception
# Verify argument is valid python regular expression
try:
r = re.compile(regexp)
except re.error:
Console.error(f"Invalid Python RegExp:{regexp}")
sys.exit()
config = Config()
path = None
section = None
# Assign information based on arguments
if arguments.secret:
path = 'cloudmesh.security.secrets'
section = "secrets"
elif arguments.exception:
path = 'cloudmesh.security.exceptions'
section = "exceptions"
# Get current list of regular expressions from related section
exps = config[path]
# Add argument to expressions in related section
if arguments.add:
if regexp not in exps:
config[path].append(regexp)
config.save()
Console.ok(f"Added {regexp} to {section}")
else:
Console.warning(f"{regexp} already in {section}")
# Remove argument from expressions in related section
elif arguments.rmv:
if regexp in exps:
config[path].remove(regexp)
config.save()
Console.ok(f"Removed {regexp} from {section}")
else:
Console.warning(f"{regexp} not in {section}")
elif arguments.get:
print()
config = Config()
clouds = config["cloudmesh.cloud"].keys()
attribute = arguments.ATTRIBUTE
try:
cloud, field = attribute.split(".", 1)
field = f".{field}"
except:
cloud = attribute
field = ""
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
print(Printer.write(value, output=arguments.output))
else:
print(f"{attribute}={value}")
except Exception as e:
print(e)
return ""
return ""
def do_admin(self, args, arguments):
"""
::
Usage:
admin mongo install [--brew] [--download=PATH] [--nosudo] [--docker] [--dryrun] [--force]
admin mongo create
admin mongo status
admin mongo stats
admin mongo version
admin mongo start
admin mongo stop
admin mongo backup FILENAME
admin mongo load FILENAME
admin mongo security
admin mongo password PASSWORD
admin mongo list [--output=OUTPUT]
admin mongo ssh
admin mongo mode [MODE]
admin status
admin system info
The admin command performs some administrative functions, such as
installing packages, software and services. It also is used to
start services and configure them.
Arguments:
FILENAME the filename for backups
Options:
-f specify the file
Description:
Mongo DB
MongoDB is managed through a number of commands.
The configuration is read from ~/.cloudmesh/cloudmesh.yaml
First, you need to create a MongoDB database with
cms admin mongo create
Second, you need to start it with
cms admin mongo start
Now you can interact with it to find out the status, the stats,
and the database listing with the commands
cms admin mongo status
cms admin mongo stats
cms admin mongo list
To stop it from running use the command
cms admin mongo stop
System information about your machine can be returned by
cms admin system info
This can be very useful in case you are filing an issue or bug.
The command
cms admin mongo ssh
is only supported for docker and allows for debugging to login
to the running container. This function may be disabled in future.
admin mongo mode native
switches configuration file to use native mode
admin mongo mode running
switches the configuration to use running mode
"""
map_parameters(arguments, "output", "nosudo", "docker", "dryrun",
"force")
arguments.output = arguments.output or "table"
VERBOSE(arguments)
# arguments.PATH = arguments['--download'] or None
result = None
if arguments.mongo:
if arguments.install and arguments.docker:
installer = MongoInstaller(dryrun=arguments.dryrun,
force=arguments.force)
r = installer.docker()
return r
elif arguments.install:
print("MongoDB install")
print(79 * "=")
# print(arguments.force)
installer = MongoInstaller(dryrun=arguments.dryrun,
force=arguments.force)
sudo = not arguments.nosudo
# if 'linux' in platform.lower() :
# print("SUDO:", sudo)
# r = installer.install(sudo=sudo)
r = installer.install()
return r
elif arguments.status:
mongo = MongoDBController()
state = mongo.status()
if "error" in state["status"]:
Console.error(state["message"])
print(Printer.attribute(state))
else:
data = dotdict()
for pid in state['output']:
entry = state['output'][pid]
data["pid"] = state['output'][pid]
data["command"] = state['output'][pid][
'command'].strip()
print(Printer.dict(data, order=["pid", "command"]))
Console.ok(str(data.pid['pid']) + " " + state["message"])
elif arguments.version:
print("MongoDB Version")
print(79 * "=")
mongo = MongoDBController()
r = mongo.version()
print(r)
elif arguments.security:
mongo = MongoDBController()
mongo.set_auth()
print()
elif arguments.create:
print("MongoDB create")
MongoDBController().create()
elif arguments.ssh:
print("MongoDB ssh")
MongoDBController().ssh()
elif arguments.start:
print("MongoDB start")
MongoDBController().start(security=True)
elif arguments.stop:
print("MongoDB stop")
MongoDBController().stop()
elif arguments.backup:
print("MongoDB backup")
MongoDBController().dump(arguments.get('FILENAME'))
elif arguments.load:
print("MongoDB backup")
MongoDBController().restore(arguments.get('FILENAME'))
elif arguments.stats:
mongo = MongoDBController()
r = mongo.stats()
if len(r) > 0:
print(Printer.attribute(r))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.list:
mongo = MongoDBController()
r = mongo.list()
if len(r) > 0:
if arguments.output == 'table':
print(
Printer.dict(r,
order=[
"name", "sizeOnDisk", "empty",
"collections"
],
output=arguments.output), )
else:
print(Printer.write(r, output=arguments.output))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.mode:
if arguments.MODE:
if arguments.MODE not in ["native", "running", "docker"]:
Console.error("The mode is not supported")
config = Config()
config["cloudmesh.data.mongo.MODE"] = arguments.MODE
config.save()
else:
config = Config()
mode = config["cloudmesh.data.mongo.MODE"]
print(mode)
return ""
elif arguments.status:
# config = Config()
# data = config["cloudmesh.data.mongo"]
print("Rest Service status")
print("MongoDB status")
try:
mongo = MongoDBController()
mongo.login()
if mongo.status()['status'] == 'ok':
Console.ok("Mongo is running")
except Exception as e:
Console.error("Mongo is not running")
print(e)
elif arguments.system:
s = OperatingSystem.get()
print(Printer.attribute(s))
return result
def add_user(cls, user, password):
config = Config()
config[cls.CONFIG_ATTRIBUTE_AUTH] = cls.CONFIG_VALUE_AUTH
config[cls.CONFIG_ATTRIBUTE_USER] = user
config[cls.CONFIG_ATTRIBUTE_PASSWORD] = cls._encode(password)
config.save()
