def test_search(self):
config = Config()
Benchmark.Start()
r = config.search("cloudmesh.cloud.*.cm.active", True)
Benchmark.Stop()
pprint(r)
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
"""
TODO: missing, also define parameters
:param service:
:param config:
"""
super().__init__(service=service, config=config)
self.config = Config()
self.storage_credentials = self.config.credentials("storage", "gdrive")
if self.storage_credentials['maxfiles'] > 1000:
Console.error("Page size must be smaller than 1000")
sys.exit(1)
self.limitFiles = self.storage_credentials['maxfiles']
self.scopes = self.storage_credentials['scopes']
self.clientSecretFile = path_expand(
self.storage_credentials['location_secret'])
self.applicationName = self.storage_credentials['application_name']
self.generate_key_json()
self.flags = self.generate_flags_json()
self.credentials = self.get_credentials()
self.http = self.credentials.authorize(httplib2.Http())
self.driveService = discovery.build('drive', 'v3', http=self.http)
self.cloud = service
self.service = service
self.fields = "nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
def json_to_yaml(cls, name, filename="~/.cloudmesh/security/google.json"):
"""
Given a json file downloaded from google, copies the content into the
cloudmesh yaml file, while overwriting or creating a new compute provider
:param cls:
:param name:
:param filename: Service Account Key file downloaded from google cloud.
:return: None
"""
path = path_expand(filename)
# Open and load the JSON file.
with open(path, "r") as file:
d = json.load(file)
# Get the project id and client email.
project_id = d["project_id"]
client_email = d["client_email"]
# Format the sample with json file details.
format_sample = cls.sample.format_map(locals())
# Convert the yaml sample to JSON.
google_yaml = yaml.load(format_sample, Loader=yaml.SafeLoader)
# Extract the google compute section
google_config = google_yaml["cloudmesh"]["cloud"]
# Update the google cloud section of cloudmesh.yaml config file.
config = Config()
config["cloudmesh"]["cloud"][name] = google_config
config.save()
banner("Result")
pprint(config["cloudmesh"]["cloud"][name])
def config(self, config_path='~/.cloudmesh/cloudmesh.yaml'):
"""
Use `cloudmesh.yaml` file to configure.
"""
self._conf = Config(config_path).get("data")
# Set DB provider. There should only be one.
db_provider = self._conf.get('default.db')
if db_provider == 'local':
db_path = self._conf.get('db.local.CMDATA_DB_FOLDER')
self._db = LocalDBProvider(db_path)
# Check for local storage provider.
storage_path = self._conf.get('service.local.CMDATA_STORAGE_FOLDER')
if storage_path:
self._providers['local'] = LocalStorageProvider(storage_path)
# Check for Azure provider.
az_conf = self._conf.get('service.azure')
if az_conf:
az_act = az_conf.get('credentials.AZURE_STORAGE_ACCOUNT')
az_key = az_conf.get('credentials.AZURE_STORAGE_KEY')
az_container = az_conf.get('container')
if az_act and az_key:
self._providers['azure'] = AzureStorageProvider(
az_act, az_key, az_container)
# Set a default storage provider.
default_storage_provider = self._conf.get('default.service')
self._providers['default'] = self._providers[default_storage_provider]
def init_cloud_params(self, cloudname):
from cloudmesh.configuration.Config import Config
self.conf = Config()["cloudmesh"]
self.user = Config()["cloudmesh"]["profile"]["user"]
self.spec = self.conf["cloud"][cloudname]
self.cloudname = cloudname
print(self.spec)
self.default = self.spec["default"]
self.cloudtype = self.spec["cm"]["kind"]
if self.cloudtype == 'aws':
self.bucket_name = self.spec["cm"]["bucket_name"]
self.region_name = self.spec["cm"]["region_name"]
self.forecast = self.spec["cm"]["forecast_srv"]
self.forecastquery_srv = self.spec["cm"]["forecastquery_srv"]
self.s3_srv = self.spec["cm"]["s3_srv"]
self.role_arn = self.spec["cm"]["iam_role_arn"]
self.algorithmArn = self.spec["cm"]["algorithmArn"]
import boto3
self.session = boto3.Session(region_name=self.region_name)
self.forecast_srv = self.session.client(service_name=self.forecast)
self.forecastquery = self.session.client(
service_name=self.forecastquery_srv)
print("AWS Cloud was requested")
elif self.cloudname == 'azure':
print("Azure cloud was requested")
else:
print("Supported cloud services at this time : ")
def test_search(self):
config = Config()
StopWatch.start("search")
r = config.search("cloudmesh.cloud.*.cm.active", True)
StopWatch.stop("search")
pprint(r)
def test_dictreplace(self):
HEADING()
config = Config()
spec = textwrap.dedent("""
cloudmesh:
profile:
name: Gregor
unordered:
name: "{cloudmesh.other.name}.postfix"
other:
name: "{cloudmesh.profile.name}"
""")
print(spec)
# spec = spec.replace("{", "{{")
# spec = spec.replace("}", "}}")
# print(spec)
StopWatch.start("dictreplace")
result = config.spec_replace(spec)
StopWatch.stop("dictreplace")
print(result)
data = yaml.load(result, Loader=yaml.SafeLoader)
pprint(data)
assert data["cloudmesh"]["unordered"]["name"] == "Gregor.postfix"
assert data["cloudmesh"]["other"]["name"] == "Gregor"
class Driver(object):
def __init__(self, name="~/.cloudmesh/cloudmesh.yaml"):
name = os.path.expanduser(name)
self.config = Config(name=name)
# noinspection PyPep8Naming
def get(self, name=None):
connection = None
if name is None:
variables = Variables()
# noinspection PyUnusedLocal
cloudname = variables['cloud']
kind = self.config.get(
"cloudmesh.cloud.{name}.cm.kind".format(name=name))
credentials = self.config.get(
"cloudmesh.cloud.{name}.credentials".format(name=name))
# BUG FROM HERE ON WRONG
if kind == 'azure':
AZDriver = get_driver(Provider.AZURE)
connection = AZDriver(
subscription_id=credentials['AZURE_SUBSCRIPTION_ID'],
key_file=credentials['AZURE_MANAGEMENT_CERT_PATH'])
elif kind == 'aws':
EC2Driver = get_driver(Provider.EC2)
connection = EC2Driver(credentials['EC2_ACCESS_ID'],
credentials['EC2_SECRET_KEY'])
return connection
def __init__(self):
"""
Initializes the Queue class
"""
self.cm_config = Config()
self.info = {
'uid': None,
"cloud": None,
"kind": "batch-queue",
"name": None,
"cm": {},
"queue": {
'policy': None,
'status': None,
'active': False,
'charge': None,
'unit': None,
"numJobs": 0,
"numRunningJobs": 0,
'joblist': []
}
}
self.database = CmDatabase()
def __init__(self):
"""
Initializes the Queue class
"""
self.cm_config = Config()
self.info = munch.munchify({
'uid': None,
"cloud": None,
"kind": "batch-queue",
"name": None,
"cm": {},
"queue": {
'policy': None,
'status': None,
'active': False,
'charge': None,
'unit': None,
"numJobs": 0,
"numRunningJobs": 0,
'joblist': []
}
})
# list of parameters that can be set
self.settable_params = ['policy', 'charge', 'unit']
self.database = CmDatabase()
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh.profile.user"]
self.key_path = path_expand(
Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
self.clouduser = '******'
self.name_generator = Name(schema=f"{self.user}-vm", counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = GCloudProvider(name="google")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def edit(key, caps=True, show=False):
global gui_enabled
if not gui_enabled:
Console.Warning(
"Cloudmesh Gui not supported, can not find tkinter")
return ""
config = Config()
entry = dict(FlatDict(config[key], sep='.'))
layout = [[gui.Text(f'Cloudmesh Configuration Editor: {key}')]]
length = 1
for _key, _value in entry.items():
length = max(length, len(_key))
length = length + 3
for _key, _value in entry.items():
if caps:
label = _key.capitalize()
else:
label = _key
secrets = Config.secrets()
if _key in secrets and not show:
field = [
gui.Text(label, size=(length, 1)),
gui.InputText(key=f"{key}.{_key}",
password_char="*",
default_text=_value)
]
else:
field = [
gui.Text(label, size=(length, 1)),
gui.InputText(key=f"{key}.{_key}", default_text=_value)
]
layout.append(field)
layout.append([gui.Submit(), gui.Cancel()])
window = gui.Window('Cloudmesh Configuration Editor',
layout,
background_color="white")
event, values = window.Read()
window.Close()
if event == "Submit":
for _key, _value in values.items():
config[_key] = _value
if show:
Console.ok(f"{_key}={_value}")
config.save()
else:
print(event)
def __init__(self, name="azure", configuration=None, credentials=None):
"""
Initializes the provider. The default parameters are read from the
configuration file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration file
"""
# configuration = configuration if configuration is not None \
# else CLOUDMESH_YAML_PATH
conf = Config(configuration)["cloudmesh"]
self.user = Config()["cloudmesh"]["profile"]["user"]
self.spec = conf["volume"][name]
self.cloud = name
self.location = 'eastus'
self.size = 1
self.group_name = self.spec["default"]['group']
cred = self.spec["credentials"]
self.default = self.spec["default"]
self.cloudtype = self.spec["cm"]["kind"]
super().__init__(name, configuration)
# update credentials with the passed dict
if credentials is not None:
cred.update(credentials)
# VERBOSE(cred, verbose=10)
if self.cloudtype != 'azure':
Console.error("This class is meant for azure cloud")
# ServicePrincipalCredentials related Variables to configure in
# cloudmesh.yaml file
# AZURE_APPLICATION_ID = '<Application ID from Azure Active Directory
# App Registration Process>'
# AZURE_SECRET_KEY = '<Secret Key from Application configured in
# Azure>'
# AZURE_TENANT_ID = '<Directory ID from Azure Active Directory
# section>'
credentials = ServicePrincipalCredentials(
client_id=cred['AZURE_APPLICATION_ID'],
# application and client id are same thing
secret=cred['AZURE_SECRET_KEY'],
tenant=cred['AZURE_TENANT_ID']
)
subscription = cred['AZURE_SUBSCRIPTION_ID']
# Management Clients
self.compute_client = ComputeManagementClient(
credentials, subscription)
def test_search_openstack(self):
HEADING()
config = Config()
Benchmark.Start()
r = config.search("cloudmesh.cloud.*.cm.kind", "openstack")
Benchmark.Stop()
pprint(r)
def test_search_active_all(self):
HEADING()
config = Config()
Benchmark.Start()
r = config.search("cloudmesh.cloud.*.cm.active")
Benchmark.Stop()
pprint(r)
def test_doesnotexist_get(self):
config = Config()
key = "cloudmesh.doesnotexist"
StopWatch.start(f"not exists get({key})")
value = config.get(key, default="Hallo")
StopWatch.stop(f"not exists get({key})")
assert value == "Hallo"
def test_dict(self):
HEADING()
config = Config()
Benchmark.Start()
result = config.dict()
Benchmark.Stop()
print(type(config.data))
pprint(config['cloudmesh.profile'])
assert config is not None
def activate():
global gui_enabled
if not gui_enabled:
Console.Warning(
"Cloudmesh Gui not supported, can not find tkinter")
return ""
config = Config()
clouds = list(config["cloudmesh.cloud"].keys())
gui.SetOptions(text_justification='right')
layout = [[
gui.Text('Cloudmesh Cloud Activation', font=('Helvetica', 16))
], [gui.Text('Compute Services')]]
layout.append([gui.Text('_' * 100, size=(65, 1))])
for cloud in clouds:
tbd = "TBD" in str(config[f"cloudmesh.cloud.{cloud}.credentials"])
active = config[f"cloudmesh.cloud.{cloud}.cm.active"]
if tbd:
color = 'red'
else:
color = "green"
choice = [
gui.Checkbox(cloud,
key=cloud,
text_color=color,
default=active)
]
layout.append(choice)
layout.append([gui.Text('_' * 100, size=(65, 1))])
layout.append([gui.Submit(), gui.Cancel()])
window = gui.Window('Cloudmesh Configuration',
layout,
font=("Helvetica", 12))
event, values = window.Read()
if event == "Submit":
for cloud in values:
active = values[cloud] or False
config[f"cloudmesh.cloud.{cloud}.cm.active"] = str(active)
if active:
Console.ok(f"Cloud {cloud} is active")
config.save()
else:
print(event)
def __init__(self,
name=None,
configuration="~/.cloudmesh/.cloudmesh.yaml"):
VERBOSE(f"Init Docker {name}")
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
def __init__(self, name):
"""
Initialize provider. The default parameters are read from the
configuration file that is defined in yaml format.
:param name: name of cloud
"""
self.cloud = name
self.config = Config()["cloudmesh.volume.openstack.credentials"]
self.defaults = Config()["cloudmesh.volume.openstack.default"]
self.cm = CmDatabase()
def test_dict(self):
HEADING()
config = Config()
Benchmark.Start()
result = config.dict()
Benchmark.Stop()
pprint(result)
print(config)
print(type(config.data))
assert config is not None
def test_dict(self):
HEADING()
config = Config()
StopWatch.start("dict")
result = config.dict()
StopWatch.stop("dict")
pprint(result)
print(config)
print(type(config.data))
assert config is not None
class AzRegister(object):
def __init__(self, cloud='azure'):
self.config = Config()
self.credentials = self.config[f'cloudmesh.cloud.{cloud}.credentials']
def set_credentials(self, creds):
self.credentials['AZURE_TENANT_ID'] = creds['Access key ID']
self.credentials['AZURE_SUBSCRIPTION_ID'] = creds['Secret access key']
self.credentials['AZURE_APPLICATION_ID'] = creds['Secret access key']
self.credentials['AZURE_SECRET_KEY'] = creds['Secret access key']
self.config.save()
def azString2Dict(self, azString):
azDict = dict()
for i in azString.strip().splitlines():
x = i.split(":")
if len(x) == 2:
azDict[x[0].strip(' ,"')] = x[1].strip(' ,"')
return azDict
def register(self, cloud='azure'):
# Opens web browser and prompts user to login
subprocess.Popen('az login')
# once user has logged in, collects account information, such as subscription id
accountInfo = subprocess.getoutput('az account show')
print(accountInfo)
azoutput = self.azString2Dict(accountInfo)
AZURE_SUBSCRIPTION_ID = azoutput['id']
AZURE_TENANT_ID = azoutput['tenantId']
# WARNING: FOLLOWING CODE WILL RENDER OLD SECRET KEY INVALID
azAppKeyStr = subprocess.getoutput(
'az ad sp create-for-rbac --name http://cloudmesh')
azAppKeyDict = self.azString2Dict(azAppKeyStr)
AZURE_APPLICATION_ID = azAppKeyDict['appId']
AZURE_SECRET_KEY = azAppKeyDict['password']
creds = {
'AZURE_SUBSCRIPTION_ID': AZURE_SUBSCRIPTION_ID,
'AZURE_TENANT_ID': AZURE_TENANT_ID,
'AZURE_APPLICATION_ID': AZURE_APPLICATION_ID,
'AZURE_SECRET_KEY': AZURE_SECRET_KEY
}
self.set_credentials(creds)
Console.info(
"Azure Tenant, Subscription, Application, and Secret Key have been added to the cloudmesh.yaml file."
)
def __init__(self,
name=None,
configuration="~/.cloudmesh/cloudmesh.yaml"):
# noinspection PyPep8
try:
super().__init__(name, configuration)
self.kind = Config(configuration)[f"cloudmesh.cloud.{name}.cm.kind"]
self.credentials = Config(configuration)[
f"cloudmesh.cloud.{name}.credentials"]
self.name = name
except:
Console.error(f"provider {name} not found in {configuration}")
raise ValueError(f"provider {name} not found in {configuration}")
provider = None
providers = ProviderList()
if self.kind in ['openstack',
'azure',
'docker',
"aws",
"azureaz",
"virtualbox"]:
provider = providers[self.kind]
elif self.kind in ["awslibcloud", "google"]:
from cloudmesh.compute.libcloud.Provider import \
Provider as LibCloudProvider
provider = LibCloudProvider
elif self.kind in ['oracle']:
from cloudmesh.oracle.compute.Provider import \
Provider as OracleComputeProvider
provider = OracleComputeProvider
# elif self.kind in ["vagrant", "virtualbox"]:
# from cloudmesh.compute.virtualbox.Provider import \
# Provider as VirtualboxCloudProvider
# provider = VirtualboxCloudProvider
# elif self.kind in ["azureaz"]:
# from cloudmesh.compute.azure.AzProvider import \
# Provider as AzAzureProvider
# provider = AzAzureProvider
if provider is None:
Console.error(f"provider {name} not supported")
raise ValueError(f"provider {name} not supported")
self.p = provider(name=name, configuration=configuration)
def __init__(self, name=None):
"""
Initializes the provider. The default parameters are read from the
configuration file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration file
"""
self.config = Config()
conf = self.config["cloudmesh"]
super().__init__(name)
self.user = self.config["cloudmesh.profile.user"]
self.spec = conf["cloud"][name]
self.cloud = name
self.default = self.spec["default"]
self.cloudtype = self.spec["cm"]["kind"]
self.cred = self.config[f"cloudmesh.cloud.{name}.credentials"]
fields = ["user",
"fingerprint",
"key_file",
"pass_phrase",
"tenancy",
"compartment_id",
"region"]
for field in fields:
if self.cred[field] == 'TBD':
Console.error(
f"The credential for Oracle cloud is incomplete. {field} "
"must not be TBD")
self.credential = self._get_credentials(self.cred)
self.compute = oci.core.ComputeClient(self.credential)
self.virtual_network = oci.core.VirtualNetworkClient(self.credential)
self.identity_client = oci.identity.IdentityClient(self.credential)
self.compartment_id = self.credential["compartment_id"]
try:
self.public_key_path = conf["profile"]["publickey"]
self.key_path = path_expand(
Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
except:
raise ValueError("the public key location is not set in the "
"profile of the yaml file.")
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
super().__init__(service=service, config=config)
self.config = Config()
self.storage_credentials = self.config.credentials("storage", "gdrive")
google_json = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
"service_account"]
self.google_client = storage.Client.from_service_account_json(
google_json)
self.local_dir = self.config["cloudmesh"]["storage"]["local"]["dir"]
self.bucket = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
"bucket"]
def __init__(self, name=None,
configuration="~/.cloudmesh/.cloudmesh.yaml"):
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
if platform.system().lower() == "darwin":
self.vboxmanage = \
"/Applications/VirtualBox.app/Contents/MacOS/VBoxManage"
else:
self.vboxmanage = "VBoxManage"
def check_yaml(self):
config = Config()
errors = False
if "chameleon" in config["cloudmesh.cloud"]:
credentials = config["cloudmesh.cloud.chameleon.credentials"]
if "OS_PASSWORD" in credentials:
Console.error("You are using an old version of the cloudmesh.yaml file")
else:
Console.ok("your yaml file seems ok")
location = config.location
print(f"Location of cloudmesh.yaml: {location}")
print("we run a simple yamllint test. Not all issues reported need to be corrected.")
yamlfile = Path(f"{location}/cloudmesh.yaml")
result = Shell.run(f"yamllint {yamlfile}")
for line in result.splitlines():
line = line.strip()
if "missing document start" in line:
continue
if "line too long" in line:
continue
if "error" in line:
errors = True
Console.error (line)
else:
print(line)
if errors:
Console.error("We found issues in your yaml file")
else:
Console.ok(f"OK. Your yaml file {yamlfile} seems valid")
def __init__(self, cloud, path):
config = Config(config_path=path)["cloudmesh"]
self.cm = config["cloud"][cloud]["cm"]
self.default = config["cloud"][cloud]["default"]
self.credentials = config["cloud"][cloud]["credentials"]
self.group = config["default"]["group"]
self.experiment = config["default"]["experiment"]
def test_config_subscriptable(self):
HEADING()
config = Config()
StopWatch.start("config_subscriptable")
data = config["cloudmesh"]["data"]["mongo"]
StopWatch.stop("config_subscriptable")
assert data is not None