def config(self, config_path='~/.cloudmesh/cloudmesh4.yaml'):
"""
Use `cloudmesh4.yaml` file to configure.
"""
self._conf = Config(config_path).get("data")
# Set DB provider. There should only be one.
db_provider = self._conf.get('default.db')
if db_provider == 'local':
db_path = self._conf.get('db.local.CMDATA_DB_FOLDER')
self._db = LocalDBProvider(db_path)
# Check for local storage provider.
storage_path = self._conf.get('service.local.CMDATA_STORAGE_FOLDER')
if storage_path:
self._providers['local'] = LocalStorageProvider(storage_path)
# Check for Azure provider.
az_conf = self._conf.get('service.azure')
if az_conf:
az_act = az_conf.get('credentials.AZURE_STORAGE_ACCOUNT')
az_key = az_conf.get('credentials.AZURE_STORAGE_KEY')
az_container = az_conf.get('container')
if az_act and az_key:
self._providers['azure'] = AzureStorageProvider(az_act, az_key, az_container)
# Set a default storage provider.
default_storage_provider = self._conf.get('default.service')
self._providers['default'] = self._providers[default_storage_provider]
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh"]["profile"]["user"]
self.key_path = path_expand(
Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = GCloudProvider(name="google")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def __init__(self):
"""
Initializes the Queue class
"""
self.cm_config = Config()
self.info = munch.munchify({
'uid': None,
"cloud": None,
"kind": "batch-queue",
"name": None,
"cm": {},
"queue": {
'policy': None,
'status': None,
'active': False,
'charge': None,
'unit': None,
"numJobs": 0,
"numRunningJobs": 0,
'joblist': []
}
})
# list of parameters that can be set
self.settable_params = ['policy', 'charge', 'unit']
self.database = CmDatabase()
class Driver(object):
def __init__(self, name="~/.cloudmesh/cloudmesh4.yaml"):
name = os.path.expanduser(name)
self.config = Config(name=name)
def get(self, name=None):
connection = None
if name is None:
variables = Variables()
cloudname = variables['cloud']
kind = self.config.get(
"cloudmesh.cloud.{name}.cm.kind".format(name=name))
credentials = self.config.get(
"cloudmesh.cloud.{name}.credentials".format(name=name))
# BUG FROM HERE ON WRONG
if kind == 'azure':
AZURE_SUBSCRIPTION_ID = credentials['AZURE_SUBSCRIPTION_ID']
AZURE_MANAGEMENT_CERT_PATH = credentials[
'AZURE_MANAGEMENT_CERT_PATH']
AZDriver = get_driver(Provider.AZURE)
connection = AZDriver(subscription_id=AZURE_SUBSCRIPTION_ID,
key_file=AZURE_MANAGEMENT_CERT_PATH)
elif kind == 'aws':
EC2_ACCESS_ID = credentials['EC2_ACCESS_ID']
EC2_SECRET_KEY = credentials['EC2_SECRET_KEY']
EC2Driver = get_driver(Provider.EC2)
connection = EC2Driver(EC2_ACCESS_ID, EC2_SECRET_KEY)
return connection
def set_cloud(self, cloud):
"""
switch to another cloud provider
:param cloud: target provider
:return:
"""
self.cloud = cloud
self.os_config = Config().get('cloud.{}'.format(cloud))
def test_set(self):
before = self._conf.get("default.cloud")
self._conf.set("default.cloud", "testcloud")
new_config = Config()
after = new_config.get("default.cloud")
assert before != after
new_config.set("default.cloud", before)
def __init__(self,
name=None,
configuration="~/.cloudmesh/.cloudmesh4.yaml"):
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
def __init__(self,
name=None,
configuration="~/.cloudmesh/.cloudmesh4.yaml"):
VERBOSE.print(f"Init Docker {name}", verbose=9)
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
class TestConfig:
def setup(self):
self.config = Config()
def test_00_config(self):
HEADING()
pprint(self.config.dict())
print(self.config)
print(type(self.config.data))
assert self.config is not None
def test_20_config_subscriptable(self):
HEADING()
data = self.config["cloudmesh"]["data"]["mongo"]
assert data is not None
def test_30_dictreplace(self):
HEADING()
spec = textwrap.dedent("""
cloudmesh:
profile:
name: Gregor
unordered:
name: "{cloudmesh.other.name}.postfix"
other:
name: "{cloudmesh.profile.name}"
""")
print(spec)
# spec = spec.replace("{", "{{")
# spec = spec.replace("}", "}}")
# print(spec)
result = self.config.spec_replace(spec)
print(result)
data = yaml.load(result)
pprint(data)
assert data["cloudmesh"]["unordered"]["name"] == "Gregor.postfix"
assert data["cloudmesh"]["other"]["name"] == "Gregor"
def test_31_configreplace(self):
HEADING()
self.config = Config()
pprint(self.config["cloudmesh"]["profile"])
def __init__(self, name=None, configuration="~/.cloudmesh/cloudmesh.yaml"):
"""
Initializes the provider. The default parameters are read from the configutation
file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration filw
"""
HEADING(c=".")
conf = Config(configuration)["cloudmesh"]
#self.user = conf["profile"]
self.user = Config()["cloudmesh"]["profile"]["user"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
super().__init__(name, conf)
self.name_generator = Name(
experiment="exp",
group="grp",
user = "******",
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.name = str(self.name_generator)
self.key_path = path_expand(Config()["cloudmesh"]["profile"]["publickey"])
f = open(self.key_path, 'r')
self.key_val = f.read()
self.testnode = None
if self.cloudtype in Provider.ProviderMapper:
self.driver = get_driver(
Provider.ProviderMapper[self.cloudtype])
if self.cloudtype == 'google':
self.cloudman = self.driver(
cred["client_email"],
cred["path_to_json_file"],
project = cred["project"]
)
else:
print("Specified provider not available")
self.cloudman = None
self.default_image = None
self.default_size = None
self.public_key_path = conf["profile"]["publickey"]
def __init__(self, name=None,
configuration="~/.cloudmesh/.cloudmesh4.yaml"):
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
if platform.system().lower() == "darwin":
self.vboxmanage = "/Applications/VirtualBox.app/Contents/MacOS/VBoxManage"
else:
self.vboxmanage = "VBoxManage"
class TestConfig:
def setup(self):
self.config = Config()
def test_00_config(self):
HEADING(myself())
pprint(self.config.dict())
print(self.config)
print(type(self.config.data))
# pprint(config.credentials('local'))
assert self.config is not None
# assert 'cloud' in config.cloud
def test_10_config_print(self):
HEADING(myself())
print(self.config)
assert True is True
def test_20_config_subscriptable(self):
HEADING(myself())
data = self.config["cloudmesh"]["data"]["mongo"]
assert data is not None
def __init__(self, service='bigquery'):
VERBOSE("initialize google big query manager")
self.service_account_file = None
self.access_key = None
self.region = None
self.config = Config()
print(self.service_account_file)
self.service_account_file = self.config[
'cloudmesh.cloud.bigquery.credentials.service_account_file_path']
self.credentials = service_account.Credentials.from_service_account_file(
self.service_account_file)
self.client = bigquery.Client(credentials=self.credentials)
# print(self.key_id)
self.ec2_client = bigquery.Client(
'ec2',
region_name=self.region,
aws_access_key_id=self.key_id,
aws_secret_access_key=self.access_key)
self.ec2_resource = bigquery.Client(
'ec2',
region_name=self.region,
aws_access_key_id=self.key_id,
aws_secret_access_key=self.access_key)
def test_ec2_region(self):
config = Config()
data = config['cloudmesh.cloud.aws.credentials.EC2_SECRET_KEY']
assert data is not None
assert data != ""
def _get_specification(self, cloud=None, name=None, port=None,
image=None, **kwargs):
arg = dotdict(kwargs)
arg.port = port
config = Config()
pprint(self.config)
if cloud is None:
#
# TOD read default cloud
#
cloud = "vagrant" # TODO must come through parameter or set cloud
spec = config.data["cloudmesh"]["cloud"][cloud]
default = spec["default"]
pprint(default)
if name is not None:
arg.name = name
else:
# TODO get new name
pass
if image is not None:
arg.image = image
else:
arg.image = default["image"]
pass
arg.path = default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
arg.vagrantfile = "{directory}/Vagrantfile".format(**arg)
return arg
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
variables = Variables()
# this gives the current default cloud
cloud = variables['cloud']
# specify the cloud name to make sure this test
# is done for the openstack cloud
#
self.p = Provider(name="chameleon")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def setup(self):
self.status_id = "test-id-1"
config = Config().data["cloudmesh"]["data"]["mongo"]
self.mongo = MongoDB(host=config["MONGO_HOST"],
username=config["MONGO_USERNAME"],
password=config["MONGO_PASSWORD"],
port=config["MONGO_PORT"])
def __init__(self,
name=None,
configuration="~/.cloudmesh/.cloudmesh4.yaml"):
try:
self.kind = Config(
configuration)["cloudmesh"]["cloud"][name]["cm"]["kind"]
self.name = name
except:
Console.error(f"proider {name} not found in {configuration}")
raise ValueError(f"proider {name} not found in {configuration}")
provider = None
if self.kind in ["openstack", "aws", "google"]:
provider = LibCloudProvider
elif self.kind in ["vagrant", "virtualbox"]:
provider = VirtualboxCloudProvider
elif self.kind in ["docker"]:
provider = DockerProvider
elif self.kind in ["azure"]:
provider = AzAzureProvider
if provider is None:
Console.error(f"proider {name} not supported")
raise ValueError(f"proider {name} not supported")
self.p = provider(name=name, configuration=configuration)
def test_ec2_access_id(self):
config = Config()
data = config['cloudmesh.cloud.aws.credentials.EC2_ACCESS_ID']
assert data is not None
assert data != ""
def __init__(self,
service='redshift',
config="~/.cloudmesh/cloudmesh.yaml"):
VERBOSE("initialize redshift manager")
self.key_id = None
self.access_key = None
self.region = None
self.config = Config(config_path=config)
print(self.key_id)
self.key_id = self.config[
'cloudmesh.cloud.aws.credentials.EC2_ACCESS_ID']
self.access_key = self.config[
'cloudmesh.cloud.aws.credentials.EC2_SECRET_KEY']
self.region = self.config['cloudmesh.cloud.aws.credentials.region']
self.client = boto3.client(service,
region_name=self.region,
aws_access_key_id=self.key_id,
aws_secret_access_key=self.access_key)
# print(self.key_id)
self.ec2_client = boto3.client('ec2',
region_name=self.region,
aws_access_key_id=self.key_id,
aws_secret_access_key=self.access_key)
self.ec2_resource = boto3.resource(
'ec2',
region_name=self.region,
aws_access_key_id=self.key_id,
aws_secret_access_key=self.access_key)
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
self.p = Provider(name="azure")
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
def setup(self):
print()
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
variables = Variables()
cloud = variables['cloud']
self.p = Provider(name=cloud)
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
print("\n")
def __init__(self, host=None, username=None, password=None, port=None):
"""
create a cloudmesh database in the specified mongodb
:param host: the host
:param username: the username
:param password: the password
:param port: the port
"""
self.__dict__ = self.__shared_state
if "cnfig" not in self.__dict__:
self.config = Config().data["cloudmesh"]
self.mongo = self.config["data"]["mongo"]
self.mongo["MONGO_PASSWORD"] = str(self.mongo["MONGO_PASSWORD"])
self.database = self.mongo["MONGO_DBNAME"]
self.host = host or self.mongo["MONGO_HOST"]
self.password = urllib.parse.quote_plus(
str(password or self.mongo["MONGO_PASSWORD"]))
self.username = urllib.parse.quote_plus(
str(username or self.mongo["MONGO_USERNAME"]))
if port is None:
self.port = int(self.mongo["MONGO_PORT"])
else:
self.port = int(port)
self.client = None
self.db = None
self.connect()
def __init__(self, cloud=None, config="~/.cloudmesh/cloudmesh4.yaml"):
try:
self.config = Config()
self.credentials = config['cloudmesh']['storage'][cloud][
'credentials']
self.kind = config['cloudmesh']['storage']['kind']
except:
raise ValueError(f"storage service {cloud} not specified")
def __init__(self, debug):
"""
Initializes the SlurmCluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path,
"batch_workspace/slurm_batch.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.batch_config = GenericConfig(self.workspace)
self.debug = debug
self.all_jobIDs = []
self.slurm_cluster = {}
self.job_metadata = {}
def __init__(self):
"""
Initialization of the MOngo installer
"""
self.config = Config()
self.data = self.config.data["cloudmesh"]["data"]["mongo"]
self.expanduser()
def __init__(self, cloud=None):
config = Config().data
self.cloud = cloud
self.driver = None
self.key = None
if cloud:
data = config.get('cloudmesh')
self.os_config = data.get('cloud').get(cloud)
self.os_config = config.get('cloudmesh').get('cloud').get(cloud)
self.driver = self.get_driver(cloud)
self.key = self.os_config.get('credentials').get(
'OS_KEY_PATH') # credentials.target return null string
# if we don't find OS_KEY_PATH in yaml, go to os.environ instead which can be set in .bashrc
if self.key is None:
self.key = os.environ['OS_KEY_PATH']
else:
self.os_config = config
def __init__(self,
name=None,
configuration="~/.cloudmesh/cloudmesh4.yaml"):
"""
Initializes the provider. The default parameters are read from the configutation
file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration file
"""
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
super().__init__(name, conf)
VERBOSE.print(cred, verbose=8)
if self.cloudtype in Provider.ProviderMapper:
self.driver = get_driver(Provider.ProviderMapper[self.cloudtype])
if self.cloudtype == 'openstack':
if cred["OS_PASSWORD"] == 'TBD':
Console.error("The password TBD is not allowed")
self.cloudman = self.driver(
cred["OS_USERNAME"],
cred["OS_PASSWORD"],
ex_force_auth_url=cred['OS_AUTH_URL'],
ex_force_auth_version='2.0_password',
ex_tenant_name=cred['OS_TENANT_NAME'])
elif self.cloudtype == 'azure_asm':
self.cloudman = self.driver(
subscription_id=cred['AZURE_SUBSCRIPTION_ID'],
key_file=path_expand(cred['AZURE_KEY_FILE']))
elif self.cloudtype == 'aws':
self.cloudman = self.driver(cred["EC2_ACCESS_ID"],
cred["EC2_SECRET_KEY"],
region=cred["EC2_REGION"])
if self.cloudtype == 'google':
self.cloudman = self.driver(
cred["client_email"],
cred[
"path_to_json_file"], # should be placed in .cloudmesh
project=cred["project"])
else:
print("Specified provider not available")
self.cloudman = False
self.default_image = None
self.default_size = None
self.public_key_path = conf["profile"]["publickey"]
def __init__(self, debug):
"""
Initializes the virtualcluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path,
"vcluster_workspace/vcluster.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.vcluster_config = GenericConfig(self.workspace)
self.debug = debug
self.all_pids = []
self.virt_cluster = {}
self.runtime_config = {}
self.job_metadata = {}
def __init__(self):
"""
Initializes the Queue class
"""
self.cm_config = Config()
self.queue = {
'uid': None,
'queue_name': None,
'cluster_name': None,
'policy': None,
'status': None,
'active': None,
'charge': None,
'unit': None,
}
self.database = CmDatabase()