Ejemplo n.º 1
0
    def test_search(self):
        config = Config()

        Benchmark.Start()
        r = config.search("cloudmesh.cloud.*.cm.active", True)
        Benchmark.Stop()
        pprint(r)
Ejemplo n.º 2
0
    def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
        """
        TODO: missing, also define parameters

        :param service:
        :param config:
        """
        super().__init__(service=service, config=config)
        self.config = Config()
        self.storage_credentials = self.config.credentials("storage", "gdrive")
        if self.storage_credentials['maxfiles'] > 1000:
            Console.error("Page size must be smaller than 1000")
            sys.exit(1)
        self.limitFiles = self.storage_credentials['maxfiles']
        self.scopes = self.storage_credentials['scopes']
        self.clientSecretFile = path_expand(
            self.storage_credentials['location_secret'])
        self.applicationName = self.storage_credentials['application_name']
        self.generate_key_json()
        self.flags = self.generate_flags_json()
        self.credentials = self.get_credentials()
        self.http = self.credentials.authorize(httplib2.Http())
        self.driveService = discovery.build('drive', 'v3', http=self.http)
        self.cloud = service
        self.service = service

        self.fields = "nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
Ejemplo n.º 3
0
    def json_to_yaml(cls, name, filename="~/.cloudmesh/security/google.json"):
        """
        Given a json file downloaded from google, copies the content into the
        cloudmesh yaml file, while overwriting or creating a new compute provider
        :param cls:
        :param name:
        :param filename: Service Account Key file downloaded from google cloud.
        :return: None
        """
        path = path_expand(filename)

        # Open and load the JSON file.
        with open(path, "r") as file:
            d = json.load(file)

        # Get the project id and client email.
        project_id = d["project_id"]
        client_email = d["client_email"]

        # Format the sample with json file details.
        format_sample = cls.sample.format_map(locals())
        # Convert the yaml sample to JSON.
        google_yaml = yaml.load(format_sample, Loader=yaml.SafeLoader)
        # Extract the google compute section
        google_config = google_yaml["cloudmesh"]["cloud"]

        # Update the google cloud section of cloudmesh.yaml config file.
        config = Config()
        config["cloudmesh"]["cloud"][name] = google_config
        config.save()
        banner("Result")
        pprint(config["cloudmesh"]["cloud"][name])
Ejemplo n.º 4
0
    def config(self, config_path='~/.cloudmesh/cloudmesh.yaml'):
        """
        Use `cloudmesh.yaml` file to configure.
        """
        self._conf = Config(config_path).get("data")

        # Set DB provider. There should only be one.
        db_provider = self._conf.get('default.db')

        if db_provider == 'local':
            db_path = self._conf.get('db.local.CMDATA_DB_FOLDER')
            self._db = LocalDBProvider(db_path)

        # Check for local storage provider.
        storage_path = self._conf.get('service.local.CMDATA_STORAGE_FOLDER')
        if storage_path:
            self._providers['local'] = LocalStorageProvider(storage_path)

        # Check for Azure provider.
        az_conf = self._conf.get('service.azure')
        if az_conf:
            az_act = az_conf.get('credentials.AZURE_STORAGE_ACCOUNT')
            az_key = az_conf.get('credentials.AZURE_STORAGE_KEY')
            az_container = az_conf.get('container')
            if az_act and az_key:
                self._providers['azure'] = AzureStorageProvider(
                    az_act, az_key, az_container)

        # Set a default storage provider.
        default_storage_provider = self._conf.get('default.service')
        self._providers['default'] = self._providers[default_storage_provider]
Ejemplo n.º 5
0
    def init_cloud_params(self, cloudname):

        from cloudmesh.configuration.Config import Config
        self.conf = Config()["cloudmesh"]
        self.user = Config()["cloudmesh"]["profile"]["user"]
        self.spec = self.conf["cloud"][cloudname]
        self.cloudname = cloudname
        print(self.spec)
        self.default = self.spec["default"]
        self.cloudtype = self.spec["cm"]["kind"]

        if self.cloudtype == 'aws':
            self.bucket_name = self.spec["cm"]["bucket_name"]
            self.region_name = self.spec["cm"]["region_name"]
            self.forecast = self.spec["cm"]["forecast_srv"]
            self.forecastquery_srv = self.spec["cm"]["forecastquery_srv"]
            self.s3_srv = self.spec["cm"]["s3_srv"]
            self.role_arn = self.spec["cm"]["iam_role_arn"]
            self.algorithmArn = self.spec["cm"]["algorithmArn"]

            import boto3
            self.session = boto3.Session(region_name=self.region_name)
            self.forecast_srv = self.session.client(service_name=self.forecast)
            self.forecastquery = self.session.client(
                service_name=self.forecastquery_srv)

            print("AWS Cloud was requested")
        elif self.cloudname == 'azure':
            print("Azure cloud was requested")
        else:
            print("Supported cloud services at this time : ")
Ejemplo n.º 6
0
    def test_search(self):
        config = Config()

        StopWatch.start("search")
        r = config.search("cloudmesh.cloud.*.cm.active", True)
        StopWatch.stop("search")
        pprint(r)
Ejemplo n.º 7
0
    def test_dictreplace(self):
        HEADING()
        config = Config()
        spec = textwrap.dedent("""
        cloudmesh:
          profile:
            name: Gregor
          unordered:
            name: "{cloudmesh.other.name}.postfix"
          other:
            name: "{cloudmesh.profile.name}"
        
        """)

        print(spec)

        # spec = spec.replace("{", "{{")
        # spec = spec.replace("}", "}}")

        # print(spec)
        StopWatch.start("dictreplace")
        result = config.spec_replace(spec)
        StopWatch.stop("dictreplace")
        print(result)
        data = yaml.load(result, Loader=yaml.SafeLoader)
        pprint(data)

        assert data["cloudmesh"]["unordered"]["name"] == "Gregor.postfix"
        assert data["cloudmesh"]["other"]["name"] == "Gregor"
Ejemplo n.º 8
0
class Driver(object):
    def __init__(self, name="~/.cloudmesh/cloudmesh.yaml"):
        name = os.path.expanduser(name)
        self.config = Config(name=name)

    # noinspection PyPep8Naming
    def get(self, name=None):
        connection = None

        if name is None:
            variables = Variables()
            # noinspection PyUnusedLocal
            cloudname = variables['cloud']

        kind = self.config.get(
            "cloudmesh.cloud.{name}.cm.kind".format(name=name))
        credentials = self.config.get(
            "cloudmesh.cloud.{name}.credentials".format(name=name))

        # BUG FROM HERE ON WRONG

        if kind == 'azure':
            AZDriver = get_driver(Provider.AZURE)
            connection = AZDriver(
                subscription_id=credentials['AZURE_SUBSCRIPTION_ID'],
                key_file=credentials['AZURE_MANAGEMENT_CERT_PATH'])
        elif kind == 'aws':
            EC2Driver = get_driver(Provider.EC2)
            connection = EC2Driver(credentials['EC2_ACCESS_ID'],
                                   credentials['EC2_SECRET_KEY'])

        return connection
Ejemplo n.º 9
0
    def __init__(self):
        """
        Initializes the Queue class

        """

        self.cm_config = Config()
        self.info = {
            'uid': None,
            "cloud": None,
            "kind": "batch-queue",
            "name": None,
            "cm": {},
            "queue": {
                'policy': None,
                'status': None,
                'active': False,
                'charge': None,
                'unit': None,
                "numJobs": 0,
                "numRunningJobs": 0,
                'joblist': []
            }
        }
        self.database = CmDatabase()
Ejemplo n.º 10
0
    def __init__(self):
        """
        Initializes the Queue class

        """

        self.cm_config = Config()
        self.info = munch.munchify({
            'uid': None,
            "cloud": None,
            "kind": "batch-queue",
            "name": None,
            "cm": {},
            "queue": {
                'policy': None,
                'status': None,
                'active': False,
                'charge': None,
                'unit': None,
                "numJobs": 0,
                "numRunningJobs": 0,
                'joblist': []
            }
        })
        # list of parameters that can be set
        self.settable_params = ['policy', 'charge', 'unit']
        self.database = CmDatabase()
Ejemplo n.º 11
0
    def setup(self):
        banner("setup", c="-")
        self.user = Config()["cloudmesh.profile.user"]
        self.key_path = path_expand(
            Config()["cloudmesh"]["profile"]["publickey"])
        f = open(self.key_path, 'r')
        self.key_val = f.read()

        self.clouduser = '******'
        self.name_generator = Name(schema=f"{self.user}-vm", counter=1)

        self.name = str(self.name_generator)
        self.name_generator.incr()

        self.new_name = str(self.name_generator)

        self.p = GCloudProvider(name="google")

        self.secgroupname = "CM4TestSecGroup"
        self.secgrouprule = {
            "ip_protocol": "tcp",
            "from_port": 8080,
            "to_port": 8088,
            "ip_range": "129.79.0.0/16"
        }
        self.testnode = None
Ejemplo n.º 12
0
    def edit(key, caps=True, show=False):

        global gui_enabled

        if not gui_enabled:
            Console.Warning(
                "Cloudmesh Gui not supported, can not find tkinter")
            return ""

        config = Config()

        entry = dict(FlatDict(config[key], sep='.'))

        layout = [[gui.Text(f'Cloudmesh Configuration Editor: {key}')]]

        length = 1
        for _key, _value in entry.items():
            length = max(length, len(_key))

        length = length + 3

        for _key, _value in entry.items():
            if caps:
                label = _key.capitalize()
            else:
                label = _key

            secrets = Config.secrets()

            if _key in secrets and not show:
                field = [
                    gui.Text(label, size=(length, 1)),
                    gui.InputText(key=f"{key}.{_key}",
                                  password_char="*",
                                  default_text=_value)
                ]
            else:
                field = [
                    gui.Text(label, size=(length, 1)),
                    gui.InputText(key=f"{key}.{_key}", default_text=_value)
                ]
            layout.append(field)

        layout.append([gui.Submit(), gui.Cancel()])

        window = gui.Window('Cloudmesh Configuration Editor',
                            layout,
                            background_color="white")
        event, values = window.Read()
        window.Close()

        if event == "Submit":
            for _key, _value in values.items():
                config[_key] = _value
                if show:
                    Console.ok(f"{_key}={_value}")
            config.save()
        else:
            print(event)
Ejemplo n.º 13
0
    def __init__(self, name="azure", configuration=None, credentials=None):
        """
        Initializes the provider. The default parameters are read from the
        configuration file that is defined in yaml format.

        :param name: The name of the provider as defined in the yaml file
        :param configuration: The location of the yaml configuration file
        """
        # configuration = configuration if configuration is not None \
        # else CLOUDMESH_YAML_PATH

        conf = Config(configuration)["cloudmesh"]

        self.user = Config()["cloudmesh"]["profile"]["user"]

        self.spec = conf["volume"][name]
        self.cloud = name
        self.location = 'eastus'
        self.size = 1
        self.group_name = self.spec["default"]['group']

        cred = self.spec["credentials"]
        self.default = self.spec["default"]
        self.cloudtype = self.spec["cm"]["kind"]
        super().__init__(name, configuration)

        # update credentials with the passed dict
        if credentials is not None:
            cred.update(credentials)

        # VERBOSE(cred, verbose=10)

        if self.cloudtype != 'azure':
            Console.error("This class is meant for azure cloud")

        # ServicePrincipalCredentials related Variables to configure in
        # cloudmesh.yaml file

        # AZURE_APPLICATION_ID = '<Application ID from Azure Active Directory
        # App Registration Process>'

        # AZURE_SECRET_KEY = '<Secret Key from Application configured in
        # Azure>'

        # AZURE_TENANT_ID = '<Directory ID from Azure Active Directory
        # section>'

        credentials = ServicePrincipalCredentials(
            client_id=cred['AZURE_APPLICATION_ID'],
            # application and client id are same thing
            secret=cred['AZURE_SECRET_KEY'],
            tenant=cred['AZURE_TENANT_ID']
        )

        subscription = cred['AZURE_SUBSCRIPTION_ID']

        # Management Clients
        self.compute_client = ComputeManagementClient(
            credentials, subscription)
Ejemplo n.º 14
0
    def test_search_openstack(self):
        HEADING()
        config = Config()

        Benchmark.Start()
        r = config.search("cloudmesh.cloud.*.cm.kind", "openstack")
        Benchmark.Stop()
        pprint(r)
Ejemplo n.º 15
0
    def test_search_active_all(self):
        HEADING()
        config = Config()

        Benchmark.Start()
        r = config.search("cloudmesh.cloud.*.cm.active")
        Benchmark.Stop()
        pprint(r)
Ejemplo n.º 16
0
    def test_doesnotexist_get(self):
        config = Config()

        key = "cloudmesh.doesnotexist"
        StopWatch.start(f"not exists get({key})")
        value = config.get(key, default="Hallo")
        StopWatch.stop(f"not exists get({key})")

        assert value == "Hallo"
Ejemplo n.º 17
0
 def test_dict(self):
     HEADING()
     config = Config()
     Benchmark.Start()
     result = config.dict()
     Benchmark.Stop()
     print(type(config.data))
     pprint(config['cloudmesh.profile'])
     assert config is not None
Ejemplo n.º 18
0
    def activate():

        global gui_enabled

        if not gui_enabled:
            Console.Warning(
                "Cloudmesh Gui not supported, can not find tkinter")
            return ""

        config = Config()
        clouds = list(config["cloudmesh.cloud"].keys())

        gui.SetOptions(text_justification='right')

        layout = [[
            gui.Text('Cloudmesh Cloud Activation', font=('Helvetica', 16))
        ], [gui.Text('Compute Services')]]

        layout.append([gui.Text('_' * 100, size=(65, 1))])

        for cloud in clouds:
            tbd = "TBD" in str(config[f"cloudmesh.cloud.{cloud}.credentials"])
            active = config[f"cloudmesh.cloud.{cloud}.cm.active"]
            if tbd:
                color = 'red'
            else:
                color = "green"

            choice = [
                gui.Checkbox(cloud,
                             key=cloud,
                             text_color=color,
                             default=active)
            ]
            layout.append(choice)

        layout.append([gui.Text('_' * 100, size=(65, 1))])

        layout.append([gui.Submit(), gui.Cancel()])

        window = gui.Window('Cloudmesh Configuration',
                            layout,
                            font=("Helvetica", 12))

        event, values = window.Read()

        if event == "Submit":
            for cloud in values:

                active = values[cloud] or False
                config[f"cloudmesh.cloud.{cloud}.cm.active"] = str(active)
                if active:
                    Console.ok(f"Cloud {cloud} is active")

            config.save()
        else:
            print(event)
Ejemplo n.º 19
0
 def __init__(self,
              name=None,
              configuration="~/.cloudmesh/.cloudmesh.yaml"):
     VERBOSE(f"Init Docker {name}")
     self.config = Config()
     conf = Config(configuration)["cloudmesh"]
     self.user = conf["profile"]
     self.spec = conf["cloud"][name]
     self.cloud = name
     cred = self.spec["credentials"]
     self.cloudtype = self.spec["cm"]["kind"]
Ejemplo n.º 20
0
    def __init__(self, name):
        """
        Initialize provider. The default parameters are read from the
        configuration file that is defined in yaml format.

        :param name: name of cloud
        """
        self.cloud = name
        self.config = Config()["cloudmesh.volume.openstack.credentials"]
        self.defaults = Config()["cloudmesh.volume.openstack.default"]
        self.cm = CmDatabase()
Ejemplo n.º 21
0
    def test_dict(self):
        HEADING()
        config = Config()
        Benchmark.Start()
        result = config.dict()
        Benchmark.Stop()
        pprint(result)
        print(config)
        print(type(config.data))

        assert config is not None
Ejemplo n.º 22
0
    def test_dict(self):
        HEADING()
        config = Config()
        StopWatch.start("dict")
        result = config.dict()
        StopWatch.stop("dict")
        pprint(result)
        print(config)
        print(type(config.data))

        assert config is not None
Ejemplo n.º 23
0
class AzRegister(object):
    def __init__(self, cloud='azure'):

        self.config = Config()
        self.credentials = self.config[f'cloudmesh.cloud.{cloud}.credentials']

    def set_credentials(self, creds):
        self.credentials['AZURE_TENANT_ID'] = creds['Access key ID']
        self.credentials['AZURE_SUBSCRIPTION_ID'] = creds['Secret access key']
        self.credentials['AZURE_APPLICATION_ID'] = creds['Secret access key']
        self.credentials['AZURE_SECRET_KEY'] = creds['Secret access key']
        self.config.save()

    def azString2Dict(self, azString):
        azDict = dict()
        for i in azString.strip().splitlines():
            x = i.split(":")
            if len(x) == 2:
                azDict[x[0].strip(' ,"')] = x[1].strip(' ,"')
        return azDict

    def register(self, cloud='azure'):
        # Opens web browser and prompts user to login
        subprocess.Popen('az login')

        # once user has logged in, collects account information, such as subscription id
        accountInfo = subprocess.getoutput('az account show')
        print(accountInfo)

        azoutput = self.azString2Dict(accountInfo)

        AZURE_SUBSCRIPTION_ID = azoutput['id']
        AZURE_TENANT_ID = azoutput['tenantId']

        # WARNING: FOLLOWING CODE WILL RENDER OLD SECRET KEY INVALID
        azAppKeyStr = subprocess.getoutput(
            'az ad sp create-for-rbac --name http://cloudmesh')
        azAppKeyDict = self.azString2Dict(azAppKeyStr)

        AZURE_APPLICATION_ID = azAppKeyDict['appId']
        AZURE_SECRET_KEY = azAppKeyDict['password']

        creds = {
            'AZURE_SUBSCRIPTION_ID': AZURE_SUBSCRIPTION_ID,
            'AZURE_TENANT_ID': AZURE_TENANT_ID,
            'AZURE_APPLICATION_ID': AZURE_APPLICATION_ID,
            'AZURE_SECRET_KEY': AZURE_SECRET_KEY
        }

        self.set_credentials(creds)

        Console.info(
            "Azure Tenant, Subscription, Application, and Secret Key have been added to the cloudmesh.yaml file."
        )
Ejemplo n.º 24
0
    def __init__(self,
                 name=None,
                 configuration="~/.cloudmesh/cloudmesh.yaml"):
        # noinspection PyPep8
        try:
            super().__init__(name, configuration)
            self.kind = Config(configuration)[f"cloudmesh.cloud.{name}.cm.kind"]
            self.credentials = Config(configuration)[
                f"cloudmesh.cloud.{name}.credentials"]
            self.name = name
        except:
            Console.error(f"provider {name} not found in {configuration}")
            raise ValueError(f"provider {name} not found in {configuration}")

        provider = None

        providers = ProviderList()

        if self.kind in ['openstack',
                         'azure',
                         'docker',
                         "aws",
                         "azureaz",
                         "virtualbox"]:

            provider = providers[self.kind]

        elif self.kind in ["awslibcloud", "google"]:

            from cloudmesh.compute.libcloud.Provider import \
                Provider as LibCloudProvider
            provider = LibCloudProvider

        elif self.kind in ['oracle']:
            from cloudmesh.oracle.compute.Provider import \
                Provider as OracleComputeProvider
            provider = OracleComputeProvider

        # elif self.kind in ["vagrant", "virtualbox"]:
        #    from cloudmesh.compute.virtualbox.Provider import \
        #        Provider as VirtualboxCloudProvider
        #    provider = VirtualboxCloudProvider
        # elif self.kind in ["azureaz"]:
        #    from cloudmesh.compute.azure.AzProvider import \
        #        Provider as AzAzureProvider
        #    provider = AzAzureProvider

        if provider is None:
            Console.error(f"provider {name} not supported")
            raise ValueError(f"provider {name} not supported")

        self.p = provider(name=name, configuration=configuration)
Ejemplo n.º 25
0
    def __init__(self, name=None):
        """
        Initializes the provider. The default parameters are read from the
        configuration file that is defined in yaml format.

        :param name: The name of the provider as defined in the yaml file
        :param configuration: The location of the yaml configuration file
        """

        self.config = Config()
        conf = self.config["cloudmesh"]
        super().__init__(name)

        self.user = self.config["cloudmesh.profile.user"]
        self.spec = conf["cloud"][name]
        self.cloud = name

        self.default = self.spec["default"]
        self.cloudtype = self.spec["cm"]["kind"]

        self.cred = self.config[f"cloudmesh.cloud.{name}.credentials"]

        fields = ["user",
                  "fingerprint",
                  "key_file",
                  "pass_phrase",
                  "tenancy",
                  "compartment_id",
                  "region"]

        for field in fields:
            if self.cred[field] == 'TBD':
                Console.error(
                    f"The credential for Oracle cloud is incomplete. {field} "
                    "must not be TBD")
        self.credential = self._get_credentials(self.cred)

        self.compute = oci.core.ComputeClient(self.credential)
        self.virtual_network = oci.core.VirtualNetworkClient(self.credential)
        self.identity_client = oci.identity.IdentityClient(self.credential)
        self.compartment_id = self.credential["compartment_id"]

        try:
            self.public_key_path = conf["profile"]["publickey"]
            self.key_path = path_expand(
                Config()["cloudmesh"]["profile"]["publickey"])
            f = open(self.key_path, 'r')
            self.key_val = f.read()
        except:
            raise ValueError("the public key location is not set in the "
                             "profile of the yaml file.")
    def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
        super().__init__(service=service, config=config)
        self.config = Config()

        self.storage_credentials = self.config.credentials("storage", "gdrive")

        google_json = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
            "service_account"]
        self.google_client = storage.Client.from_service_account_json(
            google_json)

        self.local_dir = self.config["cloudmesh"]["storage"]["local"]["dir"]
        self.bucket = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
            "bucket"]
Ejemplo n.º 27
0
    def __init__(self, name=None,
                 configuration="~/.cloudmesh/.cloudmesh.yaml"):
        self.config = Config()
        conf = Config(configuration)["cloudmesh"]
        self.user = conf["profile"]
        self.spec = conf["cloud"][name]
        self.cloud = name
        cred = self.spec["credentials"]
        self.cloudtype = self.spec["cm"]["kind"]

        if platform.system().lower() == "darwin":
            self.vboxmanage = \
                "/Applications/VirtualBox.app/Contents/MacOS/VBoxManage"
        else:
            self.vboxmanage = "VBoxManage"
Ejemplo n.º 28
0
    def check_yaml(self):
        config = Config()
        errors = False
        if "chameleon" in config["cloudmesh.cloud"]:
            credentials = config["cloudmesh.cloud.chameleon.credentials"]
            if "OS_PASSWORD" in credentials:
                Console.error("You are using an old version of the cloudmesh.yaml file")
            else:
                Console.ok("your yaml file seems ok")
        location = config.location
        print(f"Location of cloudmesh.yaml: {location}")
        print("we run a simple yamllint test. Not all issues reported need to be corrected.")

        yamlfile = Path(f"{location}/cloudmesh.yaml")
        result = Shell.run(f"yamllint {yamlfile}")
        for line in result.splitlines():
            line = line.strip()
            if "missing document start" in line:
                continue
            if "line too long" in line:
                continue
            if "error" in line:
                errors = True
                Console.error (line)
            else:
                print(line)
        if errors:
            Console.error("We found issues in your yaml file")
        else:
            Console.ok(f"OK. Your yaml file {yamlfile} seems valid")
Ejemplo n.º 29
0
 def __init__(self, cloud, path):
     config = Config(config_path=path)["cloudmesh"]
     self.cm = config["cloud"][cloud]["cm"]
     self.default = config["cloud"][cloud]["default"]
     self.credentials = config["cloud"][cloud]["credentials"]
     self.group = config["default"]["group"]
     self.experiment = config["default"]["experiment"]
Ejemplo n.º 30
0
 def test_config_subscriptable(self):
     HEADING()
     config = Config()
     StopWatch.start("config_subscriptable")
     data = config["cloudmesh"]["data"]["mongo"]
     StopWatch.stop("config_subscriptable")
     assert data is not None