Esempio n. 1
0
def launch_node(node_id, creds, params, init, ssh_username, ssh_private_key):
    # "pip install azure"
    sms = ServiceManagementService(
        subscription_id='69581868-8a08-4d98-a5b0-1d111c616fc3',
        cert_file='/Users/dgriffin/certs/iOSWAToolkit.pem')
    for i in sms.list_os_images():
        print 'I is ', i.name, ' -- ', i.label, ' -- ', i.location, ' -- ', i.media_link
    media_link = \
        'http://opdemandstorage.blob.core.windows.net/communityimages/' + \
        'b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu_DAILY_BUILD-' + \
        'precise-12_04_2-LTS-amd64-server-20130702-en-us-30GB.vhd'
    config = LinuxConfigurationSet(user_name="ubuntu",
                                   user_password="******")
    hard_disk = OSVirtualHardDisk(
        'b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu_DAILY_BUILD-' +
        'precise-12_04_2-LTS-amd64-server-20130702-en-us-30GB',
        media_link,
        disk_label='opdemandservice')
    ret = sms.create_virtual_machine_deployment('opdemandservice', 'deploy1',
                                                'production',
                                                'opdemandservice2',
                                                'opdemandservice3', config,
                                                hard_disk)
    # service_name, deployment_name, deployment_slot, label, role_name
    # system_config, os_virtual_hard_disk
    print 'Ret ', ret
    return sms
Esempio n. 2
0
    def _ensureBuildMachineExists(self):
        """
        Creates the VM for the build server.
        """
        service_name = self.config.getBuildServiceName()
        service_storage_name = self.config.getStorageAccountName()
        cert_thumbprint = self.config.getServiceCertificateThumbprint()
        vm_username = self.config.getVirtualMachineLogonUsername()
        vm_password = self.config.getVirtualMachineLogonPassword()
        vm_hostname = service_name

        role_instances = self._getRoleInstances(service_name)
        if vm_hostname in role_instances:
            logger.warn("Role instance %s already exists: skipping creation.", vm_hostname)
        else:
            logger.info("Role instance %s provisioning begins.", vm_hostname)
            self._assertOsImageExists(self.config.getBuildOSImageName())

            vm_diskname = '{0}.vhd'.format(vm_hostname)
            vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(service_storage_name, vm_diskname)
            os_hd = OSVirtualHardDisk(self.config.getBuildOSImageName(),
                                      vm_disk_media_link,
                                      disk_name=vm_diskname,
                                      disk_label=vm_diskname)
            linux_config = LinuxConfigurationSet(vm_hostname, vm_username, vm_password, True)
            linux_config.ssh.public_keys.public_keys.append(
                PublicKey(cert_thumbprint, u'/home/{0}/.ssh/authorized_keys'.format(vm_username))
            )
            linux_config.ssh.key_pairs.key_pairs.append(
                KeyPair(cert_thumbprint, u'/home/{0}/.ssh/id_rsa'.format(vm_username))
            )
            network_config = ConfigurationSet()
            network_config.configuration_set_type = 'NetworkConfiguration'
            ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
                                                         protocol='TCP',
                                                         port=u'22',
                                                         local_port=u'22')
            network_config.input_endpoints.input_endpoints.append(ssh_endpoint)

            result = self.sms.create_virtual_machine_deployment(service_name=service_name,
                                                                deployment_name=service_name,
                                                                deployment_slot='Production',
                                                                label=vm_hostname,
                                                                role_name=vm_hostname,
                                                                system_config=linux_config,
                                                                os_virtual_hard_disk=os_hd,
                                                                network_config=network_config,
                                                                availability_set_name=None,
                                                                data_virtual_hard_disks=None,
                                                                role_size=self.config.getBuildInstanceRoleSize())
            self._wait_for_operation_success(result.request_id, timeout=self.config.getAzureOperationTimeout())
            self._wait_for_role_instance_status(vm_hostname, service_name, 'ReadyRole',
                                                self.config.getAzureOperationTimeout())
            logger.info("Role instance %s has been created.", vm_hostname)
    def test_create_instance_initial_deployment(self, mock_os_disk):
        self.service.get_deployment_by_name.side_effect = Exception(
            '<Code>ResourceNotFound</Code><Message>No deployments were found')
        request_result = mock.Mock()
        storage_properties = mock.MagicMock()
        storage_properties.storage_service_properties.location = 'region'
        self.service.get_storage_account_properties.return_value = \
            storage_properties

        service_properties = mock.MagicMock()
        service_properties.hosted_service_properties.location = 'region'
        self.service.get_hosted_service_properties.return_value = \
            service_properties

        image_locations = mock.MagicMock()
        image_locations.location = 'region'
        self.service.get_os_image.return_value = image_locations

        os_disk = OSVirtualHardDisk('foo', 'foo')
        mock_os_disk.return_value = os_disk
        endpoint = self.vm.create_network_endpoint('SSH', 22, 22, 'TCP')
        network_config = self.vm.create_network_configuration([endpoint])
        result = self.vm.create_instance(
            cloud_service_name='cloud-service',
            disk_name='foo.vhd',
            system_config=self.system_config,
            network_config=network_config,
            label='some-label',
            reserved_ip_name='test_reserved_ip_name')
        mock_os_disk.assert_called_once_with(
            'foo.vhd',
            'https://bob.blob.test.url/foo/cloud-service_instance_some-host_image_foo.vhd'
        )
        self.service.create_virtual_machine_deployment.assert_called_once_with(
            deployment_slot='production',
            role_size='Small',
            deployment_name='cloud-service',
            service_name='cloud-service',
            os_virtual_hard_disk=os_disk,
            label='some-label',
            system_config=self.system_config,
            reserved_ip_name='test_reserved_ip_name',
            role_name=self.system_config.host_name,
            network_config=network_config,
            provision_guest_agent=True)
        assert result['instance_name'] == 'some-host'
    def get_os_virtual_hard_disk(self):
        if self.is_vm_image():
            return None

        i = self.virtual_environment[AZURE_UNIT.IMAGE]
        sa = self.virtual_environment[AZURE_UNIT.STORAGE_ACCOUNT]
        c = self.virtual_environment[AZURE_UNIT.CONTAINER]
        now = self.util.get_now()
        blob = self.BLOB_BASE % (i[AZURE_UNIT.IMAGE_NAME], str(
            now.year), str(now.month), str(now.day), str(
                now.hour), str(now.minute), str(
                    now.second), str(current_thread().ident))
        media_link = self.MEDIA_BASE % (
            sa[AZURE_UNIT.STORAGE_ACCOUNT_SERVICE_NAME],
            sa[AZURE_UNIT.STORAGE_ACCOUNT_URL_BASE], c, blob)
        os_virtual_hard_disk = OSVirtualHardDisk(i[AZURE_UNIT.IMAGE_NAME],
                                                 media_link)
        return os_virtual_hard_disk
Esempio n. 5
0
 def get_os_virtual_hard_disk(self):
     """
     Return None if image type is vm
     Media link should be unique
     :return:
     """
     if self.is_vm_image():
         return None
     i = self.virtual_environment[self.T_I]
     sa = self.virtual_environment[self.T_SA]
     c = self.virtual_environment[self.T_C]
     now = self.util.get_now()
     blob = self.BLOB_BASE % (i[self.T_I_N], str(now.year), str(
         now.month), str(now.day), str(now.hour), str(
             now.minute), str(now.second), str(current_thread().ident))
     media_link = self.MEDIA_BASE % (sa[self.T_SA_SN], sa[self.T_SA_UB], c,
                                     blob)
     os_virtual_hard_disk = OSVirtualHardDisk(i[self.T_I_N], media_link)
     return os_virtual_hard_disk
Esempio n. 6
0
    def create_machine(self, name, region='West US',
                       image=None, role_size='Small',
                       min_count=1, max_count=1,
                       media='storage_url_blob_cloudrunner',
                       username='', password='', ssh_pub_key='',
                       server=CR_SERVER,
                       cleanup=None, **kwargs):
        self.log.info("Registering Azure machine [%s::%s] for [%s]" %
                      (name, image, CR_SERVER))
        try:
            sms = ServiceManagementService(self.profile.username,
                                           self._cert_path)
            server_config = LinuxConfigurationSet('myhostname', 'myuser',
                                                  'mypassword', True)
            media_link = "%s__%s" % (media, name)
            os_hd = OSVirtualHardDisk(image, media_link)

            res = sms.create_virtual_machine_deployment(
                service_name=name,
                deployment_name=name,
                deployment_slot='production',
                label=name,
                role_name=name,
                system_config=server_config,
                os_virtual_hard_disk=os_hd,
                role_size='Small')

            instance_ids = []
            meta = {}
            if not res:
                return self.FAIL, [], {}
            meta['deployment_name'] = name
            meta['cleanup_service'] = cleanup in ['1', 'True', 'true']
            return self.OK, instance_ids, meta
        except Exception, ex:
            self.log.exception(ex)
            return self.FAIL, [], {}
    def test_create_instance_add_role(self, mock_os_disk):
        storage_properties = mock.MagicMock()
        storage_properties.storage_service_properties.location = 'region'
        self.service.get_storage_account_properties.return_value = \
            storage_properties

        service_properties = mock.MagicMock()
        service_properties.hosted_service_properties.location = 'region'
        self.service.get_hosted_service_properties.return_value = \
            service_properties

        image_locations = mock.MagicMock()
        image_locations.location = 'region'
        self.service.get_os_image.return_value = image_locations

        os_disk = OSVirtualHardDisk('foo', 'foo')
        mock_os_disk.return_value = os_disk
        endpoint = self.vm.create_network_endpoint('SSH', 22, 22, 'TCP')
        network_config = self.vm.create_network_configuration([endpoint])
        result = self.vm.create_instance(cloud_service_name='cloud-service',
                                         disk_name='foo.vhd',
                                         system_config=self.system_config,
                                         network_config=network_config)
        mock_os_disk.assert_called_once_with(
            'foo.vhd',
            'https://bob.blob.test.url/foo/cloud-service_instance_some-host_image_foo.vhd'
        )
        self.service.add_role.assert_called_once_with(
            role_size='Small',
            deployment_name='cloud-service',
            service_name='cloud-service',
            os_virtual_hard_disk=os_disk,
            system_config=self.system_config,
            role_name=self.system_config.host_name,
            network_config=network_config,
            provision_guest_agent=True)
        assert result['instance_name'] == 'some-host'
    def create_virtual_machine(self, service_name, vm_name, image_name,
                               role_size):
        media_link = self._get_media_link(vm_name)
        # Linux VM configuration
        hostname = '-'.join((vm_name, 'host'))
        linux_config = LinuxConfigurationSet(hostname, self.linux_user,
                                             self.linux_pass, True)

        # Hard disk for the OS
        os_hd = OSVirtualHardDisk(image_name, media_link)

        # Create vm
        result = self.sms.create_virtual_machine_deployment(
            service_name=service_name,
            deployment_name=vm_name,
            deployment_slot='production',
            label=vm_name,
            role_name=vm_name,
            system_config=linux_config,
            os_virtual_hard_disk=os_hd,
            role_size=role_size)
        request_id = result.request_id

        return {'request_id': request_id, 'media_link': media_link}
Esempio n. 9
0
    def createInstances(self, sms, name, region, imageID, instanceSize, pkfile,
                        count, thumbprint, cert_data, num_instances,
                        certPasswd, storagename, master_size):
        hostname = name
        mainPath = os.path.dirname(os.path.abspath(__file__))
        username = "******"
        password = None
        affGrp = self.createAffinityGrp(sms, storagename, storagename,
                                        storagename, region)
        storage_name = self.StorageCreate(sms, storagename, storagename,
                                          storagename, region, affGrp)
        account_key = self.getStorageAccountKey(sms, storage_name)
        permission = None
        container_name = self.get_vm_name()
        account_name = storage_name
        container_name = self.create_containers(sms, storage_name, account_key,
                                                container_name.lower(), None)
        time.sleep(5)
        #print "Container Name:"+disks
        medialink = "http://" + storage_name + ".blob.core.windows.net/" + container_name + "/"
        # --------------------------------------------------------
        blobService = BlobService(account_name=storage_name,
                                  account_key=account_key)
        blobName = container_name + "blob.vhd"
        try:
            image = sms.get_os_image(imageID)
        except:
            if (self.copy_image_vmDepot(blobService, container_name, blobName,
                                        imageID, sms)):
                print "INFO -- The Disk Blob has been copied"
            media_link = "http://" + storage_name + ".blob.core.windows.net/" + container_name + "/" + blobName
            if (self.make_os_image(sms, media_link, imageID)):
                print "INFO -- The image '" + imageID + "' is ready now!!"

            else:
                print "Error: Can not complete creating The image"
                exit(0)

        #-----------------------------------------------------------
        medialink = "http://" + storage_name + ".blob.core.windows.net/" + container_name + "/"
        media_link = ""
        # Configuring EndPoint "Firwall Configuration":
        endpoint_config = ConfigurationSet()
        endpoint_config.configuration_set_type = 'NetworkConfiguration'
        endpoint1 = ConfigurationSetInputEndpoint(
            name='XML',
            protocol='tcp',
            port='5000',
            local_port='5000',
            load_balanced_endpoint_set_name=None,
            enable_direct_server_return=False)
        endpoint_config.input_endpoints.input_endpoints.append(endpoint1)
        self.logger.info(
            "Configuring EndPoints 'Firwall Configuration' SHH, PBS Torque and OpenMPI ports"
        )
        # Linux VM Configuration:
        linux_config = LinuxConfigurationSet(hostname, username, password,
                                             True)
        publickey = PublicKey(thumbprint, pkfile)
        linux_config.ssh.public_keys.public_keys.append(publickey)
        self.logger.info("Linux VM Configuration")
        # Configuring Image ID:
        #----------------------
        os_hd = OSVirtualHardDisk(imageID, media_link)
        self.logger.info(
            "Configuring The Virtual Hard Disk using Image ID: %s", imageID)

        # Start Deployment of VM on Azure:
        self.logger.info("Start Deployment of Elastic hpc on Azure")

        # Configuring Certificates:
        cert_format = 'pfx'
        cert_password = certPasswd
        VMname = hostname
        vmname = hostname
        instances = []

        try:
            for num in range(count):
                name = vmname + str(num)
                vname = vmname
                Service_name = vname
                Service_url = self.newCloudService(sms, Service_name,
                                                   Service_name, Service_name,
                                                   region, affGrp)
                endpoint3 = ConfigurationSetInputEndpoint(
                    name='SSH' + str(num),
                    protocol='tcp',
                    port='220' + str(num),
                    local_port='22',
                    load_balanced_endpoint_set_name=None,
                    enable_direct_server_return=False)
                endpoint_config.input_endpoints.input_endpoints.append(
                    endpoint3)
                #endpoint4 = ConfigurationSetInputEndpoint(name="FTP", protocol='tcp', port='21', local_port='21', load_balanced_endpoint_set_name=None, enable_direct_server_return=False)
                #endpoint_config.input_endpoints.input_endpoints.append(endpoint4)
                #endpoint5 = ConfigurationSetInputEndpoint(name="FTP1", protocol='tcp', port='20', local_port='20', load_balanced_endpoint_set_name=None, enable_direct_server_return=False)
                #endpoint_config.input_endpoints.input_endpoints.append(endpoint5)
                #endpoint6 = ConfigurationSetInputEndpoint(name="FTPudp", protocol='udp', port='21', local_port='21', load_balanced_endpoint_set_name=None, enable_direct_server_return=False)
                #endpoint_config.input_endpoints.input_endpoints.append(endpoint6)
                #for i in range(6):
                #	endpointpasv = ConfigurationSetInputEndpoint(name="FTPpasv"+str(i), protocol='tcp', port='4000'+str(i), local_port='4000'+str(i), load_balanced_endpoint_set_name=None, enable_direct_server_return=False)
                #	endpoint_config.input_endpoints.input_endpoints.append(endpointpasv)
                pbs_endpoints = self.get_pbs_endpoints(0)
                for endpoint in pbs_endpoints:
                    endpoint_config.input_endpoints.input_endpoints.append(
                        endpoint)
                media_link = medialink + name[:-1] + ".vhd"
                self.logger.info("Configuring Media Link %s", media_link)
                # Configuring Image ID:
                #----------------------
                os_hd = OSVirtualHardDisk(imageID, media_link)
                self.logger.info(
                    "Configuring The Virtual Hard Disk using Image ID: %s",
                    imageID)
                self.logger.info("Deploying Node number: %d", num)
                result_cert = sms.add_service_certificate(
                    service_name=Service_name,
                    data=cert_data,
                    certificate_format=cert_format,
                    password=cert_password)
                self.logger.info("Start Deploying VM with Name: " + vname)
                try:
                    self.logger.info(vars(result_cert))
                except:
                    self.logger.info(result_cert)
                time.sleep(5)
                result = sms.create_virtual_machine_deployment(
                    service_name=Service_name,
                    deployment_name=vname,
                    deployment_slot='production',
                    label=Service_name,
                    role_name=vname,
                    system_config=linux_config,
                    os_virtual_hard_disk=os_hd,
                    network_config=endpoint_config,
                    role_size=master_size)
                #role_size="Large")
                self.logger.info("Start Deployment")
                self.wait_for_async(sms, result.request_id)
                self.wait_for_deployment_status(sms, Service_name, vname,
                                                'Running')
                self.wait_for_role_instance_status(sms, Service_name, vname,
                                                   vname, 'ReadyRole')
                instances.append(Service_name + ".cloudapp.net")
                instances.append(container_name)
        except WindowsAzureConflictError:
            self.logger.info("Error: Can not Create VM")
            sys.exit(1)
        return instances
Esempio n. 10
0
def createInstances(sms, name, region, imageID, instanceSize, pkfile, count):
    hostname = name
    username = "******"
    password = None
    affGrp = affinity_group = createAffinityGrp(sms, name, name, name, region)
    storage_name = StorageCreate(sms, name, name, name, region, affGrp)
    media_link = "http://" + storage_name + ".blob.core.windows.net/disks/" + name + ".vhd"
    log.info("Configuring Media Link %s", media_link)
    Service_url = newCloudService(sms, name, name, name, region, affGrp)
    Service_name = name

    # Configuring EndPoint "Firwall Configuration":
    endpoint_config = ConfigurationSet()
    endpoint_config.configuration_set_type = 'NetworkConfiguration'
    endpoint1 = ConfigurationSetInputEndpoint(
        name='HTTP',
        protocol='tcp',
        port='80',
        local_port='80',
        load_balanced_endpoint_set_name=None,
        enable_direct_server_return=False)
    endpoint2 = ConfigurationSetInputEndpoint(
        name='SSH',
        protocol='tcp',
        port='22',
        local_port='22',
        load_balanced_endpoint_set_name=None,
        enable_direct_server_return=False)
    endpoint_config.input_endpoints.input_endpoints.append(endpoint1)
    endpoint_config.input_endpoints.input_endpoints.append(endpoint2)
    log.info("Configuring EndPoints 'Firwall Configuration' port 22 and 80")

    # Linux VM Configuration:
    linux_config = LinuxConfigurationSet(hostname, username, password, True)
    publickey = PublicKey(thumbprint, pkfile)
    linux_config.ssh.public_keys.public_keys.append(publickey)
    log.info("Linux VM Configuration")

    # Configuring Image ID:
    #----------------------
    os_hd = OSVirtualHardDisk(imageID, media_link)
    log.info("Configuring The Virtual Hard Disk using Image ID: %s", imageID)

    # Start Deployment of VM on Azure:
    log.info("Start Deployment of Elastic hpc on Azure")

    # Configuring Certificates:
    cert_format = 'pfx'
    cert_password = '******'

    try:
        for num in range(count):
            log.info("Deploying Node number: %d", num)
            name = name + "-" + str(num)
            result_cert = sms.add_service_certificate(
                service_name=Service_name,
                data=cert_data,
                certificate_format=cert_format,
                password=cert_password)
            log.info("Start Deploying VM with Name: " + name)
            try:
                log.info(vars(result_cert))
            except:
                log.info(result_cert)
            result = sms.create_virtual_machine_deployment(
                service_name=Service_name,
                deployment_name=name,
                deployment_slot='production',
                label=name,
                role_name=name,
                system_config=linux_config,
                os_virtual_hard_disk=os_hd,
                network_config=endpoint_config,
                role_size=instanceSize)
            operation_result = sms.get_operation_status(result.request_id)
            log.info("Start Deployment .............")
            for i in range(100):
                time.sleep(2)
                sys.stdout.write("Deployment Status Progress \r%d .%%" % i)
                sys.stdout.flush()
            print "\n"
            log.info("ssh -i " + certificate_path + " " + username + "@" +
                     Service_name + ".cloudapp.net")
    except WindowsAzureConflictError:
        log.info("Error: Can not Create VM")
        sys.exit(1)
Esempio n. 11
0
        network_config.subnet_names = []
        network_config.public_ips = None
        for port in endpoints:
            network_config.input_endpoints.append(
                ConfigurationSetInputEndpoint(name='TCP-%s' % port,
                                              protocol='TCP',
                                              port=port,
                                              local_port=port))

        # First determine where to store disk
        today = datetime.date.today().strftime('%Y-%m-%d')
        disk_prefix = u'%s-%s' % (name, name)
        media_link = u'http://%s.blob.core.windows.net/vhds/%s-%s.vhd' % (
            storage_account, disk_prefix, today)
        # Create system hard disk
        os_hd = OSVirtualHardDisk(image, media_link)

        # Spin up virtual machine
        try:
            result = azure.create_virtual_machine_deployment(
                service_name=name,
                deployment_name=name,
                deployment_slot='production',
                label=name,
                role_name=name,
                system_config=linux_config,
                network_config=network_config,
                os_virtual_hard_disk=os_hd,
                role_size=role_size,
                role_type='PersistentVMRole',
                virtual_network_name=virtual_network_name)
Esempio n. 12
0
blob_service = BlobService(account_name=storage_acc_name,
                           account_key=storage_acc_key)

container_name = "vm-container"

result = blob_service.create_container(container_name)

container_url_template = "http://{}.blob.core.windows.net/{}"

container_url = container_url_template.format(storage_acc_name, container_name)

image_name = "b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140414-en-us-30GB"

blob_url = container_url + "/ubuntu.vhd"

os_hd = OSVirtualHardDisk(image_name, blob_url)

vm_name = name_generator()

linux_config = LinuxConfigurationSet(vm_name, 'rohan', 'qwerty12#', True)

SERVICE_CERT_THUMBPRINT = config_params["vm_cert_thumbprint"]

vm_public_key_path = config_params["vm_pub_key_path"]

pk = PublicKey(SERVICE_CERT_THUMBPRINT, vm_public_key_path)

pair = KeyPair(SERVICE_CERT_THUMBPRINT, vm_public_key_path)

linux_config.ssh = SSH()
Esempio n. 13
0
    def create_instance(
        self, cloud_service_name, disk_name, system_config,
        network_config=None, label=None, group='production',
        machine_size='Small', reserved_ip_name=None
    ):
        """
            create a virtual disk image instance
        """
        if not self.__storage_reachable_by_cloud_service(cloud_service_name):
            message = [
                'The cloud service "%s" and the storage account "%s"',
                'are not in the same region, cannot launch an instance.'
            ]
            raise AzureStorageNotReachableByCloudServiceError(
                ' '.join(message) % (
                    cloud_service_name, self.account.storage_name()
                )
            )

        if not self.__image_reachable_by_cloud_service(
            cloud_service_name, disk_name
        ):
            message = [
                'The selected image "%s" is not available',
                'in the region of the selected cloud service "%s",',
                'cannot launch instance'
            ]
            raise AzureImageNotReachableByCloudServiceError(
                ' '.join(message) % (
                    disk_name, cloud_service_name
                )
            )

        deployment_exists = self.__get_deployment(
            cloud_service_name
        )

        if label and deployment_exists:
            message = [
                'A deployment of the name: %s already exists.',
                'Assignment of a label can only happen for the',
                'initial deployment.'
            ]
            raise AzureVmCreateError(
                ' '.join(message) % cloud_service_name
            )

        if reserved_ip_name and deployment_exists:
            message = [
                'A deployment of the name: %s already exists.',
                'Assignment of a reserved IP name can only happen for the',
                'initial deployment.'
            ]
            raise AzureVmCreateError(
                ' '.join(message) % cloud_service_name
            )

        storage = BaseBlobService(
            self.account.storage_name(),
            self.account.storage_key(),
            endpoint_suffix=self.account.get_blob_service_host_base()
        )
        media_link = storage.make_blob_url(
            self.account.storage_container(), ''.join(
                [
                    cloud_service_name,
                    '_instance_', system_config.host_name,
                    '_image_', disk_name
                ]
            )
        )
        instance_disk = OSVirtualHardDisk(disk_name, media_link)
        instance_record = {
            'deployment_name': cloud_service_name,
            'network_config': network_config,
            'role_name': system_config.host_name,
            'role_size': machine_size,
            'service_name': cloud_service_name,
            'system_config': system_config,
            'os_virtual_hard_disk': instance_disk,
            'provision_guest_agent': True
        }
        if network_config:
            instance_record['network_config'] = network_config

        try:
            if deployment_exists:
                result = self.service.add_role(
                    **instance_record
                )
            else:
                instance_record['deployment_slot'] = group
                if reserved_ip_name:
                    instance_record['reserved_ip_name'] = reserved_ip_name
                if label:
                    instance_record['label'] = label
                else:
                    instance_record['label'] = cloud_service_name
                result = self.service.create_virtual_machine_deployment(
                    **instance_record
                )
            return {
                'request_id': format(result.request_id),
                'cloud_service_name': cloud_service_name,
                'instance_name': system_config.host_name
            }
        except Exception as e:
            raise AzureVmCreateError(
                '%s: %s' % (type(e).__name__, format(e))
            )
Esempio n. 14
0
def create_virtual_machine(module, azure):
    """
    Create new virtual machine

    module : AnsibleModule object
    azure: authenticated azure ServiceManagementService object

    Returns:
        True if a new virtual machine was created, false otherwise
    """
    name = module.params.get('name')
    hostname = module.params.get('hostname') or name + ".cloudapp.net"
    endpoints = module.params.get('endpoints').split(',')
    ssh_cert_path = module.params.get('ssh_cert_path')
    user = module.params.get('user')
    password = module.params.get('password')
    location = module.params.get('location')
    role_size = module.params.get('role_size')
    storage_account = module.params.get('storage_account')
    image = module.params.get('image')
    vm_image = module.params.get('vm_image')
    virtual_network_name = module.params.get('virtual_network_name')
    wait = module.params.get('wait')
    wait_timeout = int(module.params.get('wait_timeout'))

    use_vm_image = vm_image is not None and image is None

    # Check if a deployment with the same name already exists
    cloud_service_name_available = azure.check_hosted_service_name_availability(
        name)
    if not cloud_service_name_available.result:
        changed = False
    else:
        changed = True
        # Create cloud service if necessary
        try:
            result = azure.create_hosted_service(service_name=name,
                                                 label=name,
                                                 location=location)
            _wait_for_completion(azure, result, wait_timeout,
                                 "create_hosted_service")
        except WindowsAzureError as e:
            module.fail_json(
                msg=
                "failed to create the new service name, it already exists: %s"
                % str(e))

        # Create linux configuration
        if not use_vm_image:
            disable_ssh_password_authentication = not password
            linux_config = LinuxConfigurationSet(
                hostname, user, password, disable_ssh_password_authentication)

            # Add ssh certificates if specified
            if ssh_cert_path:
                fingerprint, pkcs12_base64 = get_ssh_certificate_tokens(
                    module, ssh_cert_path)
                # Add certificate to cloud service
                result = azure.add_service_certificate(name, pkcs12_base64,
                                                       'pfx', '')
                _wait_for_completion(azure, result, wait_timeout,
                                     "add_service_certificate")

                # Create ssh config
                ssh_config = SSH()
                ssh_config.public_keys = PublicKeys()
                authorized_keys_path = u'/home/%s/.ssh/authorized_keys' % user
                ssh_config.public_keys.public_keys.append(
                    PublicKey(path=authorized_keys_path,
                              fingerprint=fingerprint))
                # Append ssh config to linux machine config
                linux_config.ssh = ssh_config
        else:
            # When using vm_image, you can't specify an OS configuration set (burned into the image)
            linux_config = None

        # Create network configuration
        network_config = ConfigurationSetInputEndpoints()
        network_config.configuration_set_type = 'NetworkConfiguration'
        network_config.subnet_names = []
        network_config.public_ips = None
        for port in endpoints:
            network_config.input_endpoints.append(
                ConfigurationSetInputEndpoint(name='TCP-%s' % port,
                                              protocol='TCP',
                                              port=port,
                                              local_port=port))

        # First determine where to store disk
        today = datetime.date.today().strftime('%Y-%m-%d')
        disk_prefix = u'%s-%s' % (name, name)
        media_link = u'http://%s.blob.core.windows.net/vhds/%s-%s.vhd' % (
            storage_account, disk_prefix, today)
        # Create system hard disk
        if image:
            os_hd = OSVirtualHardDisk(image, media_link)
        else:
            # When using vm_image_name, you can't specify an OS Virtual Hard Disk,
            # since the image already contains the VHD configuration
            os_hd = None

        # Spin up virtual machine
        try:
            result = azure.create_virtual_machine_deployment(
                service_name=name,
                deployment_name=name,
                deployment_slot='production',
                label=name,
                role_name=name,
                system_config=linux_config,
                network_config=network_config,
                os_virtual_hard_disk=os_hd,
                vm_image_name=vm_image,
                role_size=role_size,
                role_type='PersistentVMRole',
                virtual_network_name=virtual_network_name)
            _wait_for_completion(azure, result, wait_timeout,
                                 "create_virtual_machine_deployment")
        except WindowsAzureError as e:
            module.fail_json(
                msg="failed to create the new virtual machine, error was: %s" %
                str(e))

    try:
        deployment = azure.get_deployment_by_name(service_name=name,
                                                  deployment_name=name)
        return (changed, urlparse(deployment.url).hostname, deployment)
    except WindowsAzureError as e:
        module.fail_json(
            msg=
            "failed to lookup the deployment information for %s, error was: %s"
            % (name, str(e)))
Esempio n. 15
0
    def _ensureVirtualMachinesExist(self):
        """
        Creates the VMs for the web site.
        """
        service_name = self.config.getServiceName()
        cert_thumbprint = self.config.getServiceCertificateThumbprint()
        vm_username = self.config.getVirtualMachineLogonUsername()
        vm_password = self.config.getVirtualMachineLogonPassword()
        vm_role_size = self.config.getServiceInstanceRoleSize()
        vm_numbers = self.config.getServiceInstanceCount()
        if vm_numbers < 1:
            raise Exception(
                "Detected an invalid number of instances: {0}.".format(
                    vm_numbers))

        self._assertOsImageExists(self.config.getServiceOSImageName())

        role_instances = self._getRoleInstances(service_name)
        for vm_number in range(1, vm_numbers + 1):
            vm_hostname = '{0}-{1}'.format(service_name, vm_number)
            if vm_hostname in role_instances:
                logger.warn(
                    "Role instance %s already exists: skipping creation.",
                    vm_hostname)
                continue

            logger.info("Role instance %s provisioning begins.", vm_hostname)
            vm_diskname = '{0}.vhd'.format(vm_hostname)
            vm_disk_media_link = 'http://{0}.blob.core.windows.net/vhds/{1}'.format(
                self.config.getServiceStorageAccountName(), vm_diskname)
            ssh_port = str(self.config.getServiceInstanceSshPort() + vm_number)

            os_hd = OSVirtualHardDisk(self.config.getServiceOSImageName(),
                                      vm_disk_media_link,
                                      disk_name=vm_diskname,
                                      disk_label=vm_diskname)
            linux_config = LinuxConfigurationSet(vm_hostname, vm_username,
                                                 vm_password, True)
            linux_config.ssh.public_keys.public_keys.append(
                PublicKey(
                    cert_thumbprint,
                    u'/home/{0}/.ssh/authorized_keys'.format(vm_username)))
            linux_config.ssh.key_pairs.key_pairs.append(
                KeyPair(cert_thumbprint,
                        u'/home/{0}/.ssh/id_rsa'.format(vm_username)))
            network_config = ConfigurationSet()
            network_config.configuration_set_type = 'NetworkConfiguration'
            ssh_endpoint = ConfigurationSetInputEndpoint(name='SSH',
                                                         protocol='TCP',
                                                         port=ssh_port,
                                                         local_port=u'22')
            network_config.input_endpoints.input_endpoints.append(ssh_endpoint)
            http_endpoint = ConfigurationSetInputEndpoint(
                name='HTTP',
                protocol='TCP',
                port=u'80',
                local_port=u'80',
                load_balanced_endpoint_set_name=service_name)
            http_endpoint.load_balancer_probe.port = '80'
            http_endpoint.load_balancer_probe.protocol = 'TCP'
            network_config.input_endpoints.input_endpoints.append(
                http_endpoint)

            if vm_number == 1:
                result = self.sms.create_virtual_machine_deployment(
                    service_name=service_name,
                    deployment_name=service_name,
                    deployment_slot='Production',
                    label=vm_hostname,
                    role_name=vm_hostname,
                    system_config=linux_config,
                    os_virtual_hard_disk=os_hd,
                    network_config=network_config,
                    availability_set_name=service_name,
                    data_virtual_hard_disks=None,
                    role_size=vm_role_size)
                self._wait_for_operation_success(
                    result.request_id,
                    timeout=self.config.getAzureOperationTimeout())
                self._wait_for_role_instance_status(
                    vm_hostname, service_name, 'ReadyRole',
                    self.config.getAzureOperationTimeout())
            else:
                result = self.sms.add_role(service_name=service_name,
                                           deployment_name=service_name,
                                           role_name=vm_hostname,
                                           system_config=linux_config,
                                           os_virtual_hard_disk=os_hd,
                                           network_config=network_config,
                                           availability_set_name=service_name,
                                           role_size=vm_role_size)
                self._wait_for_operation_success(
                    result.request_id,
                    timeout=self.config.getAzureOperationTimeout())
                self._wait_for_role_instance_status(
                    vm_hostname, service_name, 'ReadyRole',
                    self.config.getAzureOperationTimeout())

            logger.info("Role instance %s has been created.", vm_hostname)
Esempio n. 16
0
def create_virtual_machine(module, azure):
    """
    Create new virtual machine

    module : AnsibleModule object
    azure: authenticated azure ServiceManagementService object

    Returns:
        True if a new virtual machine and/or cloud service was created, false otherwise
    """
    name = module.params.get('name')
    os_type = module.params.get('os_type')
    hostname = module.params.get('hostname') or name + ".cloudapp.net"
    endpoints = module.params.get('endpoints').split(',')
    ssh_cert_path = module.params.get('ssh_cert_path')
    user = module.params.get('user')
    password = module.params.get('password')
    location = module.params.get('location')
    role_size = module.params.get('role_size')
    storage_account = module.params.get('storage_account')
    image = module.params.get('image')
    virtual_network_name = module.params.get('virtual_network_name')
    wait = module.params.get('wait')
    wait_timeout = int(module.params.get('wait_timeout'))

    changed = False

    # Check if a deployment with the same name already exists
    cloud_service_name_available = azure.check_hosted_service_name_availability(
        name)
    if cloud_service_name_available.result:
        # cloud service does not exist; create it
        try:
            result = azure.create_hosted_service(service_name=name,
                                                 label=name,
                                                 location=location)
            _wait_for_completion(azure, result, wait_timeout,
                                 "create_hosted_service")
            changed = True
        except AzureException as e:
            module.fail_json(
                msg="failed to create the new service, error was: %s" % str(e))

    try:
        # check to see if a vm with this name exists; if so, do nothing
        azure.get_role(name, name, name)
    except AzureMissingException:
        # vm does not exist; create it

        if os_type == 'linux':
            # Create linux configuration
            disable_ssh_password_authentication = not password
            vm_config = LinuxConfigurationSet(
                hostname, user, password, disable_ssh_password_authentication)
        else:
            # Create Windows Config
            vm_config = WindowsConfigurationSet(
                hostname, password, None, module.params.get('auto_updates'),
                None, user)
            vm_config.domain_join = None
            if module.params.get('enable_winrm'):
                listener = Listener('Http')
                vm_config.win_rm.listeners.listeners.append(listener)
            else:
                vm_config.win_rm = None

        # Add ssh certificates if specified
        if ssh_cert_path:
            fingerprint, pkcs12_base64 = get_ssh_certificate_tokens(
                module, ssh_cert_path)
            # Add certificate to cloud service
            result = azure.add_service_certificate(name, pkcs12_base64, 'pfx',
                                                   '')
            _wait_for_completion(azure, result, wait_timeout,
                                 "add_service_certificate")

            # Create ssh config
            ssh_config = SSH()
            ssh_config.public_keys = PublicKeys()
            authorized_keys_path = u'/home/%s/.ssh/authorized_keys' % user
            ssh_config.public_keys.public_keys.append(
                PublicKey(path=authorized_keys_path, fingerprint=fingerprint))
            # Append ssh config to linux machine config
            vm_config.ssh = ssh_config

        # Create network configuration
        network_config = ConfigurationSetInputEndpoints()
        network_config.configuration_set_type = 'NetworkConfiguration'
        network_config.subnet_names = []
        network_config.public_ips = None
        for port in endpoints:
            network_config.input_endpoints.append(
                ConfigurationSetInputEndpoint(name='TCP-%s' % port,
                                              protocol='TCP',
                                              port=port,
                                              local_port=port))

        # First determine where to store disk
        today = datetime.date.today().strftime('%Y-%m-%d')
        disk_prefix = u'%s-%s' % (name, name)
        media_link = u'http://%s.blob.core.windows.net/vhds/%s-%s.vhd' % (
            storage_account, disk_prefix, today)
        # Create system hard disk
        os_hd = OSVirtualHardDisk(image, media_link)

        # Spin up virtual machine
        try:
            result = azure.create_virtual_machine_deployment(
                service_name=name,
                deployment_name=name,
                deployment_slot='production',
                label=name,
                role_name=name,
                system_config=vm_config,
                network_config=network_config,
                os_virtual_hard_disk=os_hd,
                role_size=role_size,
                role_type='PersistentVMRole',
                virtual_network_name=virtual_network_name)
            _wait_for_completion(azure, result, wait_timeout,
                                 "create_virtual_machine_deployment")
            changed = True
        except AzureException as e:
            module.fail_json(
                msg="failed to create the new virtual machine, error was: %s" %
                str(e))

    try:
        deployment = azure.get_deployment_by_name(service_name=name,
                                                  deployment_name=name)
        return (changed, urlparse(deployment.url).hostname, deployment)
    except AzureException as e:
        module.fail_json(
            msg=
            "failed to lookup the deployment information for %s, error was: %s"
            % (name, str(e)))
Esempio n. 17
0
    def create_docker_host_vm(self, hackathon_id):
        """
        create docker host VM for hackathon whose id is hackathon_id

        :param hackathon_id: the id of hackathon in DB table:hackathon
        :type hackathon_id: Integer

        :return: True if send an Azure creating VM request via API successfully after validating storage, container and
         service
         Otherwise, False
        :rtype: bool
        """
        sms = self.__get_sms_object(hackathon_id)
        if sms is None:
            self.log.error('No Azure account found for Hackathon:%d' % hackathon_id)
            return False
        # get storage and container
        res, storage_account_name, container_name = self.__get_available_storage_account_and_container(hackathon_id)
        if not res:
            return False
        # get service
        res, service_name = self.__get_available_cloud_service(hackathon_id)
        if not res:
            return False
        # set host_name to ensure its uniqueness
        host_name = str(uuid1())[0:9] + strftime("%Y%m%d%H%M%S")
        # set vm os image and hard disk storage
        image_name_default = 'b549f4301d0b4295b8e76ceb65df47d4__Ubuntu-14_04-LTS-amd64-server-20140606.1-en-us-30GB'
        image_name = self.util.safe_get_config('dockerhostserver.vm.image_name', image_name_default)
        media_link = 'https://%s.blob.core.chinacloudapi.cn/%s/%s.vhd' % (storage_account_name, container_name,
                                                                          host_name)
        os_hd = OSVirtualHardDisk(image_name, media_link)
        # set linux_config and endpoint_config
        customdata = self.__get_customdata_from_local_file()
        linux_config = LinuxConfigurationSet(host_name,
                                             self.util.safe_get_config('dockerhostserver.vm.username', 'opentech'),
                                             self.util.safe_get_config('dockerhostserver.vm.password', 'Password01!'),
                                             False, custom_data=customdata)
        endpoint_config = self.__set_endpoint_config(service_name, hackathon_id)
        deployment_exist = True
        deployment_slot = ServiceDeploymentSlot.PRODUCTION
        try:
            deployment_exist = self.__deployment_exists(service_name, deployment_slot, hackathon_id)
        except Exception as e:
            self.log.error(e)
            return False
        if not deployment_exist:
            try:
                result = sms.create_virtual_machine_deployment(service_name=service_name,
                                                               network_config=endpoint_config,
                                                               deployment_name=service_name,
                                                               deployment_slot=deployment_slot,
                                                               label=service_name,
                                                               role_name=host_name,
                                                               system_config=linux_config,
                                                               os_virtual_hard_disk=os_hd,
                                                               role_size=AzureVMSize.MEDIUM_SIZE)
                self.log.debug('To create VM:%s in service:%s.(deployment)' % (host_name, service_name))
            except Exception as e:
                self.log.error(e)
                return False
        else:
            try:
                result = sms.add_role(service_name=service_name,
                                      deployment_name=service_name,
                                      role_name=host_name,
                                      system_config=linux_config,
                                      os_virtual_hard_disk=os_hd,
                                      network_config=endpoint_config,
                                      role_size=AzureVMSize.MEDIUM_SIZE)
                self.log.debug('To create VM:%s in service:%s.(add role)' % (host_name, service_name))
            except Exception as e:
                self.log.error(e)
                return False
        # storage parameters in context
        context = Context(hackathon_id=hackathon_id, request_id=result.request_id,
                          service_name=service_name, role_name=host_name,
                          deployment_name=service_name, deployment_slot=deployment_slot,
                          host_name=host_name)
        # start schedule
        self.sche.add_once('docker_host_manager', 'check_vm_status', context=context, minutes=5)
        return True