コード例 #1
0
ファイル: openstack.py プロジェクト: LiMinggang/buildbot
    def __init__(self, name, password,
                 flavor,
                 os_username,
                 os_password,
                 os_tenant_name,
                 os_auth_url,
                 block_devices=None,
                 image=None,
                 meta=None,
                 # Have a nova_args parameter to allow passing things directly
                 # to novaclient v1.1.
                 nova_args=None,
                 **kwargs):

        if not client or not nce:
            config.error("The python module 'novaclient' is needed  "
                         "to use a OpenStackLatentWorker")

        if not block_devices and not image:
            raise ValueError('One of block_devices or image must be given')

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        self.flavor = flavor
        self.os_username = os_username
        self.os_password = os_password
        self.os_tenant_name = os_tenant_name
        self.os_auth_url = os_auth_url
        if block_devices is not None:
            self.block_devices = [self._parseBlockDevice(bd) for bd in block_devices]
        else:
            self.block_devices = None
        self.image = image
        self.meta = meta
        self.nova_args = nova_args if nova_args is not None else {}
コード例 #2
0
ファイル: libvirt.py プロジェクト: LiMinggang/buildbot
    def __init__(self, name, password, connection, hd_image, base_image=None, xml=None,
                 **kwargs):
        AbstractLatentWorker.__init__(self, name, password, **kwargs)
        if not libvirt:
            config.error("The python module 'libvirt' is needed to use a LibVirtWorker")

        self.connection = connection
        self.image = hd_image
        self.base_image = base_image
        self.xml = xml

        self.cheap_copy = True
        self.graceful_shutdown = False

        self.domain = None

        self.ready = False
        self._find_existing_deferred = self._find_existing_instance()
コード例 #3
0
ファイル: libvirt.py プロジェクト: LiMinggang/buildbot
    def canStartBuild(self):
        if not self.ready:
            log.msg("Not accepting builds as existing domains not iterated")
            return False

        if self.domain and not self.isConnected():
            log.msg("Not accepting builds as existing domain but worker not connected")
            return False

        return AbstractLatentWorker.canStartBuild(self)
コード例 #4
0
ファイル: openstack.py プロジェクト: rajul/buildbot
    def __init__(
            self,
            name,
            password,
            flavor,
            os_username,
            os_password,
            os_tenant_name,
            os_auth_url,
            block_devices=None,
            image=None,
            meta=None,
            # Have a nova_args parameter to allow passing things directly
            # to novaclient v1.1.
            nova_args=None,
            client_version='1.1',
            **kwargs):

        if not client or not nce:
            config.error("The python module 'novaclient' is needed  "
                         "to use a OpenStackLatentWorker")

        if not block_devices and not image:
            raise ValueError('One of block_devices or image must be given')

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        self.flavor = flavor
        self.os_username = os_username
        self.os_password = os_password
        self.os_tenant_name = os_tenant_name
        self.os_auth_url = os_auth_url
        self.client_version = client_version

        if block_devices is not None:
            self.block_devices = [
                self._parseBlockDevice(bd) for bd in block_devices
            ]
        else:
            self.block_devices = None
        self.image = image
        self.meta = meta
        self.nova_args = nova_args if nova_args is not None else {}
コード例 #5
0
    def canStartBuild(self):
        if not self.ready:
            log.msg("Not accepting builds as existing domains not iterated")
            return False

        if self.domain and not self.isConnected():
            log.msg(
                "Not accepting builds as existing domain but worker not connected")
            return False

        return AbstractLatentWorker.canStartBuild(self)
コード例 #6
0
ファイル: latent.py プロジェクト: kenygia/buildbot
 def reconfigService(self, name, _, **kwargs):
     AbstractLatentWorker.reconfigService(self, name, None, **kwargs)
コード例 #7
0
ファイル: latent.py プロジェクト: kenygia/buildbot
 def checkConfig(self, name, _, **kwargs):
     AbstractLatentWorker.checkConfig(self, name, None, **kwargs)
コード例 #8
0
ファイル: latent.py プロジェクト: kenygia/buildbot
 def __init__(self, name, controller, **kwargs):
     AbstractLatentWorker.__init__(self, name, None, **kwargs)
     self._controller = controller
コード例 #9
0
ファイル: latent.py プロジェクト: vicosx12/buildbot
 def reconfigService(self, name, _, **kwargs):
     AbstractLatentWorker.reconfigService(self, name, None, **kwargs)
コード例 #10
0
 def checkConfig(self, name, _):
     AbstractLatentWorker.checkConfig(self, name, None)
コード例 #11
0
ファイル: test_latent.py プロジェクト: GuoJianzhu/buildbot
 def reconfigService(self, name, _):
     AbstractLatentWorker.reconfigService(self, name, None)
コード例 #12
0
ファイル: ec2.py プロジェクト: zjtheone/buildbot
    def __init__(self,
                 name,
                 password,
                 instance_type,
                 ami=None,
                 valid_ami_owners=None,
                 valid_ami_location_regex=None,
                 elastic_ip=None,
                 identifier=None,
                 secret_identifier=None,
                 aws_id_file_path=None,
                 user_data=None,
                 region=None,
                 keypair_name=None,
                 security_name=None,
                 spot_instance=False,
                 max_spot_price=1.6,
                 volumes=None,
                 placement=None,
                 price_multiplier=1.2,
                 tags=None,
                 product_description='Linux/UNIX',
                 subnet_id=None,
                 security_group_ids=None,
                 instance_profile_name=None,
                 block_device_map=None,
                 session=None,
                 **kwargs):

        if not boto3:
            config.error("The python module 'boto3' is needed to use a "
                         "EC2LatentWorker")

        if keypair_name is None:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'keypair_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            keypair_name = 'latent_buildbot_slave'
        if security_name is None and not subnet_id:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'security_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            security_name = 'latent_buildbot_slave'

        if volumes is None:
            volumes = []

        if tags is None:
            tags = {}

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        if security_name and subnet_id:
            raise ValueError(
                'security_name (EC2 classic security groups) is not supported '
                'in a VPC.  Use security_group_ids instead.')
        if not ((ami is not None) ^ (valid_ami_owners is not None
                                     or valid_ami_location_regex is not None)):
            raise ValueError(
                'You must provide either a specific ami, or one or both of '
                'valid_ami_location_regex and valid_ami_owners')
        self.ami = ami
        if valid_ami_owners is not None:
            if isinstance(valid_ami_owners, (int, long)):
                valid_ami_owners = (valid_ami_owners, )
            else:
                for element in valid_ami_owners:
                    if not isinstance(element, (int, long)):
                        raise ValueError(
                            'valid_ami_owners should be int or iterable '
                            'of ints', element)
        if valid_ami_location_regex is not None:
            if not isinstance(valid_ami_location_regex, basestring):
                raise ValueError('valid_ami_location_regex should be a string')
            else:
                # verify that regex will compile
                re.compile(valid_ami_location_regex)
        if spot_instance and price_multiplier is None and max_spot_price is None:
            raise ValueError('You must provide either one, or both, of '
                             'price_multiplier or max_spot_price')
        self.valid_ami_owners = None
        if valid_ami_owners:
            self.valid_ami_owners = [str(o) for o in valid_ami_owners]
        self.valid_ami_location_regex = valid_ami_location_regex
        self.instance_type = instance_type
        self.keypair_name = keypair_name
        self.security_name = security_name
        self.user_data = user_data
        self.spot_instance = spot_instance
        self.max_spot_price = max_spot_price
        self.volumes = volumes
        self.price_multiplier = price_multiplier
        self.product_description = product_description

        if None not in [placement, region]:
            self.placement = '%s%s' % (region, placement)
        else:
            self.placement = None
        if identifier is None:
            assert secret_identifier is None, (
                'supply both or neither of identifier, secret_identifier')
            if aws_id_file_path is None:
                home = os.environ['HOME']
                default_path = os.path.join(home, '.ec2', 'aws_id')
                if os.path.exists(default_path):
                    aws_id_file_path = default_path
            if aws_id_file_path:
                log.msg('WARNING: EC2LatentWorker is using deprecated '
                        'aws_id file')
                with open(aws_id_file_path, 'r') as aws_file:
                    identifier = aws_file.readline().strip()
                    secret_identifier = aws_file.readline().strip()
        else:
            assert aws_id_file_path is None, \
                'if you supply the identifier and secret_identifier, ' \
                'do not specify the aws_id_file_path'
            assert secret_identifier is not None, \
                'supply both or neither of identifier, secret_identifier'

        region_found = None

        # Make the EC2 connection.
        self.session = session
        if self.session is None:
            if region is not None:
                for r in boto3.Session(aws_access_key_id=identifier,
                                       aws_secret_access_key=secret_identifier
                                       ).get_available_regions('ec2'):

                    if r == region:
                        region_found = r

                if region_found is not None:
                    self.session = boto3.Session(
                        region_name=region,
                        aws_access_key_id=identifier,
                        aws_secret_access_key=secret_identifier)
                else:
                    raise ValueError('The specified region does not exist: ' +
                                     region)

            else:
                # boto2 defaulted to us-east-1 when region was unset, we
                # mimic this here in boto3
                region = botocore.session.get_session().get_config_variable(
                    'region')
                if region is None:
                    region = 'us-east-1'
                self.session = boto3.Session(
                    aws_access_key_id=identifier,
                    aws_secret_access_key=secret_identifier,
                    region_name=region)

        self.ec2 = self.session.resource('ec2')

        # Make a keypair
        #
        # We currently discard the keypair data because we don't need it.
        # If we do need it in the future, we will always recreate the keypairs
        # because there is no way to
        # programmatically retrieve the private key component, unless we
        # generate it and store it on the filesystem, which is an unnecessary
        # usage requirement.
        try:
            self.ec2.KeyPair(self.keypair_name).load()
            # key_pair.delete() # would be used to recreate
        except ClientError as e:
            if 'InvalidKeyPair.NotFound' not in e.message:
                if 'AuthFailure' in e.message:
                    log.msg('POSSIBLE CAUSES OF ERROR:\n'
                            '  Did you supply your AWS credentials?\n'
                            '  Did you sign up for EC2?\n'
                            '  Did you put a credit card number in your AWS '
                            'account?\n'
                            'Please doublecheck before reporting a problem.\n')
                raise
            # make one; we would always do this, and stash the result, if we
            # needed the key (for instance, to SSH to the box).  We'd then
            # use paramiko to use the key to connect.
            self.ec2.create_key_pair(KeyName=keypair_name)

        # create security group
        if security_name:
            try:
                self.ec2.SecurityGroup(security_name).load()
            except ClientError as e:
                if 'InvalidGroup.NotFound' in e.message:
                    self.security_group = self.ec2.create_security_group(
                        GroupName=security_name,
                        Description=
                        'Authorization to access the buildbot instance.')
                    # Authorize the master as necessary
                    # TODO this is where we'd open the hole to do the reverse pb
                    # connect to the buildbot
                    # ip = urllib.urlopen(
                    #     'http://checkip.amazonaws.com').read().strip()
                    # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
                    # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
                else:
                    raise

        # get the image
        if self.ami is not None:
            self.image = self.ec2.Image(self.ami)
        else:
            # verify we have access to at least one acceptable image
            discard = self.get_image()
            assert discard

        # get the specified elastic IP, if any
        if elastic_ip is not None:
            # Using ec2.vpc_addresses.filter(PublicIps=[elastic_ip]) throws a
            # NotImplementedError("Filtering not supported in describe_address.") in moto
            # https://github.com/spulec/moto/blob/100ec4e7c8aa3fde87ff6981e2139768816992e4/moto/ec2/responses/elastic_ip_addresses.py#L52
            addresses = self.ec2.meta.client.describe_addresses(
                PublicIps=[elastic_ip])['Addresses']
            if not addresses:
                raise ValueError('Could not find EIP for IP: ' + elastic_ip)
            allocation_id = addresses[0]['AllocationId']
            elastic_ip = self.ec2.VpcAddress(allocation_id)
        self.elastic_ip = elastic_ip
        self.subnet_id = subnet_id
        self.security_group_ids = security_group_ids
        self.classic_security_groups = [self.security_name
                                        ] if self.security_name else None
        self.instance_profile_name = instance_profile_name
        self.tags = tags
        self.block_device_map = self.create_block_device_mapping(
            block_device_map) if block_device_map else None
コード例 #13
0
ファイル: ec2.py プロジェクト: kenygia/buildbot
    def __init__(
        self,
        name,
        password,
        instance_type,
        ami=None,
        valid_ami_owners=None,
        valid_ami_location_regex=None,
        elastic_ip=None,
        identifier=None,
        secret_identifier=None,
        aws_id_file_path=None,
        user_data=None,
        region=None,
        keypair_name=None,
        security_name=None,
        spot_instance=False,
        max_spot_price=1.6,
        volumes=None,
        placement=None,
        price_multiplier=1.2,
        tags=None,
        product_description="Linux/UNIX",
        subnet_id=None,
        security_group_ids=None,
        instance_profile_name=None,
        block_device_map=None,
        session=None,
        **kwargs
    ):

        if not boto3:
            config.error("The python module 'boto3' is needed to use a " "EC2LatentWorker")

        if keypair_name is None:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'keypair_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value"
            )
            keypair_name = "latent_buildbot_slave"
        if security_name is None and not subnet_id:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'security_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value"
            )
            security_name = "latent_buildbot_slave"

        if volumes is None:
            volumes = []

        if tags is None:
            tags = {}

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        if security_name and subnet_id:
            raise ValueError(
                "security_name (EC2 classic security groups) is not supported "
                "in a VPC.  Use security_group_ids instead."
            )
        if not ((ami is not None) ^ (valid_ami_owners is not None or valid_ami_location_regex is not None)):
            raise ValueError(
                "You must provide either a specific ami, or one or both of "
                "valid_ami_location_regex and valid_ami_owners"
            )
        self.ami = ami
        if valid_ami_owners is not None:
            if isinstance(valid_ami_owners, integer_types):
                valid_ami_owners = (valid_ami_owners,)
            else:
                for element in valid_ami_owners:
                    if not isinstance(element, integer_types):
                        raise ValueError("valid_ami_owners should be int or iterable " "of ints", element)
        if valid_ami_location_regex is not None:
            if not isinstance(valid_ami_location_regex, string_types):
                raise ValueError("valid_ami_location_regex should be a string")
            else:
                # verify that regex will compile
                re.compile(valid_ami_location_regex)
        if spot_instance and price_multiplier is None and max_spot_price is None:
            raise ValueError("You must provide either one, or both, of " "price_multiplier or max_spot_price")
        self.valid_ami_owners = None
        if valid_ami_owners:
            self.valid_ami_owners = [str(o) for o in valid_ami_owners]
        self.valid_ami_location_regex = valid_ami_location_regex
        self.instance_type = instance_type
        self.keypair_name = keypair_name
        self.security_name = security_name
        self.user_data = user_data
        self.spot_instance = spot_instance
        self.max_spot_price = max_spot_price
        self.volumes = volumes
        self.price_multiplier = price_multiplier
        self.product_description = product_description

        if None not in [placement, region]:
            self.placement = "%s%s" % (region, placement)
        else:
            self.placement = None
        if identifier is None:
            assert secret_identifier is None, "supply both or neither of identifier, secret_identifier"
            if aws_id_file_path is None:
                home = os.environ["HOME"]
                default_path = os.path.join(home, ".ec2", "aws_id")
                if os.path.exists(default_path):
                    aws_id_file_path = default_path
            if aws_id_file_path:
                log.msg("WARNING: EC2LatentWorker is using deprecated " "aws_id file")
                with open(aws_id_file_path, "r") as aws_file:
                    identifier = aws_file.readline().strip()
                    secret_identifier = aws_file.readline().strip()
        else:
            assert aws_id_file_path is None, (
                "if you supply the identifier and secret_identifier, " "do not specify the aws_id_file_path"
            )
            assert secret_identifier is not None, "supply both or neither of identifier, secret_identifier"

        region_found = None

        # Make the EC2 connection.
        self.session = session
        if self.session is None:
            if region is not None:
                for r in boto3.Session(
                    aws_access_key_id=identifier, aws_secret_access_key=secret_identifier
                ).get_available_regions("ec2"):

                    if r == region:
                        region_found = r

                if region_found is not None:
                    self.session = boto3.Session(
                        region_name=region, aws_access_key_id=identifier, aws_secret_access_key=secret_identifier
                    )
                else:
                    raise ValueError("The specified region does not exist: " + region)

            else:
                # boto2 defaulted to us-east-1 when region was unset, we
                # mimic this here in boto3
                region = botocore.session.get_session().get_config_variable("region")
                if region is None:
                    region = "us-east-1"
                self.session = boto3.Session(
                    aws_access_key_id=identifier, aws_secret_access_key=secret_identifier, region_name=region
                )

        self.ec2 = self.session.resource("ec2")

        # Make a keypair
        #
        # We currently discard the keypair data because we don't need it.
        # If we do need it in the future, we will always recreate the keypairs
        # because there is no way to
        # programmatically retrieve the private key component, unless we
        # generate it and store it on the filesystem, which is an unnecessary
        # usage requirement.
        try:
            self.ec2.KeyPair(self.keypair_name).load()
            # key_pair.delete() # would be used to recreate
        except ClientError as e:
            if "InvalidKeyPair.NotFound" not in e.message:
                if "AuthFailure" in e.message:
                    log.msg(
                        "POSSIBLE CAUSES OF ERROR:\n"
                        "  Did you supply your AWS credentials?\n"
                        "  Did you sign up for EC2?\n"
                        "  Did you put a credit card number in your AWS "
                        "account?\n"
                        "Please doublecheck before reporting a problem.\n"
                    )
                raise
            # make one; we would always do this, and stash the result, if we
            # needed the key (for instance, to SSH to the box).  We'd then
            # use paramiko to use the key to connect.
            self.ec2.create_key_pair(KeyName=keypair_name)

        # create security group
        if security_name:
            try:
                self.ec2.SecurityGroup(security_name).load()
            except ClientError as e:
                if "InvalidGroup.NotFound" in e.message:
                    self.security_group = self.ec2.create_security_group(
                        GroupName=security_name, Description="Authorization to access the buildbot instance."
                    )
                    # Authorize the master as necessary
                    # TODO this is where we'd open the hole to do the reverse pb
                    # connect to the buildbot
                    # ip = urllib.urlopen(
                    #     'http://checkip.amazonaws.com').read().strip()
                    # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
                    # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
                else:
                    raise

        # get the image
        if self.ami is not None:
            self.image = self.ec2.Image(self.ami)
        else:
            # verify we have access to at least one acceptable image
            discard = self.get_image()
            assert discard

        # get the specified elastic IP, if any
        if elastic_ip is not None:
            # Using ec2.vpc_addresses.filter(PublicIps=[elastic_ip]) throws a
            # NotImplementedError("Filtering not supported in describe_address.") in moto
            # https://github.com/spulec/moto/blob/100ec4e7c8aa3fde87ff6981e2139768816992e4/moto/ec2/responses/elastic_ip_addresses.py#L52
            addresses = self.ec2.meta.client.describe_addresses(PublicIps=[elastic_ip])["Addresses"]
            if not addresses:
                raise ValueError("Could not find EIP for IP: " + elastic_ip)
            allocation_id = addresses[0]["AllocationId"]
            elastic_ip = self.ec2.VpcAddress(allocation_id)
        self.elastic_ip = elastic_ip
        self.subnet_id = subnet_id
        self.security_group_ids = security_group_ids
        self.classic_security_groups = [self.security_name] if self.security_name else None
        self.instance_profile_name = instance_profile_name
        self.tags = tags
        self.block_device_map = self.create_block_device_mapping(block_device_map) if block_device_map else None
コード例 #14
0
 def reconfigService(self, name, _):
     AbstractLatentWorker.reconfigService(self, name, None)
コード例 #15
0
ファイル: test_latent.py プロジェクト: GuoJianzhu/buildbot
 def __init__(self, name, controller):
     AbstractLatentWorker.__init__(self, name, None)
     self._controller = controller
コード例 #16
0
ファイル: test_latent.py プロジェクト: GuoJianzhu/buildbot
 def checkConfig(self, name, _):
     AbstractLatentWorker.checkConfig(self, name, None)
コード例 #17
0
ファイル: latent.py プロジェクト: vicosx12/buildbot
 def __init__(self, name, controller, **kwargs):
     AbstractLatentWorker.__init__(self, name, None, **kwargs)
     self._controller = controller
コード例 #18
0
ファイル: ec2.py プロジェクト: BeiNanWoo/buildbot
    def __init__(self, name, password, instance_type, ami=None,
                 valid_ami_owners=None, valid_ami_location_regex=None,
                 elastic_ip=None, identifier=None, secret_identifier=None,
                 aws_id_file_path=None, user_data=None, region=None,
                 keypair_name=None,
                 security_name=None,
                 spot_instance=False, max_spot_price=1.6, volumes=None,
                 placement=None, price_multiplier=1.2, tags=None, retry=1,
                 retry_price_adjustment=1, product_description='Linux/UNIX',
                 subnet_id=None, security_group_ids=None, instance_profile_name=None,
                 block_device_map=None,
                 **kwargs):

        if not boto:
            config.error("The python module 'boto' is needed to use a "
                         "EC2LatentWorker")

        if keypair_name is None:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'keypair_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            keypair_name = 'latent_buildbot_slave'
        if security_name is None and not subnet_id:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'security_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            security_name = 'latent_buildbot_slave'

        if volumes is None:
            volumes = []

        if tags is None:
            tags = {}

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        if security_name and subnet_id:
            raise ValueError(
                'security_name (EC2 classic security groups) is not supported '
                'in a VPC.  Use security_group_ids instead.')
        if not ((ami is not None) ^
                (valid_ami_owners is not None or
                 valid_ami_location_regex is not None)):
            raise ValueError(
                'You must provide either a specific ami, or one or both of '
                'valid_ami_location_regex and valid_ami_owners')
        self.ami = ami
        if valid_ami_owners is not None:
            if isinstance(valid_ami_owners, (int, long)):
                valid_ami_owners = (valid_ami_owners,)
            else:
                for element in valid_ami_owners:
                    if not isinstance(element, (int, long)):
                        raise ValueError(
                            'valid_ami_owners should be int or iterable '
                            'of ints', element)
        if valid_ami_location_regex is not None:
            if not isinstance(valid_ami_location_regex, basestring):
                raise ValueError(
                    'valid_ami_location_regex should be a string')
            else:
                # verify that regex will compile
                re.compile(valid_ami_location_regex)
        self.valid_ami_owners = valid_ami_owners
        self.valid_ami_location_regex = valid_ami_location_regex
        self.instance_type = instance_type
        self.keypair_name = keypair_name
        self.security_name = security_name
        self.user_data = user_data
        self.spot_instance = spot_instance
        self.max_spot_price = max_spot_price
        self.volumes = volumes
        self.price_multiplier = price_multiplier
        self.retry_price_adjustment = retry_price_adjustment
        self.retry = retry
        self.attempt = 1
        self.product_description = product_description

        if None not in [placement, region]:
            self.placement = '%s%s' % (region, placement)
        else:
            self.placement = None
        if identifier is None:
            assert secret_identifier is None, (
                'supply both or neither of identifier, secret_identifier')
            if aws_id_file_path is None:
                home = os.environ['HOME']
                default_path = os.path.join(home, '.ec2', 'aws_id')
                if os.path.exists(default_path):
                    aws_id_file_path = default_path
            if aws_id_file_path:
                log.msg('WARNING: EC2LatentWorker is using deprecated '
                        'aws_id file')
                with open(aws_id_file_path, 'r') as aws_file:
                    identifier = aws_file.readline().strip()
                    secret_identifier = aws_file.readline().strip()
        else:
            assert aws_id_file_path is None, \
                'if you supply the identifier and secret_identifier, ' \
                'do not specify the aws_id_file_path'
            assert secret_identifier is not None, \
                'supply both or neither of identifier, secret_identifier'

        region_found = None

        # Make the EC2 connection.
        if region is not None:
            for r in boto.ec2.regions(aws_access_key_id=identifier,
                                      aws_secret_access_key=secret_identifier):

                if r.name == region:
                    region_found = r

            if region_found is not None:
                self.ec2_conn = boto.ec2.connect_to_region(region,
                                                           aws_access_key_id=identifier,
                                                           aws_secret_access_key=secret_identifier)
            else:
                raise ValueError(
                    'The specified region does not exist: ' + region)

        else:
            self.ec2_conn = boto.connect_ec2(identifier, secret_identifier)

        # Make a keypair
        #
        # We currently discard the keypair data because we don't need it.
        # If we do need it in the future, we will always recreate the keypairs
        # because there is no way to
        # programmatically retrieve the private key component, unless we
        # generate it and store it on the filesystem, which is an unnecessary
        # usage requirement.
        try:
            key_pair = self.ec2_conn.get_all_key_pairs(keypair_name)[0]
            assert key_pair
            # key_pair.delete() # would be used to recreate
        except boto.exception.EC2ResponseError as e:
            if 'InvalidKeyPair.NotFound' not in e.body:
                if 'AuthFailure' in e.body:
                    log.msg('POSSIBLE CAUSES OF ERROR:\n'
                            '  Did you supply your AWS credentials?\n'
                            '  Did you sign up for EC2?\n'
                            '  Did you put a credit card number in your AWS '
                            'account?\n'
                            'Please doublecheck before reporting a problem.\n')
                raise
            # make one; we would always do this, and stash the result, if we
            # needed the key (for instance, to SSH to the box).  We'd then
            # use paramiko to use the key to connect.
            self.ec2_conn.create_key_pair(keypair_name)

        # create security group
        if security_name:
            try:
                group = self.ec2_conn.get_all_security_groups(security_name)[0]
                assert group
            except boto.exception.EC2ResponseError as e:
                if 'InvalidGroup.NotFound' in e.body:
                    self.security_group = self.ec2_conn.create_security_group(
                        security_name,
                        'Authorization to access the buildbot instance.')
                    # Authorize the master as necessary
                    # TODO this is where we'd open the hole to do the reverse pb
                    # connect to the buildbot
                    # ip = urllib.urlopen(
                    #     'http://checkip.amazonaws.com').read().strip()
                    # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
                    # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
                else:
                    raise

        # get the image
        if self.ami is not None:
            self.image = self.ec2_conn.get_image(self.ami)
        else:
            # verify we have access to at least one acceptable image
            discard = self.get_image()
            assert discard

        # get the specified elastic IP, if any
        if elastic_ip is not None:
            elastic_ip = self.ec2_conn.get_all_addresses([elastic_ip])[0]
        self.elastic_ip = elastic_ip
        self.subnet_id = subnet_id
        self.security_group_ids = security_group_ids
        self.classic_security_groups = [self.security_name] if self.security_name else None
        self.instance_profile_name = instance_profile_name
        self.tags = tags
        self.block_device_map = self.create_block_device_mapping(block_device_map)
コード例 #19
0
ファイル: latent.py プロジェクト: vicosx12/buildbot
 def checkConfig(self, name, _, **kwargs):
     AbstractLatentWorker.checkConfig(self, name, None, **kwargs)
コード例 #20
0
ファイル: ec2.py プロジェクト: peterwillcn/buildbot
    def __init__(self,
                 name,
                 password,
                 instance_type,
                 ami=None,
                 valid_ami_owners=None,
                 valid_ami_location_regex=None,
                 elastic_ip=None,
                 identifier=None,
                 secret_identifier=None,
                 aws_id_file_path=None,
                 user_data=None,
                 region=None,
                 keypair_name=None,
                 security_name=None,
                 spot_instance=False,
                 max_spot_price=1.6,
                 volumes=None,
                 placement=None,
                 price_multiplier=1.2,
                 tags=None,
                 retry=1,
                 retry_price_adjustment=1,
                 product_description='Linux/UNIX',
                 subnet_id=None,
                 security_group_ids=None,
                 instance_profile_name=None,
                 block_device_map=None,
                 **kwargs):

        if not boto:
            config.error("The python module 'boto' is needed to use a "
                         "EC2LatentWorker")

        if keypair_name is None:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'keypair_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            keypair_name = 'latent_buildbot_slave'
        if security_name is None and not subnet_id:
            reportDeprecatedWorkerNameUsage(
                "Use of default value of 'security_name' of EC2LatentWorker "
                "constructor is deprecated. Please explicitly specify value")
            security_name = 'latent_buildbot_slave'

        if volumes is None:
            volumes = []

        if tags is None:
            tags = {}

        AbstractLatentWorker.__init__(self, name, password, **kwargs)

        if security_name and subnet_id:
            raise ValueError(
                'security_name (EC2 classic security groups) is not supported '
                'in a VPC.  Use security_group_ids instead.')
        if not ((ami is not None) ^ (valid_ami_owners is not None
                                     or valid_ami_location_regex is not None)):
            raise ValueError(
                'You must provide either a specific ami, or one or both of '
                'valid_ami_location_regex and valid_ami_owners')
        self.ami = ami
        if valid_ami_owners is not None:
            if isinstance(valid_ami_owners, (int, long)):
                valid_ami_owners = (valid_ami_owners, )
            else:
                for element in valid_ami_owners:
                    if not isinstance(element, (int, long)):
                        raise ValueError(
                            'valid_ami_owners should be int or iterable '
                            'of ints', element)
        if valid_ami_location_regex is not None:
            if not isinstance(valid_ami_location_regex, basestring):
                raise ValueError('valid_ami_location_regex should be a string')
            else:
                # verify that regex will compile
                re.compile(valid_ami_location_regex)
        self.valid_ami_owners = valid_ami_owners
        self.valid_ami_location_regex = valid_ami_location_regex
        self.instance_type = instance_type
        self.keypair_name = keypair_name
        self.security_name = security_name
        self.user_data = user_data
        self.spot_instance = spot_instance
        self.max_spot_price = max_spot_price
        self.volumes = volumes
        self.price_multiplier = price_multiplier
        self.retry_price_adjustment = retry_price_adjustment
        self.retry = retry
        self.attempt = 1
        self.product_description = product_description

        if None not in [placement, region]:
            self.placement = '%s%s' % (region, placement)
        else:
            self.placement = None
        if identifier is None:
            assert secret_identifier is None, (
                'supply both or neither of identifier, secret_identifier')
            if aws_id_file_path is None:
                home = os.environ['HOME']
                default_path = os.path.join(home, '.ec2', 'aws_id')
                if os.path.exists(default_path):
                    aws_id_file_path = default_path
            if aws_id_file_path:
                log.msg('WARNING: EC2LatentWorker is using deprecated '
                        'aws_id file')
                with open(aws_id_file_path, 'r') as aws_file:
                    identifier = aws_file.readline().strip()
                    secret_identifier = aws_file.readline().strip()
        else:
            assert aws_id_file_path is None, \
                'if you supply the identifier and secret_identifier, ' \
                'do not specify the aws_id_file_path'
            assert secret_identifier is not None, \
                'supply both or neither of identifier, secret_identifier'

        region_found = None

        # Make the EC2 connection.
        if region is not None:
            for r in boto.ec2.regions(aws_access_key_id=identifier,
                                      aws_secret_access_key=secret_identifier):

                if r.name == region:
                    region_found = r

            if region_found is not None:
                self.ec2_conn = boto.ec2.connect_to_region(
                    region,
                    aws_access_key_id=identifier,
                    aws_secret_access_key=secret_identifier)
            else:
                raise ValueError('The specified region does not exist: ' +
                                 region)

        else:
            self.ec2_conn = boto.connect_ec2(identifier, secret_identifier)

        # Make a keypair
        #
        # We currently discard the keypair data because we don't need it.
        # If we do need it in the future, we will always recreate the keypairs
        # because there is no way to
        # programmatically retrieve the private key component, unless we
        # generate it and store it on the filesystem, which is an unnecessary
        # usage requirement.
        try:
            key_pair = self.ec2_conn.get_all_key_pairs(keypair_name)[0]
            assert key_pair
            # key_pair.delete() # would be used to recreate
        except boto.exception.EC2ResponseError as e:
            if 'InvalidKeyPair.NotFound' not in e.body:
                if 'AuthFailure' in e.body:
                    log.msg('POSSIBLE CAUSES OF ERROR:\n'
                            '  Did you supply your AWS credentials?\n'
                            '  Did you sign up for EC2?\n'
                            '  Did you put a credit card number in your AWS '
                            'account?\n'
                            'Please doublecheck before reporting a problem.\n')
                raise
            # make one; we would always do this, and stash the result, if we
            # needed the key (for instance, to SSH to the box).  We'd then
            # use paramiko to use the key to connect.
            self.ec2_conn.create_key_pair(keypair_name)

        # create security group
        if security_name:
            try:
                group = self.ec2_conn.get_all_security_groups(security_name)[0]
                assert group
            except boto.exception.EC2ResponseError as e:
                if 'InvalidGroup.NotFound' in e.body:
                    self.security_group = self.ec2_conn.create_security_group(
                        security_name,
                        'Authorization to access the buildbot instance.')
                    # Authorize the master as necessary
                    # TODO this is where we'd open the hole to do the reverse pb
                    # connect to the buildbot
                    # ip = urllib.urlopen(
                    #     'http://checkip.amazonaws.com').read().strip()
                    # self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
                    # self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
                else:
                    raise

        # get the image
        if self.ami is not None:
            self.image = self.ec2_conn.get_image(self.ami)
        else:
            # verify we have access to at least one acceptable image
            discard = self.get_image()
            assert discard

        # get the specified elastic IP, if any
        if elastic_ip is not None:
            elastic_ip = self.ec2_conn.get_all_addresses([elastic_ip])[0]
        self.elastic_ip = elastic_ip
        self.subnet_id = subnet_id
        self.security_group_ids = security_group_ids
        self.classic_security_groups = [self.security_name
                                        ] if self.security_name else None
        self.instance_profile_name = instance_profile_name
        self.tags = tags
        self.block_device_map = self.create_block_device_mapping(
            block_device_map)
コード例 #21
0
 def __init__(self, name, controller):
     AbstractLatentWorker.__init__(self, name, None)
     self._controller = controller