def __init__(self, name, password,
flavor,
image,
os_username,
os_password,
os_tenant_name,
os_auth_url,
meta=None,
max_builds=None, notify_on_missing=[], missing_timeout=60*20,
build_wait_timeout=60*10, properties={}, locks=None):
if not client or not nce:
config.error("The python module 'novaclient' is needed "
"to use a OpenStackLatentBuildSlave")
AbstractLatentBuildSlave.__init__(
self, name, password, max_builds, notify_on_missing,
missing_timeout, build_wait_timeout, properties, locks)
self.flavor = flavor
self.image = image
self.os_username = os_username
self.os_password = os_password
self.os_tenant_name = os_tenant_name
self.os_auth_url = os_auth_url
self.meta = meta
def __init__(self,
name,
password,
connection,
hd_image,
base_image=None,
xml=None,
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={},
locks=None):
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing, missing_timeout,
build_wait_timeout, properties,
locks)
if not libvirt:
config.error(
"The python module 'libvirt' is needed to use a LibVirtSlave")
self.name = name
self.connection = connection
self.image = hd_image
self.base_image = base_image
self.xml = xml
self.cheap_copy = True
self.graceful_shutdown = False
self.domain = None
self.ready = False
self._find_existing_deferred = self._find_existing_instance()
def __init__(
self,
name,
password,
connection,
hd_image,
base_image=None,
xml=None,
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={},
locks=None,
):
AbstractLatentBuildSlave.__init__(
self, name, password, max_builds, notify_on_missing, missing_timeout, build_wait_timeout, properties, locks
)
if not libvirt:
config.error("The python module 'libvirt' is needed to use a LibVirtSlave")
self.name = name
self.connection = connection
self.image = hd_image
self.base_image = base_image
self.xml = xml
self.cheap_copy = True
self.graceful_shutdown = False
self.domain = None
self.ready = False
self._find_existing_deferred = self._find_existing_instance()
def __init__(self,
name,
password,
docker_host,
image=None,
command=None,
max_builds=None,
notify_on_missing=None,
missing_timeout=(60 * 20),
build_wait_timeout=(60 * 10),
properties={},
locks=None,
volumes=None,
dockerfile=None):
if not client:
config.error("The python module 'docker-py' is needed "
"to use a DockerLatentBuildSlave")
if not image:
config.error("DockerLatentBuildSlave: You need to specify an"
" image name")
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing or [],
missing_timeout, build_wait_timeout,
properties, locks)
self.docker_host = docker_host
self.image = image
self.command = command or []
self.volumes = volumes or []
self.dockerfile = dockerfile
def __init__(self,
name,
password,
connection,
hd_image,
base_image=None,
xml=None,
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={},
locks=None):
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing, missing_timeout,
build_wait_timeout, properties,
locks)
self.name = name
self.connection = connection
self.image = hd_image
self.base_image = base_image
self.xml = xml
self.insubstantiate_after_build = True
self.cheap_copy = True
self.graceful_shutdown = False
self.domain = None
def buildFinished(self, *args, **kwargs):
"""
I insubstantiate a slave after it has done a build, if that is
desired behaviour.
"""
AbstractLatentBuildSlave.buildFinished(self, *args, **kwargs)
if self.insubstantiate_after_build:
log.msg("Got buildFinished notification - attempting to insubstantiate")
self.insubstantiate()
def __init__(self,
name,
password,
docker_host,
image=None,
command=None,
volumes=None,
dockerfile=None,
version=None,
tls=None,
followStartupLogs=False,
masterFQDN=None,
**kwargs):
if not client:
config.error("The python module 'docker-py' is needed to use a"
" DockerLatentBuildSlave")
if not image and not dockerfile:
config.error("DockerLatentBuildSlave: You need to specify at least"
" an image name, or a dockerfile")
self.volumes = []
self.binds = {}
self.followStartupLogs = followStartupLogs
for volume_string in (volumes or []):
try:
volume, bind = volume_string.split(":", 1)
except ValueError:
config.error("Invalid volume definition for docker "
"%s. Skipping..." % volume_string)
self.volumes.append(volume)
ro = False
if bind.endswith(':ro') or bind.endswith(':rw'):
ro = bind[-2:] == 'ro'
bind = bind[:-3]
self.binds[volume] = {'bind': bind, 'ro': ro}
# Set build_wait_timeout to 0 if not explicitely set: Starting a
# container is almost immediate, we can affort doing so for each build.
if 'build_wait_timeout' not in kwargs:
kwargs['build_wait_timeout'] = 0
AbstractLatentBuildSlave.__init__(self, name, password, **kwargs)
self.image = image
self.command = command or []
self.dockerfile = dockerfile
if masterFQDN is None:
masterFQDN = socket.getfqdn()
self.masterFQDN = masterFQDN
# Prepare the parameters for the Docker Client object.
self.client_args = {'base_url': docker_host}
if version is not None:
self.client_args['version'] = version
if tls is not None:
self.client_args['tls'] = tls
def __init__(self, name, password, cloudservers_username,
cloudservers_apikey, image, flavor=1, files=None, **kwargs):
AbstractLatentBuildSlave.__init__(self, name, password, **kwargs)
self.conn = cloudservers.CloudServers(cloudservers_username, cloudservers_apikey)
self.conn.client = RetryingCloudServersClient(cloudservers_username, cloudservers_apikey)
self.image = image
self.flavor = flavor
self.files = files
self.instance = None
def buildFinished(self, *args, **kwargs):
"""
I insubstantiate a slave after it has done a build, if that is
desired behaviour.
"""
AbstractLatentBuildSlave.buildFinished(self, *args, **kwargs)
if self.insubstantiate_after_build:
log.msg(
"Got buildFinished notification - attempting to insubstantiate"
)
self.insubstantiate()
def __init__(self, name, password, connection, hd_image, base_image = None, xml=None, max_builds=None, notify_on_missing=[],
missing_timeout=60*20, build_wait_timeout=60*10, properties={}, locks=None):
AbstractLatentBuildSlave.__init__(self, name, password, max_builds, notify_on_missing,
missing_timeout, build_wait_timeout, properties, locks)
self.name = name
self.connection = connection
self.image = hd_image
self.base_image = base_image
self.xml = xml
self.insubstantiate_after_build = True
self.cheap_copy = True
self.graceful_shutdown = False
self.domain = None
def attached(self, bot):
if self.substantiation_deferred is None:
msg = 'Slave %s received connection while not trying to ' \
'substantiate. Ignoring.' % (self.slavename,)
log.msg(msg)
return defer.succeed(None)
return AbstractLatentBuildSlave.attached(self, bot)
def __init__(self, name, password, docker_host, image=None, command=None,
volumes=None, dockerfile=None, version=None, tls=None,
followStartupLogs=False,
create_container_args=None,
**kwargs):
if not client:
config.error("The python module 'docker-py' is needed to use a"
" DockerLatentBuildSlave")
if not image and not dockerfile:
config.error("DockerLatentBuildSlave: You need to specify at least"
" an image name, or a dockerfile")
self.volumes = []
self.binds = {}
self.followStartupLogs = followStartupLogs
for volume_string in (volumes or []):
try:
volume, bind = volume_string.split(":", 1)
except ValueError:
config.error("Invalid volume definition for docker "
"%s. Skipping..." % volume_string)
self.volumes.append(volume)
ro = False
if bind.endswith(':ro') or bind.endswith(':rw'):
ro = bind[-2:] == 'ro'
bind = bind[:-3]
self.binds[volume] = {'bind': bind, 'ro': ro}
# Set build_wait_timeout to 0 if not explicitely set: Starting a
# container is almost immediate, we can affort doing so for each build.
if 'build_wait_timeout' not in kwargs:
kwargs['build_wait_timeout'] = 0
AbstractLatentBuildSlave.__init__(self, name, password, **kwargs)
self.image = image
self.command = command or []
self.dockerfile = dockerfile
# Prepare the parameters for the Docker Client object.
self.client_args = {'base_url': docker_host}
if version is not None:
self.client_args['version'] = version
if tls is not None:
self.client_args['tls'] = tls
self.create_container_args = create_container_args or {}
def __init__(self,
name,
password,
start_script,
stop_script,
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={},
locks=None):
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing, missing_timeout,
build_wait_timeout, properties,
locks)
self.name = name
self.start_script = shlex.split(start_script)
self.stop_script = shlex.split(stop_script)
def canStartBuild(self):
if not self.ready:
log.msg("Not accepting builds as existing domains not iterated")
return False
if self.domain and not self.isConnected():
log.msg("Not accepting builds as existing domain but slave not connected")
return False
return AbstractLatentBuildSlave.canStartBuild(self)
def __init__(self, name, password,
docker_host,
image,
command,
env={},
max_builds=None, notify_on_missing=[], missing_timeout=60*20,
build_wait_timeout=60*10, properties={}, locks=None):
if not client:
config.error("The python module 'docker-py' is needed "
"to use a DockerLatentBuildSlave")
AbstractLatentBuildSlave.__init__(
self, name, password, max_builds, notify_on_missing,
missing_timeout, build_wait_timeout, properties, locks)
self.docker_host = docker_host
self.image = image
self.command = command
self.env = env
def canStartBuild(self):
if not self.ready:
log.msg("Not accepting builds as existing domains not iterated")
return False
if self.domain and not self.isConnected():
log.msg("Not accepting builds as existing domain but slave not connected")
return False
return AbstractLatentBuildSlave.canStartBuild(self)
def __init__(self, name, password, docker_host, image=None, command=None,
max_builds=None, notify_on_missing=None,
missing_timeout=(60 * 20), build_wait_timeout=(60 * 10),
properties={}, locks=None, volumes=None, dockerfile=None):
if not client:
config.error("The python module 'docker-py' is needed "
"to use a DockerLatentBuildSlave")
if not image:
config.error("DockerLatentBuildSlave: You need to specify an"
" image name")
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing or [],
missing_timeout, build_wait_timeout,
properties, locks)
self.docker_host = docker_host
self.image = image
self.command = command or []
self.volumes = volumes or []
self.dockerfile = dockerfile
def __init__(self, name, password, instance_type, ami=None,
valid_ami_owners=None, valid_ami_location_regex=None,
elastic_ip=None, identifier=None, secret_identifier=None,
aws_id_file_path=None, user_data=None,
keypair_name='latent_buildbot_slave',
security_name='latent_buildbot_slave',
max_builds=None, notify_on_missing=[], missing_timeout=60*20,
build_wait_timeout=60*10, properties={}):
AbstractLatentBuildSlave.__init__(
self, name, password, max_builds, notify_on_missing,
missing_timeout, build_wait_timeout, properties)
if not ((ami is not None) ^
(valid_ami_owners is not None or
valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, (int, long)):
valid_ami_owners = (valid_ami_owners,)
else:
for element in valid_ami_owners:
if not isinstance(element, (int, long)):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, basestring):
raise ValueError(
'valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
self.valid_ami_owners = valid_ami_owners
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
aws_id_file_path = os.path.join(home, '.ec2', 'aws_id')
if not os.path.exists(aws_id_file_path):
raise ValueError(
"Please supply your AWS access key identifier and secret "
"access key identifier either when instantiating this %s "
"or in the %s file (on two lines).\n" %
(self.__class__.__name__, aws_id_file_path))
aws_file = open(aws_id_file_path, 'r')
try:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
finally:
aws_file.close()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
# Make the EC2 connection.
self.conn = boto.connect_ec2(identifier, secret_identifier)
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.conn.get_all_key_pairs(keypair_name)[0]
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError, e:
if e.code != 'InvalidKeyPair.NotFound':
if e.code == 'AuthFailure':
print ('POSSIBLE CAUSES OF ERROR:\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.conn.create_key_pair(keypair_name)
def __init__(self,
name,
password,
instance_type,
ami=None,
valid_ami_owners=None,
valid_ami_location_regex=None,
elastic_ip=None,
identifier=None,
secret_identifier=None,
aws_id_file_path=None,
user_data=None,
keypair_name='latent_buildbot_slave',
security_name='latent_buildbot_slave',
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={}):
AbstractLatentBuildSlave.__init__(self, name, password, max_builds,
notify_on_missing, missing_timeout,
build_wait_timeout, properties)
if not ((ami is not None) ^ (valid_ami_owners is not None
or valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, (int, long)):
valid_ami_owners = (valid_ami_owners, )
else:
for element in valid_ami_owners:
if not isinstance(element, (int, long)):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, basestring):
raise ValueError('valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
self.valid_ami_owners = valid_ami_owners
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
aws_id_file_path = os.path.join(home, '.ec2', 'aws_id')
if not os.path.exists(aws_id_file_path):
raise ValueError(
"Please supply your AWS access key identifier and secret "
"access key identifier either when instantiating this %s "
"or in the %s file (on two lines).\n" %
(self.__class__.__name__, aws_id_file_path))
aws_file = open(aws_id_file_path, 'r')
try:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
finally:
aws_file.close()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
# Make the EC2 connection.
self.conn = boto.connect_ec2(identifier, secret_identifier)
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.conn.get_all_key_pairs(keypair_name)[0]
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError, e:
if e.code != 'InvalidKeyPair.NotFound':
if e.code == 'AuthFailure':
print('POSSIBLE CAUSES OF ERROR:\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.conn.create_key_pair(keypair_name)
def __init__(
self,
name,
password,
instance_type,
ami=None,
valid_ami_owners=None,
valid_ami_location_regex=None,
elastic_ip=None,
identifier=None,
secret_identifier=None,
aws_id_file_path=None,
user_data=None,
region=None,
keypair_name="latent_buildbot_slave",
security_name="latent_buildbot_slave",
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
build_wait_timeout=60 * 10,
properties={},
locks=None,
):
AbstractLatentBuildSlave.__init__(
self, name, password, max_builds, notify_on_missing, missing_timeout, build_wait_timeout, properties, locks
)
if not ((ami is not None) ^ (valid_ami_owners is not None or valid_ami_location_regex is not None)):
raise ValueError(
"You must provide either a specific ami, or one or both of "
"valid_ami_location_regex and valid_ami_owners"
)
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, (int, long)):
valid_ami_owners = (valid_ami_owners,)
else:
for element in valid_ami_owners:
if not isinstance(element, (int, long)):
raise ValueError("valid_ami_owners should be int or iterable " "of ints", element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, basestring):
raise ValueError("valid_ami_location_regex should be a string")
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
self.valid_ami_owners = valid_ami_owners
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
if identifier is None:
assert secret_identifier is None, "supply both or neither of identifier, secret_identifier"
if aws_id_file_path is None:
home = os.environ["HOME"]
aws_id_file_path = os.path.join(home, ".ec2", "aws_id")
if not os.path.exists(aws_id_file_path):
raise ValueError(
"Please supply your AWS access key identifier and secret "
"access key identifier either when instantiating this %s "
"or in the %s file (on two lines).\n" % (self.__class__.__name__, aws_id_file_path)
)
aws_file = open(aws_id_file_path, "r")
try:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
finally:
aws_file.close()
else:
assert aws_id_file_path is None, (
"if you supply the identifier and secret_identifier, " "do not specify the aws_id_file_path"
)
assert secret_identifier is not None, "supply both or neither of identifier, secret_identifier"
region_found = None
# Make the EC2 connection.
if region is not None:
for r in boto.ec2.regions(aws_access_key_id=identifier, aws_secret_access_key=secret_identifier):
if r.name == region:
region_found = r
if region_found is not None:
self.conn = boto.ec2.connect_to_region(
region, aws_access_key_id=identifier, aws_secret_access_key=secret_identifier
)
else:
raise ValueError("The specified region does not exist: {0}".format(region))
else:
self.conn = boto.connect_ec2(identifier, secret_identifier)
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.conn.get_all_key_pairs(keypair_name)[0]
assert key_pair
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError, e:
if "InvalidKeyPair.NotFound" not in e.body:
if "AuthFailure" in e.body:
print(
"POSSIBLE CAUSES OF ERROR:\n"
" Did you sign up for EC2?\n"
" Did you put a credit card number in your AWS "
"account?\n"
"Please doublecheck before reporting a problem.\n"
)
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.conn.create_key_pair(keypair_name)
