def load_workers(self, filename, config_dict):
config_valid = True
deprecated_workers = config_dict.get('slaves')
if deprecated_workers is not None:
reportDeprecatedWorkerNameUsage(
"c['slaves'] key is deprecated, use c['workers'] instead",
filename=filename)
if not self._check_workers(deprecated_workers, "c['slaves']"):
config_valid = False
workers = config_dict.get('workers')
if workers is not None:
if not self._check_workers(workers, "c['workers']"):
config_valid = False
if deprecated_workers is not None and workers is not None:
error("Use of c['workers'] and c['slaves'] at the same time is "
"not supported. Use only c['workers'] instead")
return
if not config_valid:
return
elif deprecated_workers is not None or workers is not None:
self.workers = []
if deprecated_workers is not None:
self.workers.extend(deprecated_workers)
if workers is not None:
self.workers.extend(workers)
else:
pass
def __init__(self, s, workerdest=None,
workdir=None, maxsize=None, blocksize=16 * 1024, mode=None,
slavedest=None, # deprecated, use `workerdest` instead
**buildstep_kwargs):
# Deprecated API support.
if slavedest is not None:
reportDeprecatedWorkerNameUsage(
"'slavedest' keyword argument is deprecated, "
"use 'workerdest' instead")
assert workerdest is None
workerdest = slavedest
# Emulate that first two arguments are positional.
if workerdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.s = s
self.workerdest = workerdest
self._registerOldWorkerAttr("workerdest")
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
config.error(
"StringDownload step's mode must be an integer or None,"
" got '%s'" % mode)
self.mode = mode
def __init__(
self,
name,
maxCount=1,
maxCountForWorker=None,
# deprecated, use `maxCountForWorker` instead
maxCountForSlave=None):
# Deprecated API support.
if maxCountForSlave is not None:
reportDeprecatedWorkerNameUsage(
"'maxCountForSlave' keyword argument is deprecated, "
"use 'maxCountForWorker' instead")
assert maxCountForWorker is None
maxCountForWorker = maxCountForSlave
self.name = name
self.maxCount = maxCount
if maxCountForWorker is None:
maxCountForWorker = {}
self.maxCountForWorker = maxCountForWorker
self._registerOldWorkerAttr("maxCountForWorker")
# for comparison purposes, turn this dictionary into a stably-sorted
# list of tuples
self._maxCountForWorkerList = tuple(
sorted(self.maxCountForWorker.items()))
def __init__(self, workersrc=None, masterdest=None,
workdir=None, maxsize=None, blocksize=16 * 1024,
compress=None, url=None,
slavesrc=None, # deprecated, use `workersrc` instead
**buildstep_kwargs
):
# Deprecated API support.
if slavesrc is not None:
reportDeprecatedWorkerNameUsage(
"'slavesrc' keyword argument is deprecated, "
"use 'workersrc' instead")
assert workersrc is None
workersrc = slavesrc
# Emulate that first two arguments are positional.
if workersrc is None or masterdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.workersrc = workersrc
self._registerOldWorkerAttr("workersrc")
self.masterdest = masterdest
self.maxsize = maxsize
self.blocksize = blocksize
if compress not in (None, 'gz', 'bz2'):
config.error(
"'compress' must be one of None, 'gz', or 'bz2'")
self.compress = compress
self.url = url
def __init__(self, workersrc=None, masterdest=None,
workdir=None, maxsize=None, blocksize=16 * 1024, mode=None,
keepstamp=False, url=None,
slavesrc=None, # deprecated, use `workersrc` instead
**buildstep_kwargs):
# Deprecated API support.
if slavesrc is not None:
reportDeprecatedWorkerNameUsage(
"'slavesrc' keyword argument is deprecated, "
"use 'workersrc' instead")
assert workersrc is None
workersrc = slavesrc
# Emulate that first two arguments are positional.
if workersrc is None or masterdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.workersrc = workersrc
self._registerOldWorkerAttr("workersrc")
self.masterdest = masterdest
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
config.error(
'mode must be an integer or None')
self.mode = mode
self.keepstamp = keepstamp
self.url = url
def __init__(
self,
s,
workerdest=None,
workdir=None,
maxsize=None,
blocksize=16 * 1024,
mode=None,
slavedest=None, # deprecated, use `workerdest` instead
**buildstep_kwargs):
# Deprecated API support.
if slavedest is not None:
reportDeprecatedWorkerNameUsage(
"'slavedest' keyword argument is deprecated, "
"use 'workerdest' instead")
assert workerdest is None
workerdest = slavedest
# Emulate that first two arguments are positional.
if workerdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.s = s
self.workerdest = workerdest
self._registerOldWorkerAttr("workerdest")
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
config.error(
"StringDownload step's mode must be an integer or None,"
" got '%s'" % mode)
self.mode = mode
def __init__(self, workersrcs=None, masterdest=None,
workdir=None, maxsize=None, blocksize=16 * 1024, glob=False,
mode=None, compress=None, keepstamp=False, url=None,
slavesrcs=None, # deprecated, use `workersrcs` instead
**buildstep_kwargs):
# Deprecated API support.
if slavesrcs is not None:
reportDeprecatedWorkerNameUsage(
"'slavesrcs' keyword argument is deprecated, "
"use 'workersrcs' instead")
assert workersrcs is None
workersrcs = slavesrcs
# Emulate that first two arguments are positional.
if workersrcs is None or masterdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.workersrcs = workersrcs if isinstance(workersrcs, list) else [workersrcs]
self._registerOldWorkerAttr("workersrcs")
self.masterdest = masterdest
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
config.error(
'mode must be an integer or None')
self.mode = mode
if compress not in (None, 'gz', 'bz2'):
config.error(
"'compress' must be one of None, 'gz', or 'bz2'")
self.compress = compress
self.glob = glob
self.keepstamp = keepstamp
self.url = url
def load_workers(self, filename, config_dict):
config_valid = True
deprecated_workers = config_dict.get('slaves')
if deprecated_workers is not None:
reportDeprecatedWorkerNameUsage(
"c['slaves'] key is deprecated, use c['workers'] instead",
filename=filename)
if not self._check_workers(deprecated_workers, "c['slaves']"):
config_valid = False
workers = config_dict.get('workers')
if workers is not None:
if not self._check_workers(workers, "c['workers']"):
config_valid = False
if deprecated_workers is not None and workers is not None:
error("Use of c['workers'] and c['slaves'] at the same time is "
"not supported. Use only c['workers'] instead")
return
if not config_valid:
return
elif deprecated_workers is not None or workers is not None:
self.workers = []
if deprecated_workers is not None:
self.workers.extend(deprecated_workers)
if workers is not None:
self.workers.extend(workers)
else:
pass
def __init__(self, workersrc=None, masterdest=None,
workdir=None, maxsize=None, blocksize=16 * 1024, mode=None,
keepstamp=False, url=None,
slavesrc=None, # deprecated, use `workersrc` instead
**buildstep_kwargs):
# Deprecated API support.
if slavesrc is not None:
reportDeprecatedWorkerNameUsage(
"'slavesrc' keyword argument is deprecated, "
"use 'workersrc' instead")
assert workersrc is None
workersrc = slavesrc
# Emulate that first two arguments are positional.
if workersrc is None or masterdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.workersrc = workersrc
self._registerOldWorkerAttr("workersrc")
self.masterdest = masterdest
self.maxsize = maxsize
self.blocksize = blocksize
if not isinstance(mode, (int, type(None))):
config.error(
'mode must be an integer or None')
self.mode = mode
self.keepstamp = keepstamp
self.url = url
def __getattr__(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you accessed 'buildslave.{0}' name).".format(name))
return _Plugins.__getattr__(self, name)
def __init__(
self,
o,
workerdest=None,
slavedest=None, # deprecated, use `workerdest` instead
**buildstep_kwargs):
# Deprecated API support.
if slavedest is not None:
reportDeprecatedWorkerNameUsage(
"'slavedest' keyword argument is deprecated, "
"use 'workerdest' instead")
assert workerdest is None
workerdest = slavedest
# Emulate that first two arguments are positional.
if workerdest is None:
raise TypeError("__init__() takes at least 3 arguments")
if 's' in buildstep_kwargs:
del buildstep_kwargs['s']
s = json.dumps(o)
StringDownload.__init__(self,
s=s,
workerdest=workerdest,
**buildstep_kwargs)
def __getattr__(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you accessed 'buildslave.{0}' name).".format(name))
return _Plugins.__getattr__(self, name)
def __init__(
self,
workersrc=None,
masterdest=None,
workdir=None,
maxsize=None,
blocksize=16 * 1024,
compress=None,
url=None,
slavesrc=None, # deprecated, use `workersrc` instead
**buildstep_kwargs):
# Deprecated API support.
if slavesrc is not None:
reportDeprecatedWorkerNameUsage(
"'slavesrc' keyword argument is deprecated, "
"use 'workersrc' instead")
assert workersrc is None
workersrc = slavesrc
# Emulate that first two arguments are positional.
if workersrc is None or masterdest is None:
raise TypeError("__init__() takes at least 3 arguments")
_TransferBuildStep.__init__(self, workdir=workdir, **buildstep_kwargs)
self.workersrc = workersrc
self._registerOldWorkerAttr("workersrc")
self.masterdest = masterdest
self.maxsize = maxsize
self.blocksize = blocksize
if compress not in (None, 'gz', 'bz2'):
config.error("'compress' must be one of None, 'gz', or 'bz2'")
self.compress = compress
self.url = url
def get(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you requested '{0}' name of "
"'buildbot.plugins.buildslave' plugin).".format(name))
return _Plugins.get(self, name)
def __contains__(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you checked is '{0}' name inside "
"'buildbot.plugins.buildslave').".format(name))
return _Plugins.__contains__(self, name)
def value(self):
reportDeprecatedWorkerNameUsage(
"'{group}.{compat_name}' is deprecated, "
"use '{group}.{new_name}' instead".format(
group=self.group,
compat_name=self._compat_name,
new_name=self._new_name))
return self._plugin_entry.value
def value(self):
reportDeprecatedWorkerNameUsage(
"'{group}.{compat_name}' is deprecated, "
"use '{group}.{new_name}' instead".format(
group=self.group,
compat_name=self._compat_name,
new_name=self._new_name))
return self._plugin_entry.value
def __contains__(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you checked is '{0}' name inside "
"'buildbot.plugins.buildslave').".format(name))
return _Plugins.__contains__(self, name)
def get(self, name):
reportDeprecatedWorkerNameUsage(
"'buildbot.plugins.buildslave' plugins namespace is deprecated, "
"use 'buildbot.plugins.worker' instead "
"(you requested '{0}' name of "
"'buildbot.plugins.buildslave' plugin).".format(name))
return _Plugins.get(self, name)
def __init__(self, workdir, command, env=None,
want_stdout=1, want_stderr=1,
timeout=20 * 60, maxTime=None, sigtermTime=None,
logfiles=None, usePTY=None, logEnviron=True,
collectStdout=False, collectStderr=False,
interruptSignal=None,
initialStdin=None, decodeRC=None,
stdioLogName='stdio'):
if logfiles is None:
logfiles = {}
if decodeRC is None:
decodeRC = {0: SUCCESS}
self.command = command # stash .command, set it later
if isinstance(self.command, string_types):
# Single string command doesn't support obfuscation.
self.fake_command = command
else:
# Try to obfuscate command.
def obfuscate(arg):
if isinstance(arg, tuple) and len(arg) == 3 and arg[0] == 'obfuscated':
return arg[2]
else:
return arg
self.fake_command = [obfuscate(c) for c in self.command]
if env is not None:
# avoid mutating the original master.cfg dictionary. Each
# ShellCommand gets its own copy, any start() methods won't be
# able to modify the original.
env = env.copy()
if usePTY == 'slave-config':
reportDeprecatedWorkerNameUsage(
"'slave-config' value of 'usePTY' attribute is deprecated, "
"use None instead.")
usePTY = None
args = {'workdir': workdir,
'env': env,
'want_stdout': want_stdout,
'want_stderr': want_stderr,
'logfiles': logfiles,
'timeout': timeout,
'maxTime': maxTime,
'sigtermTime': sigtermTime,
'usePTY': usePTY,
'logEnviron': logEnviron,
'initial_stdin': initialStdin
}
if interruptSignal is not None:
args['interruptSignal'] = interruptSignal
RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout,
collectStderr=collectStderr,
decodeRC=decodeRC,
stdioLogName=stdioLogName)
def create_block_device_mapping(self, mapping_definitions):
if isinstance(mapping_definitions, list):
for mapping_definition in mapping_definitions:
ebs = mapping_definition.get('Ebs')
if ebs:
ebs.setdefault('DeleteOnTermination', True)
return mapping_definitions
reportDeprecatedWorkerNameUsage(
"Use of dict value to 'block_device_map' of EC2LatentWorker "
"constructor is deprecated. Please use a list matching the AWS API "
"https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html"
)
return self._convert_deprecated_block_device_mapping(mapping_definitions)
def create_block_device_mapping(self, mapping_definitions):
if isinstance(mapping_definitions, list):
for mapping_definition in mapping_definitions:
ebs = mapping_definition.get('Ebs')
if ebs:
ebs.setdefault('DeleteOnTermination', True)
return mapping_definitions
reportDeprecatedWorkerNameUsage(
"Use of dict value to 'block_device_map' of EC2LatentWorker "
"constructor is deprecated. Please use a list matching the AWS API "
"https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html"
)
return self._convert_deprecated_block_device_mapping(mapping_definitions)
def _on_property_usage(prop_name, stacklevel):
"""Handle deprecated properties after worker-name transition.
:param stacklevel: stack level relative to the caller's frame.
Defaults to caller of the caller of this function.
"""
# "Remove" current frame
stacklevel += 1
deprecated_to_new_props = {"slavename": "workername"}
if prop_name in deprecated_to_new_props:
reportDeprecatedWorkerNameUsage(
"Property '{old_name}' is deprecated, "
"use '{new_name}' instead.".format(old_name=prop_name, new_name=deprecated_to_new_props[prop_name]),
stacklevel=stacklevel,
)
def _on_property_usage(prop_name, stacklevel):
"""Handle deprecated properties after worker-name transition.
:param stacklevel: stack level relative to the caller's frame.
Defaults to caller of the caller of this function.
"""
# "Remove" current frame
stacklevel += 1
deprecated_to_new_props = {'slavename': 'workername'}
if prop_name in deprecated_to_new_props:
reportDeprecatedWorkerNameUsage(
"Property '{old_name}' is deprecated, "
"use '{new_name}' instead.".format(
old_name=prop_name,
new_name=deprecated_to_new_props[prop_name]),
stacklevel=stacklevel)
def __init__(self, workerdest=None,
slavedest=None, # deprecated, use `workerdest` instead
**buildstep_kwargs):
# Deprecated API support.
if slavedest is not None:
reportDeprecatedWorkerNameUsage(
"'slavedest' keyword argument is deprecated, "
"use 'workerdest' instead")
assert workerdest is None
workerdest = slavedest
# Emulate that first two arguments are positional.
if workerdest is None:
raise TypeError("__init__() takes at least 2 arguments")
self.super_class = StringDownload
if 's' in buildstep_kwargs:
del buildstep_kwargs['s']
StringDownload.__init__(self, s=None, workerdest=workerdest, **buildstep_kwargs)
def __init__(self, name, maxCount=1, maxCountForWorker=None,
maxCountForSlave=None # deprecated, use `maxCountForWorker` instead
):
# Deprecated API support.
if maxCountForSlave is not None:
reportDeprecatedWorkerNameUsage(
"'maxCountForSlave' keyword argument is deprecated, "
"use 'maxCountForWorker' instead")
assert maxCountForWorker is None
maxCountForWorker = maxCountForSlave
self.name = name
self.maxCount = maxCount
if maxCountForWorker is None:
maxCountForWorker = {}
self.maxCountForWorker = maxCountForWorker
self._registerOldWorkerAttr("maxCountForWorker")
# for comparison purposes, turn this dictionary into a stably-sorted
# list of tuples
self._maxCountForWorkerList = tuple(sorted(self.maxCountForWorker.items()))
def __init__(self, name, password, instance_type, ami=None,
valid_ami_owners=None, valid_ami_location_regex=None,
elastic_ip=None, identifier=None, secret_identifier=None,
aws_id_file_path=None, user_data=None, region=None,
keypair_name=None,
security_name=None,
spot_instance=False, max_spot_price=1.6, volumes=None,
placement=None, price_multiplier=1.2, tags=None, retry=1,
retry_price_adjustment=1, product_description='Linux/UNIX',
subnet_id=None, security_group_ids=None, instance_profile_name=None,
block_device_map=None,
**kwargs):
if not boto:
config.error("The python module 'boto' is needed to use a "
"EC2LatentWorker")
if keypair_name is None:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'keypair_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
keypair_name = 'latent_buildbot_slave'
if security_name is None and not subnet_id:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'security_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
security_name = 'latent_buildbot_slave'
if volumes is None:
volumes = []
if tags is None:
tags = {}
AbstractLatentWorker.__init__(self, name, password, **kwargs)
if security_name and subnet_id:
raise ValueError(
'security_name (EC2 classic security groups) is not supported '
'in a VPC. Use security_group_ids instead.')
if not ((ami is not None) ^
(valid_ami_owners is not None or
valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, (int, long)):
valid_ami_owners = (valid_ami_owners,)
else:
for element in valid_ami_owners:
if not isinstance(element, (int, long)):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, basestring):
raise ValueError(
'valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
self.valid_ami_owners = valid_ami_owners
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
self.spot_instance = spot_instance
self.max_spot_price = max_spot_price
self.volumes = volumes
self.price_multiplier = price_multiplier
self.retry_price_adjustment = retry_price_adjustment
self.retry = retry
self.attempt = 1
self.product_description = product_description
if None not in [placement, region]:
self.placement = '%s%s' % (region, placement)
else:
self.placement = None
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
default_path = os.path.join(home, '.ec2', 'aws_id')
if os.path.exists(default_path):
aws_id_file_path = default_path
if aws_id_file_path:
log.msg('WARNING: EC2LatentWorker is using deprecated '
'aws_id file')
with open(aws_id_file_path, 'r') as aws_file:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
region_found = None
# Make the EC2 connection.
if region is not None:
for r in boto.ec2.regions(aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier):
if r.name == region:
region_found = r
if region_found is not None:
self.ec2_conn = boto.ec2.connect_to_region(region,
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier)
else:
raise ValueError(
'The specified region does not exist: ' + region)
else:
self.ec2_conn = boto.connect_ec2(identifier, secret_identifier)
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.ec2_conn.get_all_key_pairs(keypair_name)[0]
assert key_pair
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError as e:
if 'InvalidKeyPair.NotFound' not in e.body:
if 'AuthFailure' in e.body:
log.msg('POSSIBLE CAUSES OF ERROR:\n'
' Did you supply your AWS credentials?\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.ec2_conn.create_key_pair(keypair_name)
# create security group
if security_name:
try:
group = self.ec2_conn.get_all_security_groups(security_name)[0]
assert group
except boto.exception.EC2ResponseError as e:
if 'InvalidGroup.NotFound' in e.body:
self.security_group = self.ec2_conn.create_security_group(
security_name,
'Authorization to access the buildbot instance.')
# Authorize the master as necessary
# TODO this is where we'd open the hole to do the reverse pb
# connect to the buildbot
# ip = urllib.urlopen(
# 'http://checkip.amazonaws.com').read().strip()
# self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
# self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
else:
raise
# get the image
if self.ami is not None:
self.image = self.ec2_conn.get_image(self.ami)
else:
# verify we have access to at least one acceptable image
discard = self.get_image()
assert discard
# get the specified elastic IP, if any
if elastic_ip is not None:
elastic_ip = self.ec2_conn.get_all_addresses([elastic_ip])[0]
self.elastic_ip = elastic_ip
self.subnet_id = subnet_id
self.security_group_ids = security_group_ids
self.classic_security_groups = [self.security_name] if self.security_name else None
self.instance_profile_name = instance_profile_name
self.tags = tags
self.block_device_map = self.create_block_device_mapping(block_device_map)
def __init__(self, name, password, instance_type, ami=None,
valid_ami_owners=None, valid_ami_location_regex=None,
elastic_ip=None, identifier=None, secret_identifier=None,
aws_id_file_path=None, user_data=None, region=None,
keypair_name=None,
security_name=None,
spot_instance=False, max_spot_price=1.6, volumes=None,
placement=None, price_multiplier=1.2, tags=None,
product_description='Linux/UNIX',
subnet_id=None, security_group_ids=None, instance_profile_name=None,
block_device_map=None, session=None,
**kwargs):
if not boto3:
config.error("The python module 'boto3' is needed to use a "
"EC2LatentWorker")
if keypair_name is None:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'keypair_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
keypair_name = 'latent_buildbot_slave'
if security_name is None and not subnet_id:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'security_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
security_name = 'latent_buildbot_slave'
if volumes is None:
volumes = []
if tags is None:
tags = {}
AbstractLatentWorker.__init__(self, name, password, **kwargs)
if security_name and subnet_id:
raise ValueError(
'security_name (EC2 classic security groups) is not supported '
'in a VPC. Use security_group_ids instead.')
if not ((ami is not None) ^
(valid_ami_owners is not None or
valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, integer_types):
valid_ami_owners = (valid_ami_owners,)
else:
for element in valid_ami_owners:
if not isinstance(element, integer_types):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, string_types):
raise ValueError(
'valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
if spot_instance and price_multiplier is None and max_spot_price is None:
raise ValueError('You must provide either one, or both, of '
'price_multiplier or max_spot_price')
self.valid_ami_owners = None
if valid_ami_owners:
self.valid_ami_owners = [str(o) for o in valid_ami_owners]
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
self.spot_instance = spot_instance
self.max_spot_price = max_spot_price
self.volumes = volumes
self.price_multiplier = price_multiplier
self.product_description = product_description
if None not in [placement, region]:
self.placement = '%s%s' % (region, placement)
else:
self.placement = None
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
default_path = os.path.join(home, '.ec2', 'aws_id')
if os.path.exists(default_path):
aws_id_file_path = default_path
if aws_id_file_path:
log.msg('WARNING: EC2LatentWorker is using deprecated '
'aws_id file')
with open(aws_id_file_path, 'r') as aws_file:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
region_found = None
# Make the EC2 connection.
self.session = session
if self.session is None:
if region is not None:
for r in boto3.Session(
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier).get_available_regions('ec2'):
if r == region:
region_found = r
if region_found is not None:
self.session = boto3.Session(
region_name=region,
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier)
else:
raise ValueError(
'The specified region does not exist: ' + region)
else:
# boto2 defaulted to us-east-1 when region was unset, we
# mimic this here in boto3
region = botocore.session.get_session().get_config_variable('region')
if region is None:
region = 'us-east-1'
self.session = boto3.Session(
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier,
region_name=region
)
self.ec2 = self.session.resource('ec2')
self.ec2_client = self.session.client('ec2')
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
self.ec2.KeyPair(self.keypair_name).load()
# key_pair.delete() # would be used to recreate
except ClientError as e:
if 'InvalidKeyPair.NotFound' not in str(e):
if 'AuthFailure' in str(e):
log.msg('POSSIBLE CAUSES OF ERROR:\n'
' Did you supply your AWS credentials?\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.ec2.create_key_pair(KeyName=keypair_name)
# create security group
if security_name:
try:
self.ec2_client.describe_security_groups(GroupNames=[security_name])
except ClientError as e:
if 'InvalidGroup.NotFound' in str(e):
self.security_group = self.ec2.create_security_group(
GroupName=security_name,
Description='Authorization to access the buildbot instance.')
# Authorize the master as necessary
# TODO this is where we'd open the hole to do the reverse pb
# connect to the buildbot
# ip = urllib.urlopen(
# 'http://checkip.amazonaws.com').read().strip()
# self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
# self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
else:
raise
# get the image
if self.ami is not None:
self.image = self.ec2.Image(self.ami)
else:
# verify we have access to at least one acceptable image
discard = self.get_image()
assert discard
# get the specified elastic IP, if any
if elastic_ip is not None:
# Using ec2.vpc_addresses.filter(PublicIps=[elastic_ip]) throws a
# NotImplementedError("Filtering not supported in describe_address.") in moto
# https://github.com/spulec/moto/blob/100ec4e7c8aa3fde87ff6981e2139768816992e4/moto/ec2/responses/elastic_ip_addresses.py#L52
addresses = self.ec2.meta.client.describe_addresses(
PublicIps=[elastic_ip])['Addresses']
if not addresses:
raise ValueError(
'Could not find EIP for IP: ' + elastic_ip)
allocation_id = addresses[0]['AllocationId']
elastic_ip = self.ec2.VpcAddress(allocation_id)
self.elastic_ip = elastic_ip
self.subnet_id = subnet_id
self.security_group_ids = security_group_ids
self.classic_security_groups = [
self.security_name] if self.security_name else None
self.instance_profile_name = instance_profile_name
self.tags = tags
self.block_device_map = self.create_block_device_mapping(
block_device_map) if block_device_map else None
def __init__(self,
workdir,
command,
env=None,
want_stdout=1,
want_stderr=1,
timeout=20 * 60,
maxTime=None,
sigtermTime=None,
logfiles=None,
usePTY=None,
logEnviron=True,
collectStdout=False,
collectStderr=False,
interruptSignal=None,
initialStdin=None,
decodeRC=None,
stdioLogName='stdio'):
if logfiles is None:
logfiles = {}
if decodeRC is None:
decodeRC = {0: SUCCESS}
self.command = command # stash .command, set it later
if isinstance(self.command, string_types):
# Single string command doesn't support obfuscation.
self.fake_command = command
else:
# Try to obfuscate command.
def obfuscate(arg):
if isinstance(
arg,
tuple) and len(arg) == 3 and arg[0] == 'obfuscated':
return arg[2]
else:
return arg
self.fake_command = [obfuscate(c) for c in self.command]
if env is not None:
# avoid mutating the original master.cfg dictionary. Each
# ShellCommand gets its own copy, any start() methods won't be
# able to modify the original.
env = env.copy()
if usePTY == 'slave-config':
reportDeprecatedWorkerNameUsage(
"'slave-config' value of 'usePTY' attribute is deprecated, "
"use None instead.")
usePTY = None
args = {
'workdir': workdir,
'env': env,
'want_stdout': want_stdout,
'want_stderr': want_stderr,
'logfiles': logfiles,
'timeout': timeout,
'maxTime': maxTime,
'sigtermTime': sigtermTime,
'usePTY': usePTY,
'logEnviron': logEnviron,
'initial_stdin': initialStdin
}
if interruptSignal is not None:
args['interruptSignal'] = interruptSignal
RemoteCommand.__init__(self,
"shell",
args,
collectStdout=collectStdout,
collectStderr=collectStderr,
decodeRC=decodeRC,
stdioLogName=stdioLogName)
def __init__(self, name, password, instance_type, ami=None,
valid_ami_owners=None, valid_ami_location_regex=None,
elastic_ip=None, identifier=None, secret_identifier=None,
aws_id_file_path=None, user_data=None, region=None,
keypair_name=None,
security_name=None,
spot_instance=False, max_spot_price=1.6, volumes=None,
placement=None, price_multiplier=1.2, tags=None,
product_description='Linux/UNIX',
subnet_id=None, security_group_ids=None, instance_profile_name=None,
block_device_map=None, session=None,
**kwargs):
if not boto3:
config.error("The python module 'boto3' is needed to use a "
"EC2LatentWorker")
if keypair_name is None:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'keypair_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
keypair_name = 'latent_buildbot_slave'
if security_name is None and not subnet_id:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'security_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
security_name = 'latent_buildbot_slave'
if volumes is None:
volumes = []
if tags is None:
tags = {}
AbstractLatentWorker.__init__(self, name, password, **kwargs)
if security_name and subnet_id:
raise ValueError(
'security_name (EC2 classic security groups) is not supported '
'in a VPC. Use security_group_ids instead.')
if not ((ami is not None) ^
(valid_ami_owners is not None or
valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, integer_types):
valid_ami_owners = (valid_ami_owners,)
else:
for element in valid_ami_owners:
if not isinstance(element, integer_types):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, string_types):
raise ValueError(
'valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
if spot_instance and price_multiplier is None and max_spot_price is None:
raise ValueError('You must provide either one, or both, of '
'price_multiplier or max_spot_price')
self.valid_ami_owners = None
if valid_ami_owners:
self.valid_ami_owners = [str(o) for o in valid_ami_owners]
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
self.spot_instance = spot_instance
self.max_spot_price = max_spot_price
self.volumes = volumes
self.price_multiplier = price_multiplier
self.product_description = product_description
if None not in [placement, region]:
self.placement = '%s%s' % (region, placement)
else:
self.placement = None
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
default_path = os.path.join(home, '.ec2', 'aws_id')
if os.path.exists(default_path):
aws_id_file_path = default_path
if aws_id_file_path:
log.msg('WARNING: EC2LatentWorker is using deprecated '
'aws_id file')
with open(aws_id_file_path, 'r') as aws_file:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
region_found = None
# Make the EC2 connection.
self.session = session
if self.session is None:
if region is not None:
for r in boto3.Session(
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier).get_available_regions('ec2'):
if r == region:
region_found = r
if region_found is not None:
self.session = boto3.Session(
region_name=region,
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier)
else:
raise ValueError(
'The specified region does not exist: ' + region)
else:
# boto2 defaulted to us-east-1 when region was unset, we
# mimic this here in boto3
region = botocore.session.get_session().get_config_variable('region')
if region is None:
region = 'us-east-1'
self.session = boto3.Session(
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier,
region_name=region
)
self.ec2 = self.session.resource('ec2')
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
self.ec2.KeyPair(self.keypair_name).load()
# key_pair.delete() # would be used to recreate
except ClientError as e:
if 'InvalidKeyPair.NotFound' not in str(e):
if 'AuthFailure' in str(e):
log.msg('POSSIBLE CAUSES OF ERROR:\n'
' Did you supply your AWS credentials?\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.ec2.create_key_pair(KeyName=keypair_name)
# create security group
if security_name:
try:
self.ec2.SecurityGroup(security_name).load()
except ClientError as e:
if 'InvalidGroup.NotFound' in str(e):
self.security_group = self.ec2.create_security_group(
GroupName=security_name,
Description='Authorization to access the buildbot instance.')
# Authorize the master as necessary
# TODO this is where we'd open the hole to do the reverse pb
# connect to the buildbot
# ip = urllib.urlopen(
# 'http://checkip.amazonaws.com').read().strip()
# self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
# self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
else:
raise
# get the image
if self.ami is not None:
self.image = self.ec2.Image(self.ami)
else:
# verify we have access to at least one acceptable image
discard = self.get_image()
assert discard
# get the specified elastic IP, if any
if elastic_ip is not None:
# Using ec2.vpc_addresses.filter(PublicIps=[elastic_ip]) throws a
# NotImplementedError("Filtering not supported in describe_address.") in moto
# https://github.com/spulec/moto/blob/100ec4e7c8aa3fde87ff6981e2139768816992e4/moto/ec2/responses/elastic_ip_addresses.py#L52
addresses = self.ec2.meta.client.describe_addresses(
PublicIps=[elastic_ip])['Addresses']
if not addresses:
raise ValueError(
'Could not find EIP for IP: ' + elastic_ip)
allocation_id = addresses[0]['AllocationId']
elastic_ip = self.ec2.VpcAddress(allocation_id)
self.elastic_ip = elastic_ip
self.subnet_id = subnet_id
self.security_group_ids = security_group_ids
self.classic_security_groups = [
self.security_name] if self.security_name else None
self.instance_profile_name = instance_profile_name
self.tags = tags
self.block_device_map = self.create_block_device_mapping(
block_device_map) if block_device_map else None
def load_global(self, filename, config_dict):
def copy_param(name, alt_key=None,
check_type=None, check_type_name=None):
if name in config_dict:
v = config_dict[name]
elif alt_key and alt_key in config_dict:
v = config_dict[alt_key]
else:
return
if v is not None and check_type and not isinstance(v, check_type):
error("c['%s'] must be %s" %
(name, check_type_name))
else:
setattr(self, name, v)
def copy_int_param(name, alt_key=None):
copy_param(name, alt_key=alt_key,
check_type=int, check_type_name='an int')
def copy_str_param(name, alt_key=None):
copy_param(name, alt_key=alt_key,
check_type=string_types, check_type_name='a string')
copy_str_param('title', alt_key='projectName')
copy_str_param('titleURL', alt_key='projectURL')
copy_str_param('buildbotURL')
copy_int_param('changeHorizon')
copy_int_param('logHorizon')
copy_int_param('buildHorizon')
if 'eventHorizon' in config_dict:
warnDeprecated(
'0.9.0',
'`eventHorizon` is deprecated and will be removed in a future version.',
)
copy_int_param('logCompressionLimit')
self.logCompressionMethod = config_dict.get(
'logCompressionMethod', 'gz')
if self.logCompressionMethod not in ('raw', 'bz2', 'gz', 'lz4'):
error(
"c['logCompressionMethod'] must be 'raw', 'bz2', 'gz' or 'lz4'")
if self.logCompressionMethod == "lz4":
try:
import lz4
[lz4]
except ImportError:
error(
"To set c['logCompressionMethod'] to 'lz4' you must install the lz4 library ('pip install lz4')")
copy_int_param('logMaxSize')
copy_int_param('logMaxTailSize')
copy_param('logEncoding')
properties = config_dict.get('properties', {})
if not isinstance(properties, dict):
error("c['properties'] must be a dictionary")
else:
self.properties.update(properties, filename)
collapseRequests = config_dict.get('collapseRequests')
if (collapseRequests not in (None, True, False)
and not callable(collapseRequests)):
error("collapseRequests must be a callable, True, or False")
else:
self.collapseRequests = collapseRequests
codebaseGenerator = config_dict.get('codebaseGenerator')
if (codebaseGenerator is not None and
not callable(codebaseGenerator)):
error(
"codebaseGenerator must be a callable accepting a dict and returning a str")
else:
self.codebaseGenerator = codebaseGenerator
prioritizeBuilders = config_dict.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(prioritizeBuilders):
error("prioritizeBuilders must be a callable")
else:
self.prioritizeBuilders = prioritizeBuilders
protocols = config_dict.get('protocols', {})
if isinstance(protocols, dict):
for proto, options in iteritems(protocols):
if not isinstance(proto, str):
error("c['protocols'] keys must be strings")
if not isinstance(options, dict):
error("c['protocols']['%s'] must be a dict" % proto)
return
if (proto == "pb" and options.get("port") and
'slavePortnum' in config_dict):
error("Both c['slavePortnum'] and c['protocols']['pb']['port']"
" defined, recommended to remove slavePortnum and leave"
" only c['protocols']['pb']['port']")
if proto == "wamp":
self.check_wamp_proto(options)
else:
error("c['protocols'] must be dict")
return
self.protocols = protocols
# saved for backward compatability
if 'slavePortnum' in config_dict:
reportDeprecatedWorkerNameUsage(
"c['slavePortnum'] key is deprecated, use "
"c['protocols']['pb']['port'] instead",
filename=filename)
port = config_dict.get('slavePortnum')
if isinstance(port, int):
port = "tcp:%d" % port
pb_options = self.protocols.get('pb', {})
pb_options['port'] = port
self.protocols['pb'] = pb_options
if 'multiMaster' in config_dict:
self.multiMaster = config_dict["multiMaster"]
if 'debugPassword' in config_dict:
log.msg(
"the 'debugPassword' parameter is unused and can be removed from the configuration flie")
if 'manhole' in config_dict:
# we don't check that this is a manhole instance, since that
# requires importing buildbot.manhole for every user, and currently
# that will fail if pycrypto isn't installed
self.manhole = config_dict['manhole']
if 'revlink' in config_dict:
revlink = config_dict['revlink']
if not callable(revlink):
error("revlink must be a callable")
else:
self.revlink = revlink
def __init__(self, name=None, workername=None, workernames=None,
builddir=None, workerbuilddir=None, factory=None,
tags=None, category=None,
nextWorker=None, nextBuild=None, locks=None, env=None,
properties=None, collapseRequests=None, description=None,
canStartBuild=None,
slavename=None, # deprecated, use `workername` instead
slavenames=None, # deprecated, use `workernames` instead
# deprecated, use `workerbuilddir` instead
slavebuilddir=None,
nextSlave=None, # deprecated, use `nextWorker` instead
):
# Deprecated API support.
if slavename is not None:
reportDeprecatedWorkerNameUsage(
"'slavename' keyword argument is deprecated, "
"use 'workername' instead")
assert workername is None
workername = slavename
if slavenames is not None:
reportDeprecatedWorkerNameUsage(
"'slavenames' keyword argument is deprecated, "
"use 'workernames' instead")
assert workernames is None
workernames = slavenames
if slavebuilddir is not None:
reportDeprecatedWorkerNameUsage(
"'slavebuilddir' keyword argument is deprecated, "
"use 'workerbuilddir' instead")
assert workerbuilddir is None
workerbuilddir = slavebuilddir
if nextSlave is not None:
reportDeprecatedWorkerNameUsage(
"'nextSlave' keyword argument is deprecated, "
"use 'nextWorker' instead")
assert nextWorker is None
nextWorker = nextSlave
# name is required, and can't start with '_'
if not name or type(name) not in (bytes, text_type):
error("builder's name is required")
name = '<unknown>'
elif name[0] == '_' and name not in RESERVED_UNDERSCORE_NAMES:
error(
"builder names must not start with an underscore: '%s'" % name)
try:
self.name = util.ascii2unicode(name)
except UnicodeDecodeError:
error("builder names must be unicode or ASCII")
# factory is required
if factory is None:
error("builder '%s' has no factory" % name)
from buildbot.process.factory import BuildFactory
if factory is not None and not isinstance(factory, BuildFactory):
error("builder '%s's factory is not a BuildFactory instance" %
name)
self.factory = factory
# workernames can be a single worker name or a list, and should also
# include workername, if given
if isinstance(workernames, str):
workernames = [workernames]
if workernames:
if not isinstance(workernames, list):
error("builder '%s': workernames must be a list or a string" %
(name,))
else:
workernames = []
if workername:
if not isinstance(workername, str):
error("builder '%s': workername must be a string but it is %r" % (name, workername))
workernames = workernames + [workername]
if not workernames:
error("builder '%s': at least one workername is required" %
(name,))
self.workernames = workernames
self._registerOldWorkerAttr("workernames")
# builddir defaults to name
if builddir is None:
builddir = safeTranslate(name)
builddir = bytes2NativeString(builddir)
self.builddir = builddir
# workerbuilddir defaults to builddir
if workerbuilddir is None:
workerbuilddir = builddir
self.workerbuilddir = workerbuilddir
self._registerOldWorkerAttr("workerbuilddir")
# remainder are optional
if category and tags:
error("builder '%s': builder categories are deprecated and "
"replaced by tags; you should only specify tags" % (name,))
if category:
warnDeprecated("0.9", "builder '%s': builder categories are "
"deprecated and should be replaced with "
"'tags=[cat]'" % (name,))
if not isinstance(category, str):
error("builder '%s': category must be a string" % (name,))
tags = [category]
if tags:
if not isinstance(tags, list):
error("builder '%s': tags must be a list" % (name,))
bad_tags = any((tag for tag in tags if not isinstance(tag, str)))
if bad_tags:
error(
"builder '%s': tags list contains something that is not a string" % (name,))
if len(tags) != len(set(tags)):
dupes = " ".join(set([x for x in tags if tags.count(x) > 1]))
error(
"builder '%s': tags list contains duplicate tags: %s" % (name, dupes))
else:
tags = []
self.tags = tags
self.nextWorker = nextWorker
self._registerOldWorkerAttr("nextWorker")
if nextWorker and not callable(nextWorker):
error('nextWorker must be a callable')
# Keeping support of the previous nextWorker API
if nextWorker:
argCount = self._countFuncArgs(nextWorker)
if (argCount == 2 or (isinstance(nextWorker, MethodType) and
argCount == 3)):
warnDeprecated(
"0.9", "nextWorker now takes a "
"3rd argument (build request)")
self.nextWorker = lambda x, y, z: nextWorker(
x, y) # pragma: no cover
self.nextBuild = nextBuild
if nextBuild and not callable(nextBuild):
error('nextBuild must be a callable')
self.canStartBuild = canStartBuild
if canStartBuild and not callable(canStartBuild):
error('canStartBuild must be a callable')
self.locks = locks or []
self.env = env or {}
if not isinstance(self.env, dict):
error("builder's env must be a dictionary")
self.properties = properties or {}
self.collapseRequests = collapseRequests
self.description = description
def load_global(self, filename, config_dict):
def copy_param(name,
alt_key=None,
check_type=None,
check_type_name=None,
can_be_callable=False):
if name in config_dict:
v = config_dict[name]
elif alt_key and alt_key in config_dict:
v = config_dict[alt_key]
else:
return
if v is not None and check_type and not (isinstance(
v, check_type) or (can_be_callable and callable(v))):
error("c['%s'] must be %s" % (name, check_type_name))
else:
setattr(self, name, v)
def copy_int_param(name, alt_key=None):
copy_param(name,
alt_key=alt_key,
check_type=int,
check_type_name='an int')
def copy_str_param(name, alt_key=None):
copy_param(name,
alt_key=alt_key,
check_type=string_types,
check_type_name='a string')
copy_str_param('title', alt_key='projectName')
copy_str_param('titleURL', alt_key='projectURL')
copy_str_param('buildbotURL')
def copy_str_or_callable_param(name, alt_key=None):
copy_param(name,
alt_key=alt_key,
check_type=string_types,
check_type_name='a string or callable',
can_be_callable=True)
if "buildbotNetUsageData" not in config_dict:
warnDeprecated(
'0.9.0',
'`buildbotNetUsageData` is not configured and defaults to basic\n'
'This parameter helps the buildbot development team to understand the installation base\n'
'No personal information is collected.\n'
'Only installation software version info and plugin usage is sent\n'
'You can `opt-out` by setting this variable to None\n'
'Or `opt-in` for more information by setting it to "full"\n')
copy_str_or_callable_param('buildbotNetUsageData')
copy_int_param('changeHorizon')
copy_int_param('logHorizon')
copy_int_param('buildHorizon')
if 'eventHorizon' in config_dict:
warnDeprecated(
'0.9.0',
'`eventHorizon` is deprecated and will be removed in a future version.',
)
copy_int_param('logCompressionLimit')
self.logCompressionMethod = config_dict.get('logCompressionMethod',
'gz')
if self.logCompressionMethod not in ('raw', 'bz2', 'gz', 'lz4'):
error(
"c['logCompressionMethod'] must be 'raw', 'bz2', 'gz' or 'lz4'"
)
if self.logCompressionMethod == "lz4":
try:
import lz4
[lz4]
except ImportError:
error(
"To set c['logCompressionMethod'] to 'lz4' you must install the lz4 library ('pip install lz4')"
)
copy_int_param('logMaxSize')
copy_int_param('logMaxTailSize')
copy_param('logEncoding')
properties = config_dict.get('properties', {})
if not isinstance(properties, dict):
error("c['properties'] must be a dictionary")
else:
self.properties.update(properties, filename)
collapseRequests = config_dict.get('collapseRequests')
if (collapseRequests not in (None, True, False)
and not callable(collapseRequests)):
error("collapseRequests must be a callable, True, or False")
else:
self.collapseRequests = collapseRequests
codebaseGenerator = config_dict.get('codebaseGenerator')
if (codebaseGenerator is not None and not callable(codebaseGenerator)):
error(
"codebaseGenerator must be a callable accepting a dict and returning a str"
)
else:
self.codebaseGenerator = codebaseGenerator
prioritizeBuilders = config_dict.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(prioritizeBuilders):
error("prioritizeBuilders must be a callable")
else:
self.prioritizeBuilders = prioritizeBuilders
protocols = config_dict.get('protocols', {})
if isinstance(protocols, dict):
for proto, options in iteritems(protocols):
if not isinstance(proto, str):
error("c['protocols'] keys must be strings")
if not isinstance(options, dict):
error("c['protocols']['%s'] must be a dict" % proto)
return
if (proto == "pb" and options.get("port")
and 'slavePortnum' in config_dict):
error(
"Both c['slavePortnum'] and c['protocols']['pb']['port']"
" defined, recommended to remove slavePortnum and leave"
" only c['protocols']['pb']['port']")
if proto == "wamp":
self.check_wamp_proto(options)
else:
error("c['protocols'] must be dict")
return
self.protocols = protocols
# saved for backward compatability
if 'slavePortnum' in config_dict:
reportDeprecatedWorkerNameUsage(
"c['slavePortnum'] key is deprecated, use "
"c['protocols']['pb']['port'] instead",
filename=filename)
port = config_dict.get('slavePortnum')
if isinstance(port, int):
port = "tcp:%d" % port
pb_options = self.protocols.get('pb', {})
pb_options['port'] = port
self.protocols['pb'] = pb_options
if 'multiMaster' in config_dict:
self.multiMaster = config_dict["multiMaster"]
if 'debugPassword' in config_dict:
log.msg(
"the 'debugPassword' parameter is unused and can be removed from the configuration flie"
)
if 'manhole' in config_dict:
# we don't check that this is a manhole instance, since that
# requires importing buildbot.manhole for every user, and currently
# that will fail if pycrypto isn't installed
self.manhole = config_dict['manhole']
if 'revlink' in config_dict:
revlink = config_dict['revlink']
if not callable(revlink):
error("revlink must be a callable")
else:
self.revlink = revlink
def load_global(self, filename, config_dict):
def copy_param(name, alt_key=None,
check_type=None, check_type_name=None, can_be_callable=False):
if name in config_dict:
v = config_dict[name]
elif alt_key and alt_key in config_dict:
v = config_dict[alt_key]
else:
return
if v is not None and check_type and not (
isinstance(v, check_type) or (can_be_callable and callable(v))):
error("c['%s'] must be %s" %
(name, check_type_name))
else:
setattr(self, name, v)
def copy_int_param(name, alt_key=None):
copy_param(name, alt_key=alt_key,
check_type=int, check_type_name='an int')
def copy_str_param(name, alt_key=None):
copy_param(name, alt_key=alt_key,
check_type=string_types, check_type_name='a string')
copy_str_param('title', alt_key='projectName')
copy_str_param('titleURL', alt_key='projectURL')
copy_str_param('buildbotURL')
def copy_str_or_callable_param(name, alt_key=None):
copy_param(name, alt_key=alt_key,
check_type=string_types, check_type_name='a string or callable', can_be_callable=True)
if "buildbotNetUsageData" not in config_dict:
if _in_unit_tests:
self.buildbotNetUsageData = None
else:
warnDeprecated(
'0.9.0',
'`buildbotNetUsageData` is not configured and defaults to basic.\n'
'This parameter helps the buildbot development team to understand'
' the installation base.\n'
'No personal information is collected.\n'
'Only installation software version info and plugin usage is sent.\n'
'You can `opt-out` by setting this variable to None.\n'
'Or `opt-in` for more information by setting it to "full".\n'
)
copy_str_or_callable_param('buildbotNetUsageData')
for horizon in ('logHorizon', 'buildHorizon', 'eventHorizon'):
if horizon in config_dict:
warnDeprecated(
'0.9.0',
"NOTE: `{}` is deprecated and ignored "
"They are replaced by util.JanitorConfigurator".format(horizon))
copy_int_param('changeHorizon')
copy_int_param('logCompressionLimit')
self.logCompressionMethod = config_dict.get(
'logCompressionMethod', 'gz')
if self.logCompressionMethod not in ('raw', 'bz2', 'gz', 'lz4'):
error(
"c['logCompressionMethod'] must be 'raw', 'bz2', 'gz' or 'lz4'")
if self.logCompressionMethod == "lz4":
try:
import lz4
[lz4]
except ImportError:
error(
"To set c['logCompressionMethod'] to 'lz4' you must install the lz4 library ('pip install lz4')")
copy_int_param('logMaxSize')
copy_int_param('logMaxTailSize')
copy_param('logEncoding')
properties = config_dict.get('properties', {})
if not isinstance(properties, dict):
error("c['properties'] must be a dictionary")
else:
self.properties.update(properties, filename)
collapseRequests = config_dict.get('collapseRequests')
if (collapseRequests not in (None, True, False)
and not callable(collapseRequests)):
error("collapseRequests must be a callable, True, or False")
else:
self.collapseRequests = collapseRequests
codebaseGenerator = config_dict.get('codebaseGenerator')
if (codebaseGenerator is not None and
not callable(codebaseGenerator)):
error(
"codebaseGenerator must be a callable accepting a dict and returning a str")
else:
self.codebaseGenerator = codebaseGenerator
prioritizeBuilders = config_dict.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(prioritizeBuilders):
error("prioritizeBuilders must be a callable")
else:
self.prioritizeBuilders = prioritizeBuilders
protocols = config_dict.get('protocols', {})
if isinstance(protocols, dict):
for proto, options in iteritems(protocols):
if not isinstance(proto, str):
error("c['protocols'] keys must be strings")
if not isinstance(options, dict):
error("c['protocols']['%s'] must be a dict" % proto)
return
if (proto == "pb" and options.get("port") and
'slavePortnum' in config_dict):
error("Both c['slavePortnum'] and c['protocols']['pb']['port']"
" defined, recommended to remove slavePortnum and leave"
" only c['protocols']['pb']['port']")
if proto == "wamp":
self.check_wamp_proto(options)
else:
error("c['protocols'] must be dict")
return
self.protocols = protocols
# saved for backward compatibility
if 'slavePortnum' in config_dict:
reportDeprecatedWorkerNameUsage(
"c['slavePortnum'] key is deprecated, use "
"c['protocols']['pb']['port'] instead",
filename=filename)
port = config_dict.get('slavePortnum')
if isinstance(port, int):
port = "tcp:%d" % port
pb_options = self.protocols.get('pb', {})
pb_options['port'] = port
self.protocols['pb'] = pb_options
if 'multiMaster' in config_dict:
self.multiMaster = config_dict["multiMaster"]
if 'debugPassword' in config_dict:
log.msg(
"the 'debugPassword' parameter is unused and can be removed from the configuration flie")
if 'manhole' in config_dict:
# we don't check that this is a manhole instance, since that
# requires importing buildbot.manhole for every user, and currently
# that will fail if cryptography isn't installed
self.manhole = config_dict['manhole']
if 'revlink' in config_dict:
revlink = config_dict['revlink']
if not callable(revlink):
error("revlink must be a callable")
else:
self.revlink = revlink
def __init__(self,
name,
password,
instance_type,
ami=None,
valid_ami_owners=None,
valid_ami_location_regex=None,
elastic_ip=None,
identifier=None,
secret_identifier=None,
aws_id_file_path=None,
user_data=None,
region=None,
keypair_name=None,
security_name=None,
spot_instance=False,
max_spot_price=1.6,
volumes=None,
placement=None,
price_multiplier=1.2,
tags=None,
retry=1,
retry_price_adjustment=1,
product_description='Linux/UNIX',
subnet_id=None,
security_group_ids=None,
instance_profile_name=None,
block_device_map=None,
**kwargs):
if not boto:
config.error("The python module 'boto' is needed to use a "
"EC2LatentWorker")
if keypair_name is None:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'keypair_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
keypair_name = 'latent_buildbot_slave'
if security_name is None and not subnet_id:
reportDeprecatedWorkerNameUsage(
"Use of default value of 'security_name' of EC2LatentWorker "
"constructor is deprecated. Please explicitly specify value")
security_name = 'latent_buildbot_slave'
if volumes is None:
volumes = []
if tags is None:
tags = {}
AbstractLatentWorker.__init__(self, name, password, **kwargs)
if security_name and subnet_id:
raise ValueError(
'security_name (EC2 classic security groups) is not supported '
'in a VPC. Use security_group_ids instead.')
if not ((ami is not None) ^ (valid_ami_owners is not None
or valid_ami_location_regex is not None)):
raise ValueError(
'You must provide either a specific ami, or one or both of '
'valid_ami_location_regex and valid_ami_owners')
self.ami = ami
if valid_ami_owners is not None:
if isinstance(valid_ami_owners, (int, long)):
valid_ami_owners = (valid_ami_owners, )
else:
for element in valid_ami_owners:
if not isinstance(element, (int, long)):
raise ValueError(
'valid_ami_owners should be int or iterable '
'of ints', element)
if valid_ami_location_regex is not None:
if not isinstance(valid_ami_location_regex, basestring):
raise ValueError('valid_ami_location_regex should be a string')
else:
# verify that regex will compile
re.compile(valid_ami_location_regex)
self.valid_ami_owners = valid_ami_owners
self.valid_ami_location_regex = valid_ami_location_regex
self.instance_type = instance_type
self.keypair_name = keypair_name
self.security_name = security_name
self.user_data = user_data
self.spot_instance = spot_instance
self.max_spot_price = max_spot_price
self.volumes = volumes
self.price_multiplier = price_multiplier
self.retry_price_adjustment = retry_price_adjustment
self.retry = retry
self.attempt = 1
self.product_description = product_description
if None not in [placement, region]:
self.placement = '%s%s' % (region, placement)
else:
self.placement = None
if identifier is None:
assert secret_identifier is None, (
'supply both or neither of identifier, secret_identifier')
if aws_id_file_path is None:
home = os.environ['HOME']
default_path = os.path.join(home, '.ec2', 'aws_id')
if os.path.exists(default_path):
aws_id_file_path = default_path
if aws_id_file_path:
log.msg('WARNING: EC2LatentWorker is using deprecated '
'aws_id file')
with open(aws_id_file_path, 'r') as aws_file:
identifier = aws_file.readline().strip()
secret_identifier = aws_file.readline().strip()
else:
assert aws_id_file_path is None, \
'if you supply the identifier and secret_identifier, ' \
'do not specify the aws_id_file_path'
assert secret_identifier is not None, \
'supply both or neither of identifier, secret_identifier'
region_found = None
# Make the EC2 connection.
if region is not None:
for r in boto.ec2.regions(aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier):
if r.name == region:
region_found = r
if region_found is not None:
self.ec2_conn = boto.ec2.connect_to_region(
region,
aws_access_key_id=identifier,
aws_secret_access_key=secret_identifier)
else:
raise ValueError('The specified region does not exist: ' +
region)
else:
self.ec2_conn = boto.connect_ec2(identifier, secret_identifier)
# Make a keypair
#
# We currently discard the keypair data because we don't need it.
# If we do need it in the future, we will always recreate the keypairs
# because there is no way to
# programmatically retrieve the private key component, unless we
# generate it and store it on the filesystem, which is an unnecessary
# usage requirement.
try:
key_pair = self.ec2_conn.get_all_key_pairs(keypair_name)[0]
assert key_pair
# key_pair.delete() # would be used to recreate
except boto.exception.EC2ResponseError as e:
if 'InvalidKeyPair.NotFound' not in e.body:
if 'AuthFailure' in e.body:
log.msg('POSSIBLE CAUSES OF ERROR:\n'
' Did you supply your AWS credentials?\n'
' Did you sign up for EC2?\n'
' Did you put a credit card number in your AWS '
'account?\n'
'Please doublecheck before reporting a problem.\n')
raise
# make one; we would always do this, and stash the result, if we
# needed the key (for instance, to SSH to the box). We'd then
# use paramiko to use the key to connect.
self.ec2_conn.create_key_pair(keypair_name)
# create security group
if security_name:
try:
group = self.ec2_conn.get_all_security_groups(security_name)[0]
assert group
except boto.exception.EC2ResponseError as e:
if 'InvalidGroup.NotFound' in e.body:
self.security_group = self.ec2_conn.create_security_group(
security_name,
'Authorization to access the buildbot instance.')
# Authorize the master as necessary
# TODO this is where we'd open the hole to do the reverse pb
# connect to the buildbot
# ip = urllib.urlopen(
# 'http://checkip.amazonaws.com').read().strip()
# self.security_group.authorize('tcp', 22, 22, '%s/32' % ip)
# self.security_group.authorize('tcp', 80, 80, '%s/32' % ip)
else:
raise
# get the image
if self.ami is not None:
self.image = self.ec2_conn.get_image(self.ami)
else:
# verify we have access to at least one acceptable image
discard = self.get_image()
assert discard
# get the specified elastic IP, if any
if elastic_ip is not None:
elastic_ip = self.ec2_conn.get_all_addresses([elastic_ip])[0]
self.elastic_ip = elastic_ip
self.subnet_id = subnet_id
self.security_group_ids = security_group_ids
self.classic_security_groups = [self.security_name
] if self.security_name else None
self.instance_profile_name = instance_profile_name
self.tags = tags
self.block_device_map = self.create_block_device_mapping(
block_device_map)
def __init__(
self,
name=None,
workername=None,
workernames=None,
builddir=None,
workerbuilddir=None,
factory=None,
tags=None,
category=None,
nextWorker=None,
nextBuild=None,
locks=None,
env=None,
properties=None,
collapseRequests=None,
description=None,
canStartBuild=None,
slavename=None, # deprecated, use `workername` instead
slavenames=None, # deprecated, use `workernames` instead
# deprecated, use `workerbuilddir` instead
slavebuilddir=None,
nextSlave=None, # deprecated, use `nextWorker` instead
):
# Deprecated API support.
if slavename is not None:
reportDeprecatedWorkerNameUsage(
"'slavename' keyword argument is deprecated, "
"use 'workername' instead")
assert workername is None
workername = slavename
if slavenames is not None:
reportDeprecatedWorkerNameUsage(
"'slavenames' keyword argument is deprecated, "
"use 'workernames' instead")
assert workernames is None
workernames = slavenames
if slavebuilddir is not None:
reportDeprecatedWorkerNameUsage(
"'slavebuilddir' keyword argument is deprecated, "
"use 'workerbuilddir' instead")
assert workerbuilddir is None
workerbuilddir = slavebuilddir
if nextSlave is not None:
reportDeprecatedWorkerNameUsage(
"'nextSlave' keyword argument is deprecated, "
"use 'nextWorker' instead")
assert nextWorker is None
nextWorker = nextSlave
# name is required, and can't start with '_'
if not name or type(name) not in (str, text_type):
error("builder's name is required")
name = '<unknown>'
elif name[0] == '_':
error("builder names must not start with an underscore: '%s'" %
name)
try:
self.name = util.ascii2unicode(name)
except UnicodeDecodeError:
error("builder names must be unicode or ASCII")
# factory is required
if factory is None:
error("builder '%s' has no factory" % name)
from buildbot.process.factory import BuildFactory
if factory is not None and not isinstance(factory, BuildFactory):
error("builder '%s's factory is not a BuildFactory instance" %
name)
self.factory = factory
# workernames can be a single worker name or a list, and should also
# include workername, if given
if isinstance(workernames, str):
workernames = [workernames]
if workernames:
if not isinstance(workernames, list):
error("builder '%s': workernames must be a list or a string" %
(name, ))
else:
workernames = []
if workername:
if not isinstance(workername, str):
error("builder '%s': workername must be a string" % (name, ))
workernames = workernames + [workername]
if not workernames:
error("builder '%s': at least one workername is required" %
(name, ))
self.workernames = workernames
self._registerOldWorkerAttr("workernames")
# builddir defaults to name
if builddir is None:
builddir = safeTranslate(name)
self.builddir = builddir
# workerbuilddir defaults to builddir
if workerbuilddir is None:
workerbuilddir = builddir
self.workerbuilddir = workerbuilddir
self._registerOldWorkerAttr("workerbuilddir")
# remainder are optional
if category and tags:
error("builder '%s': builder categories are deprecated and "
"replaced by tags; you should only specify tags" % (name, ))
if category:
warnDeprecated(
"0.9", "builder '%s': builder categories are "
"deprecated and should be replaced with "
"'tags=[cat]'" % (name, ))
if not isinstance(category, str):
error("builder '%s': category must be a string" % (name, ))
tags = [category]
if tags:
if not isinstance(tags, list):
error("builder '%s': tags must be a list" % (name, ))
bad_tags = any((tag for tag in tags if not isinstance(tag, str)))
if bad_tags:
error(
"builder '%s': tags list contains something that is not a string"
% (name, ))
if len(tags) != len(set(tags)):
dupes = " ".join(set([x for x in tags if tags.count(x) > 1]))
error("builder '%s': tags list contains duplicate tags: %s" %
(name, dupes))
else:
tags = []
self.tags = tags
self.nextWorker = nextWorker
self._registerOldWorkerAttr("nextWorker")
if nextWorker and not callable(nextWorker):
error('nextWorker must be a callable')
# Keeping support of the previous nextWorker API
if nextWorker and (nextWorker.func_code.co_argcount == 2 or
(isinstance(nextWorker, MethodType)
and nextWorker.func_code.co_argcount == 3)):
warnDeprecated(
"0.9", "nextWorker now takes a 3rd argument (build request)")
self.nextWorker = lambda x, y, z: nextWorker(x, y
) # pragma: no cover
self.nextBuild = nextBuild
if nextBuild and not callable(nextBuild):
error('nextBuild must be a callable')
self.canStartBuild = canStartBuild
if canStartBuild and not callable(canStartBuild):
error('canStartBuild must be a callable')
self.locks = locks or []
self.env = env or {}
if not isinstance(self.env, dict):
error("builder's env must be a dictionary")
self.properties = properties or {}
self.collapseRequests = collapseRequests
self.description = description
