def _thd_start_instance(self):
docker_client = client.Client(**self.client_args)
found = False
if self.image is not None:
found = self._image_exists(docker_client)
image = self.image
else:
image = '%s_%s_image' % (self.slavename, id(self))
if (not found) and (self.dockerfile is not None):
log.msg("Image '%s' not found, building it from scratch" % image)
for line in docker_client.build(fileobj=BytesIO(
self.dockerfile.encode('utf-8')),
tag=image):
for streamline in handle_stream_line(line):
log.msg(streamline)
if (not self._image_exists(docker_client, image)):
log.msg("Image '%s' not found" % image)
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
'Image "%s" not found on docker host.' % image)
instance = docker_client.create_container(
image,
self.command,
name='%s_%s' % (self.slavename, id(self)),
volumes=self.volumes,
)
if instance.get('Id') is None:
log.msg('Failed to create the container')
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
'Failed to start container')
shortid = instance['Id'][:6]
log.msg('Container created, Id: %s...' % (shortid, ))
instance['image'] = image
self.instance = instance
docker_client.start(instance, binds=self.binds)
log.msg('Container started')
if self.followStartupLogs:
logs = docker_client.attach(container=instance,
stdout=True,
stderr=True,
stream=True)
for line in logs:
log.msg("docker VM %s: %s" % (shortid, line.strip()))
if self.conn:
break
del logs
return [instance['Id'], self.image]
def _start_instance(self):
# Authenticate to OpenStack.
os_client = client.Client(self.os_username, self.os_password,
self.os_tenant_name, self.os_auth_url)
image_uuid = self._getImage(os_client)
flavor_id = self.flavor
boot_args = [self.slavename, image_uuid, flavor_id]
boot_kwargs = {}
if self.meta is not None:
boot_kwargs['meta'] = self.meta
instance = os_client.servers.create(*boot_args, **boot_kwargs)
self.instance = instance
log.msg(
'%s %s starting instance %s (image %s)' %
(self.__class__.__name__, self.slavename, instance.id, image_uuid))
duration = 0
interval = self._poll_resolution
inst = instance
while inst.status.startswith(BUILD):
time.sleep(interval)
duration += interval
if duration % 60 == 0:
log.msg('%s %s has waited %d minutes for instance %s' %
(self.__class__.__name__, self.slavename,
duration // 60, instance.id))
try:
inst = os_client.servers.get(instance.id)
except nce.NotFound:
log.msg('%s %s instance %s (%s) went missing' %
(self.__class__.__name__, self.slavename, instance.id,
instance.name))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
instance.id, instance.status)
if inst.status == ACTIVE:
minutes = duration // 60
seconds = duration % 60
log.msg('%s %s instance %s (%s) started '
'in about %d minutes %d seconds' %
(self.__class__.__name__, self.slavename, instance.id,
instance.name, minutes, seconds))
return [
instance.id, image_uuid,
'%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds)
]
else:
log.msg('%s %s failed to start instance %s (%s)' %
(self.__class__.__name__, self.slavename, instance.id,
inst.status))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
instance.id, inst.status)
def _wait_for_instance(self, image):
log.msg('%s %s waiting for instance %s to start' %
(self.__class__.__name__, self.slavename, self.instance.id))
duration = 0
interval = self._poll_resolution
while self.instance.state == PENDING:
time.sleep(interval)
duration += interval
if duration % 60 == 0:
log.msg('%s %s has waited %d minutes for instance %s' %
(self.__class__.__name__, self.slavename,
duration // 60, self.instance.id))
self.instance.update()
if self.instance.state == RUNNING:
self.output = self.instance.get_console_output()
minutes = duration // 60
seconds = duration % 60
log.msg('%s %s instance %s started on %s '
'in about %d minutes %d seconds (%s)' %
(self.__class__.__name__, self.slavename, self.instance.id,
self.dns, minutes, seconds, self.output.output))
if self.elastic_ip is not None:
self.instance.use_ip(self.elastic_ip)
start_time = '%02d:%02d:%02d' % (minutes // 60, minutes % 60,
seconds)
if len(self.volumes) > 0:
self._attach_volumes()
return self.instance.id, image.id, start_time
else:
log.msg('%s %s failed to start instance %s (%s)' %
(self.__class__.__name__, self.slavename, self.instance.id,
self.instance.state))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
self.instance.id, self.instance.state)
def _thd_start_instance(self):
docker_client = client.Client(base_url=self.docker_host)
found = self._image_exists(docker_client)
if (not found) and (self.dockerfile is not None):
log.msg("Image '%s' not found, building it from scratch" %
self.image)
for line in docker_client.build(fileobj=BytesIO(
self.dockerfile.encode('utf-8')),
tag=self.image):
log.msg(line)
if not self._image_exists(docker_client):
log.msg("Image '%s' not found" % self.image)
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
'Image "%s" not found on docker host.' % self.image)
volumes = {}
binds = {}
for volume_string in self.volumes:
try:
volume = volume_string.split(":")[1]
except IndexError:
log.err("Invalid volume definition for docker "
"{0}. Skipping...".format(volume_string))
continue
volumes[volume] = {}
volume, bind = volume_string.split(':', 1)
binds[volume] = bind
instance = docker_client.create_container(
self.image,
self.command,
volumes=volumes,
)
if instance.get('Id') is None:
log.msg('Failed to create the container')
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
'Failed to start container')
log.msg('Container created, Id: %s...' % instance['Id'][:6])
self.instance = instance
docker_client.start(instance['Id'], binds=binds)
return [instance['Id'], self.image]
def _request_spot_instance(self):
if self.retry > 1:
for attempt in range(1, self.retry + 1):
self.attempt = attempt
instance_id, image_id, start_time, success = self._submit_request(
)
if success:
break
if attempt >= self.retry:
self.attempt = 0
log.msg(
'%s %s failed to substantiate after %d requests' %
(self.__class__.__name__, self.slavename, self.retry))
raise interfaces.LatentBuildSlaveFailedToSubstantiate()
else:
instance_id, image_id, start_time, success = self._submit_request()
if not success:
raise interfaces.LatentBuildSlaveFailedToSubstantiate()
return instance_id, image_id, start_time
def _submit_request(self):
timestamp_yesterday = time.gmtime(int(time.time() - 86400))
spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ',
timestamp_yesterday)
spot_prices = self.conn.get_spot_price_history(
start_time=spot_history_starttime,
product_description=self.product_description,
availability_zone=self.placement)
price_sum = 0.0
price_count = 0
for price in spot_prices:
if price.instance_type == self.instance_type:
price_sum += price.price
price_count += 1
if price_count == 0:
self.current_spot_price = 0.02
else:
self.current_spot_price = (price_sum /
price_count) * self.price_multiplier
if self.current_spot_price > self.max_spot_price:
log.msg('%s %s calculated spot price %0.3f exceeds '
'configured maximum of %0.3f' %
(self.__class__.__name__, self.slavename,
self.current_spot_price, self.max_spot_price))
raise interfaces.LatentBuildSlaveFailedToSubstantiate()
else:
if self.retry > 1:
log.msg(
'%s %s requesting spot instance with price %0.4f, attempt %d of %d'
% (self.__class__.__name__, self.slavename,
self.current_spot_price, self.attempt, self.retry))
else:
log.msg('%s %s requesting spot instance with price %0.4f' %
(self.__class__.__name__, self.slavename,
self.current_spot_price))
reservations = self.conn.request_spot_instances(
self.current_spot_price,
self.ami,
key_name=self.keypair_name,
security_groups=[self.security_name],
instance_type=self.instance_type,
user_data=self.user_data,
placement=self.placement)
request, success = self._wait_for_request(reservations[0])
if not success:
return request, None, None, False
else:
instance_id = request.instance_id
reservations = self.conn.get_all_instances(
instance_ids=[instance_id])
self.instance = reservations[0].instances[0]
instance_id, image_id, start_time = self._wait_for_instance(
self.get_image())
return instance_id, image_id, start_time, True
def _wait_for_request(self, reservation):
log.msg('%s %s requesting spot instance' %
(self.__class__.__name__, self.slavename))
duration = 0
interval = self._poll_resolution
requests = self.conn.get_all_spot_instance_requests(
request_ids=[reservation.id])
request = requests[0]
request_status = request.status.code
while request_status in SPOT_REQUEST_PENDING_STATES:
time.sleep(interval)
duration += interval
if duration % 60 == 0:
log.msg('%s %s has waited %d minutes for spot request %s' %
(self.__class__.__name__, self.slavename,
duration // 60, request.id))
requests = self.conn.get_all_spot_instance_requests(
request_ids=[request.id])
request = requests[0]
request_status = request.status.code
if request_status == FULFILLED:
minutes = duration // 60
seconds = duration % 60
log.msg('%s %s spot request %s fulfilled '
'in about %d minutes %d seconds' %
(self.__class__.__name__, self.slavename, request.id,
minutes, seconds))
return request
elif request_status == PRICE_TOO_LOW:
log.msg('%s %s spot request rejected, spot price too low' %
(self.__class__.__name__, self.slavename))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
request.id, request.status)
else:
log.msg('%s %s failed to fulfill spot request %s with status %s' %
(self.__class__.__name__, self.slavename, request.id,
request_status))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
request.id, request.status)
def _start_instance(self):
image = self.get_image()
reservation = image.run(key_name=self.keypair_name,
security_groups=[self.security_name],
instance_type=self.instance_type,
user_data=self.user_data,
placement=self.placement)
self.instance = reservation.instances[0]
instance_id, image_id, start_time = self._wait_for_instance(
reservation)
if None not in [instance_id, image_id, start_time]:
return [instance_id, image_id, start_time]
else:
log.msg('%s %s failed to start instance %s (%s)' %
(self.__class__.__name__, self.slavename, self.instance.id,
self.instance.state))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
self.instance.id, self.instance.state)
def _request_spot_instance(self):
timestamp_yesterday = time.gmtime(int(time.time() - 86400))
spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ',
timestamp_yesterday)
spot_prices = self.conn.get_spot_price_history(
start_time=spot_history_starttime,
product_description='Linux/UNIX (Amazon VPC)',
availability_zone=self.placement)
price_sum = 0.0
price_count = 0
for price in spot_prices:
if price.instance_type == self.instance_type:
price_sum += price.price
price_count += 1
if price_count == 0:
target_price = 0.02
else:
target_price = (price_sum / price_count) * self.price_multiplier
if target_price > self.max_spot_price:
log.msg('%s %s calculated spot price %0.2f exceeds '
'configured maximum of %0.2f' %
(self.__class__.__name__, self.slavename, target_price,
self.max_spot_price))
raise interfaces.LatentBuildSlaveFailedToSubstantiate()
else:
log.msg('%s %s requesting spot instance with price %0.2f.' %
(self.__class__.__name__, self.slavename, target_price))
reservations = self.conn.request_spot_instances(
target_price,
self.ami,
key_name=self.keypair_name,
security_groups=[self.security_name],
instance_type=self.instance_type,
user_data=self.user_data,
placement=self.placement)
request = self._wait_for_request(reservations[0])
instance_id = request.instance_id
reservations = self.conn.get_all_instances(instance_ids=[instance_id])
self.instance = reservations[0].instances[0]
return self._wait_for_instance(self.get_image())
def _start_instance(self):
image = self.get_image()
reservation = image.run(key_name=self.keypair_name,
security_groups=[self.security_name],
instance_type=self.instance_type,
user_data=self.user_data)
self.instance = reservation.instances[0]
log.msg('%s %s starting instance %s' %
(self.__class__.__name__, self.slavename, self.instance.id))
duration = 0
interval = self._poll_resolution
while self.instance.state == PENDING:
time.sleep(interval)
duration += interval
if duration % 60 == 0:
log.msg('%s %s has waited %d minutes for instance %s' %
(self.__class__.__name__, self.slavename,
duration // 60, self.instance.id))
self.instance.update()
if self.instance.state == RUNNING:
self.output = self.instance.get_console_output()
minutes = duration // 60
seconds = duration % 60
log.msg('%s %s instance %s started on %s '
'in about %d minutes %d seconds (%s)' %
(self.__class__.__name__, self.slavename, self.instance.id,
self.dns, minutes, seconds, self.output.output))
if self.elastic_ip is not None:
self.instance.use_ip(self.elastic_ip)
return [
self.instance.id, image.id,
'%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds)
]
else:
log.msg('%s %s failed to start instance %s (%s)' %
(self.__class__.__name__, self.slavename, self.instance.id,
self.instance.state))
raise interfaces.LatentBuildSlaveFailedToSubstantiate(
self.instance.id, self.instance.state)
