def _thd_start_instance(self): docker_client = client.Client(**self.client_args) found = False if self.image is not None: found = self._image_exists(docker_client) image = self.image else: image = '%s_%s_image' % (self.slavename, id(self)) if (not found) and (self.dockerfile is not None): log.msg("Image '%s' not found, building it from scratch" % image) for line in docker_client.build(fileobj=BytesIO( self.dockerfile.encode('utf-8')), tag=image): for streamline in handle_stream_line(line): log.msg(streamline) if (not self._image_exists(docker_client, image)): log.msg("Image '%s' not found" % image) raise interfaces.LatentBuildSlaveFailedToSubstantiate( 'Image "%s" not found on docker host.' % image) instance = docker_client.create_container( image, self.command, name='%s_%s' % (self.slavename, id(self)), volumes=self.volumes, ) if instance.get('Id') is None: log.msg('Failed to create the container') raise interfaces.LatentBuildSlaveFailedToSubstantiate( 'Failed to start container') shortid = instance['Id'][:6] log.msg('Container created, Id: %s...' % (shortid, )) instance['image'] = image self.instance = instance docker_client.start(instance, binds=self.binds) log.msg('Container started') if self.followStartupLogs: logs = docker_client.attach(container=instance, stdout=True, stderr=True, stream=True) for line in logs: log.msg("docker VM %s: %s" % (shortid, line.strip())) if self.conn: break del logs return [instance['Id'], self.image]
def _start_instance(self): # Authenticate to OpenStack. os_client = client.Client(self.os_username, self.os_password, self.os_tenant_name, self.os_auth_url) image_uuid = self._getImage(os_client) flavor_id = self.flavor boot_args = [self.slavename, image_uuid, flavor_id] boot_kwargs = {} if self.meta is not None: boot_kwargs['meta'] = self.meta instance = os_client.servers.create(*boot_args, **boot_kwargs) self.instance = instance log.msg( '%s %s starting instance %s (image %s)' % (self.__class__.__name__, self.slavename, instance.id, image_uuid)) duration = 0 interval = self._poll_resolution inst = instance while inst.status.startswith(BUILD): time.sleep(interval) duration += interval if duration % 60 == 0: log.msg('%s %s has waited %d minutes for instance %s' % (self.__class__.__name__, self.slavename, duration // 60, instance.id)) try: inst = os_client.servers.get(instance.id) except nce.NotFound: log.msg('%s %s instance %s (%s) went missing' % (self.__class__.__name__, self.slavename, instance.id, instance.name)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( instance.id, instance.status) if inst.status == ACTIVE: minutes = duration // 60 seconds = duration % 60 log.msg('%s %s instance %s (%s) started ' 'in about %d minutes %d seconds' % (self.__class__.__name__, self.slavename, instance.id, instance.name, minutes, seconds)) return [ instance.id, image_uuid, '%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds) ] else: log.msg('%s %s failed to start instance %s (%s)' % (self.__class__.__name__, self.slavename, instance.id, inst.status)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( instance.id, inst.status)
def _wait_for_instance(self, image): log.msg('%s %s waiting for instance %s to start' % (self.__class__.__name__, self.slavename, self.instance.id)) duration = 0 interval = self._poll_resolution while self.instance.state == PENDING: time.sleep(interval) duration += interval if duration % 60 == 0: log.msg('%s %s has waited %d minutes for instance %s' % (self.__class__.__name__, self.slavename, duration // 60, self.instance.id)) self.instance.update() if self.instance.state == RUNNING: self.output = self.instance.get_console_output() minutes = duration // 60 seconds = duration % 60 log.msg('%s %s instance %s started on %s ' 'in about %d minutes %d seconds (%s)' % (self.__class__.__name__, self.slavename, self.instance.id, self.dns, minutes, seconds, self.output.output)) if self.elastic_ip is not None: self.instance.use_ip(self.elastic_ip) start_time = '%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds) if len(self.volumes) > 0: self._attach_volumes() return self.instance.id, image.id, start_time else: log.msg('%s %s failed to start instance %s (%s)' % (self.__class__.__name__, self.slavename, self.instance.id, self.instance.state)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( self.instance.id, self.instance.state)
def _thd_start_instance(self): docker_client = client.Client(base_url=self.docker_host) found = self._image_exists(docker_client) if (not found) and (self.dockerfile is not None): log.msg("Image '%s' not found, building it from scratch" % self.image) for line in docker_client.build(fileobj=BytesIO( self.dockerfile.encode('utf-8')), tag=self.image): log.msg(line) if not self._image_exists(docker_client): log.msg("Image '%s' not found" % self.image) raise interfaces.LatentBuildSlaveFailedToSubstantiate( 'Image "%s" not found on docker host.' % self.image) volumes = {} binds = {} for volume_string in self.volumes: try: volume = volume_string.split(":")[1] except IndexError: log.err("Invalid volume definition for docker " "{0}. Skipping...".format(volume_string)) continue volumes[volume] = {} volume, bind = volume_string.split(':', 1) binds[volume] = bind instance = docker_client.create_container( self.image, self.command, volumes=volumes, ) if instance.get('Id') is None: log.msg('Failed to create the container') raise interfaces.LatentBuildSlaveFailedToSubstantiate( 'Failed to start container') log.msg('Container created, Id: %s...' % instance['Id'][:6]) self.instance = instance docker_client.start(instance['Id'], binds=binds) return [instance['Id'], self.image]
def _request_spot_instance(self): if self.retry > 1: for attempt in range(1, self.retry + 1): self.attempt = attempt instance_id, image_id, start_time, success = self._submit_request( ) if success: break if attempt >= self.retry: self.attempt = 0 log.msg( '%s %s failed to substantiate after %d requests' % (self.__class__.__name__, self.slavename, self.retry)) raise interfaces.LatentBuildSlaveFailedToSubstantiate() else: instance_id, image_id, start_time, success = self._submit_request() if not success: raise interfaces.LatentBuildSlaveFailedToSubstantiate() return instance_id, image_id, start_time
def _submit_request(self): timestamp_yesterday = time.gmtime(int(time.time() - 86400)) spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday) spot_prices = self.conn.get_spot_price_history( start_time=spot_history_starttime, product_description=self.product_description, availability_zone=self.placement) price_sum = 0.0 price_count = 0 for price in spot_prices: if price.instance_type == self.instance_type: price_sum += price.price price_count += 1 if price_count == 0: self.current_spot_price = 0.02 else: self.current_spot_price = (price_sum / price_count) * self.price_multiplier if self.current_spot_price > self.max_spot_price: log.msg('%s %s calculated spot price %0.3f exceeds ' 'configured maximum of %0.3f' % (self.__class__.__name__, self.slavename, self.current_spot_price, self.max_spot_price)) raise interfaces.LatentBuildSlaveFailedToSubstantiate() else: if self.retry > 1: log.msg( '%s %s requesting spot instance with price %0.4f, attempt %d of %d' % (self.__class__.__name__, self.slavename, self.current_spot_price, self.attempt, self.retry)) else: log.msg('%s %s requesting spot instance with price %0.4f' % (self.__class__.__name__, self.slavename, self.current_spot_price)) reservations = self.conn.request_spot_instances( self.current_spot_price, self.ami, key_name=self.keypair_name, security_groups=[self.security_name], instance_type=self.instance_type, user_data=self.user_data, placement=self.placement) request, success = self._wait_for_request(reservations[0]) if not success: return request, None, None, False else: instance_id = request.instance_id reservations = self.conn.get_all_instances( instance_ids=[instance_id]) self.instance = reservations[0].instances[0] instance_id, image_id, start_time = self._wait_for_instance( self.get_image()) return instance_id, image_id, start_time, True
def _wait_for_request(self, reservation): log.msg('%s %s requesting spot instance' % (self.__class__.__name__, self.slavename)) duration = 0 interval = self._poll_resolution requests = self.conn.get_all_spot_instance_requests( request_ids=[reservation.id]) request = requests[0] request_status = request.status.code while request_status in SPOT_REQUEST_PENDING_STATES: time.sleep(interval) duration += interval if duration % 60 == 0: log.msg('%s %s has waited %d minutes for spot request %s' % (self.__class__.__name__, self.slavename, duration // 60, request.id)) requests = self.conn.get_all_spot_instance_requests( request_ids=[request.id]) request = requests[0] request_status = request.status.code if request_status == FULFILLED: minutes = duration // 60 seconds = duration % 60 log.msg('%s %s spot request %s fulfilled ' 'in about %d minutes %d seconds' % (self.__class__.__name__, self.slavename, request.id, minutes, seconds)) return request elif request_status == PRICE_TOO_LOW: log.msg('%s %s spot request rejected, spot price too low' % (self.__class__.__name__, self.slavename)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( request.id, request.status) else: log.msg('%s %s failed to fulfill spot request %s with status %s' % (self.__class__.__name__, self.slavename, request.id, request_status)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( request.id, request.status)
def _start_instance(self): image = self.get_image() reservation = image.run(key_name=self.keypair_name, security_groups=[self.security_name], instance_type=self.instance_type, user_data=self.user_data, placement=self.placement) self.instance = reservation.instances[0] instance_id, image_id, start_time = self._wait_for_instance( reservation) if None not in [instance_id, image_id, start_time]: return [instance_id, image_id, start_time] else: log.msg('%s %s failed to start instance %s (%s)' % (self.__class__.__name__, self.slavename, self.instance.id, self.instance.state)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( self.instance.id, self.instance.state)
def _request_spot_instance(self): timestamp_yesterday = time.gmtime(int(time.time() - 86400)) spot_history_starttime = time.strftime('%Y-%m-%dT%H:%M:%SZ', timestamp_yesterday) spot_prices = self.conn.get_spot_price_history( start_time=spot_history_starttime, product_description='Linux/UNIX (Amazon VPC)', availability_zone=self.placement) price_sum = 0.0 price_count = 0 for price in spot_prices: if price.instance_type == self.instance_type: price_sum += price.price price_count += 1 if price_count == 0: target_price = 0.02 else: target_price = (price_sum / price_count) * self.price_multiplier if target_price > self.max_spot_price: log.msg('%s %s calculated spot price %0.2f exceeds ' 'configured maximum of %0.2f' % (self.__class__.__name__, self.slavename, target_price, self.max_spot_price)) raise interfaces.LatentBuildSlaveFailedToSubstantiate() else: log.msg('%s %s requesting spot instance with price %0.2f.' % (self.__class__.__name__, self.slavename, target_price)) reservations = self.conn.request_spot_instances( target_price, self.ami, key_name=self.keypair_name, security_groups=[self.security_name], instance_type=self.instance_type, user_data=self.user_data, placement=self.placement) request = self._wait_for_request(reservations[0]) instance_id = request.instance_id reservations = self.conn.get_all_instances(instance_ids=[instance_id]) self.instance = reservations[0].instances[0] return self._wait_for_instance(self.get_image())
def _start_instance(self): image = self.get_image() reservation = image.run(key_name=self.keypair_name, security_groups=[self.security_name], instance_type=self.instance_type, user_data=self.user_data) self.instance = reservation.instances[0] log.msg('%s %s starting instance %s' % (self.__class__.__name__, self.slavename, self.instance.id)) duration = 0 interval = self._poll_resolution while self.instance.state == PENDING: time.sleep(interval) duration += interval if duration % 60 == 0: log.msg('%s %s has waited %d minutes for instance %s' % (self.__class__.__name__, self.slavename, duration // 60, self.instance.id)) self.instance.update() if self.instance.state == RUNNING: self.output = self.instance.get_console_output() minutes = duration // 60 seconds = duration % 60 log.msg('%s %s instance %s started on %s ' 'in about %d minutes %d seconds (%s)' % (self.__class__.__name__, self.slavename, self.instance.id, self.dns, minutes, seconds, self.output.output)) if self.elastic_ip is not None: self.instance.use_ip(self.elastic_ip) return [ self.instance.id, image.id, '%02d:%02d:%02d' % (minutes // 60, minutes % 60, seconds) ] else: log.msg('%s %s failed to start instance %s (%s)' % (self.__class__.__name__, self.slavename, self.instance.id, self.instance.state)) raise interfaces.LatentBuildSlaveFailedToSubstantiate( self.instance.id, self.instance.state)