class OpsworksController(object): def __init__(self, cluster, force=False, dry_run=False): self.force = force self.dry_run = dry_run self.opsworks = boto3.client('opsworks') self.ec2 = boto3.resource('ec2') stacks = self.opsworks.describe_stacks()['Stacks'] try: self.stack = next(x for x in stacks if x['Name'] == cluster) except StopIteration: raise OpsworksControllerException( "No opsworks stack named '%s' found" % cluster ) instances = self.opsworks.describe_instances( StackId=self.stack['StackId'])['Instances'] try: mh_admin = next(x for x in instances if x['Hostname'].startswith('admin')) except StopIteration: raise OpsworksControllerException("No admin node found") self.mhorn = MatterhornController(mh_admin['PublicDns']) self._instances = [OpsworksInstance(x, self) for x in instances] def __repr__(self): return "%s (%s)" % (self.__class__, self.stack['Name']) @property def instances(self): return [x for x in self._instances if not x.is_autoscale()] @property def workers(self): return [x for x in self.instances if x.is_worker()] @property def online_workers(self): return [x for x in self.workers if x.is_online()] @property def pending_workers(self): return [x for x in self.workers if x.is_pending()] @property def online_or_pending_workers(self): return self.online_workers + self.pending_workers @property def idle_workers(self): return self.mhorn.filter_idle(self.online_workers) @property def stopped_workers(self): return [x for x in self.workers if x.is_stopped()] @property def online_instances(self): return [x for x in self.instances if x.is_online()] @property def admin(self): try: return next(x for x in self.instances if x.is_admin()) except StopIteration: raise OpsworksControllerException("No admin node found") def status(self): status = { "cluster": self.stack['Name'], "matterhorn_online": self.mhorn.is_online(), "instances": len(self.instances), "instances_online": len(self.online_instances), "workers": len(self.workers), "workers_online": len(self.online_workers), "workers_pending": len(self.pending_workers), "worker_details": [] } status['job_status'] = self.mhorn.job_status() for inst in self.workers: inst_status = { "state": inst.Status, "opsworks_id": inst.InstanceId, "ec2_id": inst.Ec2InstanceId, "hostname": inst.Hostname, "mh_host_url": inst.mh_host_url, "uptime": inst.uptime(), "billed_minutes": inst.billed_minutes(), } inst_status.update(self.mhorn.node_status(inst)) # if hasattr(inst, 'Ec2InstanceId'): # inst_status['ec2_id'] = inst.Ec2InstanceId status['worker_details'].append(inst_status) return status def actions(self): stopped = [x for x in self.workers if x.action_taken == 'stopped'] started = [x for x in self.workers if x.action_taken == 'started'] return { 'total_stopped': len(stopped), 'stopped': '; '.join("%r" % x for x in stopped), 'total_started': len(started), 'started': ', '.join("%s" % x for x in started) } def start_instance(self, inst): LOGGER.info("Starting %r", inst) if not self.dry_run: self.opsworks.start_instance(InstanceId=inst.InstanceId) def stop_instance(self, inst): LOGGER.info("Stopping %r", inst) if not self.dry_run: self.opsworks.stop_instance(InstanceId=inst.InstanceId) def scale_to(self, num_workers, scale_available=False): current_workers = len(self.online_or_pending_workers) if current_workers == num_workers: raise OpsworksControllerException( "Cluster already at %d online or pending workers!" % num_workers ) elif current_workers > num_workers: self.scale('down', current_workers - num_workers) else: self.scale('up', num_workers - current_workers, scale_available) def scale(self, direction, num_workers=None, scale_available=False): if direction == "up": LOGGER.info("Attempting to scale up %d workers", num_workers) self._scale_up(num_workers, scale_available) elif direction == "auto": LOGGER.info("Initiating auto-scaling") self._scale_auto() else: with self.mhorn.in_maintenance(self.online_workers, dry_run=self.dry_run): if direction == "down": LOGGER.info("Attempting to scale down %d workers", num_workers) self._scale_down(num_workers) def _scale_auto(self): AUTOSCALE_TYPE = env('AUTOSCALE_TYPE') if not AUTOSCALE_TYPE: raise OpsworksScalingException("No autoscaling type defined") from autoscalers import create_autoscaler try: autoscaler = create_autoscaler(AUTOSCALE_TYPE, self) except Exception, e: raise OpsworksControllerException( "Failed loading autoscale type '%s'" % AUTOSCALE_TYPE ) try: LOGGER.info("Executing autoscale type: %s" , AUTOSCALE_TYPE) autoscaler.scale() except Exception, e: raise OpsworksScalingException( "Autoscale aborted: %s" % str(e) )
class OpsworksController(object): def __init__(self, cluster, force=False, dry_run=False): self.force = force self.dry_run = dry_run self.opsworks = boto3.client("opsworks") self.ec2 = boto3.resource("ec2") stacks = self.opsworks.describe_stacks()["Stacks"] try: self.stack = next(x for x in stacks if x["Name"] == cluster) except StopIteration: raise OpsworksControllerException("No opsworks stack named '%s' found" % cluster) instances = self.opsworks.describe_instances(StackId=self.stack["StackId"])["Instances"] try: mh_admin = next(x for x in instances if x["Hostname"].startswith("admin")) except StopIteration: raise OpsworksControllerException("No admin node found") self.mhorn = MatterhornController(mh_admin["PublicDns"]) self._instances = [OpsworksInstance(x, self) for x in instances] def __repr__(self): return "%s (%s)" % (self.__class__, self.stack["Name"]) @property def instances(self): return [x for x in self._instances if not x.is_autoscale()] @property def workers(self): return [x for x in self.instances if x.is_worker()] @property def online_workers(self): return [x for x in self.workers if x.is_online()] @property def pending_workers(self): return [x for x in self.workers if x.is_pending()] @property def online_or_pending_workers(self): return self.online_workers + self.pending_workers @property def idle_workers(self): return self.mhorn.filter_idle(self.online_workers) @property def stopped_workers(self): return [x for x in self.workers if x.is_stopped()] @property def online_instances(self): return [x for x in self.instances if x.is_online()] @property def admin(self): try: return next(x for x in self.instances if x.is_admin()) except StopIteration: raise OpsworksControllerException("No admin node found") def get_layer_id(self, layer_name): layers = self.opsworks.describe_layers(StackId=self.stack["StackId"])["Layers"] try: layer = next(x for x in layers if x["Name"] == layer_name) return layer["LayerId"] except StopIteration: raise OpsworksControllerException("Could not find layer '%s'" % layer_name) def get_ec2_id(self, instance_name): return next(x.Ec2InstanceId for x in self.instances if x.Hostname == instance_name) def status(self): status = { "cluster": self.stack["Name"], "matterhorn_online": self.mhorn.is_online(), "instances": len(self.instances), "instances_online": len(self.online_instances), "workers": len(self.workers), "workers_online": len(self.online_workers), "workers_pending": len(self.pending_workers), "worker_details": [], } status["job_status"] = self.mhorn.job_status() for inst in self.workers: inst_status = { "state": inst.Status, "opsworks_id": inst.InstanceId, "ec2_id": inst.Ec2InstanceId, "hostname": inst.Hostname, "mh_host_url": inst.mh_host_url, "uptime": inst.uptime(), "billed_minutes": inst.billed_minutes(), } inst_status.update(self.mhorn.node_status(inst)) # if hasattr(inst, 'Ec2InstanceId'): # inst_status['ec2_id'] = inst.Ec2InstanceId status["worker_details"].append(inst_status) return status def actions(self): stopped = [x for x in self.workers if x.action_taken == "stopped"] started = [x for x in self.workers if x.action_taken == "started"] return { "total_stopped": len(stopped), "stopped": "; ".join("%r" % x for x in stopped), "total_started": len(started), "started": ", ".join("%s" % x for x in started), } def start_instance(self, inst): LOGGER.info("Starting %r", inst) if not self.dry_run: self.opsworks.start_instance(InstanceId=inst.InstanceId) def stop_instance(self, inst): LOGGER.info("Stopping %r", inst) if not self.dry_run: self.opsworks.stop_instance(InstanceId=inst.InstanceId) def scale_to(self, num_workers, scale_available=False): current_workers = len(self.online_or_pending_workers) if current_workers == num_workers: raise OpsworksControllerException("Cluster already at %d online or pending workers!" % num_workers) elif current_workers > num_workers: self.scale("down", current_workers - num_workers) else: self.scale("up", num_workers - current_workers, scale_available) def scale(self, direction, num_workers=None, scale_available=False): if direction == "up": LOGGER.info("Attempting to scale up %d workers", num_workers) self._scale_up(num_workers, scale_available) else: with self.mhorn.in_maintenance(self.online_workers, dry_run=self.dry_run): if direction == "down": LOGGER.info("Attempting to scale down %d workers", num_workers) self._scale_down(num_workers) def autoscale(self, settings): autoscaler = Autoscaler(self, settings) try: LOGGER.info("Executing autoscaler") autoscaler.execute() except Exception, e: raise OpsworksScalingException("Autoscale aborted: %s" % str(e))