def getECRrepos(self, reponames=list(), images=False):
"""Gather information about ECR repositories"""
if len(reponames) == 0:
response = self.ecr.describe_repositories()
else:
response = self.ecr.describe_repositories(
repositoryNames=reponames)
self.repos = fl.fluentWrap(response['repositories'])
for repo in self.repos:
if (images):
rresponse = self.ecr.list_images(
repositoryName=repo.repositoryName)
repo.imageIds = fl.fluentWrap(rresponse['imageIds'])
imageIds = list()
for image in repo.imageIds:
if image.checkPath("imageTag"):
imageIds.append({'imageTag': image.imageTag})
else:
imageIds.append({'imageDigest': image.imageDigest})
idetresp = self.ecr.describe_images(
repositoryName=repo.repositoryName, imageIds=imageIds)
if ('imageDetails' in idetresp):
imageDetails = fl.fluentWrap(idetresp['imageDetails'])
for image in repo.imageIds:
for candImage in imageDetails:
if candImage.imageDigest == image.imageDigest:
image = image + candImage
break
else:
image.imageDetails = fl.fluentWrap([])
else:
repo.imageIds = fl.fluentWrap([])
def getRecordSets(self, hostzoneid):
"""Get all the record sets in a hosted zone"""
response = self.r53.list_resource_record_sets(HostedZoneId=hostzoneid)
answer = self.__recordSetLoop(response)
self.rr[hostzoneid] = fl.fluentWrap(response['ResourceRecordSets'])
while answer.isTruncated:
response = self.r53.list_resource_record_sets(
HostedZoneId=hostzoneid,
StartRecordName=answer.NextRecordName,
StartRecordType=answer.NextRecordType,
NextRecordIdentifier=answer.NextRcordIdentifier)
self.rr[hostzoneid] = self.rr[hostzoneid] + fl.fluentWrap(
response['ResourceRecordSets'])
answer = self.__recordSetLoop(response)
# build a cross reference from the IPaddress to the resource record
for rr in self.rr[hostzoneid]:
if rr.Type == 'A':
for rrs in rr.ResourceRecords:
if rr.Value not in self.ipRRCrossRef:
self.ipRRCrossRef[rrs.Value] = list()
self.ipRRCrossRef[rrs.Value].append(rr)
def getCacheClusters(self):
response = self.elasticache.describe_cache_clusters()
cacheClusters = fl.fluentWrap(response['CacheClusters'])
self.cacheClusters = fl.fluentWrap()
if self.filter is not None:
for cc in cacheClusters:
if self.filterRe.match(cc.CacheClusterId):
self.cacheClusters.append(cc)
else:
self.cacheClusters = cacheClusters
def collectAsgData(self):
"""Collect information about auto-scaling-groups"""
paginator = self.asg.get_paginator('describe_auto_scaling_groups')
page_iterator = paginator.paginate()
allAsgs = page_iterator.build_full_result()['AutoScalingGroups']
self.allAsgs = fl.fluentWrap(allAsgs)
self.deployAsgs = dict()
for asg in self.allAsgs:
asg.name = self.getNameTag(asg.Tags)
if self.filterRe.match(asg.name):
asg.LaunchConfiguration = self.getLaunchConfig(asg)
self.deployAsgs[asg.name] = asg
self.deployAsgs = fl.fluentWrap(self.deployAsgs)
def getPrices(self, noTreat=False):
"""Grab the pricelist using the supplied filter list"""
client = lambda x: self.pricing.get_products(ServiceCode=self.service,
Filters=self.filterList,
FormatVersion='aws_v1',
NextToken=x)
ap = autoPaginator(client, index='PriceList', fluentWrap=False)
priceList = list()
# amazingly the returned values are string encased json, so here
# we unpack it - this looks like it comes from SAP to me but I
# could be wrong
for priceListData in ap:
for item in priceListData:
r = json.loads(item)
priceList.append(r)
self.prices = fl.fluentWrap(priceList)
if noTreat:
return
self.__treatPrices()
def getSpotPrice(self):
response = self.ec2Client.describe_spot_price_history(
InstanceTypes=['t3.small'],
MaxResults=100,
ProductDescriptions=['Linux/UNIX'],
AvailabilityZone='eu-west-2c')
self.spotHistory = fl.fluentWrap(response['SpotPriceHistory'])
def getVPCs(self, getSubnets=False, getNATGWs=False, getHostedZones=False):
""" Gather information about VPCs """
response = self.ec2Client.describe_vpcs()
self.vpc = dict()
for result in response['Vpcs']:
vpcId = result['VpcId']
vpc = self.ec2.Vpc(vpcId)
name = self.getNameTag(vpc.tags, "unnamed")
name = self.__getUniqueKey(name, self.vpc)
if getSubnets or getNATGWs:
subnets = self.getSubnets(vpc=vpc, getNATGWs=getNATGWs)
else:
subnets = list()
if getHostedZones:
hz = self.getHostedZonesByVpc(vpcId)
else:
hz = list()
self.vpc[name] = fl.fluentWrap({
'vpc':
vpc,
'id':
vpcId,
'hostedZones':
hz,
'default':
'True' if vpc.is_default else 'False',
'cidr':
vpc.cidr_block,
'subnets':
subnets
})
def getTargetGroupHealth(self):
for lbs in self.lbsByArn:
for tg in lbs.targetGroups:
response = self.elbv2.describe_target_health(
TargetGroupArn=tg.TargetGroupArn)
if 'TargetHealthDescriptions' in response:
tg.HealthDescription = fl.fluentWrap(
response['TargetHealthDescriptions'])
def getAMIData(self, filterText=["ami-*"]):
# centos account 125523088429
# amazonlinux-2-base_1579686771 2020-01-22T10:00:28.000Z ami-00b8d754e256a1884 766535289950
#images = self.ec2Client.describe_images(Owners=['self'])
#fltr = [{'Name': 'name', 'Values': [ 'ami-*' ]}]
fltr = [{'Name': 'name', 'Values': filterText}]
response = self.ec2Client.describe_images(Filters=fltr)
self.ami = fl.fluentWrap(response['Images'])
def getRDSinstances(self, cost=False):
response = self.rds.describe_db_instances()
db = fl.fluentWrap(response['DBInstances'])
if self.filter is not None:
self.db = fl.fluentWrap()
for procDb in db:
if self.filterRe.match(procDb.DBInstanceIdentifier):
self.db.append(procDb)
else:
self.db = db
if cost:
for db in self.db:
db.price = self.getRDSPrice(db.DBInstanceClass, db.Engine,
db.MultiAZ)
def getTargetGroups(self):
client = lambda x: self.elbv2.describe_target_groups(Marker=x)
ap = autoPaginator(client,
index='TargetGroups',
fluentWrap=True,
apMarker='Marker')
for tgs in ap:
self.tgs += tgs
# cross reference
for tg in tgs:
for lbarn in tg.LoadBalancerArns:
if lbarn not in self.lbsByArn:
self.lbsByArn[lbarn] = fl.fluentWrap()
self.lbsByArn[lbarn].targetGroups = fl.fluentWrap()
self.lbsByArn[lbarn].targetGroups.append(tg)
def __recordSetLoop(self, response):
for answer in [
'isTruncated', 'NextRecordName', 'NextRecordType',
'NextRecordIdentifier'
]:
part = dict()
if answer in response:
part[answer] = response[answer]
return fl.fluentWrap(part)
def data(self):
if self.index is not None:
if self.index in self.response:
data = self.response[self.index]
else:
data = []
else:
data = self.response
return fl.fluentWrap(data) if self.fluentWrap else data
def getRedisReplGroups(self):
response = self.elasticache.describe_replication_groups()
replicationGroups = fl.fluentWrap(response['ReplicationGroups'])
if self.filter is not None:
for rg in replicationGroups:
if self.filterRe.match(rg.ReplicationGroupId):
self.replicationGroups.append(rg)
else:
self.replicationGroups = replicationGroups
def getInstanceData(self, collectRdns=False, getPrice=False):
instances = self.instances
self.instanceCount = dict()
id2name = dict()
for instance in instances:
tagHostName = self.getNameTag(instance.tags)
running = (instance.state['Code'] & 255) <= 32
if running:
if tagHostName not in self.instanceCount:
self.instanceCount[tagHostName] = -1
self.instanceCount[
tagHostName] = self.instanceCount[tagHostName] + 1
upSeconds = self.upTime(instance.launch_time)
if (getPrice):
if (running):
price = self.getEC2Price(instance.instance_type)
else:
price = 0
else:
price = 0
id2name[instance.id] = {
"name": tagHostName,
"instance": instance,
'running': running,
'uptime': upSeconds,
'price': price,
'connection': None
}
# remove zeros, so we don't get name-0 extensions
for tagHostName in [
item for item in self.instanceCount
if self.instanceCount[item] == 0
]:
self.instanceCount.pop(tagHostName)
self.ec2hosts = fl.fluentWrap(id2name)
# create a shortcut lookup by instance id
for host in self.ec2hosts:
self.ec2hostsById[host.instance.id] = host
self.__setUniqueHostNames()
if collectRdns:
self.collectRdnsHostData()
return
def getSecurityGroups(self):
if (self.filter is not None):
response = self.ec2Client.describe_security_groups(
Filters=[{
'Name': 'tag:Name',
'Values': ['{}*'.format(self.filter)]
}])
else:
response = self.ec2Client.describe_security_groups()
self.sgs = fl.fluentWrap(response['SecurityGroups'])
self.sgsByGroupId = dict()
for sg in self.sgs:
self.sgsByGroupId[sg.GroupId] = sg
def getServices(self, serviceCodes=[]):
"""Get amazon service information"""
#boto3.set_stream_logger('botocore', 10)
serviceCodes = list(serviceCodes)
self.services = fl.fluentWrap()
if len(serviceCodes) == 0:
serviceCodes.append("")
for serviceCode in serviceCodes:
client = lambda x: self.pricing.describe_services(
FormatVersion='aws_v1',
MaxResults=100,
ServiceCode=serviceCode,
NextToken=x)
ap = autoPaginator(client, index='Services', fluentWrap=True)
for service in ap:
self.services += service
def getNatGWs(self, ngwFilters=list()):
natGW = self.ec2Client.describe_nat_gateways(Filters=ngwFilters)
natGWs = fl.fluentWrap(natGW['NatGateways'])
natgws = dict()
for natGW in natGWs:
name = self.getNameTag(natGW.Tags)
name = self.__getUniqueKey(name, self.natgws)
addlist = list()
for address in natGW.NatGatewayAddresses:
addlist.append({
'public': address.PublicIp,
'private': address.PrivateIp,
'allocid': address.AllocationId
})
self.natgws[name] = {
'id': natGW.NatGatewayId,
'name': name,
'state': natGW.State,
'addresses': addlist
}
natgws[name] = self.natgws[name]
return natgws
def getLoadBalancers(self):
response = self.elbv2.describe_load_balancers()
v2lbs = fl.fluentWrap(response['LoadBalancers'])
self.v2lbs = fl.fluentWrap()
if self.filter is not None:
for lbs in v2lbs:
if self.filterRe.match(lbs.LoadBalancerName):
self.v2lbs.append(lbs)
else:
self.v2lbs = v2lbs
response = self.elb.describe_load_balancers()
lbs = fl.fluentWrap(response['LoadBalancerDescriptions'])
self.lbs = fl.fluentWrap()
# adjust by adding a Type so we can recognise legacy lbs
if self.filter is not None:
for lbs in lbs:
if self.filterRe.match(lbs.LoadBalancerName):
lbs.Type = "Legacy"
self.lbs.append(lbs)
else:
for lbs in lbs:
lbs.Type = "Legacy"
self.lbs.append(lbs)
# make a cross reference
for lbs in self.v2lbs + self.lbs:
if lbs.LoadBalancerArn not in self.lbsByArn:
self.lbsByArn[lbs.LoadBalancerArn] = fl.fluentWrap()
self.lbsByArn[
lbs.LoadBalancerArn].targetGroups = fl.fluentWrap()
self.lbsByArn[lbs.LoadBalancerArn].loadBalancer = lbs
def getHostedZonesByVpc(self, vpcId, vpcRegion='eu-west-2'):
r53response = self.r53.list_hosted_zones_by_vpc(VPCId=vpcId,
VPCRegion=vpcRegion)
return (fl.fluentWrap(r53response['HostedZoneSummaries']))
def __init__(self, session, log, filter=None):
self.log = log
my_retry_config = botocore.config.Config(
retries={'max_attempts': MAX_NUM_THROTTLE_RETRIES})
# boto3 connectors, clients and resources
self.session = session
self.ec2 = session.resource('ec2', config=my_retry_config)
self.asg = boto3.client('autoscaling', config=my_retry_config)
self.ec2Client = session.client('ec2', config=my_retry_config)
self.r53 = boto3.client('route53', config=my_retry_config)
self.ssm = boto3.client('ssm', config=my_retry_config)
self.elasticache = boto3.client('elasticache', config=my_retry_config)
self.pricing = session.client('pricing',
region_name='us-east-1',
config=my_retry_config)
self.rds = session.client('rds', config=my_retry_config)
self.sts = boto3.client('sts', config=my_retry_config)
# rds instances
self.db = dict()
# pricing
self.spotHistory = dict()
self.services = fl.fluentWrap()
# price caches
self.ec2Prices = dict()
self.RDSPrices = dict()
# hosted zone management
self.dns = dict()
# resource records
self.rr = dict()
self.ipRRCrossRef = dict()
# ec2 hosts management
self.ec2hosts = dict()
self.ec2hostsById = dict()
self.instanceCount = dict()
self.instances = list()
# ec2 rr cross referencing
self.ipHostPriv = dict()
self.ipHostPub = dict()
self.hostVpc = dict()
# configuration for fabric to connect to hosts
self.jump = None
self.jumpHost = None
self.jumpUser = None
self.jumpPrivKeyFile = None
self.ec2PrivKeyFile = None
if (filter is not None):
self.setFilter(filter)
else:
self.filter = None
# asg variables
self.user_data = False
self.allAsgs = None
self.deployAsgs = None
# ssh writing config
self.config = dict()
self.IdentityFile = None
self.ProxyHost = None
self.ProxyMatch = None
self.ec2user = None
# internal state indicators
self.configLoaded = False
self.builtConnections = False
# load balancing info
self.elbv2 = boto3.client('elbv2', config=my_retry_config)
self.elb = boto3.client('elb', config=my_retry_config)
self.v2lbs = fl.fluentWrap()
self.lbs = fl.fluentWrap()
self.lbsByArn = dict()
self.tgs = fl.fluentWrap()
# vpc info
self.vpc = dict()
self.subnets = dict()
self.natgws = dict()
# ecr info
self.ecr = boto3.client('ecr')
self.repos = dict()
# security groups info
self.sgs = dict()
self.sgsByGroudId = dict()
# elastic cache
self.cacheClusters = fl.fluentWrap()
self.replicationGroups = fl.fluentWrap()
# ami
self.ami = fl.fluentWrap()
# volumes
self.volumes = fl.fluentWrap()
# snapshots
self.snapshots = fl.fluentWrap()
def getUserDataData(self, rawUd):
ud = base64.b64decode(rawUd)
md5 = hashlib.md5(str(ud).encode('utf-8')).hexdigest()
return fl.fluentWrap({"content": ud, "md5": md5})
def getLaunchConfig(self, asg):
response = self.asg.describe_launch_configurations(
LaunchConfigurationNames=[asg.LaunchConfigurationName])
if 'LaunchConfigurations' in response:
return fl.fluentWrap(response['LaunchConfigurations'])
return None
def getHostedZones(self):
response = self.r53.list_hosted_zones()
self.dns = fl.fluentWrap(response['HostedZones'])
