class ResourceAdapter(object): \
# pylint: disable=too-many-public-methods
'''
This is the base class for all resource adapters to derive from.
The default actions simply print a debug message to show that the
subclass did not implement the action.
'''
def __init__(self, addHostSession=None):
if '__adaptername__' not in self.__class__.__dict__:
raise NotImplementedError(
'Subclasses of ResourceAdapter must have __adaptername__'
' defined')
self._logger = logging.getLogger('tortuga.resourceAdapter.%s' %
(self.__adaptername__))
self._logger.addHandler(logging.NullHandler())
self.__installer_public_hostname = None
self.__installer_public_ipaddress = None
self.__private_dns_zone = None
# Initialize caches
self.__addHostApi = None
self.__nodeApi = None
self.__osObject = None
self.__sanApi = None
# Initialize abort flag (to "not" aborting)
self.__isAborted = False
self._cm = ConfigManager()
self._addHostSession = addHostSession
@property
def addHostSession(self):
return self._addHostSession
@property
def cacheCfgFilePath(self):
return os.path.join(self._cm.getRoot(), 'var',
'%s-instance.conf' % (self.__adaptername__))
@property
def cfgFileName(self):
return os.path.join(
self._cm.getKitConfigBase(),
'adapter-defaults-%s.conf' % (self.__adaptername__))
def hookAction(self, action, nodes, args=None):
# Only the 'default' resource adapter overrides the hookAction()
# method.
pass
def start(self,
addNodesRequest,
dbSession,
dbHardwareProfile,
dbSoftwareProfile=None): \
# pylint: disable=unused-argument
self.__trace(addNodesRequest, dbSession, dbHardwareProfile,
dbSoftwareProfile)
def validate_start_arguments(self, addNodesRequest, dbHardwareProfile,
dbSoftwareProfile):
self.__trace(addNodesRequest, dbHardwareProfile, dbSoftwareProfile)
def stop(self, hardwareProfileName, deviceName):
self.__trace(hardwareProfileName, deviceName)
def updateNode(self, session, node, updateNodeRequest): \
# pylint: disable=unused-argument
self.__trace(session, node, updateNodeRequest)
def suspendActiveNode(self, nodeId):
'''Change the given active node to an idle node'''
self.__trace(nodeId)
def idleActiveNode(self, nodeIds):
'''Change the given active node to an idle node'''
self.__trace(nodeIds)
def activateIdleNode(self, node, softwareProfileName,
softwareProfileChanged):
'''Change the given idle node to an active node'''
self.__trace(node, softwareProfileName, softwareProfileChanged)
def deleteNode(self, nodeIds):
'''Remove the given node (active or idle) from the system'''
self.__trace(nodeIds)
def _async_delete_nodes(self, nodes):
"""
Asynchronously delete nodes; calls "ResourceAdapter._delete_node()"
method for each deleted nodes
:param dbNodes: list of Nodes objects
:return: None
"""
greenlets = []
for node in nodes:
greenlets.append(gevent.spawn(self._delete_node, node))
# TODO: implement timeout
gevent.joinall(greenlets)
def _delete_node(self, node):
"""
Abstract method called to delete node from
"ResourceAdapter._async_delete_nodes()"
:param node: Nodes object
"""
def transferNode(self, nodeIdSoftwareProfileTuples,
newSoftwareProfileName):
'''Transfer the given idle node'''
self.__trace(nodeIdSoftwareProfileTuples, newSoftwareProfileName)
def startupNode(self, nodeIds, remainingNodeList=None, tmpBootMethod='n'): \
# pylint: disable=unused-argument
'''Start the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support starting')
def shutdownNode(self, nodes, bSoftReset=False): \
# pylint: disable=unused-argument
'''Shutdown the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support shutdown')
def rebootNode(self, nodes, bSoftReset=False): \
# pylint: disable=unused-argument
'''Reboot the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support rebooting')
def checkpointNode(self, nodeId): \
# pylint: disable=unused-argument
'''Checkpoint the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support checkpointing')
def revertNodeToCheckpoint(self, nodeId): \
# pylint: disable=unused-argument
'''Revert the given node to the checkpoint'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support checkpointing')
def migrateNode(self, nodeId, remainingNodeList, liveMigrate): \
# pylint: disable=unused-argument
'''Migrate the given node'''
# By default raise unsupported operation
raise UnsupportedOperation('Node does not support migrating')
def addVolumeToNode(self, node, volume, isDirect): \
# pylint: disable=unused-argument
'''Add a disk to a node'''
# By default raise unsupported operation
raise UnsupportedOperation(
'Node does not support dynamic disk addition')
def removeVolumeFromNode(self, node, volume): \
# pylint: disable=unused-argument
'''Remove a disk from a node'''
# By default raise unsupported operation
raise UnsupportedOperation(
'Node does not support dynamic disk deletion' % (node))
def abort(self):
'''abort node addition'''
self._logger.debug('Setting abort flag')
self.__isAborted = True
def isAborted(self):
'''Returns status of abort flag'''
return self.__isAborted
def __trace(self, *pargs, **kargs):
stack = traceback.extract_stack()
funcname = stack[-2][2]
self._logger.debug('-- (pass) %s::%s %s %s' %
(self.__adaptername__, funcname, pargs, kargs))
def getLogger(self):
return self._logger
def getResourceAdapterConfig(self, sectionName=None):
"""
Raises:
ResourceNotFound
"""
self.getLogger().debug(
'getResourceAdapterConfig(sectionName=[{0}])'.format(
sectionName if sectionName else '(none)'))
try:
# Load default values
defaultResourceAdapterConfigDict = self._loadConfigDict()
if sectionName is None or sectionName == 'default':
return defaultResourceAdapterConfigDict
except ResourceNotFound:
defaultResourceAdapterConfigDict = {}
overrideConfigDict = self._loadConfigDict(sectionName)
# Override defaults with hardware profile specific settings
return dict(
list(defaultResourceAdapterConfigDict.items()) +
list(overrideConfigDict.items()))
def _loadConfigDict(self, sectionName=None):
"""
Raises:
ResourceNotFound
"""
if sectionName is None:
sectionName = 'default'
session = DbManager().openSession()
try:
self.getLogger().debug('_loadConfigDict()')
result = ResourceAdapterCredentialsDbHandler().get(
session, self.__adaptername__, sectionName)
configDict = {}
for entry in result['configuration']:
configDict[entry['key']] = entry['value']
finally:
DbManager().closeSession()
return configDict
def getResourceAdapterConfigProfileByNodeName(self, name):
"""Get resource adapter configuration for existing node"""
self.getLogger().debug(
'getResourceAdapterConfigByNodeName(): name=[{0}]'.format(name))
instance_cache = self.instanceCacheRefresh()
return instance_cache.get(name, 'resource_adapter_configuration') \
if instance_cache.has_section(name) and instance_cache.has_option(
name, 'resource_adapter_configuration') else None
def __getAddHostApi(self):
'''Get and cache the Add Host API'''
if self.__addHostApi is None:
from tortuga.addhost.addHostServerLocal \
import AddHostServerLocal
self.__addHostApi = AddHostServerLocal()
return self.__addHostApi
def __getNodeApi(self):
'''Get and cache the Node API'''
if self.__nodeApi is None:
from tortuga.node.nodeApi import NodeApi
self.__nodeApi = NodeApi()
return self.__nodeApi
def __getOsObject(self):
'''Get and cache the OS Object Factory'''
if self.__osObject is None:
from tortuga.os_utility import osUtility
self.__osObject = osUtility.getOsObjectFactory()
return self.__osObject
def __getSanApi(self):
'''Internal: Get and cache the SAN API'''
if self.__sanApi is None:
from tortuga.san import san
self.__sanApi = san.San()
return self.__sanApi
# Properties for this object
addHostApi = property(__getAddHostApi, None, None, None)
nodeApi = property(__getNodeApi, None, None, None)
osObject = property(__getOsObject, None, None, None)
sanApi = property(__getSanApi, None, None, None)
def statusMessage(self, msg):
if self._addHostSession:
AddHostManager().updateStatus(self._addHostSession, msg)
else:
# Just print out the message...this is a stop gap for resource
# adapters running outside of the addHostManager framework
sys.stdout.write(msg + '\n')
sys.stdout.flush()
def getOptions(self, dbSoftwareProfile, dbHardwareProfile): \
# pylint: disable=unused-argument
return {}
def instanceCacheWrite(self, cfg):
# Write the instance cache back to disk
self.getLogger().debug('instanceCacheWrite()')
with open(self.cacheCfgFilePath, 'w') as fp:
cfg.write(fp)
def instanceCacheRefresh(self):
self.getLogger().debug('instanceCacheRefresh()')
cfg = configparser.ConfigParser()
cfg.read(self.cacheCfgFilePath)
return cfg
def instanceCacheSet(self, name, metadata=None):
self.getLogger().debug('instanceCacheSet(node=[%s], metadata=[%s])' %
(name, metadata))
cfg = self.instanceCacheRefresh()
if not cfg.has_section(name):
cfg.add_section(name)
# Write metadata to node section
if metadata:
for key, value in metadata.items():
cfg.set(name, key, value)
self.instanceCacheWrite(cfg)
def instanceCacheSetBulk(self, instance_ids, nodes=None):
self.getLogger().debug(
'instanceCacheSetBulk(instance_ids=[%s], nodes=[%s])' %
(' '.join(instance_ids), ' '.join(
[node.name for node in nodes or []])))
cfg = self.instanceCacheRefresh()
if not nodes:
if not cfg.has_section('unassigned'):
cfg.add_section('unassigned')
instances = set()
else:
val = cfg.get('unassigned', 'instances')
instances = set(val.split(' '))
instances |= set(instance_ids)
cfg.set('unassigned', 'instances', ' '.join(instances))
self.instanceCacheWrite(cfg)
def instanceCacheGet(self, nodeName):
self.getLogger().debug('instanceCacheGet(nodeName=[%s])' % (nodeName))
cfg = self.instanceCacheRefresh()
if not cfg.has_section(nodeName):
raise ResourceNotFound(
'No instance cache entry for [{0}]'.format(nodeName))
# Read entire section into a dict
result = {}
for key, value in cfg.items(nodeName):
result[key] = value
return result
def instanceCacheDelete(self, name):
# Clear instance from configuration
config = self.instanceCacheRefresh()
if not config.has_section(name):
self.getLogger().debug(
'Cache clear: node [{0}] not found, no action'
' taken'.format(name))
return
self.getLogger().debug('Cache clear: node [{0}]'.format(name))
config.remove_section(name)
self.instanceCacheWrite(config)
def instanceCacheUpdate(self, name, added=None, deleted=None):
"""
'added' is a list of key-value tuples to be added
'deleted' is a list of keys to be removed from the instance cache
"""
self.getLogger().debug(
'instanceCacheUpdate(): name=[{0}]'.format(name))
config = self.instanceCacheRefresh()
if not config.has_section(name):
config.add_section(name)
for key, value in added or []:
config.set(name, key, value)
for key in deleted or []:
config.remove_option(name, key)
self.instanceCacheWrite(config)
def __findNicForProvisioningNetwork(self, nics, prov_network):
"""
TODO: move this elsewhere
Raises:
NicNotFound
"""
nics = [nic for nic in nics if nic.network == prov_network]
if not nics:
raise NicNotFound(
'Unable to find NIC on provisioning network [%s]' %
(prov_network.address + '/' + prov_network.netmask))
return nics[0]
def writeLocalBootConfiguration(self, node, hardwareprofile,
softwareprofile):
"""
Raises:
NicNotFound
"""
if not hardwareprofile.nics:
# Hardware profile has no provisioning NICs defined. This
# shouldn't happen...
self.getLogger().debug(
'No provisioning nics defined in hardware profile %s' %
(hardwareprofile.name))
return
# Determine the provisioning nic for the hardware profile
hwProfileProvisioningNic = hardwareprofile.nics[0]
nic = None
if hwProfileProvisioningNic.network:
# Find the nic attached to the newly added node that is on
# the same network as the provisioning nic.
nic = self.__findNicForProvisioningNetwork(
node.nics, hwProfileProvisioningNic.network)
if not nic or not nic.mac:
self.getLogger().warning(
'MAC address not defined for nic (ip=[%s]) on node [%s]' %
(nic.ip, node.name))
return
# Set up DHCP/PXE for newly addded node
bhm = getOsObjectFactory().getOsBootHostManager()
# Write out the PXE file
bhm.writePXEFile(node,
hardwareprofile=hardwareprofile,
softwareprofile=softwareprofile,
localboot=False)
# Add a DHCP lease
bhm.addDhcpLease(node, nic)
def removeLocalBootConfiguration(self, node):
bhm = self.osObject.getOsBootHostManager()
bhm.rmPXEFile(node)
bhm.removeDhcpLease(node)
def _pre_add_host(self, name, hwprofilename, swprofilename, ip): \
# pylint: disable=unused-argument
# Perform "pre-add-host" operation
command = ('sudo %s/pre-add-host'
' --hardware-profile %s'
' --software-profile %s'
' --host-name %s' %
(self._cm.getBinDir(), hwprofilename, swprofilename, name))
if ip:
command += ' --ip %s' % (ip)
self.getLogger().debug('calling command= [%s]' % (command))
p = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
p.communicate()
p.wait()
@property
def installer_public_hostname(self):
if self.__installer_public_hostname is None:
cmd = '/opt/puppetlabs/bin/facter fqdn'
with open(os.devnull, 'w') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=devnull)
stdout, _ = p.communicate()
retval = p.wait()
if retval == 0:
self.__installer_public_hostname = stdout.decode().rstrip()
self.getLogger().debug('using installerName [%s] from Facter' %
(self.__installer_public_hostname))
else:
self.__installer_public_hostname = self._cm.getHost()
self.getLogger().debug('using installerName [%s] from system' %
(self.__installer_public_hostname))
return self.__installer_public_hostname
@property
def installer_public_ipaddress(self):
# Get installer IP
if self.__installer_public_ipaddress is None:
self.getLogger().debug('Looking up installer IP using DNS')
aiInfo = socket.getaddrinfo(self.installer_public_hostname, None,
socket.AF_INET, socket.SOCK_STREAM)
self.__installer_public_ipaddress = aiInfo[0][4][0]
return self.__installer_public_ipaddress
@property
def private_dns_zone(self):
if self.__private_dns_zone is None:
self.__private_dns_zone = \
ParameterApi().getParameter('DNSZone').getValue()
return self.__private_dns_zone
def get_node_vcpus(self, name): \
# pylint: disable=unused-argument
return 1
def get_instance_size_mapping(self, value):
"""
Helper method for matching the first field (instance size) in
the resource adapter specific CSV file
:return: instance type/size to vcpus mapping
:returntype int:
"""
fn = os.path.join(
self._cm.getKitConfigBase(),
'{0}-instance-sizes.csv'.format(self.__adaptername__))
if not os.path.exists(fn):
return 1
try:
with open(fn) as fp:
reader = csv.reader(fp)
for row in reader:
if row[0] == value:
return int(row[1])
return 1
except Exception as exc:
self.getLogger().error(
'Error processing instance type mapping'
' [{0}] (exc=[{1}]). Using default value'.format(fn, exc))
return 1
class LicenseUsageCommand(Command):
"""
Grab the usage between 2 dates
and write to CSV file.
IAM permission 'ce:GetCostAndUsage'
is required to gather the data.
"""
name = 'usage'
help = 'Create usage report'
arguments = [
Argument('--start', help='YYYY-MM-DD', required=True),
Argument('--end', help='YYYY-MM-DD', required=True),
Argument('-o',
'--output',
help='Specify output file path',
type=str,
default=None)
]
def __init__(self) -> None:
"""
Initialise client and logger.
:returns: None
"""
super(LicenseUsageCommand, self).__init__()
self._args = None
self._client = boto3.client('ce')
self._cm = ConfigManager()
def execute(self, args: argparse.Namespace) -> None:
"""
Iterate over each row and write to file.
:returns: None
"""
self._args = args
start, end = self._string_to_date()
if self._args.output:
path: str = os.path.abspath(self._args.output)
writer = PathWriter(path)
else:
writer = StreamWriter()
with writer as f:
for row in self._get_data(start, end):
f.write(json.dumps(row))
def _string_to_date(self) -> Tuple[date]:
"""
Convert the arguments to date
objects.
:returns: Tuple Date Date
"""
try:
start: date = date.fromisoformat(self._args.start)
end: date = date.fromisoformat(self._args.end)
except AttributeError: # Above method only in 3.7.
split_start: Tuple[str] = self._args.start.split('-')
split_end: Tuple[str] = self._args.end.split('-')
split_start: Tuple[int] = map(int, split_start)
split_end: Tuple[int] = map(int, split_end)
start: date = date(*split_start)
end: date = date(*split_end)
return start, end
def _get_data(self, start: date,
end: date) -> Generator[Dict[Any, Any], None, None]:
"""
Get the usage data from AWS.
:param start: Date
:param end: Date
:returns: Generator Dictionary
"""
page_token: Optional[str] = None
while True:
if page_token:
kwargs: dict = {'NextPageToken': page_token}
else:
kwargs: dict = {}
response: Dict[Any, Any] = self._client.get_cost_and_usage(
TimePeriod={
'Start': start.strftime('%Y-%m-%d'),
'End': end.strftime('%Y-%m-%d')
},
Granularity='MONTHLY',
Metrics=['UsageQuantity'],
GroupBy=[{
'Type': 'DIMENSION',
'Key': 'SERVICE',
}, {
'Type': 'DIMENSION',
'Key': 'INSTANCE_TYPE'
}],
Filter={
'And': [{
'Tags': {
'Key': 'tortuga:installer_hostname',
'Values': [self._cm.getHost()]
}
}, {
'Dimensions': {
'Key': 'USAGE_TYPE_GROUP',
'Values': ['EC2: Running Hours']
}
}]
},
**kwargs)
for result in response['ResultsByTime']:
yield result
page_token = response.get('NextPageToken', None)
if not page_token:
break
