def terminateThese( authToken, inRecs ):
logger.info( 'to terminate %d instances', len(inRecs) )
iids = [inRec['instanceId'] for inRec in inRecs]
ncs.terminateInstances( authToken, iids )
else:
iidsToTerminate.extend( badIids )
forwarders = findForwarders()
forwardersByPort = { fw['port']: fw for fw in forwarders }
for port in sorted( forwardersByPort.keys() ):
forwarder = forwardersByPort[port]
logger.debug( 'forwarding port %d to %s', forwarder['port'], forwarder['host'] )
badIids = checkForwarders( liveInstances, forwarders )
if badIids:
logger.warning( '%d not-forwarded instance(s)', len( badIids ) )
#ncs.terminateInstances( authToken, badIids )
iidsToTerminate.extend( badIids )
iidsToTerminate = set( iidsToTerminate )
terminatedIids = []
if args.terminateBad and iidsToTerminate:
logger.info( 'terminating %d bad instances', len(iidsToTerminate) )
ncs.terminateInstances( authToken, list( iidsToTerminate ) )
terminatedIids = list( iidsToTerminate )
toPurge = [inst for inst in startedInstances if inst['instanceId'] in iidsToTerminate]
purgeHostKeys( toPurge )
stillLive = [inst for inst in liveInstances if inst['instanceId'] not in terminatedIids]
logger.info( '%d instances are still live', len(stillLive) )
with open( dataDirPath + '/liveAgents.json','w' ) as outFile:
json.dump( stillLive, outFile, indent=2 )
logger.info( 'finished' )
# terminate bad instances, deauthorizing any that are also authorized
for iid in badIids:
wasSigner = (iid in savedSigners) or (iid in historicSigners)
#logger.info( 'saved signer? %s', wasSigner )
if wasSigner:
logger.info( 'will deauthorize %s', iid[0:16])
# victim is first account in savedSigners list for this instance
logger.info( 'sleeping for 90 seconds' )
time.sleep( 90 )
victimAccount = historicSigners[iid][0]
logger.info( 'deauthorizing %s account %s', iid[0:16], victimAccount )
results = ncsgeth.authorizeSigner( authorizers, configName, victimAccount, False )
logger.info( 'authorizeSigner returned: %s', results )
waitForAuth( victimAccount, False, authorizers, configName, timeLimit=15*60 )
logger.info( 'terminating %s', iid)
ncs.terminateInstances( authToken, [iid] )
terminatedIids.append( iid )
# authorize good instances that have been up for long enough
#TODO skip this if maxNewAuths <= 0
trustedThresh = 12 # hours
nNewAuths = 0
sleepAmt = 15
if badIids:
sleepAmt = 90 # longer to avoid possible trouble
authorizers = findAuthorizers( anchorInstances+liveInstances, savedSigners, badIids )
now = datetime.datetime.now( datetime.timezone.utc )
for inst in goodInstances:
# limit the loop to <= maxNewAuths auths per run
iid = inst['instanceId']
abbrevIid = iid[0:16]
rc2 = subprocess.call( [scriptDirPath()+'/plotAgentMap.py', '--dataDirPath', outDataDir],
stdout=subprocess.DEVNULL )
if rc2:
logger.warning( 'plotAgentMap exited with returnCode %d', rc2 )
# start the ssh port-forwarding
logger.info( 'would forward ports for %d instances', len(goodInstances) )
forwarders = startForwarders.startForwarders( goodInstances,
forwarderHost=forwarderHost,
portMap=portMap,
portRangeStart=portRangeStart, maxPort=portRangeStart+100,
forwardingCsvFilePath=outDataDir+'/agentForwarding.csv'
)
if len( forwarders ) < len( goodInstances ):
logger.warning( 'some instances could not be forwarded to' )
logger.debug( 'forwarders: %s', forwarders )
#TODO get iids only for successfully forwarded agents
forwardedIids = [inst['instanceId'] for inst in goodInstances ]
unusableIids = list( set(launchedIids) - set( forwardedIids) )
if unusableIids:
logger.debug( 'terminating %d unusable instances', len(unusableIids) )
ncs.terminateInstances( authToken, unusableIids )
unusableInstances = [inst for inst in launchedInstances \
if inst['instanceId'] in unusableIids]
purgeHostKeys( unusableInstances )
if launchedInstances:
print( 'when you want to terminate these instances, use %s %s "%s"'
% (sys.executable, scriptDirPath()+'/terminateAgents.py', outDataDir))
sys.exit( rc )
except KeyboardInterrupt:
logger.warning( 'an interuption occurred')